I have a use case where I need to upload csv files to workdocs. I'm using golang language and I receive the error as "The request signature we calculated does not match the signature you provided." I'm using InitiateDocumentVersionUpload with IAM user credentials. Can you please help me as what might be causing this error.
optionsWd := workdocs.Options{Credentials: credentials.NewStaticCredentialsProvider(request.AccessKeyId, request.SecretAccessKey, ""),
Region: "us-east-1"}
client := workdocs.New(optionsWd)
folderId := "e38c72c9ae6918109b573a17ece5f24e7a353374672b627b1b3b54918354cd5e"
docName := "testdoc"
docType := "text/csv"
data, err := r.S3.GetGetObject(ctx, "test-bucket", s3Path)
params := workdocs.InitiateDocumentVersionUploadInput{
ParentFolderId: &folderId,
Name: &docName,
ContentType: &docType,
}
res, err := client.InitiateDocumentVersionUpload(ctx, ¶ms)
if err != nil {
fmt.Println(err)
}
fmt.Println(res.Metadata)
resval := *res.UploadMetadata
urlVal := *resval.UploadUrl
signedHeadVal := resval.SignedHeaders
fmt.Println(urlVal)
fmt.Println(signedHeadVal)
metadata := *res.Metadata
fmt.Println(metadata)
fmt.Println(signedHeadVal)
wdclient := &http.Client{}
req, err := http.NewRequest(http.MethodPut, urlVal, strings.NewReader(data))
if err != nil {
fmt.Println(err)
}
req.Header.Set("Content-Type", "text/csv")
signer := v4.NewSigner()
credsVal := aws.Credentials{
AccessKeyID: aws.ToString(&request.AccessKeyId),
SecretAccessKey: aws.ToString(&request.SecretAccessKey),
SessionToken: "",
}
requestBodyBytes, _ := ioutil.ReadAll(req.Body)
sha := sha256.Sum256(requestBodyBytes)
payloadHash := hex.EncodeToString(sha[:])
if err != nil {
fmt.Println(err)
}
signer.SignHTTP(req.Context(), credsVal, req, payloadHash, "s3", "us-east-1", time.Now())
_, err = wdclient.Do(req)
if err != nil {
fmt.Println(err)
}
I tried the code mentioned above and unable to resolve the error. Expectation is to upload the file in workdocs.
Related
I wrote simple script to receive all data from wigle api using wigleapiv2, definitely this endpoint /api/v2/network/search. But I faced the problem, that I can receive only 1000 unique ssid's. I'm changing URL every iteration, and put in URL previous page's searchAfter. How can I fix it and receive all data from certain latitude and longitude?
Here an example of first iteration Uri (https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4)
And here an example of remaining iterations uris (https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4&searchAfter=1976621348&first=1). For every iteration I'm changing searchAfter and first.
It would be great id someone can say me where I'm doing wrong:)
I've tried to using only first or search after parameters, but it has the same result. One mark that I noticed, that when I'm using only searchAfter param I can receive only 100 unique ssids, but when I'm using both (searchAfter and first) I can receive 1000 unique ssids.
Here my main.go code
var (
wg = sync.WaitGroup{}
receiveResp = make(chan []*response.WiFiNetworkWithLocation, 100)
)
func main() {
startTime := time.Now()
viper.AddConfigPath(".")
viper.SetConfigFile("config.json")
if err := viper.ReadInConfig(); err != nil {
log.Fatal("error trying read from config: %w", err)
}
u := user.NewUser(viper.GetString("users.user.username"), viper.GetString("users.user.password"))
db, err := postgres.NewPG()
if err != nil {
log.Fatalf("Cannot create postgres connection: %v", err)
}
postgres.WG.Add(1)
go getResponse(u)
go parseResponse(db)
postgres.WG.Wait()
fmt.Printf("Execution time: %v ", time.Since(startTime))
}
func getResponse(u *user.Creds) {
url := fmt.Sprintf("%s? closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s",
viper.GetString("wigle.url"),
viper.GetString("queries.closestLat"),
viper.GetString("queries.closestLong"),
viper.GetString("queries.latrange1"),
viper.GetString("queries.latrange2"),
viper.GetString("queries.longrange1"),
viper.GetString("queries.longrange2"),
)
j := 0
i := 0
for {
i++
fmt.Println(url)
req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte("")))
if err != nil {
log.Printf("Failed wraps request: %v", err)
continue
}
req.SetBasicAuth(u.Username, u.Password)
c := http.Client{}
resp, err := c.Do(req)
if err != nil {
log.Printf("Failed send request: %v", err)
continue
}
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Failed read response body: %v", err)
continue
}
var r response.NetSearchResponse
if err := json.Unmarshal(bytes, &r); err != nil {
log.Printf("Failed unmarshal: %v", err)
continue
}
receiveResp <- r.Results
fmt.Println(r.TotalResults, r.SearchAfter)
if r.SearchAfter == "" {
postgres.WG.Done()
return
}
url = fmt.Sprintf("%s? closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s&searchAfter=%s&first=%v" ,
viper.GetString("wigle.url"),
viper.GetString("queries.closestLat"),
viper.GetString("queries.closestLong"),
viper.GetString("queries.latrange1"),
viper.GetString("queries.latrange2"),
viper.GetString("queries.longrange1"),
viper.GetString("queries.longrange2"),
r.SearchAfter,
i,
)
j++
fmt.Println(j)
}
func parseResponse(db *sql.DB) {
for {
select {
case responses := <-receiveResp:
clearResponses := make([]response.WiFiNetworkWithLocation, 0, len(responses))
for _, val := range responses {
clearResponses = append(clearResponses, *val)
}
postgres.WG.Add(1)
go postgres.SaveToDB(db, "test", clearResponses)
}
}
}
I'm getting 400 Bad Request for frappe.cloud API, when I'm trying to call it using golang code using http.NewRequest, this API is working fine when I check it using postman. following is the API
https://xxxx.frappe.cloud/api/resource/Item?fields=["name","item_name","item_group","description"]&filters=[["Item","item_group","=","xxx Product"]]
If I use the same golang code to call same API with out filters it works fine. following is the working API
https://xxxx.frappe.cloud/api/resource/Item?fields=["name","item_name","item_group","description"]
code as follows
func FetchProperties(dataChannel models.DataChannel) (map[string]interface{}, error) {
thisMap := make(map[string][]map[string]interface{})
client := &http.Client{}
req, err := http.NewRequest("GET", dataChannel.APIPath, nil)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, err
}
eds, err := GetDecryptedEDSByEDSID(dataChannel.EDSId)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", eds.DataSource.Auth.Token)
response, err := client.Do(req)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, err
}
data, err := ioutil.ReadAll(response.Body)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
if response.StatusCode == 200 {
err = json.Unmarshal(data, &thisMap)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
return thisMap["data"][0], err
} else {
return nil, &commons.RequestError{StatusCode: response.StatusCode, Err: errors.New("getting " + strconv.Itoa(response.StatusCode) + " From Data channel API")}
}
Postman has an option to convert request to programming language equivalent.
Here is a working go code for sending the request. package main
import (
"fmt"
"strings"
"net/http"
"io/ioutil"
)
func main() {
url := "https://xxx.frappe.cloud/api/resource/Item?fields=%5B%22name%22,%22item_name%22,%22item_group%22,%22description%22%5D&filters=%5B%5B%22Item%22,%22item_group%22,%22=%22,%22xxx%20Product%22%5D%5D%0A"
method := "GET"
payload := strings.NewReader(`{
"payload": {},
"url_key": "",
"req_type": ""
}`)
client := &http.Client {
}
req, err := http.NewRequest(method, url, payload)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Cookie", "full_name=foo; sid=secret_sid; system_user=yes; user_id=foobar; user_image=")
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}
This is my first experience with go, I'm coming from a python background and I'm trying to run this post api(https://reqres.in/api/users) in parallel using goroutines with the different set of post data which is variable jsonStr in this code.
Any help much appreciated for running this api in paralle with the list of data
My data
data = [{"name": "bonny gaud", "movies": ["Terminator", "Transformer"],
{"name": "Sarah palin", "movies": ["No country for old", "James Bond"] }
My Code:
package main
import (
"fmt"
"io/ioutil"
"net/http"
"time"
"bytes"
)
func main() {
start := time.Now()
url := "https://reqres.in/api/users"
fmt.Println("URL:>", url)
var jsonStr = []byte(`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
req.Header.Set("X-Custom-Header", "myvalue")
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("response Body:", string(body))
fmt.Print("Everything:", time.Since(start))
fmt.Print(string(body))
}
You can use waitGroup or errGroup. Here is the example :
package main
import (
"bytes"
"fmt"
"golang.org/x/sync/errgroup"
"io/ioutil"
"net/http"
"time"
)
func main() {
dataSets := []string{`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`,
`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`}
var eg errgroup.Group
client := http.DefaultClient
for _, jsonStr := range dataSets {
eg.Go(func() error {
return callClient(client, jsonStr)
})
}
if err := eg.Wait(); err != nil {
fmt.Printf("Encountered error: %v", err)
}
fmt.Println("Successfully finished.")
}
func callClient(client *http.Client, jsonStr string) error {
start := time.Now()
url := "https://reqres.in/api/users"
fmt.Println("URL:>", url)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(jsonStr)))
req.Header.Set("X-Custom-Header", "myvalue")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("send request: %w", err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read body: %w", err)
}
fmt.Println("response Body:", string(body))
fmt.Print("Everything:", time.Since(start))
fmt.Print(string(body))
return nil
}
I am new to working with AWS particularly s3. I am using the aws go sdk. I am trying to set bucket life cycle rules in the method below;
func SetLifecycle(svc *s3.S3, bucket , id , status, md5 string) (*s3.PutBucketLifecycleConfigurationOutput, error) {
input := &s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: []*s3.LifecycleRule{
{
ID: aws.String(id),
Status: aws.String(status),
},
},
},
}
req, resp := svc.PutBucketLifecycleConfigurationRequest(input)
req.HTTPRequest.Header.Set("Content-Md5", string(md5))
err := req.Send()
return resp, err
}
And calling the above method in a test:
func (suite *HeadSuite) TestLifecycleSet() {
assert := suite
//acl := map[string]string{"Authorization": ""}
bucket := GetBucketName()
err := CreateBucket(svc, bucket)
content := strings.NewReader("Enabled")
h := md5.New()
content.WriteTo(h)
sum := h.Sum(nil)
b := make([]byte, base64.StdEncoding.EncodedLen(len(sum)))
base64.StdEncoding.Encode(b,sum)
md5 := string(b)
_, err = SetLifecycle(svc, bucket, "rule1", "Enabled", md5)
assert.Nil(err)
}
I keep getting an error NotImplemented. Why would this be happening? I had originally not added a content-md5 header which I added after reading the putbucketlifecycle documentation. However , I still get an error.
I did not require calculation of the MD5 header. However, I noticed that I needed to set a prefix, which is specified in the documentation. In addition, what version of the SDK are you using?
Here's a working example below
input := &s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: []*s3.LifecycleRule{
{
Prefix: aws.String(prefix),
Status: aws.String(status),
ID: aws.String(id),
Expiration: &s3.LifecycleExpiration{
Days: aws.Int64(1),
},
},
},
},
}
req, resp := svc.PutBucketLifecycleConfigurationRequest(input)
if err := req.Send(); err != nil {
panic(err)
}
i am using golang for my application and in this application i tried getting attachments from Confluence, following are detail
req:="https://domain.atlassian.net/wiki/rest/api/content?expand=body.view,version&type=page&start=0&limit="
res, err := w.sendRequest(req)
if err != nil {
return nil, err
}
if strings.EqualFold(contentID, "") == false {
if len(res.Results) != 0 {
for i, _ := range res.Results {
Log.Info("files processed is:", i)
extension := filepath.Ext(res.Results[i].Title)
isExtenstionExclude := isExcludedExtenstion(sbConfig, extension)
ispathExclude := isExcludedFolder(sbConfig, res.Results[i].Links.Webui)
if sbgoclient.ExtesionMap[extension] == 0 || isExtenstionExclude == true || ispathExclude == true {
binarycount++
Log.Info("Excluded by extension" + extension + " for file" + res.Results[i].Title)
} else {
md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
md5HashInString := hex.EncodeToString(md5HashInBytes[:])
file_path := parameter[0] + "/" + md5HashInString + strings.Replace(res.Results[i].Title, " ", "", -1)
file, err := os.Create(file_path)
if err != nil {
fmt.Println(err)
panic(err)
}
url_1 := sbConfig.ConfluenceUrl + res.Results[i].Links.Download
req, err := http.NewRequest("GET", url_1, nil)
resp, _ := w.client.Do(req) // add a filter to check redirect
if err != nil {
fmt.Println(err)
panic(err)
}
// Close body on function exit
defer resp.Body.Close()
fmt.Println(resp.Status)
size, err = io.Copy(file, resp.Body)
if err != nil {
panic(err)
}
defer file.Close()
fmt.Printf("%s with %v bytes downloaded", res.Results[i].Title, size)
meta := map[string]string{
"size": strconv.FormatInt(size, 10),
}
}
}
}
} else {
if len(res.Results) != 0 {
for i, _ := range res.Results {
Log.Info("page indexing is", res.Results[i].Title, "and i value is:", i)
fmt.Println("hmtl content is", res.Results[i].Body.View.Value)
fmt.Println("page name is:", res.Results[i].Title)
md5HashInBytes := md5.Sum([]byte(res.Results[i].Title))
md5HashInString := hex.EncodeToString(md5HashInBytes[:])
file_path := parameter[0] + "/" + md5HashInString + strings.Replace(res.Results[i].Title, " ", "", -1) + ".html"
file, err := os.Create(file_path)
if err != nil {
fmt.Println(err)
panic(err)
}
defer file.Close()
html_content := "<html><body>" + res.Results[i].Body.View.Value + "</body></html>"
err = ioutil.WriteFile(file.Name(), []byte(html_content), 0777)
if err != nil {
fmt.Println("error writing into file", err)
panic(err)
}
file.Close()
}
func (w *Wiki) sendRequest(req *http.Request) (*vijay_content, error) {
var testjson vijay_content
req.Header.Add("Accept", "application/json, */*")
w.authMethod.auth(req)
resp, err := w.client.Do(req)
if err != nil {
return nil, err
}
bodyBytes, _ := ioutil.ReadAll(resp.Body)
body := string(bodyBytes)
fmt.Printf("response is %s\n", body)
err = json.Unmarshal(bodyBytes, &testjson)
if err != nil {
fmt.Println("error here is", err)
return nil, err
}
switch resp.StatusCode {
case http.StatusOK, http.StatusCreated, http.StatusPartialContent:
return &testjson, nil
case http.StatusNoContent, http.StatusResetContent:
return nil, nil
case http.StatusUnauthorized:
return nil, fmt.Errorf("Authentication failed.")
case http.StatusServiceUnavailable:
return nil, fmt.Errorf("Service is not available (%s).", resp.Status)
case http.StatusInternalServerError:
return nil, fmt.Errorf("Internal server error: %s", resp.Status)
}
return nil, fmt.Errorf("Unknown response status %s", resp.Status)
}
and here in this confluence domain actually i have more than 1000 documents but i am able to download only around 80 to 90, i don't know whats happening here please suggest any changes to be done
and following is the struct used to get values from response json
type Links struct {
Download string `json:"download,omitempty"`
Self string `json:"self,omitempty"`
Webui string `json:"webui,omitempty"`
}
type View_struct struct {
Value string `json:",innerxml"`
}
type Body_struct struct {
View View_struct `json:"view,omitempty"`
}
type Vijay_Results struct {
ID string `json:"id,omitempty"`
Links Links `json:"_links,omitempty"`
Title string `json:"title,omitempty"`
Body Body_struct `json:"body,omitempty"`
}
type vijay_content struct {
Results []Vijay_Results `json:"results,omitempty"`
Start int `json:"start,omitempty"`
Limit int `json:"limit,omitempty"`
Size int `json:"size,omitempty"`
}
The API paginates the results. You should fetch the whole list in multiple requests by specifying start and limit.
E.g. request the list of first 30 documents with start=0&limit=30, then the next 30 with start=30&limit=30, and so on, until you get a response with empty list.
You can read more details in the docs on pagination.