Golang HTTP Request returning 200 response but empty body - api

I'm doing a post request and I get a 200 OK response. I also receive the headers. However, the body keeps coming back empty.
There should be a body, when I run it in postman the body shows up. What am I missing here?
func AddHealthCheck(baseURL string, payload HealthCheck, platform string, hostname string) (string, error) {
url := fmt.Sprintf(baseURL+"add-healthcheck/%s/%s", platform, hostname)
//convert go struct to json
jsonPayload, err := json.Marshal(payload)
if err != nil {
log.Error("[ADD HEALTH CHECK] Could not convert go struct to json : ", err)
return "", err
}
// Create client & set timeout
client := &http.Client{}
client.Timeout = time.Second * 15
// Create request
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonPayload))
if err != nil {
log.Error("[ADD HEALTH CHECK] Could not create request : ", err)
return "", err
}
req.Header.Set("Content-Type", "application/json")
// Fetch Request
resp, err := client.Do(req)
if err != nil {
log.Error("[ADD HEALTH CHECK] Could not fetch request : ", err)
return "", err
}
defer resp.Body.Close()
// Read Response Body
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Error("[HEALTH CHECK] Could not read response body : ", err)
return "", err
}
fmt.Println("response Status : ", resp.Status)
fmt.Println("response Headers : ", resp.Header)
fmt.Println("response Body : ", string(respBody))
return string(respBody), nil
}

I have confirmed locally that your code, as shown, should work.
Here is the code I used:
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"time"
)
func main() {
http.HandleFunc("/", handler)
go func(){
http.ListenAndServe(":8080", nil)
}()
AddHealthCheck()
}
func handler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Hi there")
}
func panicError(err error) {
if err != nil {
panic(err)
}
}
func AddHealthCheck() (string, error) {
//convert go struct to json
payload := "bob"
jsonPayload, err := json.Marshal(payload)
panicError(err)
// Create client & set timeout
client := &http.Client{}
client.Timeout = time.Second * 15
// Create request
req, err := http.NewRequest("POST", "http://localhost:8080", bytes.NewBuffer(jsonPayload))
panicError(err)
req.Header.Set("Content-Type", "application/json")
// Fetch Request
resp, err := client.Do(req)
panicError(err)
defer resp.Body.Close()
// Read Response Body
respBody, err := ioutil.ReadAll(resp.Body)
panicError(err)
fmt.Println("response Status : ", resp.Status)
fmt.Println("response Headers : ", resp.Header)
fmt.Println("response Body : ", string(respBody))
return string(respBody), nil
}
The code above is just a slightly stripped down version of your code, and it outputs the body of the response. (Note that I provide a server here to receive the post request and return a response)
The server is simply not sending you a body. You can confirm this with something like wireshark.
If you are getting a body back using postman, you must be sending a different request in postman than in go. It can sometimes be tough to see what is the difference, as both go and postman can sometimes add headers behind the scenes that you don't see. Again, something like wireshark can help here.
Or if you have access to the server, you can add logs there.

Related

How i can fetch all data by latitude and longitude from wigle api

I wrote simple script to receive all data from wigle api using wigleapiv2, definitely this endpoint /api/v2/network/search. But I faced the problem, that I can receive only 1000 unique ssid's. I'm changing URL every iteration, and put in URL previous page's searchAfter. How can I fix it and receive all data from certain latitude and longitude?
Here an example of first iteration Uri (https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4)
And here an example of remaining iterations uris (https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4&searchAfter=1976621348&first=1). For every iteration I'm changing searchAfter and first.
It would be great id someone can say me where I'm doing wrong:)
I've tried to using only first or search after parameters, but it has the same result. One mark that I noticed, that when I'm using only searchAfter param I can receive only 100 unique ssids, but when I'm using both (searchAfter and first) I can receive 1000 unique ssids.
Here my main.go code
var (
wg = sync.WaitGroup{}
receiveResp = make(chan []*response.WiFiNetworkWithLocation, 100)
)
func main() {
startTime := time.Now()
viper.AddConfigPath(".")
viper.SetConfigFile("config.json")
if err := viper.ReadInConfig(); err != nil {
log.Fatal("error trying read from config: %w", err)
}
u := user.NewUser(viper.GetString("users.user.username"), viper.GetString("users.user.password"))
db, err := postgres.NewPG()
if err != nil {
log.Fatalf("Cannot create postgres connection: %v", err)
}
postgres.WG.Add(1)
go getResponse(u)
go parseResponse(db)
postgres.WG.Wait()
fmt.Printf("Execution time: %v ", time.Since(startTime))
}
func getResponse(u *user.Creds) {
url := fmt.Sprintf("%s? closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s",
viper.GetString("wigle.url"),
viper.GetString("queries.closestLat"),
viper.GetString("queries.closestLong"),
viper.GetString("queries.latrange1"),
viper.GetString("queries.latrange2"),
viper.GetString("queries.longrange1"),
viper.GetString("queries.longrange2"),
)
j := 0
i := 0
for {
i++
fmt.Println(url)
req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte("")))
if err != nil {
log.Printf("Failed wraps request: %v", err)
continue
}
req.SetBasicAuth(u.Username, u.Password)
c := http.Client{}
resp, err := c.Do(req)
if err != nil {
log.Printf("Failed send request: %v", err)
continue
}
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Failed read response body: %v", err)
continue
}
var r response.NetSearchResponse
if err := json.Unmarshal(bytes, &r); err != nil {
log.Printf("Failed unmarshal: %v", err)
continue
}
receiveResp <- r.Results
fmt.Println(r.TotalResults, r.SearchAfter)
if r.SearchAfter == "" {
postgres.WG.Done()
return
}
url = fmt.Sprintf("%s? closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s&searchAfter=%s&first=%v" ,
viper.GetString("wigle.url"),
viper.GetString("queries.closestLat"),
viper.GetString("queries.closestLong"),
viper.GetString("queries.latrange1"),
viper.GetString("queries.latrange2"),
viper.GetString("queries.longrange1"),
viper.GetString("queries.longrange2"),
r.SearchAfter,
i,
)
j++
fmt.Println(j)
}
func parseResponse(db *sql.DB) {
for {
select {
case responses := <-receiveResp:
clearResponses := make([]response.WiFiNetworkWithLocation, 0, len(responses))
for _, val := range responses {
clearResponses = append(clearResponses, *val)
}
postgres.WG.Add(1)
go postgres.SaveToDB(db, "test", clearResponses)
}
}
}

Go how to use cookiejar for multiple requests?

I'm trying to make a cli for a site that has csrf, needing the csrf token to be sent on headers and on a form.
I can't seem to understand net/http.Client or net/http/cookieJar
This its even good practice? There's a better way of doing csrf login on Go ?
Thx in advance ^v^
This its my code:
package main
import (
"fmt"
"log"
"net/http"
"net/http/cookiejar"
"net/url"
"strings"
"time"
)
var (
httpClient = &http.Client{}
)
func main() {
jar, err := cookiejar.New(nil)
if err != nil {
log.Fatal(err)
}
httpClient = &http.Client{
Timeout: 30 * time.Second,
Jar: jar,
}
requestURL := "https://example.com/"
res, err := httpClient.Get(requestURL)
if err != nil {
log.Fatal(err)
}
log.Println(res.Cookies())
// stdout: cookie as expected
u := &url.URL{}
u.Parse(requestURL)
log.Println(httpClient.Jar.Cookies(u))
// stdout: []
form := make(url.Values)
/* ... */
req, err := http.NewRequest(http.MethodPost, requestURL, strings.NewReader(form.Encode()))
if err != nil {
fmt.Printf("client: could not create request: %s\n", err)
}
res, err = httpClient.Do(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(req)
// stdout: cookie as expected
}

400 Bad Request for frappe.cloud API

I'm getting 400 Bad Request for frappe.cloud API, when I'm trying to call it using golang code using http.NewRequest, this API is working fine when I check it using postman. following is the API
https://xxxx.frappe.cloud/api/resource/Item?fields=["name","item_name","item_group","description"]&filters=[["Item","item_group","=","xxx Product"]]
If I use the same golang code to call same API with out filters it works fine. following is the working API
https://xxxx.frappe.cloud/api/resource/Item?fields=["name","item_name","item_group","description"]
code as follows
func FetchProperties(dataChannel models.DataChannel) (map[string]interface{}, error) {
thisMap := make(map[string][]map[string]interface{})
client := &http.Client{}
req, err := http.NewRequest("GET", dataChannel.APIPath, nil)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, err
}
eds, err := GetDecryptedEDSByEDSID(dataChannel.EDSId)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", eds.DataSource.Auth.Token)
response, err := client.Do(req)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, err
}
data, err := ioutil.ReadAll(response.Body)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
if response.StatusCode == 200 {
err = json.Unmarshal(data, &thisMap)
if err != nil {
commons.ErrorLogger.Println(err.Error())
return nil, &commons.RequestError{StatusCode: 400, Err: err}
}
return thisMap["data"][0], err
} else {
return nil, &commons.RequestError{StatusCode: response.StatusCode, Err: errors.New("getting " + strconv.Itoa(response.StatusCode) + " From Data channel API")}
}
Postman has an option to convert request to programming language equivalent.
Here is a working go code for sending the request. package main
import (
"fmt"
"strings"
"net/http"
"io/ioutil"
)
func main() {
url := "https://xxx.frappe.cloud/api/resource/Item?fields=%5B%22name%22,%22item_name%22,%22item_group%22,%22description%22%5D&filters=%5B%5B%22Item%22,%22item_group%22,%22=%22,%22xxx%20Product%22%5D%5D%0A"
method := "GET"
payload := strings.NewReader(`{
"payload": {},
"url_key": "",
"req_type": ""
}`)
client := &http.Client {
}
req, err := http.NewRequest(method, url, payload)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Cookie", "full_name=foo; sid=secret_sid; system_user=yes; user_id=foobar; user_image=")
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}

How to write a response for kubernetes admission controller

I am trying to write a simple admission controller for pod naming (validation) but for some reason I am generating a wrong response.
Here is my code:
package main
import (
"fmt"
"encoding/json"
"io/ioutil"
"net/http"
"github.com/golang/glog"
// for Kubernetes
"k8s.io/api/admission/v1beta1"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"regexp"
)
type myValidServerhandler struct {
}
// this is the handler fuction from the HTTP server
func (gs *myValidServerhandler) serve(w http.ResponseWriter, r *http.Request) {
var Body []byte
if r.Body != nil {
if data , err := ioutil.ReadAll(r.Body); err == nil {
Body = data
}
}
if len(Body) == 0 {
glog.Error("Unable to retrive Body from API")
http.Error(w,"Empty Body", http.StatusBadRequest)
return
}
glog.Info("Received Request")
// this is where I make sure the request is for the validation prefix
if r.URL.Path != "/validate" {
glog.Error("Not a Validataion String")
http.Error(w,"Not a Validataion String", http.StatusBadRequest)
return
}
// in this part the function takes the AdmissionReivew and make sure in is in the right
// JSON format
arRequest := &v1beta1.AdmissionReview{}
if err := json.Unmarshal(Body, arRequest); err != nil {
glog.Error("incorrect Body")
http.Error(w, "incorrect Body", http.StatusBadRequest)
return
}
raw := arRequest.Request.Object.Raw
pod := v1.Pod{}
if err := json.Unmarshal(raw, &pod); err != nil {
glog.Error("Error Deserializing Pod")
return
}
// this is where I make sure the pod name contains the kuku string
podnamingReg := regexp.MustCompile(`kuku`)
if podnamingReg.MatchString(string(pod.Name)) {
return
} else {
glog.Error("the pod does not contain \"kuku\"")
http.Error(w, "the pod does not contain \"kuku\"", http.StatusBadRequest)
return
}
// I think the main problem is with this part of the code because the
// error from the events I getting in the Kubernetes namespace is that
// I am sending 200 without a body response
arResponse := v1beta1.AdmissionReview{
Response: &v1beta1.AdmissionResponse{
Result: &metav1.Status{},
Allowed: true,
},
}
// generating the JSON response after the validation
resp, err := json.Marshal(arResponse)
if err != nil {
glog.Error("Can't encode response:", err)
http.Error(w, fmt.Sprintf("couldn't encode response: %v", err), http.StatusInternalServerError)
}
glog.Infof("Ready to write response ...")
if _, err := w.Write(resp); err != nil {
glog.Error("Can't write response", err)
http.Error(w, fmt.Sprintf("cloud not write response: %v", err), http.StatusInternalServerError)
}
}
The code is working as expected except for a positive output (where the pod name meets the criteria)
there is another file with a main just grabbing the TLS files and starting the HTTP service.
so after a few digging I found what was wrong with my code
first this part
if podnamingReg.MatchString(string(pod.Name)) {
return
} else {
glog.Error("the pod does not contain \"kuku\"")
http.Error(w, "the pod does not contain \"kuku\"", http.StatusBadRequest)
return
}
by writing "return" twice I discarded the rest of the code and more so I haven't attached the request UID to the response UID and because I am using the v1 and not the v1beta1 I needed to adding the APIVersion in the response
so the rest of the code looks like :
arResponse := v1beta1.AdmissionReview{
Response: &v1beta1.AdmissionResponse{
Result: &metav1.Status{},
Allowed: false,
},
}
podnamingReg := regexp.MustCompile(`kuku`)
if podnamingReg.MatchString(string(pod.Name)) {
fmt.Printf("the pod %s is up to the name standard", pod.Name)
arResponse.Response.Allowed = true
}
arResponse.APIVersion = "admission.k8s.io/v1"
arResponse.Kind = arRequest.Kind
arResponse.Response.UID = arRequest.Request.UID
so I needed to add the 2 parts and make sure that in case the pod name is not up to standard then I need to return the right response

goroutines run post api in parallel with list of data

This is my first experience with go, I'm coming from a python background and I'm trying to run this post api(https://reqres.in/api/users) in parallel using goroutines with the different set of post data which is variable jsonStr in this code.
Any help much appreciated for running this api in paralle with the list of data
My data
data = [{"name": "bonny gaud", "movies": ["Terminator", "Transformer"],
{"name": "Sarah palin", "movies": ["No country for old", "James Bond"] }
My Code:
package main
import (
"fmt"
"io/ioutil"
"net/http"
"time"
"bytes"
)
func main() {
start := time.Now()
url := "https://reqres.in/api/users"
fmt.Println("URL:>", url)
var jsonStr = []byte(`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
req.Header.Set("X-Custom-Header", "myvalue")
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("response Body:", string(body))
fmt.Print("Everything:", time.Since(start))
fmt.Print(string(body))
}
You can use waitGroup or errGroup. Here is the example :
package main
import (
"bytes"
"fmt"
"golang.org/x/sync/errgroup"
"io/ioutil"
"net/http"
"time"
)
func main() {
dataSets := []string{`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`,
`{"name": "paul rudd", "movies": ["I Love You Man", "Role Models"] }`}
var eg errgroup.Group
client := http.DefaultClient
for _, jsonStr := range dataSets {
eg.Go(func() error {
return callClient(client, jsonStr)
})
}
if err := eg.Wait(); err != nil {
fmt.Printf("Encountered error: %v", err)
}
fmt.Println("Successfully finished.")
}
func callClient(client *http.Client, jsonStr string) error {
start := time.Now()
url := "https://reqres.in/api/users"
fmt.Println("URL:>", url)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(jsonStr)))
req.Header.Set("X-Custom-Header", "myvalue")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("send request: %w", err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("read body: %w", err)
}
fmt.Println("response Body:", string(body))
fmt.Print("Everything:", time.Since(start))
fmt.Print(string(body))
return nil
}