I'm currently developing a custom scanner in go for OpenVAS. Problem is that the handshake with my custom server fails.
I traced the problem down to error -73: GNUTLS_E_ASN1_TAG_ERROR from gnutls_handshake, but I can't find any resources on that problem. I read something about the certificates being incorrect then but I can't do anything other than regenerating the OpenVAS certificates. The tls functionality in the go server just uses a simple ListenAndServeTLS and gets the server cert and key.
edit:
So this is the relevant network part on the custom scanner:
var (
port = ":1234"
cert = "/usr/local/var/lib/openvas/CA/servercert.pem"
ca = "/usr/local/var/lib/openvas/CA/cacert.pem"
key = "/usr/local/var/lib/openvas/private/CA/serverkey.pem"
)
func start_server() {
ca_file, err := ioutil.ReadFile(ca)
if err != nil {
panic(err)
}
blocks, _ := pem.Decode( ca_file )
ca, err := x509.ParseCertificate(blocks.Bytes)
if err != nil {
panic(err)
}
priv_file, _ := ioutil.ReadFile(key)
blocks2, _ := pem.Decode( priv_file )
priv, err := x509.ParsePKCS1PrivateKey(blocks2.Bytes)
if err != nil {
panic(err)
}
pool := x509.NewCertPool()
pool.AddCert(ca)
cert := tls.Certificate{
Certificate: [][]byte{ca_file},
PrivateKey: priv,
}
config := tls.Config{
ClientAuth: tls.RequireAndVerifyClientCert,
Certificates: []tls.Certificate{cert},
ClientCAs: pool,
}
config.Rand = rand.Reader
service := "0.0.0.0" + port
listener, err := tls.Listen("tcp", service, &config)
if err != nil {
log.Fatalf("server: listen: %s", err)
}
log.Print("server: listening")
for {
conn, err := listener.Accept()
if err != nil {
log.Printf("server: accept: %s", err)
break
}
defer conn.Close()
log.Printf("server: accepted from %s", conn.RemoteAddr())
go handle(conn)
}
}
func handle(conn net.Conn) {
str := "Hello"
defer conn.Close()
buf := make([]byte, 512)
for {
log.Print("server: conn: waiting")
conn.Write( ([]byte)(str) )
n, err := conn.Read(buf)
if err != nil {
if err != nil {
fmt.Println (err)
}
break
}
tlscon, ok := conn.(*tls.Conn)
if ok {
state := tlscon.ConnectionState()
sub := state.PeerCertificates[0].Subject
log.Println(sub)
}
log.Printf("server: conn: echo %q\n", string(buf[:n]))
n, err = conn.Write(buf[:n])
n, err = conn.Write(buf[:n])
log.Printf("server: conn: wrote %d bytes", n)
if err != nil {
log.Printf("server: write: %s", err)
break
}
}
log.Println("server: conn: closed")
}
func main() {
start_server()
}
It's taken from an example but it didn't work properly at first ( there was no decode before parsecertificates). Maybe the certificate is mal-formatted now because of that? Before adding the two decodes I had a similar error about the asn1 tags not matching. So also an asn1 error. I thought of generating my own certificate but I don't know if this will not break OpenVAS for the other scanners. I had the same results when just using listenandservetls from go. The error is definitely produced in gnutls_handshake. It's frustrating that I only get an error code from that.
I want to create a tls client using the net/http in GO how can I create it given the ca certificates?
package main
import (
"crypto/tls"
"crypto/x509"
"flag"
"io/ioutil"
"log"
"net/http"
)
var (
certFile = flag.String("cert", "someCertFile", "A PEM eoncoded certificate file.")
keyFile = flag.String("key", "someKeyFile", "A PEM encoded private key file.")
caFile = flag.String("CA", "someCertCAFile", "A PEM eoncoded CA's certificate file.")
)
func main() {
flag.Parse()
// Load client cert
cert, err := tls.LoadX509KeyPair(*certFile, *keyFile)
if err != nil {
log.Fatal(err)
}
// Load CA cert
caCert, err := ioutil.ReadFile(*caFile)
if err != nil {
log.Fatal(err)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
// Setup HTTPS client
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
}
tlsConfig.BuildNameToCertificate()
transport := &http.Transport{TLSClientConfig: tlsConfig}
client := &http.Client{Transport: transport}
// Do GET something
resp, err := client.Get("https://localdev.local:8443")
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
// Dump response
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
log.Println(string(data))
}
Mostly borrowed from this gist. And here is a great article to work with TLS in Go: https://ericchiang.github.io/tls/go/https/2015/06/21/go-tls.html
I'm using JWT. This is how I create the token.
func createToken(user User) (string, error) {
token := jwt.New(jwt.GetSigningMethod("RS256"))
token.Claims["Name"] = user.Name
token.Claims["Email"] = user.Email
//token.Claims["ExpDate"] = time.Now().Add(time.Hour * 1).Unix()
tokenString, err := token.SignedString([]byte(config.PrivateKey))
if err != nil {
return "", err
}
return tokenString, nil
}
This is how I verify the token.
token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
return []byte(config.PublicKey), nil
})
I generated my public and private keys with this python code
from Crypto.PublicKey import RSA
private = RSA.generate(1024)
public = private.publickey()
priv = private.exportKey()
pub = public.exportKey()
I get crypto/rsa: verification error. When I print the parsed token everything looks fine except token.Valid which is false and token.Signature is empty.
type Token struct {
Raw string // The raw token. Populated when you Parse a token
Method SigningMethod // The signing method used or to be used
Header map[string]interface{} // The first segment of the token
Claims map[string]interface{} // The second segment of the token
Signature string // The third segment of the token. Populated when you Parse a token
Valid bool // Is the token valid? Populated when you Parse/Verify a token
}
tokenString--> eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJFbWFpbCI6InRlc3RAaG90bWFpbC5jb20iLCJOYW1lIjoidGVzdE5hbWUifQ.fgd1h4LB1zzAiPFLKMOJrQu12rTLeXBDKHdnqiNc04NRn-1v7cHEQpDNawvScMIGrcQLbZo6WrldZQT9ImYWpUyy3CcD2uMO95I5PN6aXOSPb26nNGQpmIi1HNZrq5359hKZ6BWEJnW9iTg7RgmMvZGmIqlGLsOY2a6UiiwBsI0
token.Raw--> eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJFbWFpbCI6InRlc3RAaG90bWFpbC5jb20iLCJOYW1lIjoidGVzdE5hbWUifQ.fgd1h4LB1zzAiPFLKMOJrQu12rTLeXBDKHdnqiNc04NRn-1v7cHEQpDNawvScMIGrcQLbZo6WrldZQT9ImYWpUyy3CcD2uMO95I5PN6aXOSPb26nNGQpmIi1HNZrq5359hKZ6BWEJnW9iTg7RgmMvZGmIqlGLsOY2a6UiiwBsI0
token.Header--> map[alg:RS256 typ:JWT]
token.Claims--> map[Email:test#hotmail.com Name:testName]
token.Signature-->
token.Valid--> false
PS: I don't have any SSL-Certificates.
This worked for me. I generated the private.pem and public.pem using these commands.
openssl genrsa -des3 -out private.pem 1024
openssl rsa -in private.pem -outform PEM -pubout -out public.pem
Code:
package main
import (
"fmt"
"github.com/dgrijalva/jwt-go"
"io/ioutil"
"log"
"time"
)
//Claims can have user id.. etc for Identification purpose
type AppClaims struct {
UserId string `json:"userId"`
jwt.StandardClaims
}
var (
privateKey []byte
publicKey []byte
err error
)
const (
longForm = "Jan 2, 2006 at 3:04pm (MST)"
)
func errLog(err error) {
if err != nil {
log.Fatal("Error:", err.Error())
}
}
func init() {
privateKey, err = ioutil.ReadFile("../private.pem")
errLog(err)
publicKey, err = ioutil.ReadFile("../public.pem")
errLog(err)
}
func jwtTokenGen() (interface{}, error) {
privateRSA, err := jwt.ParseRSAPrivateKeyFromPEM(privateKey)
if err != nil {
return nil, err
}
claims := AppClaims{
"RAJINIS*",
jwt.StandardClaims{
ExpiresAt: time.Now().Add(time.Minute * 15).Unix(),
Issuer: "test",
},
}
token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
ss, err := token.SignedString(privateRSA)
return ss, err
}
func jwtTokenRead(inToken interface{}) (interface{}, error) {
publicRSA, err := jwt.ParseRSAPublicKeyFromPEM(publicKey)
if err != nil {
return nil, err
}
token, err := jwt.Parse(inToken.(string), func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodRSA); !ok {
return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
}
return publicRSA, err
})
if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
return claims, nil
} else {
return nil, err
}
}
func getTokenRemainingValidity(timestamp interface{}) int {
expireOffset := 0
if validity, ok := timestamp.(float64); ok {
tm := time.Unix(int64(validity), 0)
remainder := tm.Sub(time.Now())
if remainder > 0 {
fmt.Println(remainder)
return int(remainder.Seconds()) + expireOffset
}
}
return expireOffset
}
func main() {
signedString, err := jwtTokenGen()
fmt.Println(signedString, err)
claims, err := jwtTokenRead(signedString)
if err != nil {
errLog(err)
}
claimValue := claims.(jwt.MapClaims)
fmt.Println(claimValue["iss"], claimValue["exp"], claimValue["userId"])
// t, _ := time.Parse(longForm, string(claimValue["exp"].(float64)))
fmt.Println(getTokenRemainingValidity(claimValue["exp"]))
}
I'm trying to connect to an amazon AWS linux server with a key using the [ssh][1] package of Go programming language. However the package documentation is a bit cryptic/confusing. Does anyone know how to connect through ssh using a key or at least if it's possible ? What bothers me is that in the [Dial][3] example it says
// An SSH client is represented with a ClientConn. Currently only
// the "password" authentication method is supported.
I basically want to mimic the ssh -i x.pem root#server.com behavior and execute a command inside the server ( e.g. whoami )
You need to use ssh.PublicKeys to turn a list of ssh.Signers into an ssh.AuthMethod. You can use ssh.ParsePrivateKey to get a Signer from the pem bytes, or if you need to use an rsa, dsa or ecdsa private key, you can give those to ssh.NewSignerFromKey.
Here's an example fleshed out a bit with Agent support too (since using an agent is usually the next step after simply using a key file).
sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
if err != nil {
log.Fatal(err)
}
agent := agent.NewClient(sock)
signers, err := agent.Signers()
if err != nil {
log.Fatal(err)
}
// or get the signer from your private key file directly
// signer, err := ssh.ParsePrivateKey(pemBytes)
// if err != nil {
// log.Fatal(err)
// }
auths := []ssh.AuthMethod{ssh.PublicKeys(signers...)}
cfg := &ssh.ClientConfig{
User: "username",
Auth: auths,
}
cfg.SetDefaults()
client, err := ssh.Dial("tcp", "aws-hostname:22", cfg)
if err != nil {
log.Fatal(err)
}
session, err = client.NewSession()
if err != nil {
log.Fatal(err)
}
log.Println("we have a session!")
...
Here is an example to run ls remotely using your "plain private key file".
pemBytes, err := ioutil.ReadFile("/location/to/YOUR.pem")
if err != nil {
log.Fatal(err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
log.Fatalf("parse key failed:%v", err)
}
config := &ssh.ClientConfig{
User: "ubuntu",
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
conn, err := ssh.Dial("tcp", "yourhost.com:22", config)
if err != nil {
log.Fatalf("dial failed:%v", err)
}
defer conn.Close()
session, err := conn.NewSession()
if err != nil {
log.Fatalf("session failed:%v", err)
}
defer session.Close()
var stdoutBuf bytes.Buffer
session.Stdout = &stdoutBuf
err = session.Run("ls -l")
if err != nil {
log.Fatalf("Run failed:%v", err)
}
log.Printf(">%s", stdoutBuf)
I'm trying to upload images from my computer to a website using go. Usually, I use a bash script that sends a file and a key to the server:
curl -F "image"=#"IMAGEFILE" -F "key"="KEY" URL
it works fine, but I'm trying to convert this request into my golang program.
http://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
I tried this link and many others, but, for each code that I try, the response from the server is "no image sent", and I've no idea why. If someone knows what's happening with the example above.
Here's some sample code.
In short, you'll need to use the mime/multipart package to build the form.
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os"
"strings"
)
func main() {
var client *http.Client
var remoteURL string
{
//setup a mocked http client.
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, err := httputil.DumpRequest(r, true)
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
}))
defer ts.Close()
client = ts.Client()
remoteURL = ts.URL
}
//prepare the reader instances to encode
values := map[string]io.Reader{
"file": mustOpen("main.go"), // lets assume its this file
"other": strings.NewReader("hello world!"),
}
err := Upload(client, remoteURL, values)
if err != nil {
panic(err)
}
}
func Upload(client *http.Client, url string, values map[string]io.Reader) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
for key, r := range values {
var fw io.Writer
if x, ok := r.(io.Closer); ok {
defer x.Close()
}
// Add an image file
if x, ok := r.(*os.File); ok {
if fw, err = w.CreateFormFile(key, x.Name()); err != nil {
return
}
} else {
// Add other fields
if fw, err = w.CreateFormField(key); err != nil {
return
}
}
if _, err = io.Copy(fw, r); err != nil {
return err
}
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
func mustOpen(f string) *os.File {
r, err := os.Open(f)
if err != nil {
panic(err)
}
return r
}
Here's a function I've used that uses io.Pipe() to avoid reading in the entire file to memory or needing to manage any buffers. It handles only a single file, but could easily be extended to handle more by adding more parts within the goroutine. The happy path works well. The error paths have not hand much testing.
import (
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
)
func UploadMultipartFile(client *http.Client, uri, key, path string) (*http.Response, error) {
body, writer := io.Pipe()
req, err := http.NewRequest(http.MethodPost, uri, body)
if err != nil {
return nil, err
}
mwriter := multipart.NewWriter(writer)
req.Header.Add("Content-Type", mwriter.FormDataContentType())
errchan := make(chan error)
go func() {
defer close(errchan)
defer writer.Close()
defer mwriter.Close()
w, err := mwriter.CreateFormFile(key, path)
if err != nil {
errchan <- err
return
}
in, err := os.Open(path)
if err != nil {
errchan <- err
return
}
defer in.Close()
if written, err := io.Copy(w, in); err != nil {
errchan <- fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err)
return
}
if err := mwriter.Close(); err != nil {
errchan <- err
return
}
}()
resp, err := client.Do(req)
merr := <-errchan
if err != nil || merr != nil {
return resp, fmt.Errorf("http error: %v, multipart error: %v", err, merr)
}
return resp, nil
}
After having to decode the accepted answer for this question for use in my unit testing I finally ended up with the follow refactored code:
func createMultipartFormData(t *testing.T, fieldName, fileName string) (bytes.Buffer, *multipart.Writer) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
file := mustOpen(fileName)
if fw, err = w.CreateFormFile(fieldName, file.Name()); err != nil {
t.Errorf("Error creating writer: %v", err)
}
if _, err = io.Copy(fw, file); err != nil {
t.Errorf("Error with io.Copy: %v", err)
}
w.Close()
return b, w
}
func mustOpen(f string) *os.File {
r, err := os.Open(f)
if err != nil {
pwd, _ := os.Getwd()
fmt.Println("PWD: ", pwd)
panic(err)
}
return r
}
Now it should be pretty easy to use:
b, w := createMultipartFormData(t, "image","../luke.png")
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
Here is an option that works for files or strings:
package main
import (
"bytes"
"io"
"mime/multipart"
"os"
"strings"
)
func createForm(form map[string]string) (string, io.Reader, error) {
body := new(bytes.Buffer)
mp := multipart.NewWriter(body)
defer mp.Close()
for key, val := range form {
if strings.HasPrefix(val, "#") {
val = val[1:]
file, err := os.Open(val)
if err != nil { return "", nil, err }
defer file.Close()
part, err := mp.CreateFormFile(key, val)
if err != nil { return "", nil, err }
io.Copy(part, file)
} else {
mp.WriteField(key, val)
}
}
return mp.FormDataContentType(), body, nil
}
Example:
package main
import "net/http"
func main() {
form := map[string]string{"image": "#IMAGEFILE", "key": "KEY"}
ct, body, err := createForm(form)
if err != nil {
panic(err)
}
http.Post("https://stackoverflow.com", ct, body)
}
https://golang.org/pkg/mime/multipart#Writer.WriteField
Send file from one service to another:
func UploadFile(network, uri string, f multipart.File, h *multipart.FileHeader) error {
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, err := writer.CreateFormFile("file", h.Filename)
if err != nil {
log.Println(err)
return err
}
b, err := ioutil.ReadAll(f)
if err != nil {
log.Println(err)
return err
}
part.Write(b)
writer.Close()
req, _ := http.NewRequest("POST", uri, buf)
req.Header.Add("Content-Type", writer.FormDataContentType())
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
b, _ = ioutil.ReadAll(resp.Body)
if resp.StatusCode >= 400 {
return errors.New(string(b))
}
return nil
}
To extend on #attila-o answer, here is the code I went with to perform a POST HTTP req in Go with:
1 file
configurable file name (f.Name() didn't work)
extra form fields.
Curl representation:
curl -X POST \
http://localhost:9091/storage/add \
-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
-F owner=0xc916Cfe5c83dD4FC3c3B0Bf2ec2d4e401782875e \
-F password=$PWD \
-F file=#./internal/file_example_JPG_500kB.jpg
Go way:
client := &http.Client{
Timeout: time.Second * 10,
}
req, err := createStoragePostReq(cfg)
res, err := executeStoragePostReq(client, req)
func createStoragePostReq(cfg Config) (*http.Request, error) {
extraFields := map[string]string{
"owner": "0xc916cfe5c83dd4fc3c3b0bf2ec2d4e401782875e",
"password": "pwd",
}
url := fmt.Sprintf("http://localhost:%d%s", cfg.HttpServerConfig().Port(), lethstorage.AddRoute)
b, w, err := createMultipartFormData("file","./internal/file_example_JPG_500kB.jpg", "file_example_JPG_500kB.jpg", extraFields)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", w.FormDataContentType())
return req, nil
}
func executeStoragePostReq(client *http.Client, req *http.Request) (lethstorage.AddRes, error) {
var addRes lethstorage.AddRes
res, err := client.Do(req)
if err != nil {
return addRes, err
}
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return addRes, err
}
err = json.Unmarshal(data, &addRes)
if err != nil {
return addRes, err
}
return addRes, nil
}
func createMultipartFormData(fileFieldName, filePath string, fileName string, extraFormFields map[string]string) (b bytes.Buffer, w *multipart.Writer, err error) {
w = multipart.NewWriter(&b)
var fw io.Writer
file, err := os.Open(filePath)
if fw, err = w.CreateFormFile(fileFieldName, fileName); err != nil {
return
}
if _, err = io.Copy(fw, file); err != nil {
return
}
for k, v := range extraFormFields {
w.WriteField(k, v)
}
w.Close()
return
}
I have found this tutorial very helpful to clarify my confusions about file uploading in Go.
Basically you upload the file via ajax using form-data on a client and use the following small snippet of Go code on the server:
file, handler, err := r.FormFile("img") // img is the key of the form-data
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
fmt.Println("File is good")
fmt.Println(handler.Filename)
fmt.Println()
fmt.Println(handler.Header)
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
fmt.Println(err)
return
}
defer f.Close()
io.Copy(f, file)
Here r is *http.Request. P.S. this just stores the file in the same folder and does not perform any security checks.