VS Code how to format a few lines of SQL - sql

I have a main.go file with a sql query saved as a string.
defer db.Close()
id := 1
// format this query
row := db.QueryRow("SELECT f_name, l_name from users WHERE id = ?", id)
var fname, lname string
err = row.Scan(&fname, &lname)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%v %v\n", fname, lname)
Is there a way to format that line?

I have also had the same problem. For me, the best solution is to move the SQL query to an embed file and format it there. This also has the advantage of making more complex or larger SQL queries more readable.
For *.sql files can than be a plugin used. For example Language PL/SQL.
Here is an example:
Excerpt of a repository (sqlite3,postgres,...)
package repository
import (
"context"
"database/sql"
_ "embed"
)
//go:embed dml/selectUsers.sql
var selectUsers string
type Repository struct{
sqlDB *sql.DB
}
func (r *Repository) GetUsers()([]*User,error){
tx, err := s.sqlDb.BeginTx(ctx, &sql.TxOptions{ReadOnly: false})
if err != nil {
return nil, fmt.Errorf("failed to begin new transaction: %w", err)
}
defer tx.Close()
selectedUsers, err := r.selectUsers(tx, selectUsers)
if err != nil {
return nil, fmt.Errorf("failed to select users: %w", err)
}
err = tx.Commit()
if err != nil {
return nil, fmt.Errorf("failed to commit transaction: %w", err)
}
return selectedUsers, nil
}
func (r *Repository) selectUsers(tx *sql.Tx, query string, args ...interface{}) ([]*User,error) {
stmt, err := tx.Prepare(insertCompany)
if err != nil {
return nil, fmt.Errorf("failed to prepare statement: %v", err)
}
defer stmt.Close()
rows, err := stmt.Query(args...)
if err != nil {
return nil, fmt.Errorf("failed to query statement: %v", err)
}
selectedUsers := make([]*User, 0)
for rows.Next() {
selectedUser := new(User)
err = rows.Scan(
&selectedUser.ID,
&selectedUser.Surename,
&selectedUser.Familyname,
)
if err != nil {
return nil, fmt.Errorf("failed to scan row: %v", err)
}
selectedUsers = append(selectedUsers, selectedUser)
}
return selectedUsers, nil
}
dml/selectUsers.sql
SELECT
id,
surename,
familyname
FROM
users;

Related

Transaction in Golang with PGX

I am currently in the process of creating a little Go App. Right now I am working on the DB part. The Library I use is this one: https://github.com/jackc/pgx
The problem I have is that every time I try to execute the database read, it tells me that my 'conn is busy'. I read about using a pgxpool instead of a single connection, but it still does not work. What am I doing wrong?
func (postgre *PostgreClient) read(query string) (pgx.Row, error) {
client, err := postgre.client.Acquire(context.TODO())
transaction, err := client.BeginTx(context.TODO(), pgx.TxOptions{})
if err != nil {
return nil, err
}
defer transaction.Rollback(context.TODO())
rows := transaction.QueryRow(context.TODO(), query)
if err != nil {
return nil, err
}
err = transaction.Commit(context.TODO())
return rows, err
}
Thanks in advance.
You have to scan the row before you commit the transaction.
If you want the handling of the transaction to remain within the function you can pass an interface that does the scanning also inside the function.
For example:
// implemented by *sql.Row & *sql.Rows
type Row interface {
Scan(dst ...interface{}) error
}
// implemented by your "models"
type RowScanner interface {
ScanRow(r Row) error
}
type User struct {
Id int
Email string
}
func (u *User) ScanRow(r Row) error {
return r.Scan(
&u.Id,
&u.Email,
)
}
func (postgre *PostgreClient) read(query string, rs RowScanner) (err error) {
conn, err := postgre.client.Acquire(context.TODO())
if err != nil {
return err
}
defer conn.Release()
tx, err := conn.BeginTx(context.TODO(), pgx.TxOptions{})
if err != nil {
return err
}
defer func() {
if err != nil {
tx.Rollback(context.TODO())
} else {
tx.Commit(context.TODO())
}
}()
row := tx.QueryRow(context.TODO(), query)
if err != nil {
return nil, err
}
return rs.ScanRow(row)
}
u := new(User)
if err := pg.read("select id, email from users limit 1", u); err != nil {
panic(err)
}
For scanning a list of models:
type UserList []*User
func (ul *UserList) ScanRow(r Row) error {
u := new(User)
if err := u.ScanRow(r); err != nil {
return err
}
*ul = append(*ul, u)
return nil
}
func (postgre *PostgreClient) list(query string, rs RowScanner) (err error) {
conn, err := postgre.client.Acquire(context.TODO())
if err != nil {
return err
}
defer conn.Release()
tx, err := conn.BeginTx(context.TODO(), pgx.TxOptions{})
if err != nil {
return err
}
defer func() {
if err != nil {
tx.Rollback(context.TODO())
} else {
tx.Commit(context.TODO())
}
}()
rows, err := tx.Query(context.TODO(), query)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
if err := rs.ScanRow(rows); err != nil {
return err
}
}
return rows.Err()
}
ul := new(UserList)
if err := pg.list("select id, email from users", ul); err != nil {
panic(err)
}

Can I return rows result from db without scan it first in Golang func?

I have mini project using Golang, my plan is make a base function which it will be called from Model to execute sql query, then return the rows result without Scan it first. I'm using this way to prevent forget write defer rows.Close() and the code for execute the Query in model more simple. I had tried this way, but when try to print the result, I got nil without any error. here my screenshoot. The result exists when the query executed and the rows result scanned are in same function. Maybe I miss something? This is my first question, sorry it's too long. Thank you
The base model where the SQL query will be executed and return the result
package model
import "database/sql"
import "hb-backend-v1/config/database"
import "fmt"
func Query(query string) (*sql.Rows, error){
connect, err := database.Connect()
if err != nil{
fmt.Println("Connection Failed")
return nil, err
}
fmt.Println("Connection Success")
defer connect.Close()
rows, err := connect.Query(query)
defer rows.Close()
if err != nil{
return nil, err
}
return rows, nil
}
This is where the base model will be called and give the result
package product
import "database/sql"
import _"fmt"
import "hb-backend=v1/model"
type Hasil struct{
Id_alamat_store int
Id_tk int
Alamat string
Id_wil int
Latitude sql.NullString
Longitude sql.NullString
}
func ProductList() ([]Hasil, error){
rows, err := model.Query("SELECT * FROM alamat_store")
if err != nil{
return nil, err
}
var result []Hasil
for rows.Next(){
var each = Hasil{}
var err = rows.Scan(&each.Id_alamat_store, &each.Id_tk, &each.Alamat, &each.Id_wil, &each.Latitude, &each.Longitude)
if err != nil{
return nil, err
}
result = append(result, each)
}
return result, nil
}
Both connection and rows will be closed once Query exits, after those two are closed you can't use rows anymore.
One approach to get around that would be to pass a closure to Query and have Query execute it before closing the two resources:
func Query(query string, scan func(*sql.Rows) error) error {
connect, err := database.Connect()
if err != nil{
return err
}
defer connect.Close()
rows, err := connect.Query(query)
if err != nil{
return err
}
defer rows.Close()
return scan(rows)
}
func ProductList() ([]Hasil, error) {
var result []Hasil
err := model.Query("SELECT * FROM alamat_store", func(rows *sql.Rows) error {
for rows.Next() {
var each = Hasil{}
var err = rows.Scan(&each.Id_alamat_store, &each.Id_tk, &each.Alamat, &each.Id_wil, &each.Latitude, &each.Longitude)
if err != nil {
return err
}
result = append(result, each)
}
return nil
})
if err != nil {
return nil, err
}
return result, nil
}

How can i tell the PATCH Method which field i want to update

I'm working on a simple REST API and I'm having troubles with the PATCH method. I don't know how can i tell the method and the query which fields i want to update(for example which fields are passed as JSON) in the database. Here is what i have so far.
func PatchServer(c echo.Context) error {
patchedServer := new(structs.Server)
requestID := c.Param("id")
if err := c.Bind(patchedServer); err != nil {
return err
}
sql := "UPDATE servers SET server_name = CASE WHEN ? IS NOT NULL THEN ? END WHERE id = ?"
stmt, err := db.Get().Prepare(sql)
if err != nil {
panic(err)
}
_, err2 := stmt.Exec(patchedServer.Name, patchedServer.Name, requestID)
if err2 != nil {
panic(err2)
}
fmt.Println(patchedServer.ID, patchedServer.Name, patchedServer.Components)
fmt.Println("Requested id: ", requestID)
return c.JSON(http.StatusOK, "Patched!")
}

POST data using the Content-Type multipart/form-data

I'm trying to upload images from my computer to a website using go. Usually, I use a bash script that sends a file and a key to the server:
curl -F "image"=#"IMAGEFILE" -F "key"="KEY" URL
it works fine, but I'm trying to convert this request into my golang program.
http://matt.aimonetti.net/posts/2013/07/01/golang-multipart-file-upload-example/
I tried this link and many others, but, for each code that I try, the response from the server is "no image sent", and I've no idea why. If someone knows what's happening with the example above.
Here's some sample code.
In short, you'll need to use the mime/multipart package to build the form.
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os"
"strings"
)
func main() {
var client *http.Client
var remoteURL string
{
//setup a mocked http client.
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, err := httputil.DumpRequest(r, true)
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
}))
defer ts.Close()
client = ts.Client()
remoteURL = ts.URL
}
//prepare the reader instances to encode
values := map[string]io.Reader{
"file": mustOpen("main.go"), // lets assume its this file
"other": strings.NewReader("hello world!"),
}
err := Upload(client, remoteURL, values)
if err != nil {
panic(err)
}
}
func Upload(client *http.Client, url string, values map[string]io.Reader) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
for key, r := range values {
var fw io.Writer
if x, ok := r.(io.Closer); ok {
defer x.Close()
}
// Add an image file
if x, ok := r.(*os.File); ok {
if fw, err = w.CreateFormFile(key, x.Name()); err != nil {
return
}
} else {
// Add other fields
if fw, err = w.CreateFormField(key); err != nil {
return
}
}
if _, err = io.Copy(fw, r); err != nil {
return err
}
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
func mustOpen(f string) *os.File {
r, err := os.Open(f)
if err != nil {
panic(err)
}
return r
}
Here's a function I've used that uses io.Pipe() to avoid reading in the entire file to memory or needing to manage any buffers. It handles only a single file, but could easily be extended to handle more by adding more parts within the goroutine. The happy path works well. The error paths have not hand much testing.
import (
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
)
func UploadMultipartFile(client *http.Client, uri, key, path string) (*http.Response, error) {
body, writer := io.Pipe()
req, err := http.NewRequest(http.MethodPost, uri, body)
if err != nil {
return nil, err
}
mwriter := multipart.NewWriter(writer)
req.Header.Add("Content-Type", mwriter.FormDataContentType())
errchan := make(chan error)
go func() {
defer close(errchan)
defer writer.Close()
defer mwriter.Close()
w, err := mwriter.CreateFormFile(key, path)
if err != nil {
errchan <- err
return
}
in, err := os.Open(path)
if err != nil {
errchan <- err
return
}
defer in.Close()
if written, err := io.Copy(w, in); err != nil {
errchan <- fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err)
return
}
if err := mwriter.Close(); err != nil {
errchan <- err
return
}
}()
resp, err := client.Do(req)
merr := <-errchan
if err != nil || merr != nil {
return resp, fmt.Errorf("http error: %v, multipart error: %v", err, merr)
}
return resp, nil
}
After having to decode the accepted answer for this question for use in my unit testing I finally ended up with the follow refactored code:
func createMultipartFormData(t *testing.T, fieldName, fileName string) (bytes.Buffer, *multipart.Writer) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
file := mustOpen(fileName)
if fw, err = w.CreateFormFile(fieldName, file.Name()); err != nil {
t.Errorf("Error creating writer: %v", err)
}
if _, err = io.Copy(fw, file); err != nil {
t.Errorf("Error with io.Copy: %v", err)
}
w.Close()
return b, w
}
func mustOpen(f string) *os.File {
r, err := os.Open(f)
if err != nil {
pwd, _ := os.Getwd()
fmt.Println("PWD: ", pwd)
panic(err)
}
return r
}
Now it should be pretty easy to use:
b, w := createMultipartFormData(t, "image","../luke.png")
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
Here is an option that works for files or strings:
package main
import (
"bytes"
"io"
"mime/multipart"
"os"
"strings"
)
func createForm(form map[string]string) (string, io.Reader, error) {
body := new(bytes.Buffer)
mp := multipart.NewWriter(body)
defer mp.Close()
for key, val := range form {
if strings.HasPrefix(val, "#") {
val = val[1:]
file, err := os.Open(val)
if err != nil { return "", nil, err }
defer file.Close()
part, err := mp.CreateFormFile(key, val)
if err != nil { return "", nil, err }
io.Copy(part, file)
} else {
mp.WriteField(key, val)
}
}
return mp.FormDataContentType(), body, nil
}
Example:
package main
import "net/http"
func main() {
form := map[string]string{"image": "#IMAGEFILE", "key": "KEY"}
ct, body, err := createForm(form)
if err != nil {
panic(err)
}
http.Post("https://stackoverflow.com", ct, body)
}
https://golang.org/pkg/mime/multipart#Writer.WriteField
Send file from one service to another:
func UploadFile(network, uri string, f multipart.File, h *multipart.FileHeader) error {
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, err := writer.CreateFormFile("file", h.Filename)
if err != nil {
log.Println(err)
return err
}
b, err := ioutil.ReadAll(f)
if err != nil {
log.Println(err)
return err
}
part.Write(b)
writer.Close()
req, _ := http.NewRequest("POST", uri, buf)
req.Header.Add("Content-Type", writer.FormDataContentType())
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
b, _ = ioutil.ReadAll(resp.Body)
if resp.StatusCode >= 400 {
return errors.New(string(b))
}
return nil
}
To extend on #attila-o answer, here is the code I went with to perform a POST HTTP req in Go with:
1 file
configurable file name (f.Name() didn't work)
extra form fields.
Curl representation:
curl -X POST \
http://localhost:9091/storage/add \
-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW' \
-F owner=0xc916Cfe5c83dD4FC3c3B0Bf2ec2d4e401782875e \
-F password=$PWD \
-F file=#./internal/file_example_JPG_500kB.jpg
Go way:
client := &http.Client{
Timeout: time.Second * 10,
}
req, err := createStoragePostReq(cfg)
res, err := executeStoragePostReq(client, req)
func createStoragePostReq(cfg Config) (*http.Request, error) {
extraFields := map[string]string{
"owner": "0xc916cfe5c83dd4fc3c3b0bf2ec2d4e401782875e",
"password": "pwd",
}
url := fmt.Sprintf("http://localhost:%d%s", cfg.HttpServerConfig().Port(), lethstorage.AddRoute)
b, w, err := createMultipartFormData("file","./internal/file_example_JPG_500kB.jpg", "file_example_JPG_500kB.jpg", extraFields)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", w.FormDataContentType())
return req, nil
}
func executeStoragePostReq(client *http.Client, req *http.Request) (lethstorage.AddRes, error) {
var addRes lethstorage.AddRes
res, err := client.Do(req)
if err != nil {
return addRes, err
}
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return addRes, err
}
err = json.Unmarshal(data, &addRes)
if err != nil {
return addRes, err
}
return addRes, nil
}
func createMultipartFormData(fileFieldName, filePath string, fileName string, extraFormFields map[string]string) (b bytes.Buffer, w *multipart.Writer, err error) {
w = multipart.NewWriter(&b)
var fw io.Writer
file, err := os.Open(filePath)
if fw, err = w.CreateFormFile(fileFieldName, fileName); err != nil {
return
}
if _, err = io.Copy(fw, file); err != nil {
return
}
for k, v := range extraFormFields {
w.WriteField(k, v)
}
w.Close()
return
}
I have found this tutorial very helpful to clarify my confusions about file uploading in Go.
Basically you upload the file via ajax using form-data on a client and use the following small snippet of Go code on the server:
file, handler, err := r.FormFile("img") // img is the key of the form-data
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
fmt.Println("File is good")
fmt.Println(handler.Filename)
fmt.Println()
fmt.Println(handler.Header)
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
fmt.Println(err)
return
}
defer f.Close()
io.Copy(f, file)
Here r is *http.Request. P.S. this just stores the file in the same folder and does not perform any security checks.

How can I use the "compress/gzip" package to gzip a file?

I'm new to Go, and can't figure out how to use the compress/gzip package to my advantage. Basically, I just want to write something to a file, gzip it and read it directly from the zipped format through another script. I would really appreciate if someone could give me an example on how to do this.
All the compress packages implement the same interface. You would use something like this to compress:
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write([]byte("hello, world\n"))
w.Close()
And this to unpack:
r, err := gzip.NewReader(&b)
io.Copy(os.Stdout, r)
r.Close()
Pretty much the same answer as Laurent, but with the file io:
import (
"bytes"
"compress/gzip"
"io/ioutil"
)
// ...
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write([]byte("hello, world\n"))
w.Close() // You must close this first to flush the bytes to the buffer.
err := ioutil.WriteFile("hello_world.txt.gz", b.Bytes(), 0666)
For the Read part, something like the useful ioutil.ReadFile for .gz files could be :
func ReadGzFile(filename string) ([]byte, error) {
fi, err := os.Open(filename)
if err != nil {
return nil, err
}
defer fi.Close()
fz, err := gzip.NewReader(fi)
if err != nil {
return nil, err
}
defer fz.Close()
s, err := ioutil.ReadAll(fz)
if err != nil {
return nil, err
}
return s, nil
}
Here the func for unpack gzip file to destination file:
func UnpackGzipFile(gzFilePath, dstFilePath string) (int64, error) {
gzFile, err := os.Open(gzFilePath)
if err != nil {
return 0, fmt.Errorf("open file %q to unpack: %w", gzFilePath, err)
}
dstFile, err := os.OpenFile(dstFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
if err != nil {
return 0, fmt.Errorf("create destination file %q to unpack: %w", dstFilePath, err)
}
defer dstFile.Close()
ioReader, ioWriter := io.Pipe()
defer ioReader.Close()
go func() { // goroutine leak is possible here
gzReader, _ := gzip.NewReader(gzFile)
// it is important to close the writer or reading from the other end of the
// pipe or io.copy() will never finish
defer func(){
gzFile.Close()
gzReader.Close()
ioWriter.Close()
}()
io.Copy(ioWriter, gzReader)
}()
written, err := io.Copy(dstFile, ioReader)
if err != nil {
return 0, err // goroutine leak is possible here
}
return written, nil
}
I decided to combine ideas from others answers and just provide a full example program. Obviously there are many different ways to do the same thing. This is just one way:
package main
import (
"compress/gzip"
"fmt"
"io/ioutil"
"os"
)
var zipFile = "zipfile.gz"
func main() {
writeZip()
readZip()
}
func writeZip() {
handle, err := openFile(zipFile)
if err != nil {
fmt.Println("[ERROR] Opening file:", err)
}
zipWriter, err := gzip.NewWriterLevel(handle, 9)
if err != nil {
fmt.Println("[ERROR] New gzip writer:", err)
}
numberOfBytesWritten, err := zipWriter.Write([]byte("Hello, World!\n"))
if err != nil {
fmt.Println("[ERROR] Writing:", err)
}
err = zipWriter.Close()
if err != nil {
fmt.Println("[ERROR] Closing zip writer:", err)
}
fmt.Println("[INFO] Number of bytes written:", numberOfBytesWritten)
closeFile(handle)
}
func readZip() {
handle, err := openFile(zipFile)
if err != nil {
fmt.Println("[ERROR] Opening file:", err)
}
zipReader, err := gzip.NewReader(handle)
if err != nil {
fmt.Println("[ERROR] New gzip reader:", err)
}
defer zipReader.Close()
fileContents, err := ioutil.ReadAll(zipReader)
if err != nil {
fmt.Println("[ERROR] ReadAll:", err)
}
fmt.Printf("[INFO] Uncompressed contents: %s\n", fileContents)
// ** Another way of reading the file **
//
// fileInfo, _ := handle.Stat()
// fileContents := make([]byte, fileInfo.Size())
// bytesRead, err := zipReader.Read(fileContents)
// if err != nil {
// fmt.Println("[ERROR] Reading gzip file:", err)
// }
// fmt.Println("[INFO] Number of bytes read from the file:", bytesRead)
closeFile(handle)
}
func openFile(fileToOpen string) (*os.File, error) {
return os.OpenFile(fileToOpen, openFileOptions, openFilePermissions)
}
func closeFile(handle *os.File) {
if handle == nil {
return
}
err := handle.Close()
if err != nil {
fmt.Println("[ERROR] Closing file:", err)
}
}
const openFileOptions int = os.O_CREATE | os.O_RDWR
const openFilePermissions os.FileMode = 0660
Having a full example like this should be helpful for future reference.
To compress any Go object of interface type as input
func compress(obj interface{}) ([]byte, error) {
var b bytes.Buffer
objBytes, err := json.Marshal(obj)
if err != nil {
return nil, err
}
gz := gzip.NewWriter(&b)
defer gz.Close() //NOT SUFFICIENT, DON'T DEFER WRITER OBJECTS
if _, err := gz.Write(objBytes); err != nil {
return nil, err
}
// NEED TO CLOSE EXPLICITLY
if err := gz.Close(); err != nil {
return nil, err
}
return b.Bytes(), nil
}
To decompress the same,
func decompress(obj []byte) ([]byte, error) {
r, err := gzip.NewReader(bytes.NewReader(obj))
if err != nil {
return nil, err
}
defer r.Close()
res, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
return res, nil
}
Note, ioutil.ReadAll(r) returns io.EOF or io.ErrUnexpectedEOF if you do not close the Writer object after writing. I assumed defer on Close() would close the object properly, but it won't. Don't defer writer objects.