tl; dr
I have an sql table users which includes an array field. How can I Scan it to variable in golang? My approach:
var id int
var username string
var activites []string
row := db.QueryRow("SELECT id, username, activities FROM users WHERE id = 1")
err := row.Scan(&id, &username, &activites)
Works fine for id, username.
As #mkopriva already pointed out, this can either be accomplished using the StringArray method or with the more flexible Array method (as it accepts an interface as the argument), both found within the "github.com/lib/pq" package.
As an aside, it is also a good practice to use prepared statements.
Full example:
var id int
var username string
var activities []string
sqlStatement := `
SELECT
id,
username,
activities
FROM
users
WHERE
id = $1
`
stmt, err := db.Prepare(sqlStatement)
if err != nil {
// handle err
}
defer stmt.Close()
row := stmt.QueryRow(1)
err = row.Scan(
&id,
&username,
pq.Array(&activities) // used here
)
if err == sql.ErrNoRows {
// handle err
}
if err != nil {
// handle err
}
Related
I am new to Go and I'm trying to populate a struct called Reliefworker from an SQL query which I can send as a JSON payload.
Essentially I have a Reliefworker who may be assigned to many communities and the community can consist of multiple regions.
I suspect there is a clever way of doing this other than what I intend which is a primitive solution that will add a sort to the SQL (by Community) and create a function that detects if the community being added is different to the previous one, in which case I would create a new community struct type object to be appended.
type Reliefworker struct {
Name string `json:"name"`
Communities []Community `json:"community"`
Deployment_id string `json:"deployment_id"`
}
type Community struct{
Name string `json:"name"`
community_id string `json:"community_id"`
Regions []Region `json:"regions"`
}
type Region struct{
Name string `json:"name"`
Region_id string `json:"region_id"`
Reconstruction_grant string `json:"reconstruction_grant"`
Currency string `json:"currency"`
}
Currently I have created a struct that reflects what I am actually getting from SQL whilst pondering my next move. Perhaps this might be a good stepping stone instead of attempting an on-the-fly transformation ?
type ReliefWorker_community_region struct {
Deployment_id string
Community_title int
Region_name string
Reconstruction_grant int
}
func GetReliefWorkers(deployment_id string) []Reliefworker {
fmt.Printf("Confirm I have a deployment id:%v\n", deployment_id)
rows, err := middleware.Db.Query("select deployment_id, community_title, region_name, reconstruction_grant WHERE Deployment_id=$1", brand_id)
if err != nil {
return
}
for rows.Next() {
reliefworker := Reliefworker{}
err = rows.Scan(&deployment_id, &community_title, ®ion_name, &reconstruction_grant)
if err != nil {
return
}
}
rows.Close()
return
}
I think a sort makes a lot of sense, primitive solutions can be the most efficient:
func GetReliefWorkers(deployment_id string) []Reliefworker {
// Added sort to query
q := "select worker_name, community_title, region_name, reconstruction_grant WHERE deployment_id=? ORDER BY community_title"
rows, err := middleware.Db.Query(q, deployment_id)
if err != nil {
return
}
defer rows.Close() // So rows get closed even on an error
c := Community{} // To keep track of the current community
cmatrix := [][]string{[]string{}} // Matrix of communities and workers
communities := []Community{} // List of communities
workers := make(map[string]Reliefworker) // Map of workers
var ccount int // Index of community in lists
for rows.Next() {
w := Reliefworker{Deployment_id: deployment_id}
r := Region{}
var ctitle string // For comparison later
err = rows.Scan(&w.Name, &ctitle, &r.Name, &r.Reconstruction_grant)
if err != nil {
return
}
if ctitle != c.Name {
communities = append(communities, c)
c = Community{}
c.Name = ctitle
ccount++
cmatrix = append(cmatrix, []string{})
}
c.Regions = append(c.Regions, r)
cmatrix[ccount] = append(cmatrix[ccount], w.Name)
workers[w.Name] = w
}
for i, c := range communities {
for _, id := range cmatrix[i] {
w := workers[id] // To avoid error
w.Communities = append(w.Communities, c)
workers[id] = w
}
}
out := []Reliefworker{}
for _, w := range workers {
out = append(out, w)
}
return out
}
Though it might make even more sense to create seperate tables for communities, regions, and workers, then query them all with a JOIN: https://www.w3schools.com/sql/sql_join_inner.asp
UPDATE: Since you only want to retrieve one Reliefworker, would something like this work?
type ReliefWorker struct {
Name string `json:"name"`
Communities []Community `json:"community"`
}
type Community struct {
Name string `json:"name"`
Regions []Region `json:"regions"`
}
type Region struct {
Name string `json:"name"`
Region_id string `json:"region_id"`
Reconstruction_grant int `json:"reconstruction_grant"`
Currency string `json:"currency"`
}
func GetReliefWorkers(deployment_id string) Reliefworker {
reliefworker := Reliefworker{}
communities := make(map[string]Community)
rows, err := middleware.Db.Query("select name, community_title, region_name, region_id, reconstruction_grant WHERE Deployment_id=$1", deployment_id)
if err != nil {
if err == sql.ErrNoRows {
fmt.Printf("No records for ReliefWorker:%v\n", deployment_id)
}
panic(err)
}
defer rows.Close()
for rows.Next() {
c := Community{}
r := Region{}
err = rows.Scan(&reliefworker.Name, &c.Name, &r.Name, &r.Region_id, &r.Reconstruction_grant)
if err != nil {
panic(err)
}
if _, ok := communities[c.Name]; ok {
c = communities[c.Name]
}
c.Regions = append(c.Regions, r)
communities[c.Name] = c
}
for _, c := range commmunities {
reliefworker.Communities = append(reliefworker.Communities, c)
}
return reliefworker
}
Ok, my crude solution doesn't contain a shred of the intelligence #Absentbird demonstrates but I'm here to learn.
#Absentbird I love your use of maps and multidimensional arrays to hold a matrix of communities and workers. I will focus on making this part of my arsenal over the weekend.
I can accept and adapt #Absentbird's solution once I have a solution to why it gives the error "cannot assign to struct field workers[id].Communities in mapcompilerUnaddressableFieldAssign" for the line workers[id].Communities = append(workers[id].Communities, c)
Firstly apologies as I had to correct two things. Firstly I only needed to return ReliefWorkers (not an array of ReliefWorkers). Secondly ReliefWorker struct did not need to contain the Deployment_id since I already knew it.
I am new to Go so I'd really appreciate feedback on what I can do to better leverage the language and write more concise code.
My structs and solution is currently as follows:
type ReliefWorker struct {
Name string `json:"name"`
Communities []Community `json:"community"`
}
type Community struct {
Name string `json:"name"`
Regions []Region `json:"regions"`
}
type Region struct {
Name string `json:"name"`
Region_id string `json:"region_id"`
Reconstruction_grant int `json:"reconstruction_grant"`
Currency string `json:"currency"`
}
type ReliefWorker_community_region struct {
Name string
Community_title string
Region_name string
Reconstruction_grant int
}
func GetReliefWorkers(deployment_id string) Reliefworker {
var reliefworker Reliefworker
var communitiesOnly []string
var name string
var allReliefWorkerData []ReliefWorker_community_region
rows, err := middleware.Db.Query("select name, community_title, region_name, reconstruction_grant WHERE Deployment_id=$1", deployment_id)
for rows.Next() {
reliefWorker_community_region := ReliefWorker_community_region{}
err = rows.Scan(&reliefWorker_community_region.Name, &reliefWorker_community_region.Community_title, &reliefWorker_community_region.Region_name, &reliefWorker_community_region.Reconstruction_grant)
if err != nil {
panic(err)
}
name = reliefWorker_community_region.Name
allReliefWorkerData = append(allReliefWorkerData, reliefWorker_community_region)
communitiesOnly = append(communitiesOnly, reliefWorker_community_region.Community_title) //All communities go in here, even duplicates, will will create a unique set later
}
rows.Close()
if err != nil {
if err == sql.ErrNoRows {
fmt.Printf("No records for ReliefWorker:%v\n", deployment_id)
}
panic(err)
}
var unique []string //Use this to create a unique index of communities
for _, v := range communitiesOnly {
skip := false
for _, u := range unique {
if v == u {
skip = true
break
}
}
if !skip {
unique = append(unique, v)
}
}
fmt.Println(unique)
reliefworker.Name = name
var community Community
var communities []Community
for _, v := range unique {
community.Name = v
communities = append(communities, community)
}
// Go through each record from the database held within allReliefWorkerData and grab every region belonging to a Community, when done append it to Communities and finally append that to ReliefWorker
for j, u := range communities {
var regions []Region
for i, v := range allReliefWorkerData {
if v.Community_title == u.Name {
var region Region
region.Name = v.Region_name
region.Reconstruction_grant = v.Reconstruction_grant
regions = append(regions, region)
}
}
communities[j].Regions = regions
}
reliefworker.Communities = communities
return reliefworker
}
In my Golang application I make SQL request to the database. Usually, in the SQL query, I specify the columns that I want to get from the table and create a structure based on it. You can see an example of the working code below.
QUESTION:
What should I do if I don't know the number and name of columns in the table? For example, I make the SQL request like SELECT * from filters; instead of SELECT FILTER_ID, FILTER_NAME FROM filters;. How do I create a structure in this case?
var GetFilters = func(responseWriter http.ResponseWriter, request *http.Request) {
rows, err := database.ClickHouse.Query("SELECT * FROM filters;"); if err != nil {
fmt.Println(err)
return
}
defer rows.Close()
columns, err := rows.Columns(); if err != nil {
fmt.Println(err)
return
}
filters := make([]interface{}, len(columns))
for i, _ := range columns {
filters[i] = new(sql.RawBytes)
}
for rows.Next() {
if err = rows.Scan(filters...); err != nil {
fmt.Println(err)
return
}
}
utils.Response(responseWriter, http.StatusOK, filters)
}
Well, finally I found the solution. As you can see from the code below first I make SQL request where I do not specify the name of the columns. Then I take information about columns by ColumnTypes() function. This function returns column information such as column type, length and nullable. Next I will learn the name and type of columns, fill interface with these data:
for i, column := range columns {
object[column.Name()] = reflect.New(column.ScanType()).Interface()
values[i] = object[column.Name()]
}
The full code which I use looks like this:
var GetFilters = func(responseWriter http.ResponseWriter, request *http.Request) {
rows, err := database.ClickHouse.Query("SELECT * FROM table_name;"); if err != nil {
fmt.Println(err)
return
}
defer rows.Close()
var objects []map[string]interface{}
for rows.Next() {
columns, err := rows.ColumnTypes(); if err != nil {
fmt.Println(err)
return
}
values := make([]interface{}, len(columns))
object := map[string]interface{}{}
for i, column := range columns {
object[column.Name()] = reflect.New(column.ScanType()).Interface()
values[i] = object[column.Name()]
}
if err = rows.Scan(values...); err != nil {
fmt.Println(err)
return
}
objects = append(objects, object)
}
utils.Response(responseWriter, http.StatusOK, objects)
}
Use the USER_TAB_COLUMNS table to get the list of columns in the executing table query store it an array or collection. later execute the query and Scan the columns that you already know from the previous Query.
I have searched for a solution on the net and in SO, but found nothing that apply to return values. It is a simple sql query with several rows that I want to return. Error handling not included:
func Fetch(query string) (string) {
type User struct{
id string
name string
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(users)
}
I get this error when compiling:
cannot use users (type []*User) as type string in return argument
How should I do to get a correct return value?
The expected input is
JD John Doe --OR-- {id:"JD",name:"John Doe"}
Add this to your code:
type userSlice []*User
func (us userSlice) String() string{
var s []string
for _, u := range us {
if u != nil {
s = append(s, fmt.Sprintf("%s %s", u.id, u.name))
}
}
return strings.Join(s, "\n")
}
type User struct{
id string
name string
}
In your Fetch function replace the last return statement like this:
func Fetch(query string) (string) {
// Note that we declare the User type outside the function.
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(userSlice(users).String()) // Replace this line in your code
}
If you have to return a String you can use the encoding/json package to serialize your user object, but you have to use fields that begin with capital letters for them to be exported. See the full example:
import (
"encoding/json"
)
func Fetch(query string) (string) {
type User struct{
Id string // <-- CHANGED THIS LINE
Name string // <-- CHANGED THIS LINE
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(ToJSON(users)) // <-- CHANGED THIS LINE
}
func ToJSON(obj interface{}) (string) {
res, err := json.Marshal(obj)
if err != nil {
panic("error with json serialization " + err.Error())
}
return string(res)
}
In python it's a simple db.query("SELECT id,login,password FROM Users") and return list [(1,'root','password'), (2,'toor','password')]. I can simple iterate it
for user in response:print("id: %s, login: %s, password: %s", %(user[0],user[1],user[2]))
But in Golang I can't find revelant example for simple way to do this.
I understand that a python has dynamic typing, golang is static. So I'm looking for an answer, maybe some libraries provide such functionality? Hacks? Thanks for answers!
You need something like this, but there may be a problem if you are using complex mysql's data types like enum, set and etc.
var (
result [][]string
container []string
pointers []interface{}
)
rows, err := db.Query("SELECT id, login, password FROM Users")
if err != nil {
panic(err.Error())
}
cols, err := rows.Columns()
if err != nil {
panic(err.Error())
}
length := len(cols)
for rows.Next() {
pointers = make([]interface{}, length)
container = make([]string, length)
for i := range pointers {
pointers[i] = &container[i]
}
err = rows.Scan(pointers...)
if err != nil {
panic(err.Error())
}
result = append(result, container)
}
Then you can loop over results and print it using fmt package.
An awesome library that nicely extends the go standard lib sql interface is sqlx
Using this lib, you can do something similar to your example in python:
db, err := sqlx.Connect("postgres", "user=foo dbname=bar sslmode=disable")
if err != nil {
log.Fatalln(err)
}
users := []User{}
db.Select(&users, "SELECT id,login,password FROM Users")
//iterate through the users slice
Let's say I have a struct:
type User struct {
Name string
Id int
Score int
}
And a database table with the same schema. What's the easiest way to parse a database row into a struct? I've added an answer below but I'm not sure it's the best one.
Go package tests often provide clues as to ways of doing things. For example, from database/sql/sql_test.go,
func TestQuery(t *testing.T) {
/* . . . */
rows, err := db.Query("SELECT|people|age,name|")
if err != nil {
t.Fatalf("Query: %v", err)
}
type row struct {
age int
name string
}
got := []row{}
for rows.Next() {
var r row
err = rows.Scan(&r.age, &r.name)
if err != nil {
t.Fatalf("Scan: %v", err)
}
got = append(got, r)
}
/* . . . */
}
func TestQueryRow(t *testing.T) {
/* . . . */
var name string
var age int
var birthday time.Time
err := db.QueryRow("SELECT|people|age,name|age=?", 3).Scan(&age)
/* . . . */
}
Which, for your question, querying a row into a structure, would translate to something like:
var row struct {
age int
name string
}
err = db.QueryRow("SELECT|people|age,name|age=?", 3).Scan(&row.age, &row.name)
I know that looks similar to your solution, but it's important to show how to find a solution.
I recommend github.com/jmoiron/sqlx.
From the README:
sqlx is a library which provides a set of extensions on go's standard
database/sql library. The sqlx versions of sql.DB, sql.TX,
sql.Stmt, et al. all leave the underlying interfaces untouched, so
that their interfaces are a superset on the standard ones. This makes
it relatively painless to integrate existing codebases using
database/sql with sqlx.
Major additional concepts are:
Marshal rows into structs (with embedded struct support), maps, and slices
Named parameter support including prepared statements
Get and Select to go quickly from query to struct/slice
The README also includes a code snippet demonstrating scanning a row into a struct:
type Place struct {
Country string
City sql.NullString
TelephoneCode int `db:"telcode"`
}
// Loop through rows using only one struct
place := Place{}
rows, err := db.Queryx("SELECT * FROM place")
for rows.Next() {
err := rows.StructScan(&place)
if err != nil {
log.Fatalln(err)
}
fmt.Printf("%#v\n", place)
}
Note that we didn't have to manually map each column to a field of the struct. sqlx has some default mappings for struct fields to database columns, as well as being able to specify database columns using tags (note the TelephoneCode field of the Place struct above). You can read more about that in the documentation.
Here's one way to do it - just assign all of the struct values manually in the Scan function.
func getUser(name string) (*User, error) {
var u User
// this calls sql.Open, etc.
db := getConnection()
// note the below syntax only works for postgres
err := db.QueryRow("SELECT * FROM users WHERE name = $1", name).Scan(&u.Id, &u.Name, &u.Score)
if err != nil {
return &User{}, err
} else {
return &u, nil
}
}
rows, err := connection.Query("SELECT `id`, `username`, `email` FROM `users`")
if err != nil {
panic(err.Error())
}
for rows.Next() {
var user User
if err := rows.Scan(&user.Id, &user.Username, &user.Email); err != nil {
log.Println(err.Error())
}
users = append(users, user)
}
Full example
Here is a library just for that: scany.
You can use it like that:
type User struct {
Name string
Id int
Score int
}
// db is your *sql.DB instance
// ctx is your current context.Context instance
// Use sqlscan.Select to query multiple records.
var users []*User
sqlscan.Select(ctx, db, &users, `SELECT name, id, score FROM users`)
// Use sqlscan.Get to query exactly one record.
var user User
sqlscan.Get(ctx, db, &user, `SELECT name, id, score FROM users WHERE id=123`)
It's well documented and easy to work with.
Disclaimer: I am the author of this library.
there's package just for that: sqlstruct
unfortunately, last time I checked it did not support embedded structs (which are trivial to implement yourself - i had a working prototype in a few hours).
just committed the changes I made to sqlstruct
use :
go-models-mysql
sqlbuilder
val, err = m.ScanRowType(row, (*UserTb)(nil))
or the full code
import (
"database/sql"
"fmt"
lib "github.com/eehsiao/go-models-lib"
mysql "github.com/eehsiao/go-models-mysql"
)
// MyUserDao : extend from mysql.Dao
type MyUserDao struct {
*mysql.Dao
}
// UserTb : sql table struct that to store into mysql
type UserTb struct {
Name sql.NullString `TbField:"Name"`
Id int `TbField:"Id"`
Score int `TbField:"Score"`
}
// GetFirstUser : this is a data logical function, you can write more logical in there
// sample data logical function to get the first user
func (m *MyUserDao) GetFirstUser() (user *User, err error) {
m.Select("Name", "Id", "Score").From("user").Limit(1)
fmt.Println("GetFirstUser", m.BuildSelectSQL().BuildedSQL())
var (
val interface{}
row *sql.Row
)
if row, err = m.GetRow(); err == nil {
if val, err = m.ScanRowType(row, (*UserTb)(nil)); err == nil {
u, _ := val.(*UserTb)
user = &User{
Name: lib.Iif(u.Name.Valid, u.Nae.String, "").(string),
Id: u.Id,
Score: u.Score,
}
}
}
row, val = nil, nil
return
}