Let's say I have a postgres query like:
SELECT
id,
ARRAY_AGG(code) AS code
FROM
codes
WHERE id = '9252781'
GROUP BY id;
My return looks like:
id | codes
-----------+-------------
9252781 | {H01,H02}
Both id and codes are varchar.
In Golang when I scan along the rows for my result, it just freezes. No error, nothing.
If you're using the github.com/lib/pq postgres driver you can use their pq.Array helper function to scan and store postgres arrays.
var id string
var arr []string
row := db.QueryRow(`SELECT '9252781', ARRAY['H01','H02']`)
if err := row.Scan(&id, pq.Array(&arr)); err != nil {
log.Fatal(err)
}
log.Println(id, arr)
// 9252781 [H01 H02]
I don't want to use a additional driver (github.com/lib/pq), and I did not found any pgx way to do it other than creating my own type.
So I did one:
type Tags []string
func (t *Tags) Scan(v interface{}) error {
if v == nil {
*t = Tags{}
return nil
}
s, ok := v.(string)
if !ok {
return fmt.Errorf("Scan is expected to receive a string from database, but got [%+v]", v)
}
s = strings.TrimPrefix(s, "{")
s = strings.TrimSuffix(s, "}")
*t = strings.Split(s, ",")
return nil
}
func (t *Tags) Value() (driver.Value, error) {
s := fmt.Sprintf("{%v}", strings.Join(([]string)(*t), ","))
return s, nil
}
then you can scan the row returned from database like:
...
err := rows.Scan(
...
&Tags,
...
)
and you can use it directly you your Exec queries.
This code works well with constants in arrays, but you need more work if want to use it in commas and brackets.
Related
So basically I got an Array filled with 5 IDs. First the ProjectId and 4 ProductIds.
[projectId1 productid1 productid2 productid3 productid4]
I use a custom Query to get these Products by Id.
It looks like this:
func (q *Queries) GetProductsByIdsAndProjektId(ctx context.Context, arg GetProductsByIdsAndProjektIdParams) ([]Products, error) {
stmt := createGetProductsByIdsAndProjektId(len(arg.ProductIds))
args := make([]interface{}, len(arg.ProductIds)+1)
args[0] = arg.ProjektID
for i, id := range arg.ProductIds {
args[i+1] = id
}
rows, err := q.db.QueryContext(ctx, stmt, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Products
for rows.Next() {
var i Product
if err := rows.Scan(
&i.ID,
&i.ProjektID,
&i.Name,
&i.Price,
); err != nil {
return nil, err
}
items = append(items, i)
}
But when I log.Info(i) it scans the ids in a different order.
for example the items array will look like this in the end
[product4data product2data product3data product1data]
It is always the same order but not the one given to it with args. Is there a way to force rows.Next() -> rows.Scan() from the first id in an array to the last, so the items array will have all data for the 4 products in the same order the ids were given? So it should be
[product1Data product2Data product3Data product4Data]
instead.
In my Golang application I make SQL request to the database. Usually, in the SQL query, I specify the columns that I want to get from the table and create a structure based on it. You can see an example of the working code below.
QUESTION:
What should I do if I don't know the number and name of columns in the table? For example, I make the SQL request like SELECT * from filters; instead of SELECT FILTER_ID, FILTER_NAME FROM filters;. How do I create a structure in this case?
var GetFilters = func(responseWriter http.ResponseWriter, request *http.Request) {
rows, err := database.ClickHouse.Query("SELECT * FROM filters;"); if err != nil {
fmt.Println(err)
return
}
defer rows.Close()
columns, err := rows.Columns(); if err != nil {
fmt.Println(err)
return
}
filters := make([]interface{}, len(columns))
for i, _ := range columns {
filters[i] = new(sql.RawBytes)
}
for rows.Next() {
if err = rows.Scan(filters...); err != nil {
fmt.Println(err)
return
}
}
utils.Response(responseWriter, http.StatusOK, filters)
}
Well, finally I found the solution. As you can see from the code below first I make SQL request where I do not specify the name of the columns. Then I take information about columns by ColumnTypes() function. This function returns column information such as column type, length and nullable. Next I will learn the name and type of columns, fill interface with these data:
for i, column := range columns {
object[column.Name()] = reflect.New(column.ScanType()).Interface()
values[i] = object[column.Name()]
}
The full code which I use looks like this:
var GetFilters = func(responseWriter http.ResponseWriter, request *http.Request) {
rows, err := database.ClickHouse.Query("SELECT * FROM table_name;"); if err != nil {
fmt.Println(err)
return
}
defer rows.Close()
var objects []map[string]interface{}
for rows.Next() {
columns, err := rows.ColumnTypes(); if err != nil {
fmt.Println(err)
return
}
values := make([]interface{}, len(columns))
object := map[string]interface{}{}
for i, column := range columns {
object[column.Name()] = reflect.New(column.ScanType()).Interface()
values[i] = object[column.Name()]
}
if err = rows.Scan(values...); err != nil {
fmt.Println(err)
return
}
objects = append(objects, object)
}
utils.Response(responseWriter, http.StatusOK, objects)
}
Use the USER_TAB_COLUMNS table to get the list of columns in the executing table query store it an array or collection. later execute the query and Scan the columns that you already know from the previous Query.
I have searched for a solution on the net and in SO, but found nothing that apply to return values. It is a simple sql query with several rows that I want to return. Error handling not included:
func Fetch(query string) (string) {
type User struct{
id string
name string
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(users)
}
I get this error when compiling:
cannot use users (type []*User) as type string in return argument
How should I do to get a correct return value?
The expected input is
JD John Doe --OR-- {id:"JD",name:"John Doe"}
Add this to your code:
type userSlice []*User
func (us userSlice) String() string{
var s []string
for _, u := range us {
if u != nil {
s = append(s, fmt.Sprintf("%s %s", u.id, u.name))
}
}
return strings.Join(s, "\n")
}
type User struct{
id string
name string
}
In your Fetch function replace the last return statement like this:
func Fetch(query string) (string) {
// Note that we declare the User type outside the function.
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(userSlice(users).String()) // Replace this line in your code
}
If you have to return a String you can use the encoding/json package to serialize your user object, but you have to use fields that begin with capital letters for them to be exported. See the full example:
import (
"encoding/json"
)
func Fetch(query string) (string) {
type User struct{
Id string // <-- CHANGED THIS LINE
Name string // <-- CHANGED THIS LINE
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(ToJSON(users)) // <-- CHANGED THIS LINE
}
func ToJSON(obj interface{}) (string) {
res, err := json.Marshal(obj)
if err != nil {
panic("error with json serialization " + err.Error())
}
return string(res)
}
I'm trying to make a function that converts a struct in the way mysql rows.Scan function needs it, so I don't need to pass manually lots of parameters.
Note: I know the existence of sqlx and the alternative of writing manually in separate lines every pointer, but I'd like to solve it in this way as I'm learning go and want to understand what's going on.
The error I get with this solution is:
panic: sql: Scan error on column index 0: destination not a pointer
to me looks like valueField.Addr().Pointer() should be a Pointer to the value. The following is a simplification of my code.
type User struct {
Name string
Age int
}
func StrutForScan(u interface{}) []interface{} {
val := reflect.ValueOf(u).Elem()
v := make([]interface{}, val.NumField())
for i := 0; i < val.NumField(); i++ {
valueField := val.Field(i)
v[i] = valueField.Addr().Pointer()
}
return v
}
func ListUsers {
rows, err := db.Query("SELECT * FROM users")
PanicIf(err)
var user User
for rows.Next() {
err := rows.Scan(StrutForScan(&user)...)
PanicIf(err)
fmt.Printf("\nName: %s, Age: %s", user.Name, string(user.Age))
}
}
You need to use .Interface() not .Pointer()
func StrutForScan(u interface{}) []interface{} {
val := reflect.ValueOf(u).Elem()
v := make([]interface{}, val.NumField())
for i := 0; i < val.NumField(); i++ {
valueField := val.Field(i)
v[i] = valueField.Addr().Interface()
}
return v
}
The reason behind that is that .Pointer() returns an actual "pointer" to the data, you can't do much with it without using the unsafe package.
Let's say I have a struct:
type User struct {
Name string
Id int
Score int
}
And a database table with the same schema. What's the easiest way to parse a database row into a struct? I've added an answer below but I'm not sure it's the best one.
Go package tests often provide clues as to ways of doing things. For example, from database/sql/sql_test.go,
func TestQuery(t *testing.T) {
/* . . . */
rows, err := db.Query("SELECT|people|age,name|")
if err != nil {
t.Fatalf("Query: %v", err)
}
type row struct {
age int
name string
}
got := []row{}
for rows.Next() {
var r row
err = rows.Scan(&r.age, &r.name)
if err != nil {
t.Fatalf("Scan: %v", err)
}
got = append(got, r)
}
/* . . . */
}
func TestQueryRow(t *testing.T) {
/* . . . */
var name string
var age int
var birthday time.Time
err := db.QueryRow("SELECT|people|age,name|age=?", 3).Scan(&age)
/* . . . */
}
Which, for your question, querying a row into a structure, would translate to something like:
var row struct {
age int
name string
}
err = db.QueryRow("SELECT|people|age,name|age=?", 3).Scan(&row.age, &row.name)
I know that looks similar to your solution, but it's important to show how to find a solution.
I recommend github.com/jmoiron/sqlx.
From the README:
sqlx is a library which provides a set of extensions on go's standard
database/sql library. The sqlx versions of sql.DB, sql.TX,
sql.Stmt, et al. all leave the underlying interfaces untouched, so
that their interfaces are a superset on the standard ones. This makes
it relatively painless to integrate existing codebases using
database/sql with sqlx.
Major additional concepts are:
Marshal rows into structs (with embedded struct support), maps, and slices
Named parameter support including prepared statements
Get and Select to go quickly from query to struct/slice
The README also includes a code snippet demonstrating scanning a row into a struct:
type Place struct {
Country string
City sql.NullString
TelephoneCode int `db:"telcode"`
}
// Loop through rows using only one struct
place := Place{}
rows, err := db.Queryx("SELECT * FROM place")
for rows.Next() {
err := rows.StructScan(&place)
if err != nil {
log.Fatalln(err)
}
fmt.Printf("%#v\n", place)
}
Note that we didn't have to manually map each column to a field of the struct. sqlx has some default mappings for struct fields to database columns, as well as being able to specify database columns using tags (note the TelephoneCode field of the Place struct above). You can read more about that in the documentation.
Here's one way to do it - just assign all of the struct values manually in the Scan function.
func getUser(name string) (*User, error) {
var u User
// this calls sql.Open, etc.
db := getConnection()
// note the below syntax only works for postgres
err := db.QueryRow("SELECT * FROM users WHERE name = $1", name).Scan(&u.Id, &u.Name, &u.Score)
if err != nil {
return &User{}, err
} else {
return &u, nil
}
}
rows, err := connection.Query("SELECT `id`, `username`, `email` FROM `users`")
if err != nil {
panic(err.Error())
}
for rows.Next() {
var user User
if err := rows.Scan(&user.Id, &user.Username, &user.Email); err != nil {
log.Println(err.Error())
}
users = append(users, user)
}
Full example
Here is a library just for that: scany.
You can use it like that:
type User struct {
Name string
Id int
Score int
}
// db is your *sql.DB instance
// ctx is your current context.Context instance
// Use sqlscan.Select to query multiple records.
var users []*User
sqlscan.Select(ctx, db, &users, `SELECT name, id, score FROM users`)
// Use sqlscan.Get to query exactly one record.
var user User
sqlscan.Get(ctx, db, &user, `SELECT name, id, score FROM users WHERE id=123`)
It's well documented and easy to work with.
Disclaimer: I am the author of this library.
there's package just for that: sqlstruct
unfortunately, last time I checked it did not support embedded structs (which are trivial to implement yourself - i had a working prototype in a few hours).
just committed the changes I made to sqlstruct
use :
go-models-mysql
sqlbuilder
val, err = m.ScanRowType(row, (*UserTb)(nil))
or the full code
import (
"database/sql"
"fmt"
lib "github.com/eehsiao/go-models-lib"
mysql "github.com/eehsiao/go-models-mysql"
)
// MyUserDao : extend from mysql.Dao
type MyUserDao struct {
*mysql.Dao
}
// UserTb : sql table struct that to store into mysql
type UserTb struct {
Name sql.NullString `TbField:"Name"`
Id int `TbField:"Id"`
Score int `TbField:"Score"`
}
// GetFirstUser : this is a data logical function, you can write more logical in there
// sample data logical function to get the first user
func (m *MyUserDao) GetFirstUser() (user *User, err error) {
m.Select("Name", "Id", "Score").From("user").Limit(1)
fmt.Println("GetFirstUser", m.BuildSelectSQL().BuildedSQL())
var (
val interface{}
row *sql.Row
)
if row, err = m.GetRow(); err == nil {
if val, err = m.ScanRowType(row, (*UserTb)(nil)); err == nil {
u, _ := val.(*UserTb)
user = &User{
Name: lib.Iif(u.Name.Valid, u.Nae.String, "").(string),
Id: u.Id,
Score: u.Score,
}
}
}
row, val = nil, nil
return
}