When I write the code:
err := database.QueryRow("SELECT page_title,page_content,page_date FROM pages WHERE id=1").
Scan(&thisPage.Title, &thisPage.Content, &thisPage.Date)
Everything works fine. But I want it to not just get the page with id=1, but to be dynamic.
So I write:
err := database.QueryRow("SELECT page_title,page_content,page_date FROM pages WHERE id=?", pageID).
Scan(&thisPage.Title, &thisPage.Content, &thisPage.Date)
But I get an error:
GolangdProjects go run test.go
1
2017/04/13 11:29:57 Couldn't get page: 1
exit status 1
The full code:
package main
import (
"database/sql"
"fmt"
_ "github.com/lib/pq"
"github.com/gorilla/mux"
"log"
"net/http"
)
const (
DBHost = "localhost"
DBPort = ":5432"
DBUser = "nirgalon"
DBPass = ""
DBDbase = "cms"
PORT = ":8080"
)
var database *sql.DB
type Page struct {
Title string
Content string
Date string
}
func ServePage(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
pageID := vars["id"]
thisPage := Page{}
fmt.Println(pageID)
err := database.QueryRow("SELECT page_title,page_content,page_date FROM pages WHERE id=1").Scan(&thisPage.Title, &thisPage.Content, &thisPage.Date)
if err != nil {
log.Fatal("Couldn't get page: " + pageID)
log.Fatal(err.Error)
}
html := `<html><head><title>` + thisPage.Title + `</title></head><body><h1>` + thisPage.Title + `</h1><div>` + thisPage.Content + `</div></body></html>`
fmt.Fprintln(w, html)
}
func main() {
db, err := sql.Open("postgres", "user=nirgalon dbname=cms sslmode=disable")
if err != nil {
log.Println("Couldn't connnect to db")
log.Fatal(err)
}
database = db
routes := mux.NewRouter()
routes.HandleFunc("/page/{id:[0-9]+}", ServePage)
http.Handle("/", routes)
http.ListenAndServe(PORT, nil)
}
psql driver uses $1, $2, etc for params:
database.QueryRow("SELECT page_title,page_content,page_date FROM pages WHERE id = $1", pageID)
QueryRow takes var args of type interface{}. It could be that pageID is being interpolated as a string (as returned by mux.Vars), resulting in a query that's incorrectly formatted as:
"SELECT page_title,page_content,page_date FROM pages WHERE id='1'"
Try converting pageID to int:
id := vars["id"]
pageID := strconv.Atoi(id)
Related
I want to send CDP commands to WebSocket server exposed by chrome browser.
For example, I create a new session on selenium and get WebSocket url as -> ws://172.20.10.2:4444/session/335d5805e9d98f3c37af004fa91e0f6e/se/cdp
Now I want to send CDP commands to this WebSocket connection and get results.
Sample client code :
package main
import (
"encoding/json"
"fmt"
"github.com/gorilla/websocket"
"log"
"net/url"
)
type message1 struct {
Id int `json:"id"`
Result Result `json:"result"`
}
type Result struct {
TargetInfo []TargetInfo `json:"targetInfos"`
}
type TargetInfo struct {
TargetId string `json:"targetId"`
}
type message2 struct {
Id int `json:"id"`
Method string `json:"method"`
Params Parameter `json:"params"`
}
type Parameter struct {
TargetId string `json:"targetId"`
Flatten bool `json:"flatten"`
}
type message3 struct {
Id int `json:"id"`
Method string `json:"method"`
SessionId string `json:"sessionId"`
Parameters EmulationParameter `json:"params"`
}
type EmulationParameter struct {
Width int `json:"width"`
Height int `json:"height"`
DeviceScaleFactor int `json:"deviceScaleFactor"`
Mobile bool `json:"mobile"`
}
type Response2 struct {
Method string `json:"method"`
Parameter SessionResponse `json:"params"`
}
type SessionResponse struct {
SessionId string `json:"sessionId"`
}
type message4 struct {
Id int `json:"id"`
Method string `json:"method"`
SessionId string `json:"sessionId"`
Parameters ScreenshotParameter `json:"params"`
}
type ScreenshotParameter struct {
Width int `json:"width,omitempty"`
}
func main() {
hostURL := "172.20.10.2:4444"
pathURL := "/session/dd44b246de3261c405cb9fcb017ab59a/se/cdp"
u := url.URL{Scheme: "ws", Host: hostURL, Path: pathURL}
c, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
if err != nil {
log.Fatal("dial:", err)
}
defer c.Close()
messageChannel := make(chan string)
go func() {
for {
_, message, err := c.ReadMessage()
//err = c.ReadJSON(p)
if err != nil {
log.Println("read:", err)
return
}
messageChannel <- string(message)
fmt.Println(string(message))
}
}()
message := "{\"id\":1,\"method\":\"Target.getTargets\"}"
//err = c.WriteJSON(websocket.TextMessage, []byte(message))
err = c.WriteMessage(websocket.TextMessage, []byte(message))
if err != nil {
log.Println("write:", err)
return
}
response := <-messageChannel
fmt.Println(response)
decodedResponse := message1{}
json.Unmarshal([]byte(response), &decodedResponse)
fmt.Println(decodedResponse)
fmt.Println(decodedResponse.Result.TargetInfo[0].TargetId)
temp := message2{}
temp.Id = 2
temp.Method = "Target.attachToTarget"
temp.Params.Flatten = true
temp.Params.TargetId = decodedResponse.Result.TargetInfo[0].TargetId
event2, _ := json.Marshal(temp)
err = c.WriteMessage(websocket.TextMessage, event2)
if err != nil {
log.Println("write:", err)
return
}
response2 := <-messageChannel
fmt.Println(response2)
decodedResponse2 := Response2{}
json.Unmarshal([]byte(response2), &decodedResponse2)
fmt.Println("The session Id is")
fmt.Println(decodedResponse2.Parameter.SessionId)
width := 645
height := 9651
scale := 2
temp2 := message3{}
temp2.Id = 1
temp2.SessionId = decodedResponse2.Parameter.SessionId
temp2.Method = "Emulation.setDeviceMetricsOverride"
temp2.Parameters.DeviceScaleFactor = scale
temp2.Parameters.Height = height
temp2.Parameters.Width = width
temp2.Parameters.Mobile = false
event3, _ := json.Marshal(temp2)
err = c.WriteMessage(websocket.TextMessage, event3)
if err != nil {
log.Println("write:", err)
return
}
response3 := <-messageChannel
fmt.Println(response3)
temp3 := message4{}
temp3.Id = 1
temp3.SessionId = decodedResponse2.Parameter.SessionId
temp3.Method = "Page.captureScreenshot"
event4, _ := json.Marshal(temp3)
err = c.WriteMessage(websocket.TextMessage, event4)
if err != nil {
log.Println("write:", err)
return
}
response4 := <-messageChannel
fmt.Println(response4)
}
I wasn't able to reuse this part of the code to send CDP commands to the WebSocket. I want a simple example CDP command.
I am dealing with a bit over 15 billion rows of data in various text files. I am trying to insert them into MariaDB using golang. Golang is a fast language and is often used for big data but I cannot get more than 10k-15k inserts a second, at this rate its gonna take over 15 days, I need this data imported sooner than that. I have tried various batch sizes but they all give about the same results.
function I'm using to handle file data:
func handlePath(path string) {
file, err := os.Open(path)
if err != nil {
fmt.Printf("error opening %v: %v", path, err)
return
}
defer file.Close()
scanner := bufio.NewScanner(file)
var temp_lines []string
for scanner.Scan() {
if len(temp_lines) == line_batch {
insertRows(temp_lines)
temp_lines = []string{}
}
temp_lines = append(temp_lines, scanner.Text())
}
insertRows(temp_lines)
fmt.Printf("\nFormatted %v\n", path)
if err := scanner.Err(); err != nil {
fmt.Printf("\nScanner error %v\n", err)
return
}
}
function I'm using for inserting:
func insertRows(rows []string) {
var Args []string
for _, row := range rows {
line_split := strings.Split(row, "|")
if len(line_split) != 6 {return}
database_id := line_split[0]
email := line_split[1]
password := line_split[2]
username := line_split[3]
ip := line_split[4]
phone := line_split[5]
arg := fmt.Sprintf("('%v','%v','%v','%v','%v','%v')",database_id,email,password,username,ip,phone)
Args = append(Args, arg)
}
sqlQuery := fmt.Sprintf("INSERT INTO new_table (database_id, email, password, username, ip, phone_number) VALUES %s", strings.Join(Args, ","))
_, err := db.Exec(sqlQuery)
if err != nil {
//fmt.Printf("%v\n", err)
return
}
total+=line_batch
writes++
}
Server specs:
server
I have searched for a solution on the net and in SO, but found nothing that apply to return values. It is a simple sql query with several rows that I want to return. Error handling not included:
func Fetch(query string) (string) {
type User struct{
id string
name string
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(users)
}
I get this error when compiling:
cannot use users (type []*User) as type string in return argument
How should I do to get a correct return value?
The expected input is
JD John Doe --OR-- {id:"JD",name:"John Doe"}
Add this to your code:
type userSlice []*User
func (us userSlice) String() string{
var s []string
for _, u := range us {
if u != nil {
s = append(s, fmt.Sprintf("%s %s", u.id, u.name))
}
}
return strings.Join(s, "\n")
}
type User struct{
id string
name string
}
In your Fetch function replace the last return statement like this:
func Fetch(query string) (string) {
// Note that we declare the User type outside the function.
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(userSlice(users).String()) // Replace this line in your code
}
If you have to return a String you can use the encoding/json package to serialize your user object, but you have to use fields that begin with capital letters for them to be exported. See the full example:
import (
"encoding/json"
)
func Fetch(query string) (string) {
type User struct{
Id string // <-- CHANGED THIS LINE
Name string // <-- CHANGED THIS LINE
}
rows, err := db.Query(query)
users := make([]*User, 0)
for rows.Next() {
user := new(User)
err := rows.Scan(&user.id, &user.name)
users = append(users, user)
}
return(ToJSON(users)) // <-- CHANGED THIS LINE
}
func ToJSON(obj interface{}) (string) {
res, err := json.Marshal(obj)
if err != nil {
panic("error with json serialization " + err.Error())
}
return string(res)
}
I have a table t containing a lot of columns, and my sql is like this: select * from t. Now I only want to scan one column or two from the wide returned row set. However, the sql.Scan accepts dest ...interface{} as arguments. Does it mean I have to scan everything and use only the column I needed?
I know I could change the sql from select * to select my_favorite_rows, however, in this case, I have no way to change the sql.
You can make use of Rows.Columns, e.g.
package main
import (
"database/sql"
"fmt"
"github.com/lib/pq"
)
type Vehicle struct {
Id int
Name string
Wheels int
}
// VehicleCol returns a reference for a column of a Vehicle
func VehicleCol(colname string, vh *Vehicle) interface{} {
switch colname {
case "id":
return &vh.Id
case "name":
return &vh.Name
case "wheels":
return &vh.Wheels
default:
panic("unknown column " + colname)
}
}
func panicOnErr(err error) {
if err != nil {
panic(err.Error())
}
}
func main() {
conn, err := pq.ParseURL(`postgres://docker:docker#172.17.0.2:5432/pgsqltest?schema=public`)
panicOnErr(err)
var db *sql.DB
db, err = sql.Open("postgres", conn)
panicOnErr(err)
var rows *sql.Rows
rows, err = db.Query("select * from vehicle")
panicOnErr(err)
// get the column names from the query
var columns []string
columns, err = rows.Columns()
panicOnErr(err)
colNum := len(columns)
all := []Vehicle{}
for rows.Next() {
vh := Vehicle{}
// make references for the cols with the aid of VehicleCol
cols := make([]interface{}, colNum)
for i := 0; i < colNum; i++ {
cols[i] = VehicleCol(columns[i], &vh)
}
err = rows.Scan(cols...)
panicOnErr(err)
all = append(all, vh)
}
fmt.Printf("%#v\n", all)
}
For unknown length of columns but if you're sure about their type,
cols, err := rows.Columns()
if err != nil {
log.Fatal(err.Error())
}
colLen := len(cols)
vals := make([]interface{}, colLen)
for rows.Next() {
for i := 0; i < len(colLen); i++ {
vals[i] = new(string)
}
err := rows.Scan(vals...)
if err != nil {
log.Fatal(err.Error()) // if wrong type
}
fmt.Printf("Column 1: %s\n", *(vals[0].(*string))) // will panic if wrong type
}
PS: Not recommended for prod
I want to create a 10 GB file that looks like:
prefix:username:timestamp, number
So an example is like:
login:jbill:2013/3/25, 1
I want to create a 10GB file, by creating random rows like the one above.
How could I do this in Go?
I can have an array of prefixes like:
login, logout, register
And also an array of usernames:
jbill, dkennedy
For example,
package main
import (
"bufio"
"fmt"
"math/rand"
"os"
"strconv"
"time"
)
func main() {
fileSize := int64(10e9) // 10GB
f, err := os.Create("/tmp/largefile")
if err != nil {
fmt.Println(err)
return
}
w := bufio.NewWriter(f)
prefixes := []string{"login", "logout", "register"}
names := []string{"jbill", "dkennedy"}
timeStart := time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)
timeDur := timeStart.AddDate(1, 0, 0).Sub(timeStart)
rand.Seed(time.Now().UnixNano())
size := int64(0)
for size < fileSize {
// prefix:username:timestamp, number
// login:jbill:2012/3/25, 1
prefix := prefixes[int(rand.Int31n(int32(len(prefixes))))]
name := names[int(rand.Int31n(int32(len(names))))]
time := timeStart.Add(time.Duration(rand.Int63n(int64(timeDur)))).Format("2006/1/2")
number := strconv.Itoa(int(rand.Int31n(100) + 1))
line := prefix + ":" + name + ":" + time + ", " + number + "\n"
n, err := w.WriteString(line)
if err != nil {
fmt.Println(n, err)
return
}
size += int64(len(line))
}
err = w.Flush()
if err != nil {
fmt.Println(err)
return
}
err = f.Close()
if err != nil {
fmt.Println(err)
return
}
fmt.Println("Size:", size)
}
Output:
register:jbill:2012/8/24, 15
login:jbill:2012/10/7, 98
register:dkennedy:2012/8/29, 70
register:jbill:2012/6/1, 89
register:jbill:2012/5/24, 63
login:dkennedy:2012/3/29, 48
logout:jbill:2012/7/8, 93
logout:dkennedy:2012/1/12, 74
login:jbill:2012/4/12, 14
login:jbill:2012/2/5, 83
This is a naive approach (1GB):
package main
import (
"fmt"
"log"
"os"
)
func main() {
myfile, err := os.OpenFile("myfile", os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
log.Fatal(err)
}
defer myfile.Close()
var pos int
var line string
// sample: login:jbill:2013/3/25, 1
line = fmt.Sprintf("%s:%s:%s, %d\n", "login", "jbill", "2013/3/25", 1)
for pos < 1024*1024*1024 {
bytes, err := myfile.Write([]byte(line))
if err != nil {
log.Fatal(err)
}
pos = pos + bytes
}
}
which takes forever (1:16), because the output is not buffered. By adding bufio you can decrease the time dramatically
package main
import (
"bufio"
"fmt"
"log"
"os"
)
func main() {
myfile, err := os.OpenFile("myfile", os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
log.Fatal(err)
}
defer myfile.Close()
mybufferedfile := bufio.NewWriter(myfile)
var pos int
var line string
// sample: login:jbill:2013/3/25, 1
line = fmt.Sprintf("%s:%s:%s, %d\n", "login", "jbill", "2013/3/25", 1)
for pos < 1024*1024*1024 {
bytes, err := mybufferedfile.WriteString(line)
if err != nil {
log.Fatal(err)
}
pos = pos + bytes
}
err = mybufferedfile.Flush()
if err != nil {
log.Fatal(err)
}
}
Still 26 sec on my machine, I'd like to see a faster solution.
BTW: you need to do the random fileds, but that is left as an exercise to the reader :)