I'm using PostgreSQL with nodejs for the first time. I want to make several PostgreSQL queries atomic in case of error.
For example:
const group = Group.of(body);
const { rows } = await this.db.query(
`INSERT INTO groups
(\"defaultImage\", \"createdAt\", \"updatedAt\")
VALUES ($1, $2, $3)
RETURNING *`,
[group.defaultImage, group.createdAt, group.updatedAt]);
const groupId = rows[0].id;
group.images.map(image => {
return this.db.query(
`INSERT INTO groups_images
(\"groups_id\", \"uri\")
VALUES ($1, $2)`,
[groupId, image.uri]);
});
If the second query fails, I want the first to be rolled back
Using Transaction will solve the problem as explained here:
const { Pool } = require('pg')
const pool = new Pool()
;(async () => {
// note: we don't try/catch this because if connecting throws an exception
// we don't need to dispose of the client (it will be undefined)
const client = await pool.connect()
try {
await client.query('BEGIN')
const queryText = 'INSERT INTO users(name) VALUES($1) RETURNING id'
const res = await client.query(queryText, ['brianc'])
const insertPhotoText = 'INSERT INTO photos(user_id, photo_url) VALUES ($1, $2)'
const insertPhotoValues = [res.rows[0].id, 's3.bucket.foo']
await client.query(insertPhotoText, insertPhotoValues)
await client.query('COMMIT')
} catch (e) {
await client.query('ROLLBACK')
throw e
} finally {
client.release()
}
})().catch(e => console.error(e.stack))
https://node-postgres.com/features/transactions
Related
This is the query I am using:
app.get("/items/:data", async (req, res) => {
const { data } = req.params;
query = `
SELECT items.discount
FROM items
WHERE items.discount #? '$[*] ? (#.discount[*].shift == $1)'
`
try {
const obj = await pool.query(query, [data]);
res.json(obj.rows[0])
} catch(err) {
console.error(err.message);
}
});
I get this error:
error: bind message supplies 1 parameters, but prepared statement "" requires 0
I am using node-postgres package in node.js.
How can I solve this issue?
Use bracket notation instead of dot notation. So instead of obj.key use obj[key]
Updated
all them driver connectors come with their own method to do what you're looking for. node-postgres also have there own
Pool
import { Pool } from 'pg';
const pool = new Pool({
host: 'localhost',
user: 'database-user',
max: 20,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
/**
* execs the given sql statement.
*
* #param {string} sql - query to run.
* #param {Array} params - an array with the parameter.
* #example
* runQuery("SELECT * FROM users WHERE id = $1", [1]).then(result=> console.log(result))
*/
export async function runQuery (sql, params) {
const connection = await pool.connect()
try {
await connection.query('BEGIN')
const queryText = 'INSERT INTO users(name) VALUES($1) RETURNING id'
const result = await connection.query(sql,params);
// check what result has
console.log(result);
return connection.query('COMMIT').then(result)
} catch (e) {
await connection.query('ROLLBACK')
throw e;
throw e
} finally {
connection.release()
}
}
Pool Config
config = {
// all valid client config options are also valid here
// in addition here are the pool specific configuration parameters:
// number of milliseconds to wait before timing out when connecting a new client
// by default this is 0 which means no timeout
connectionTimeoutMillis?: int,
// number of milliseconds a client must sit idle in the pool and not be checked out
// before it is disconnected from the backend and discarded
// default is 10000 (10 seconds) - set to 0 to disable auto-disconnection of idle clients
idleTimeoutMillis?: int,
// maximum number of clients the pool should contain
// by default this is set to 10.
max?: int,
}
conclution
so basically the structure of a query should be like or less this
const text = 'INSERT INTO users(name, email) VALUES($1, $2) RETURNING *'
const values = ['brianc', 'brian.m.carlson#gmail.com']
connection
.query(text, values)
.then(res => {
console.log(res.rows[0])
// { name: 'brianc', email: 'brian.m.carlson#gmail.com' }
})
.catch(e => console.error(e.stack))
I have an Nodejs express function where I am trying to insert data that is stored in the browser session into my postgres database. When I have the insert statement like this, the insert works but the session-stored customer_id isn't inserted and is just left null.
On the line with "var sql = INSERT INTO journal....", the values $1 and $2 are from user input and work correctly.
How can I get value 3 of the customer_id stored in the session to insert correctly? I would appreciate any advice or greater understanding.
app.post("/addJournalEntry", addJournalEntry);
function addJournalEntry(req, res) {
console.log("Posting data");
// var id = req.query.id;
//body is for post, query is for get
const customer_id = req.session.customer_id;
const journal_entry_date = req.body.journal_entry_date;
const journal_entry = req.body.journal_entry;
const params = [journal_entry, journal_entry_date, customer_id];
addEntryFromDataLayer(params, function (error, addEntry) {
console.log("Back From the addEntryFromDataLayer:", addEntry);
if (error || addEntry == null) {
res.status(500).json({
success: false,
data: error
});
}
else {
// res.json(result);
res.status(200).json(addEntry);
}
});
}
function addEntryFromDataLayer(params, callback) {
console.log("addEntryFromDataLayer called with id");
var sql = "INSERT INTO journal (journal_entry, journal_entry_date, customer_id) VALUES($1::text, $2::text, $3)";
// var params = [id];
pool.query(sql, params, function (err, addEntry) {
if (err) {
console.log("error in database connection");
console.log(err);
callback(err, null);
}
console.log("Found DB result:" + JSON.stringify(addEntry));
callback(null, addEntry);
});
}
I'm trying to dynamically make a patch request for oracle tables through Node.JS
Here's my setup:
In my router.js file I have this:
const express = require('express');
const router = new express.Router();
const employees = require('../controllers/employees.js');
const smiCats = require('../controllers/smi/smiCats.js');
const auth = require('../controllers/auth.js');
router.route('/login/:id?')
.post(auth.getToken);
router.route('/ams/:id?')
.get(auth.verifyToken, employees.get)
.post(auth.verifyToken, employees.post)
.put(auth.verifyToken, employees.put)
.delete(auth.verifyToken, employees.delete)
.patch(auth.verifyToken, employees.patch);
router.route('/smi/cats/:id?')
.get(auth.verifyToken, smiCats.get)
.post(auth.verifyToken, smiCats.post)
.put(auth.verifyToken, smiCats.put)
.patch(auth.verifyToken, smiCats.patch);
module.exports = router;
That then calls my controller that has my patch function & gets sanitized.
//sanitizer
function sanitizeCats(req) {
const cats = {
cat_desc: req.body.cat_desc,
msg_for: req.body.msg_for,
msg_user_owner: req.body.msg_user_owner || 0,
msg_realtor_owner: req.body.msg_realtor_owner || 0
};
return cats;
}
async function patch(req, res, next) {
try {
let category = sanitizeCats(req);
category.cat_id = parseInt(req.params.id, 10);
const success = await smiCats.patch(category);
if (success) {
res.status(204).end();
} else {
res.status(404).end();
}
} catch (err) {
next(err);
}
}
module.exports.patch = patch;
When that gets executed it calls my db_api module, which assembles the sql statement
(THE NEXT CODE SECTION IS WHERE MY QUESTION COMES FROM)
const database = require('../../services/database.js');
const oracledb = require('oracledb');
const patchSql =
`BEGIN
DECLARE
BEGIN
IF nvl(:cat_desc,'zzz') != 'zzz' THEN
UPDATE smi_contact_cats
SET cat_desc = :cat_desc
WHERE cat_id = :cat_id;
END IF;
IF nvl(:msg_for,'zzz') != 'zzz' THEN
UPDATE smi_contact_cats
SET msg_for = :msg_for
WHERE cat_id = :cat_id;
END IF;
IF nvl(:msg_user_owner,-1) > -1 THEN
UPDATE smi_contact_cats
SET msg_user_owner = :msg_user_owner
WHERE cat_id = :cat_id;
END IF;
IF nvl(:msg_realtor_owner,-1) > -1 THEN
UPDATE smi_contact_cats
SET msg_realtor_owner = :msg_realtor_owner
WHERE cat_id = :cat_id;
END IF;
:rowcount := sql%rowcount;
END;
END;`;
async function patch(cats) {
const category = Object.assign({}, cats);
//add binds
category.rowcount = {
dir: oracledb.BIND_OUT,
type: oracledb.NUMBER
};
const result = await database.simpleExecute(patchSql, category);
return result.outBinds.rowcount === 1;
}
module.exports.patch = patch;
This then calls the database function to actually execute & assemble the sql with the bind variables:
const oracledb = require('oracledb');
const dbConfig = require('../config/database.js');
async function initialize() {
const pool = await oracledb.createPool(dbConfig.beta);
}
module.exports.initialize = initialize;
async function close() {
await oracledb.getPool().close();
}
module.exports.close = close;
function simpleExecute(statement, binds = [], opts = {}) {
return new Promise(async (resolve, reject) => {
let conn;
opts.outFormat = oracledb.OBJECT;
opts.autoCommit = true;
try {
conn = await oracledb.getConnection();
const result = await conn.execute(statement, binds, opts);
resolve(result);
} catch (err) {
reject(err);
} finally {
if (conn) { // conn assignment worked, need to close
try {
await conn.close();
} catch (err) {
console.log(err);
}
}
}
});
}
module.exports.simpleExecute = simpleExecute;
So all of this works... but it's not dynamic enough for me to build our company api. How do I make a more dynamic patch request in Node.JS without having to type out every single column & put an nvl around it to check if it's there. As a side not if there's a better way to dynamically sanitize as well, I'm all ears, but the main question is on how to dynamically build the patch request better.
The current code is suboptimal in that is does one update per property. Here's a more dynamic solution...
Given the following:
create table smi_contact_cats (
cat_id number,
cat_desc varchar2(50),
msg_for varchar2(50),
msg_user_owner varchar2(50),
msg_realtor_owner varchar2(50)
);
insert into smi_contact_cats (
cat_id,
cat_desc,
msg_for,
msg_user_owner,
msg_realtor_owner
) values (
1,
'cat_desc orginal value',
'msg_for orginal value',
'msg_user_owner orginal value',
'msg_realtor_owner orginal value'
);
commit;
You can use logic like this. updatableColumns is the whitelist of columns that can be updated. Note that you can comment and uncomment some of the lines toward the bottom to test various input.
const oracledb = require('oracledb');
const config = require('./db-config.js');
async function patch(cat) {
let conn;
try {
const category = Object.assign({}, cat);
const categoryProps = Object.getOwnPropertyNames(category);
const updatableColumns = ['cat_desc', 'msg_for', 'msg_user_owner'];
// Validate that the pk was passed in
if (!categoryProps.includes('cat_id')) {
throw new Error('cat_id is required');
}
// Now remove the pk col from categoryProps
categoryProps.splice(categoryProps.indexOf('cat_id'), 1);
if (categoryProps.length === 0) {
throw new Error('At least one property must be specified');
}
let sql = 'update smi_contact_cats\nset ';
for (let propIdx = 0; propIdx < categoryProps.length; propIdx++) {
// Here's the whitelist check
if (!updatableColumns.includes(categoryProps[propIdx])) {
throw new Error('Invalid "update" column');
} else {
if (propIdx > 0 && propIdx < categoryProps.length) {
sql += ',\n ';
}
sql += categoryProps[propIdx] + ' = :' + categoryProps[propIdx];
}
}
sql += '\nwhere cat_id = :cat_id';
console.log('here is the sql', sql);
conn = await oracledb.getConnection(config);
const result = await conn.execute(
sql,
category,
{
autoCommit: true
}
);
if (result.rowsAffected && result.rowsAffected === 1) {
return category;
} else {
return null;
}
} catch (err) {
console.error(err);
} finally {
if (conn) {
try {
await conn.close();
} catch (err) {
console.error(err);
}
}
}
}
const patchObj = {
cat_id: 1
};
// Comment and uncomment the following to see various dynamic statements
patchObj.cat_desc = 'cat_desc value';
patchObj.msg_for = 'msg_for value';
patchObj.msg_user_owner = 'msg_user_owner value';
// Uncomment the following line to add a column that's not whitelisted
//patchObj.msg_realtor_owner = 'msg_realtor_owner value';
patch(patchObj)
.then(function(cat) {
console.log('Updated succeeded', cat);
})
.catch(function(err) {
console.log(err);
});
I am trying to right a function that copies some fields from several company databases into my own database once a day. What I have so far is below. I am wondering if where I console.log(rs) I can open another sql connection to my database and write the data or if I have to store the results somewhere and then open a new connection and send the stored results.
function updateJobs() {
var query = "SELECT JobStart, JobEnd FROM JobData";
sql.connect(Config, (err) => {
if (err) {
console.log("Error while connecting database :- " + err);
} else {
var request = new sql.Request();
request.query(query, function (err, rs) {
if (err) {
console.log("Error while querying database :- " + err);
sql.close();
} else {
console.log(rs);
sql.close();
}
})
}
})
}
This might help
// Source Database
sourceDB.each(`select * from ${Sourcetable}`, (error, row) => {
console.log(row);
const keys = Object.keys(row); // ['columnA', 'columnB']
const columns = keys.toString(); // 'columnA,columnB'
let parameters = {};
let values = '';
// Generate values and named parameters
Object.keys(row).forEach((r) => {
var key = '$' + r;
// Generates '$columnA,$columnB'
values = values.concat(',', key);
// Generates { $columnA: 'ABC', $columnB: 'GHK' }
parameters[key] = row[r];
});
// Insert into another database into OneTable (columnA,columnB) values ($columnA,$columnB)
// Parameters: { $columnA: 'ABC', $columnB: 'GHK' }
destDB.run(`insert into ${Desttable} (${columns}) values (${values})`, parameters);
})
})
I am trying to figure out how to correctly set this block of code up so that the commit function will wait until all of the rows are inserted. Currently I am reading a csv and need to insert a new row per column into a table. I also need to add a row into a parent table for that. I need all of that to finish before I call commit. I have yet to master callbacks so please be gentle.
db.beginTransaction(function (err) {
if (err)
{
//could not begin a transaction for some reason.
logger.error("beginTransaction error: " +err);
}
//need to wrap this *************
db.query("INSERT INTO TABLE VALUES ('" + unique_id + "','real_time','" + msg + "',1,1,LOCALTIMESTAMP)", function(err){
if(err)
{
logger.error("error insert into parent table: "+err);
}
});
for(var i = 0; i < headers.length; i++)
{
//replaces single quote (') with two single quotes to escape it ('')
values[i] = values[i].replace("'","''");
db.query("INSERT INTO TABLE VALUES ('" + unique_id + "','" + headers[i] + "',0,'" + values[i] + "')", function(err){
if(err)
{
logger.error("error insert into child table: "+err);
}
});
}
//To here ************
db.commitTransaction(function (err) {
if (err)
{
//error during commit
logger.error("Commit error: "+err);
}
}); //end of commitTransaction
callback();
});//End of beginTransaction
There's three basic ways of tackling this synchronization problem which I'll demonstrate here using new style arrow functions. The traditional Node way is with callbacks:
a((err, resultA) => {
// Fires when A is done or errored out
if (err) {
// Log, panic, etc.
return;
}
b((err, resultB) => {
// Fires when A and B are done or A is done and B errored out
if (err) {
// Log, panic, etc.
return;
}
c((err, resultC) => {
// Fires when A, B and C are done or A and B are done and C errored out
if (err) {
// Log, panic, etc.
return;
}
});
});
});
This is what people refer to as "callback hell" because the nesting and error propagation code gets more and more ridiculous as your dependencies grow in complexity. I find this style unsustainable for any non-trivial application.
The next style is Promise-driven:
a().then(resultA => {
// Fires when A is done
return b();
}).then(resultB => {
// Fires when B is done
return c();
}).then(resultC => {
// Fires when C is done
}).catch(err => {
// Fires if any of the previous calls produce an error
});
This tends to be a lot "flatter" and easier to follow, but it's still a lot of heavy syntax for what should be simple. The newer async/await style builds on promises by adding support for them to the JavaScript syntax:
try {
let resultA = await a();
let resultB = await a();
let resultC = await a();
} catch(err) {
// Fires when any error occurs
}
This works inside any function tagged async like:
async function runQueries() {
// async code
}
Which can make your life a lot easier. You can also use conventional try/catch notation for errors, they're propagated accordingly.
As tadman states, it's very important that you don't manually escape values and use parameterized queries instead. Make sure you fix this first.
Unfortunately it doesn't look like node-odbc supports promises. You may be able to get it to work with something like Bluebird.promisify. Currently what you want is to keep track of how many successful inserts were completed and then commit the transaction if they all complete successfully.
let successfulInsertions = 0;
let insertionAttempts = 0;
for(var i = 0; i < headers.length; i++) {
db.query("INSERT INTO TABLE VALUES (?, ?, ?, ?)", params, err => {
if (err) {
logger.error("error insert into child table: "+err);
}
else {
successfulInsertions++;
}
insertionAttempts++;
if (insertionAttempts === headers.length) {
if (successfulInsertions === insertionAttempts) {
db.commitTransaction();
}
else {
db.rollbackTransaction();
}
callback();
}
});
}
There are libraries out there that will help with this but they would require rewriting your code a bit. If you can use Bluebird's promisifyAll on the node-odbc library I would rewrite it using async/await (this is still asynchronous and functions the same):
await db.beginTransactionAsync();
try {
await db.queryAsync("INSERT INTO TABLE VALUES (?, ?, ?, ?)", params);
await Promise.all(headers.map((header, i) =>
db.queryAsync("INSERT INTO TABLE VALUES (?, ?, ?, ?)", [unique_id, header, 0, values[i])
);
await db.commitTransactionAsync();
} catch (err) {
logger.error(err);
db.rollbackTransaction();
}
Note that if beginTransaction throws an error for some reason this code will throw an error.
I don't really understand your code, why don't you return or stop your code if there is an error?
Here for example, you should have your code like this
db.beginTransaction(function (err) {
if (err)
return logger.error("beginTransaction error: " +err), db.rollback(/*...*/)
//....
}
Second, you should change your whole code with the async/await syntax.
async function foo () {
await new Promise((next, err)=> {
db.beginTransaction(e => e ? err(e) : next())
})
await new Promise((next, err)=> {
db.query(`INSERT INTO TABLE VALUES ('${unique_id}','real_time','${msg}',1,1,LOCALTIMESTAMP)`, e => e ? err(e) : next())
})
for (var i = 0; i < headers.length; i++) {
await new Promise((next, err)=> {
db.query(`INSERT INTO TABLE VALUES ('${unique_id}','${headers[i]}',0,'${values[i]}')`, e => e ? err(e) : next())
})
}
await new Promise((next, err)=> {
db.commitTransaction(e => e ? err(e) : next())
})
}
foo()
.then(()=> callback())
.catch(e=> logger.error(`Error: ${e}`))