How to execute large SQL statement - sql

I'm trying to extract data from a MongoDb database and insert them into a SQL Server table, via a NodeJS program.
However, when I run it, it gives me almost 1 million lines and a huge SQL statement (with multiple "INSERT INTO", one for each row to avoid any limit) but it doesn't update my table.
I also tried to save the whole query in a text file and import it as a script in SSMS, but the file is too big (~300 mo) so SSMS is crashing.
What can I do? How can I maybe divide my query into smaller batches (like each 100 000 records) or pause it?
Here's my code :
var sql = require('mssql');
const fs = require('fs');
var config = { user: 'xxxxxx',
password: 'xxxxx',
server: 'xxxxx',
database: 'xxxx',
stream: true,
requestTimeout: 2000000,
};
exports.insertElementsInSql = function(elements, callback) {
let dateStr = new Date().toISOString().slice(0, 10);
sql.connect(config, function(err, conn) {
if (err) {
callback(err);
} else {
var request = new sql.Request();
request.stream = true;
var query = "";
var count=0;
query = query + "SET ANSI_WARNINGS OFF;";
query = query + "DELETE FROM [xxxx].[dbo].[xxxxx];";
for(var i in elements){
query = query + "INSERT INTO [xxxx].[dbo].[xxxxx] VALUES ";
query = query +"('" + dateStr + "'";
query = query + ",'"+elements[i].OrgaCode+ "'";
query = query + ","+elements[i].nbWidgets;
query = query + ","+elements[i].nbTabs;
query = query + ",'"+elements[i].Segment+ "'";
query = query + ",'"+elements[i].SitesList+ "'";
query = query + ",'"+elements[i].Columns+ "'";
query = query + ") ";
count++;
};
query = query + "SET ANSI_WARNINGS ON;";
console.log(query);
console.log("The query INSERT INTO is complete. Nb of lines : "+count);
fs.writeFile('request.txt', query, (err) => {
if (err) throw err;
console.log('Query saved!');
});
request.query(query, function(err, results) {
if (err) {
console.log("Failure ");
console.log(err);
callback(err);
} else {
callback(null);
console.log("Yay");
}
});
}
});
};
I get :
The query INSERT INTO is complete. Nb of lines: 919045
Thanks for any help!

Related

Reading from one db into another

I am trying to right a function that copies some fields from several company databases into my own database once a day. What I have so far is below. I am wondering if where I console.log(rs) I can open another sql connection to my database and write the data or if I have to store the results somewhere and then open a new connection and send the stored results.
function updateJobs() {
var query = "SELECT JobStart, JobEnd FROM JobData";
sql.connect(Config, (err) => {
if (err) {
console.log("Error while connecting database :- " + err);
} else {
var request = new sql.Request();
request.query(query, function (err, rs) {
if (err) {
console.log("Error while querying database :- " + err);
sql.close();
} else {
console.log(rs);
sql.close();
}
})
}
})
}
This might help
// Source Database
sourceDB.each(`select * from ${Sourcetable}`, (error, row) => {
console.log(row);
const keys = Object.keys(row); // ['columnA', 'columnB']
const columns = keys.toString(); // 'columnA,columnB'
let parameters = {};
let values = '';
// Generate values and named parameters
Object.keys(row).forEach((r) => {
var key = '$' + r;
// Generates '$columnA,$columnB'
values = values.concat(',', key);
// Generates { $columnA: 'ABC', $columnB: 'GHK' }
parameters[key] = row[r];
});
// Insert into another database into OneTable (columnA,columnB) values ($columnA,$columnB)
// Parameters: { $columnA: 'ABC', $columnB: 'GHK' }
destDB.run(`insert into ${Desttable} (${columns}) values (${values})`, parameters);
})
})

how to input nodejs variable into oracle sql query

I am trying to pass a nodejs variable to a sql query
Below is my code:
var b= 101;
connection = await oracledb.getConnection( {
user : dbConfig.user,
password : dbConfig.password,
connectString : dbConfig.connectString
});
sql = 'SELECT * FROM mytab where id= b';
binds = {};
options = {
outFormat: oracledb.OBJECT // query result format
};
result = await connection.execute(sql, binds, options);
try something along these lines:
sql = 'SELECT * FROM mytab where id= $1::int';
options = {
outFormat: oracledb.OBJECT // query result format
};
result = await connection.execute(sql, [ b ], options);

SQL tedious add array as parameter

I'm running this SQL query with tedious.js using parameters:
var query = "select * from table_name where id in (#ids)";
request = new sql.Request(query, function(err, rowCount) {
if (err) {
}
});
request.on('row', function(columns) {
});
var id = [1, 2, 3];
request.addParameters('ids', TYPES.Int, id);
connection.execSql(request);
because I am looking for items that matches the ID provided with where ... in ... clause, I need to pass in an array. However, there is no TYPES.Array. How do I this properly?
for this query, i think you'll just have to manually build the entire sql string. the TYPES enum values are for the datatypes in the database, not in your JavaScript code.
//you can like this:
var userIds = result.map(function (el) {
return el.UserId;
}).join(',');
var params = [{
name: 'userIds',
type: TYPES.VarChar,
value: userIds,
options: null}];
var querySql = ['SELECT COUNT([MomentId]) FROM [T_Moment]',
'WHERE [RecordStatus] = ', sysConst.recordStatus.activation, " AND CHARINDEX(','+RTRIM([UserId])+',' , ','+ #userIds +',')>0 "].join(' ');
dbHelper.count(querySql, params, function (err, result) {
if (err) {
callback('error--');
} else {
callback(null, result);
}
});
Try creating the in clause parameters for query dynamically.
// create connection
let ids = [1, 2, 3];
let inClauseParamters = createInClauseParameters();
let query = `select * from table_name where id in (${inClauseParamters})`;
let request = new Request(query, (err, rowCount) => {
if (err) { /* handle error */ }
});
request.on('row', (columns) => { /* get row */});
request = addRequestParameters(ids, request);
connection.execSql(request);
function createInClauseParameters(values) {
return values.map((val, index) => `#Value${index}`).join(',');
}
function addRequestParameters(values, request) {
values.forEach((val, index) => {
request.addParameter(`Value${index}`, TYPES.VarChar, val);
});
return request;
}

SQL Error 42S22 when updating multiple records using query from server script in Azure Mobile Services

I'm currently running into problems while trying to customize the update script for a table of User Stories on Azure Mobile Services. My intention is to have the update script receive an item that contains an array of UserStory objects, construct a SQL string using that array, and then use mssql.query with that string against the UserStory table to update the individual records.
The following SQL achieves what I'm looking to do and works correctly when executed in Visual Studio:
UPDATE
masterstorylist.UserStory
SET
UserStory.relativepriority =
CASE UserStory.id
WHEN 'C36DC45B-170B-49F4-A747-6F4D989C1859' THEN '24'
WHEN '7EC413C3-17A8-410A-A394-ABF334364226' THEN '25'
WHEN '99890AFE-13C2-4E1A-8376-B501CB07080D' THEN '26'
END
Here's the server script that I have created in an attempt to achieve the same result:
function update(item, user, request) {
if(item.stories.length > 0){
var sql = "UPDATE masterstorylist.UserStory SET UserStory.relativepriority = CASE UserStory.id ";
for(var i = 0; i < item.stories.length; ++i){
sql+= ("WHEN '" + item.stories[i].id + "' THEN " + item.stories[i].relativepriority + " ");
}
sql+="END";
mssql.query(sql, {
success: function(results) {
request.respond();
},
error: function(err) {
request.respond(err);
}
});
}
else{
request.respond(statusCodes.NO_CONTENT, 'No records specified for update in request.');
}
}
The error I get back is
"sqlstate":"42S22","code:207"
which I think means that SQL can't find the relativepriority or id column. I've tried different syntax, such as qualifying the column names more or less or using [] around the columns, but the result is always the same.
I'm not sure what else to try, and details around creating and executing queries with the mssql object are hard to come by. I've been working off the examples here and here.
What am I missing?
EDIT: In case it helps, I reworked the code to see if using mssql.open would help. I modeled after the examples from the "MS Drivers for Node.js for SQL Server guide" (which I can't link to because I have low rep). The net result is the exact same error :/ Here's the new code in case it gives folks any ideas:
function update(item, user, request) {
if(item.stories.length > 0){
var sql = "UPDATE UserStory SET relativepriority = CASE id ";
for(var i = 0; i < item.stories.length; ++i){
sql+= ("WHEN '" + item.stories[i].id + "' THEN " + item.stories[i].relativepriority + " ");
}
sql+="END ";
console.log("opening connection...");
mssql.open({
success: function(connection){
console.log("mssql.open success");
console.log("executing query...");
connection.query(sql, function(err,results){
if(err){
console.log("query failed");
request.respond(err)
}
console.log("query successful");
request.respond();
});
},
error: function(err) {
console.log("fail on open: " + err);
request.respond(err);
}
});
}
else{
request.respond(statusCodes.OK, 'No records specified for update in request.');
}
}
P.S. This is my first post on Stack Overflow! :)
Ok, I figured out the answer to my own question. It turns out that the JSON object that the Mobile Service SDK was passing up wasn't formatted to the liking of Node.js. The item object coming into the script had an array of objects in it item.stories, which looked ok to me when it was logged to the console with console.log(item.stories); but apparently wasn't formatted well enough for me to access the individual objects in the 'item.stories' array using array notation.
I was able to fix both of the scripts above by adding the line var storiesToUpdate = JSON.parse(item.stories); and then using storiesToUpdate[i] instead of item.stories[i]. That seems to have done the trick. Ideally I'll find a way to fix the JSON generated on my clients so that I don't need this extra JSON.parse.
Here are three now working examples on how to update multiple records at once.
Simplest way to do what I wanted:
var storiesToUpdate;
var returnItem;
function update(item, user, request) {
storiesToUpdate = JSON.parse(item.stories);
returnItem = item;
if(storiesToUpdate.length > 0){
var sql = "UPDATE UserStory SET relativepriority = CASE id ";
for(var i = 0; i < storiesToUpdate.length; ++i){
sql+= ("WHEN '" + storiesToUpdate[i].id + "' THEN " + storiesToUpdate[i].relativepriority + " ");
}
sql+="END ";
mssql.query(sql,{
success: function(connection){
request.respond(statusCodes.OK, returnItem);
},
error: function(err) {
console.log("fail on open: " + err);
request.respond(err);
}
});
}
else{
request.respond(statusCodes.OK, returnItem);
}
}
Another way using mssql.open as well (not sure why you'd ever want to do this...):
var storiesToUpdate;
var returnItem;
function update(item, user, request) {
storiesToUpdate = JSON.parse(item.stories);
returnItem = item;
if(storiesToUpdate.length > 0){
var sql = "UPDATE UserStory SET relativepriority = CASE id ";
for(var i = 0; i < storiesToUpdate.length; ++i){
sql+= ("WHEN '" + storiesToUpdate[i].id + "' THEN " + storiesToUpdate[i].relativepriority + " ");
}
sql+="END ";
console.log("opening connection...");
mssql.open({
success: function(connection){
console.log("mssql.open success");
console.log("executing query...");
connection.query(sql, function(err,results){
if(err){
console.log("query failed");
request.respond(err)
}
console.log("query successful");
request.respond(statusCodes.OK, returnItem);
//request.respond(statusCodes.OK);
});
},
error: function(err) {
console.log("fail on open: " + err);
request.respond(err);
}
});
}
else{
request.respond(statusCodes.OK, returnItem);
}
}
And lastly, here's how to update multiple records without using mssql using the recommended batching techniques (this is rough and probably needs to be cleaned up):
var UserStoryTable = tables.getTable('UserStory');
var batchSize = 10;
var startIndex = 0;
var endIndex = 0;
var totalCount = 0;
var errorCount = 0;
var g_item;
var g_request;
var storiesToUpdate;
function update(item, user, request) {
//the json array has to be parsed first
storiesToUpdate = JSON.parse(item.stories);
g_item = item;
g_request = request;
if(item.stories.length > 0){
updateItems();
}
else{
console.log("empy update request");
request.respond(statusCodes.OK);
}
}
function updateItems(){
var batchCompletedCount = 0;
var updateComplete = function() {
batchCompletedCount++;
totalCount++;
if(batchCompletedCount === batchSize || totalCount === storiesToUpdate.length) {
if(totalCount < storiesToUpdate.length) {
// kick off the next batch
updateItems();
} else {
// or we are done, report the status of the job
// to the log and don't do any more processing
console.log("Update complete. %d Records processed. There were %d errors.", totalCount, errorCount);
g_request.respond(statusCodes.OK);
}
}
};
var errorHandler = function(err) {
errorCount++;
console.warn("Ignoring insert failure as part of batch.", err);
updateComplete();
};
var startIndex = totalCount;
var endIndex = totalCount + batchSize - 1;
if(endIndex >= storiesToUpdate.length) endIndex = storiesToUpdate.length - 1;
for(var i = startIndex; i <= endIndex; i++) {
console.log("Updating: " + storiesToUpdate[totalCount]);
UserStoryTable.update(storiesToUpdate[i],{
success: updateComplete,
error: errorHandler
});
}
}

Phonegap Database error

I am trying to set up a database for my phone gap application. the problem is all the transactions apart from the ones setting up the table produce an error saying
"the SQLTransactionCallback was null or threw an exception"
here is the code
function Datasetup()
{
db=window.openDatabase("PracticeData","1.0","saveData",300000);
alert("1");
db.transaction(getDB,onDBError,onDBSuccess);
}
function onDBError(error)
{
alert("Database Error"+error.message);
}
function onDBSuccess(tx,results)
{
alert("successfull");
}
function getDB(tx)
{
alert("2");
tx.executeSql("CREATE TABLE IF NOT EXISTS session(date,length,activity,pieces)");
alert("3");
tx.executeSql("CREATE TABLE IF NOT EXISTS pieces(newpiece,name,composer,youtube,images_src,date_Added)");
alert("4");
tx.executeSql('SELECT * FROM session', [], onSelectSessionSuccess, onDBError());
tx.executeSql("SELECT * FROM session",[],onSelectSessionSuccess,onDBError());
alert("5");
tx.executeSql("SELECT * FROM pieces",[],onSelectPiecesSuccess,onDBError());
}
function savepiece(tx)
{
tx.executeSql("INSERT INTO NOTES(newpiece,name,composer,youtube,images_src,date_Added)VALUES(?,?,?,?,?,?)",[true,pieceData.name,pieceData.composer,"tube","images",date()]);
}
I don't get the error meseges for this or the create table
db.transaction(getDB,onDBError,onDBSuccess)
I get error messages for
tx.executeSql('SELECT * FROM session', [], onSelectSessionSuccess, onDBError());
tx.executeSql("SELECT * FROM session",[],onSelectSessionSuccess,onDBError());
alert("5");
tx.executeSql("SELECT * FROM pieces",[],onSelectPiecesSuccess,onDBError());
that was a great help now one of my selects work the other one however comes up with
"the statement callback raised an exception or statement error callback did not return false"
here's the sql that's not working
tx.executeSql('SELECT * FROM session', [],onSelectPiecesSuccess, onDBError);
here is the updated code
function Datasetup(){
db=window.openDatabase("PracticeData","1.0","PracticeData",300000);
db.transaction(getDB,onDBError,onDBSuccess);
}
function onDBError(error){
alert("Database Error "+error.message);
}
function onDBSuccess(tx,results){
//db.transaction(query,onDBError);
db.transaction(query,onDBError);
alert("before");
}
function getDB(tx){
//alert("dropping")
//tx.executeSql("DROP TABLE pieces");
tx.executeSql("CREATE TABLE IF NOT EXISTS session(date,length,activity,pieces)");
tx.executeSql("CREATE TABLE IF NOT EXISTS pieces(newpiece,name,composer,youtube,images_src,date_added)");
//tx.executeSql('INSERT INTO session(date, length, activity,pieces) VALUES ("10-2-12", "15","2","11")');
//tx.executeSql('INSERT INTO session (date, length, activity,pieces) VALUES ("11-2-12", "15","2","11")');
//tx.executeSql('INSERT INTO session (date, length, activity,pieces) VALUES ("12-2-12", "15","2","11")');*/
tx.executeSql('INSERT INTO session (date, length, activity,pieces) VALUES ("13-2-12", "15","2","violin")');
tx.executeSql('INSERT INTO pieces (newpiece, name, youtube,images_src,date_Added) VALUES ("true", "15","tube","11",13-9-13)');
//tx.executeSql("DROP TABLE pieces");
//tx.executeSql("DROP TABLE session");
//alert("vi");
}
/**sessions**/
function query(tx){
tx.executeSql('SELECT * FROM session', [], onSelectSessionSuccess, onDBError);
}
function onSelectSessionSuccess(tx,results){
dbResult = results;
var len= results.rows.length;
var sessionList="";
for(var i=0;i<len;i++)
{
sessionList = sessionList+"<li>"+results.rows.item(i).date+"</li>"
}
alert(sessionList);
//tx.executeSql("SELECT * FROM pieces",[],onSelectPiecesSuccess,onDBError());
db.transaction(piecesquery,onDBError);
}
/**pieces**/
function piecesquery(tx){
//alert("piecesquery");
tx.executeSql('SELECT * FROM session', [],onSelectPiecesSuccess, onDBError);
}
function onSelectPiecesSuccess(tx,results){
var len= results.rows.length;
var PiecesList="";
var newPiecesList="";
var res;
alert(len);
for(var i=0;i<len;i++)
{
newPiecesList=newPiecesList+"<li>"+results.rows.item(i).newpiece + results.rows.item(i).composer +"</li>"
}
alert(newPiecesList);
$('#newPiecesList').innerHTML(newPiecesList);
}
my flow:
the tables are created and populated getDB.
session data is pulled on the success of getDb.
pieces data is pulled when the session page initiates. But i can replace the code to pull the pieces table with the code to pull from session table with no problems.
for some reason it seems that the data going into the pieces table isn't accessible
I have done table creation,insert and select process.Hope it will be useful.
var db=null;
document.addEventListener("deviceready", onDeviceReady, false);
function onDeviceReady() {
db = window.openDatabase("TestDatabase","1.0","TestingDatabase",'200000000');
alert("db1");
db.transaction(populateDatabase,errorDb,successDb);
}
function populateDatabase(tx){
tx.executeSql('DROP TABLE IF EXISTS TestTable');
tx.executeSql('CREATE TABLE IF NOT EXISTS TestTable(RollNo INT PRIMARY KEY,FirstName text,LastName text,MobileNo text)');
tx.executeSql('INSERT INTO TESTTABLE (RollNo, FirstName, LastName, MobileNo) VALUES (1, "Nisari", "Balakrishnan", "8891924207")');
tx.executeSql('INSERT INTO TESTTABLE (RollNo, FirstName, LastName, MobileNo) VALUES (2, "Mikhil", "Anandan", "9605432101")');
}
function queryDB(tx) {
tx.executeSql('SELECT * FROM TESTTABLE', [], querySuccess, errorCB);
}
function querySuccess(tx, results) {
var len = results.rows.length;
alert(len);
console.log("DEMO table: " + len + " rows found.");
for (var i=0; i<len; i++){
//console.log("Row = " + i + " ID = " + results.rows.item(i).id + " Data = " + results.rows.item(i).data);
alert("Row = " + i + " ID = " + results.rows.item(i).RollNo + " FirstName = " + results.rows.item(i).FirstName + " LastName = " + results.rows.item(i).LastName + " MobileNo = " + results.rows.item(i).MobileNo);
// db = window.openDatabase("TestDatabase","1.0","TestingDatabase",'200000000');
// db.transaction(updateDB, errorCB);
// console.log("After Open DB");
}
}
function errorCB(err) {
console.log("Error processing SQL: "+err.code);
}
function errorDb()
{
alert("Error on Database creation:" + Error);
}
function successDb()
{
alert("Database is created successfully");
db = window.openDatabase("TestDatabase","1.0","TestingDatabase",'200000000');
db.transaction(queryDB, errorCB);
}
executeSql for SELECT query is an asynchronous command. The 'onSelectSessionSuccess' is the callback function in which you'll receive the result from the table.
Since these are async calls, execution won't wait for the data to be returned by the query.
Place your next SELECT call as a part of the success callback of the previous call.
i.e you should select pieces in the success callback 'onSelectSessionSuccess' and things would work absolutely fine.
Hope that helps.