redis v 4.0.6 is in use.
Is there an error if the crawler connects repeatedly?
My code is as follows,
private setRedisClient() {
const client = createClient({
url: `redis://${process.env.REDIS_URL}:${process.env.REDIS_PORT}`,
isolationPoolOptions: {
max: 10,
min: 0,
maxWaitingClients: 10,
fifo: true,
autostart: true,
idleTimeoutMillis: 30000,
},
});
client.on('error', (e: any) => {
throw e;
});
return client;
}
try {
this.setRedisClient();
await this.client.connect();
const res = await this.client.hGetAll(key);
await this.client.quit();
return res;
} catch (error) {
console.log(error);
return undefined;
}
Repeated refreshing of a page results in an error.
ClientClosedError: The client is closed
at RedisSocket.quit (/home/ec2-user/buyer-service/node_modules/#redis/client/dist/lib/client/socket.js:72:19)
at Commander.QUIT (/home/ec2-user/buyer-service/node_modules/#redis/client/dist/lib/client/index.js:222:71)
at RedisClient.getAll (/home/ec2-user/buyer-service/dist/infra/cache/redis/cache.redis.client.js:31:31)
I am not good at speaking English.
I ask for your help me.
public async getAll(key: string): Promise<any> {
try {
const client = this.setRedisClient();
const [_, hGetAll, _a] = await Promise.all([client.connect(), client.hGetAll(key), client.quit()]);
return hGetAll;
} catch (error) {
console.log(error);
return undefined;
}
}
Related
I am new to Node.js (3 days total experience). I am using Node.js and the tedious package to query a database (azure SQL). I use the example as explained here: https://learn.microsoft.com/en-us/azure/azure-sql/database/connect-query-nodejs?tabs=macos
const connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on("connect", err => {
if (err) {
console.error(err.message);
} else {
console.log("Reading rows from the Table...");
// Read all rows from table
const request = new Request(
"SELECT * FROM clients",
(err, rowCount, columns) => {
if (err) {
console.error(err.message);
} else {
console.log(`${rowCount} row(s) returned`);
}
}
);
request.on("row", columns => {
columns.forEach(column => {
console.log("%s\t%s", column.metadata.colName, column.value);
});
});
connection.execSql(request);
}
});
I have two issues:
I do not know how to get the queried data into an object and
If I run the script it does print the items to the console, but it doesn't close the connection after it has done so. If I add a connection.close() at the bottom, it will close the connection before its done. I get the feeling that node.js executes everything at the same time (I am used to Python..).
Update
I found a way to close the connection, to my understanding the request object has several "events" that are predefined by the library. It seems I need to add the event "done" through request.on('done', ...) in order to make sure that it can even BE done. My updated code looks like this:
var connection = new Connection(config);
connection.connect(function(err) {
// If no error, then good to go...
executeStatement();
}
);
connection.on('debug', function(text) {
//remove commenting below to get full debugging.
//console.log(text);
}
);
function executeStatement() {
request = new Request("SELECT * FROM clients", function(err, rowCount) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
connection.close();
});
request.on('row', function(rows) {
_.forEach(rows, function(value, collection){
console.log(value)
console.log(value.value);
console.log(value.metadata.colName)
console.log(collection)
})
});
request.on('done', function(rowCount, more) {
console.log(rowCount + ' rows returned');
});
// In SQL Server 2000 you may need: connection.execSqlBatch(request);
connection.execSql(request);
}
Anyways, your help would be much appreciated!
Regards
Pieter
The package tedious is synchronous package, it uses the callback to return results. So when we call connection.close(), it will disable connection and stop the callback function. If will want to close the connection, I suggest you use async package to implement it.
For example
const { Connection, Request } = require("tedious");
const async = require("async");
const config = {
authentication: {
options: {
userName: "username", // update me
password: "password", // update me
},
type: "default",
},
server: "your_server.database.windows.net", // update me
options: {
database: "your_database", //update me
encrypt: true,
validateBulkLoadParameters: true,
},
};
const connection = new Connection(config);
let results=[]
function queryDatabase(callback) {
console.log("Reading rows from the Table...");
// Read all rows from table
const request = new Request("SELECT * FROM Person", (err, rowCount) => {
if (err) {
callback(err);
} else {
console.log(`${rowCount} row(s) returned`);
callback(null);
}
});
request.on("row", (columns) => {
let result={}
columns.forEach((column) => {
result[column.metadata.colName]=column.value
console.log("%s\t%s", column.metadata.colName, column.value);
});
// save result into an array
results.push(result)
});
connection.execSql(request);
}
function Complete(err, result) {
if (err) {
callback(err);
} else {
connection.close();
console.log("close connection");
}
}
connection.on("connect", function (err) {
if (err) {
console.log(err);
} else {
console.log("Connected");
// Execute all functions in the array serially
async.waterfall([queryDatabase], Complete);
}
});
connection.connect();
Besides, you also can use the package mssql. It supports asynchronous methods and depends on package tedious. We can directly call close after querying.
For example
const mssql = require("mssql");
const config = {
user: "username",
password: "password",
server: "your_server.database.windows.net",
database: "your_database",
options: {
encrypt: true,
enableArithAbort: true,
},
};
let pool = new mssql.ConnectionPool(config);
async function query() {
try {
await pool.connect();
const request = pool.request();
const result = await request.query("SELECT * FROM Person");
console.dir(result.recordset);
await pool.close();
console.log(pool.connected);
} catch (error) {
throw error;
}
}
query().catch((err) => {
throw err;
});
You can custom a class first and declare an Array to save ojects such as:
let sales = new Array();
class SalesLT{
constructor(catagryName,productName){
this.catagryName = catagryName;
this.productName = productName;
}
Here my sql statement returns 2 properties, so every time the loop takes out two elements from the ColumnValue[].
request.on("row", columns => {
for(let i=0; i<columns.length; i=i+2){
let sale = new SalesLT(columns[i].value,columns[i+1].value);
sales.push(sale);
}
sales.forEach( item => {
console.log("%s\t%s",item.catagryName, item.productName)
})
});
The code is as follows:
const { Connection, Request } = require("tedious");
let sales = new Array();
class SalesLT{
constructor(catagryName,productName){
this.catagryName = catagryName;
this.productName = productName;
}
}
// Create connection to database
const config = {
authentication: {
options: {
userName: "<***>", // update me
password: "<***>" // update me
},
type: "default"
},
server: "<****>.database.windows.net", // update me
options: {
database: "<***>", //update me
encrypt: true
}
};
const connection = new Connection(config);
// Attempt to connect and execute queries if connection goes through
connection.on ("connect", err => {
if (err) {
console.error(err.message);
} else {
queryDatabase();
}
});
function queryDatabase() {
console.log("Reading rows from the Table...");
// Read all rows from table
const request = new Request(
`SELECT TOP 2 pc.Name as CategoryName,
p.name as ProductName
FROM [SalesLT].[ProductCategory] pc
JOIN [SalesLT].[Product] p ON pc.productcategoryid = p.productcategoryid`,
(err, rowCount) => {
if (err) {
console.error(err.message);
} else {
console.log(`${rowCount} row(s) returned`);
}
connection.close();
}
);
request.on("row", columns => {
for(let i=0; i<columns.length; i=i+2){
let sale = new SalesLT(columns[i].value,columns[i+1].value);
sales.push(sale);
}
sales.forEach( item => {
console.log("%s\t%s",item.catagryName, item.productName)
})
});
connection.execSql(request);
}
this article should help you, to solve all the issues you are facing...which were the same I had when I started using Node :)
https://devblogs.microsoft.com/azure-sql/promises-node-tedious-azure-sql-oh-my/
How I can test in jest error case?
This is what I do:
I don't know if exist a method how to test this.
it ('the fetch fails and throw an error', async () => {
let response = {
status: 400,
body:
{
base : "RON",
date: "2019-08-01",
rates: {"error": 'error'}
}
};
fetch.mockReject(response)
try {
await fetchData();
} catch (e) {
expect(e).toEqual(response);
expect(await fetchData()).rejects.toThrow(e);
}
});
This is the code:
fetchData = async () => {
try {
const response = await fetch('https://api.exo/latest?base=RON');
const data = await response.json();
return data;
} catch (e) {
throw e;
}
};
expect.assertions to the rescue
it ('the fetch fails and throw an error', async () => {
expect.assertions(1);
let response = {
status: 400,
body: {
base : "RON",
date: "2019-08-01",
rates: {"error": 'error'}
}
};
fetch.mockReject(response)
try {
await fetchData();
} catch (e) {
expect(e).toEqual(response);
}
});
Test will fail once no exception is thrown. It has advantages over expect().toThrown:
you don't have to return Promise in your it() to make it work
it's easier to assert several related exceptions or sequential actions failed
it's easier to run partial matching over error caught(say with expect(e).toMatchObject({}) to skip some data you don't care about in current test case)
As for disadvantages - you have to update number manually after adding new assertions
You can do it in the following way:
async function throws () {
throw new Error('error')
}
test('promise throws', async () => {
await expect(throws()).rejects.toThrow()
})
test('the fetch fails with an error', async () => {
await expect(throws()).rejects.toThrow('error');
});
test('the fetch fails with an error', () => {
return expect(throws()).rejects.toMatch('error');
});
Read more docs.
I have a problem that I do not know how to solve...
I have an api token saved in AsyncStorage, and when I want do fetch to rest I need this api token, but I do not know how to do it.
I have file Functions.js with AsyncStorage functions.
async retrieveItem(key) {
try {
const retrievedItem = await AsyncStorage.getItem(key);
const item = JSON.parse(retrievedItem);
return item;
} catch (error) {
console.warn(error.message);
}
},
getApiToken: function(){
try {
return Functions.retrieveItem('api_token');
} catch (error) {
console.warn(error.message);
}
},
File with fetch functions. (Api.js)
I tested with an asynchronous function, but not found...
async get(url) {
try {
var api = await Functions.getApiToken();
if (!api)
api = "";
let opt = {
method: 'get',
headers: new Headers({
'Content-Type': 'application/x-www-form-urlencoded'
}),
};
return fetch(baseUrl + url + api, opt);
} catch (error){
console.warn(error);
}
},
When I did the fetch function, it worked for me without problems
And the screen file Home.js
componentDidMount() {
Api.get('home').then( response => {
this.setState({
profiles: response.profiles,
});
});
}
Please modify your Functions.js code a bit -
async retrieveItem(key) {
try {
const retrievedItem = await AsyncStorage.getItem(key);
const item = JSON.parse(retrievedItem);
return item;
} catch (error) {
console.warn(error.message);
}
},
async getApiToken{
try {
let token = await retrieveItem('api_token');
return token;
} catch (error) {
console.warn(error.message);
}
}
Hope this helps you!
I am trying to use a simple suite of functions built utilizing the Tedious library to access a Microsoft SQL Server. Here is my "tools" file:
'use strict';
const tedious = require('tedious');
const q = require('q');
var Connection = tedious.Connection;
var Request = tedious.Request;
module.exports = {
connectSQL : function(config) {
var connection = new Connection(config);
connection.on('connect', function(err) {
if (err) {
console.log('FAIL ON CONNECT');
console.log(err);
} else {
try {
/* ----- */
return connection;
} catch (err) {
console.log(err);
return;
}
}
});
connection.on('error', function(err) {
if (err) {
console.log('FAIL ON ERROR');
console.log(err);
} else {
console.log("Error called with no err object.");
}
});
},
executeSQL: function(connection, requestString) {
var results = [];
var request = new Request( requestString , function(err, data) {
if (err) {
console.log(err);
} else {
console.log( data );
}
});
request.on('row', function(row) {
//console.log(row);
results.push( row );
});
request.on('requestCompleted', function(){
console.log('Finished');
return results;
});
connection.execSql(request);
}
}
I call these functions as follows in my server file.
const sqlTools = require('./sqlTools.js');
var connection = sqlTools.connectSQL(config);
sqlTools.executeSQL(connection, "select * from dbo.test");
However, I get the error "TypeError: Cannot read property 'execSql' of undefined", even if I make the program sleep for 10 seconds before calling my function sqlTools.executeSQL (obviously not ideal).
I was able to get this to work by calling the request within the sqlTools.connectSQL function (at the "/* ----- */"), but I want to re-use the Tedious connection to make multiple calls. Any suggestions? Thanks!
~~~~~~~EDIT~~~~~~~~~~
With help from akinjide I was able to implement callbacks that allow me to make a single call to my SQL database. However, I am struggling to implement promises to make subsequent calls. I changed my "tools" file as such:
'use strict';
const tedious = require('tedious');
const q = require('q');
var Connection = tedious.Connection;
var Request = tedious.Request;
module.exports = {
connectSQL: function(config) {
var deferred = q.defer();
var connection = new Connection(config);
connection.on('connect', function(err) {
if (err) {
deferred.reject( err );
} else {
deferred.resolve( connection );
}
});
connection.on('error', function(err) {
deferred.reject(err);
});
return deferred.promise;
},
executeSQL: function(connection, requestString, callback) {
var results = [];
const request = new Request(requestString, function(err) {
callback(err);
});
request.on('row', function(row) {
results.push(row);
});
request.on('requestCompleted', function() {
console.log('request completed!');
callback(null, results);
});
connection.execSql(request);
}
}
and I call this code like this...
var promise = sqlTools.connectSQL(config);
promise.then(function (connection) {
sqlTools.executeSQL(connection, "select * from dbo.test", function(err, results) {
if (err) {
console.log(err);
}
console.log(results);
});
}).catch(function (err) {
console.log(err);
}).then(function (connection) {
sqlTools.executeSQL(connection, "select * from dbo.test2", function(err, results) {
if (err) {
console.log(err);
}
console.log(results);
});
}).catch(function(err) {
console.log(err);
});
This returns the first call's results correctly, but unfortunately returns this error "TypeError: Cannot read property 'execSql' of undefined" for the second call as it is not recognizing the connection the second time around. Any suggestions?
A better approach would be to pass a node.js callback style function as an argument to connectSQL.
return keyword won't work within an asynchronous program.
'use strict';
const tedious = require('tedious');
const Connection = tedious.Connection;
const Request = tedious.Request;
module.exports = {
connectSQL: function(config, callback) {
const connection = new Connection(config);
connection.on('connect', function(err) {
if (err) {
callback(err);
} else {
callback(null, connection);
}
});
connection.on('error', function (err) {
callback(err);
});
},
executeSQL: function(connection, requestString, callback) {
let results = [];
const request = new Request(requestString, function(err) {
callback(err);
});
request.on('row', function(row) {
results.push(row);
});
request.on('requestCompleted', function(){
console.log('Finished');
callback(null, results);
});
connection.execSql(request);
}
}
Then you can require, use sqlTools.connectSQL passing two parameters config and function(err, connection) {}
const sqlTools = require('./sqlTools');
sqlTools.connectSQL(config, function(err, connection) {
if (err) {
console.log('FAIL ON CONNECT');
console.log(err);
}
sqlTools.executeSQL(connection, "select * from dbo.test", function (err, results) {
if (err) {
console.log(err);
}
console.log(results);
});
});
i am getting error in npm mssql 3.0.0 with sqlserver 2012
i am creating single page application where i used restful using express .
there are 4 method which executing the query and returning the data to response.
for each method i am opening the connection and closing the connection.
but when savedquery is calling then connection close error occurs.
each method code is similar to savedquery method (copy pasted code only queries are changed) but they are executing savedquery is not executing
{ [ConnectionError: Connection is closed.]
name: 'ConnectionError',
message: 'Connection is closed.',
code: 'ECONNCLOSED' }
var savedquery=function(req,res){
dbConfig= {
user: 'XXX',
password: 'XXXXXXXXXX',
server: 'localhost', // You can use 'localhost\\instance' to connect to named instance
database: 'DEMO_ODS',
options: {
encrypt: true
}
};
sql.connect(dbConfig).then(function (err) {
var sqlrequest = new sql.Request();
sqlrequest.query("SELECT * from SavedQuery").then(function (recordset) {
sql.close(function (value) {
console.log("connection6 closed");
});
return res.status(200).send(recordset);
}).catch(function (err) {
console.log(err);
});
}).catch(function (err) {
console.log(err);
});
};
}
I know it is an old questionm but this answer is for the others who are facing the same isue. I had the same problem, What I did is, used promises as below.
function getData() {
try {
sqlInstance.connect(setUp)
.then(function () {
// Function to retrieve all the data - Start
new sqlInstance.Request()
.query("select * from Course")
.then(function (dbData) {
if (dbData == null || dbData.length === 0)
return;
console.dir('All the courses');
console.dir(dbData);
})
.catch(function (error) {
console.dir(error);
});
// Function to retrieve all the data - End
// To retrieve specicfic data - Start
var value = 1;
new sqlInstance.Request()
.input("param", sqlInstance.Int, value)
.query("select * from Course where CourseID = #param")
.then(function (dbData) {
if (dbData == null || dbData.length === 0)
return;
console.dir('Course with ID = 1');
console.dir(dbData);
})
.catch(function (error) {
console.dir(error);
});
// To retrieve specicfic data - End
}).catch(function (error) {
console.dir(error);
});
} catch (error) {
console.dir(error);
}
}
This solved my issue. You can find the fix here.
You should remove
options: {
encrypt: true
}
from your dbConfig
I just use promise to handle concurrent request:
const executeQuery = function (res, query, dbName) {
dbConfig = {
user: "********",
password: "********",
server: "*******",
database: dbName
}
sql.connect(dbConfig).then(pool => {
return pool.request().query(query)
}).then(result => {
res.send(result);
}).catch(err => {
res.send(err);
});
}
Hope it's help someone.