I need to automatically generate the routes on an expressjs app, based on the vimeo api
I thought that I need to loop through the api data, save the data in the db and then retrieve that data in a middleware. For example:
Api request:
const Vimeo = require("vimeo").Vimeo;
let client = new Vimeo("CLIENT_ID", "CLIENT_SECRET", "TOKEN");
client.request(
{
method: "GET",
path: "/my/path/videos"
},
function(error, body, status_code, headers) {
if (error) {
console.log(error);
}
let data = body.data;
for (var i = 0; i < data.length; i++) {
// save data in the db
}
});
Middleware:
app.use('/videos/:name', (req, res, next) {
if (req.params.name === myDBdata) {
console.log('It works!');
next();
} else {
// error code
}
});
Is this a good way to proceed? Thanks in advance
Make a function which takes two paremter like:
function makeRoute(path, handler) {
return app.use(path, handler)
}
And then call this for every data
makeRoute('test', (req, res) => { })
Related
I'm trying to build a scraping script to get a bunch of Discord server's total members. I actually did that with Puppeteer like below but I think my IP address has been banned because I'm getting "Invite Invalid" error from Discord even though invite links are working.
My question is that does Discord have APIs to get any server's total member count? Or is there any 3rd party library for that purpose? Or any other method?
const puppeteer = require('puppeteer')
const discordMembers = async ({ server, browser }) => {
if (!server) return
let totalMembers
const page = await browser.newPage()
try {
await page.goto(`https://discord.com/invite/${server}`, {
timeout: 3000
})
const selector = '.pill-qMtBTq'
await page.waitForSelector(selector, {
timeout: 3000
})
const totalMembersContent = await page.evaluate(selector => {
return document.querySelectorAll(selector)[1].textContent
}, selector)
if (totalMembersContent) {
totalMembers = totalMembersContent
.replace(/ Members/, '')
.replace(/,/g, '')
totalMembers = parseInt(totalMembers)
}
} catch (err) {
console.log(err.message)
}
await page.close()
if (totalMembers) return totalMembers
}
const asyncForEach = async (array, callback) => {
for (let i = 0; i < array.length; i++) {
await callback(array[i], i, array)
}
}
const run = async () => {
const browser = await puppeteer.launch({
headless: true,
args: ['--no-sandbox']
})
const servers = ['tQp4pSE', '3P5K3dzgdB']
await asyncForEach(servers, async server => {
const members = await discordMembers({ server, browser })
console.log({ server, members })
// result
// { server: 'tQp4pSE', members: 57600 }
// { server: '3P5K3dzgdB', members: 159106 }
})
await browser.close()
}
run()
Update: Mar 22, 2022
Thanks for #Vaviloff's answer we can actually access Discord's private APIs but the problem is it's only accessible over browser. I'm getting Request failed with status code 400 issue from Axios. Is it a CORS issue? How do we get the results in a Node.js app?
const axios = require('axios')
const discordMembers = async ({ server }) => {
try {
const apiResult = await axios({
data: {},
method: 'get',
url: `https://discord.com/api/v9/invites/${server}?with_counts=true&with_expiration=true`
})
console.log(apiResult)
} catch (err) {
console.log(err)
}
}
discordMembers({ server: 'tQp4pSE' })
A lot of modern web applications have their own internal APIs. Oftentimes you can spot frontend making requests to it, by using Networking tab in Devtools (filter by Fetch/XHR type):
Such API endpoints can change any time of course, but usually the last for a long time and is a rather convenient way of scraping
Currently Discord uses this URL for basic instance description:
https://discord.com/api/v9/invites/tQp4pSE?with_counts=true&with_expiration=true
By accessing it you get the desired data:
Update
To make your code work don't send any data in the request:
const apiResult = await axios({
method: 'get',
url: `https://discord.com/api/v9/invites/${server}?with_counts=true&with_expiration=true`
})
Sending a logout request to my server but I'm never getting a reply. The logout function is being called and the userID key is being deleted from my redis cache but I never get a response. Here's my code.
export const logout = async (req, res) => {
console.log("logout called");
const { userID } = req.user;
client.del(userID.toString, (err, reply) => {
console.log("inside client.del");
if (err) {
return res.status(500);
} else {
return res.status(200);
}
});
};
Because of callback, you should use promise
export const logout = async (req, res) => {
return new Promise((resolve,reject) => {
console.log("logout called");
const { userID } = req.user;
client.del(userID.toString, (err, reply) => {
console.log("inside client.del");
if (err) {
reject(res.status(500));
} else {
resolve(res.status(200));
}
});
});
}
res.status() does not send a response from the server. All it does is set the status as a property on the response object that will go with some future call that actually sends the response.
It is meant to be used in something like this:
res.status(500).send("Database error");
If you look at the Express doc for res.status(), you will see these examples:
res.status(403).end()
res.status(400).send('Bad Request')
res.status(404).sendFile('/absolute/path/to/404.png')
And, see that they all are followed by some other method that actually causes the response to be sent.
And, if you still had any doubt, you can look in the Express code repository and see this:
res.status = function status(code) {
this.statusCode = code;
return this;
};
Which shows that it's just setting a property on the response object and not actually sending the response yet.
You can use res.sendStatus() instead which will BOTH set the status and send the response:
export const logout = (req, res) => {
console.log("logout called");
const { userID } = req.user;
client.del(userID.toString, (err, reply) => {
console.log("inside client.del");
if (err) {
res.sendStatus(500);
} else {
res.sendStatus(200);
}
});
};
Note, I removed the two return keywords since they don't accomplish anything useful in this particular context.
I also removed the async keyword from the function definition since it was not doing anything useful in this context.
I'm trying to modify a graphql query variable using express-gateway.
The code on the gateway is as below,
const axios = require("axios");
const jsonParser = require("express").json();
const { PassThrough } = require("stream");
module.exports = {
name: 'gql-transform',
schema: {
... // removed for brevity sakes
},
policy: (actionParams) => {
return (req, res, next) => {
req.egContext.requestStream = new PassThrough();
req.pipe(req.egContext.requestStream);
return jsonParser(req, res, () => {
req.body = JSON.stringify({
...req.body,
variables: {
...req.body.variables,
clientID: '1234'
}
});
console.log(req.body); // "clientID": "1234" is logged in the body.variables successfully here
return next();
});
};
}
};
Now, when I hit the request from POSTMAN, the request goes through and returns a 200OK only when I include clientID, otherwise, it throws as error
"message": "Variable "$clientID" of required type "ID!" was not provided."
Any idea what could be going wrong here?
The only way I could get this working was by using node-fetch and then making a fetch request to the graphql-sever from my middleware instead of doing a return next() and following the middleware chain.
My setup is something like the following,
Client (vue.js w/ apollo-client) ---> Gateway (express-gateway) ---> Graphql (apollo-server) ---> Backend REST API (*)
When my client makes a graphql request to my gateway, I've modified my middleware to do the following (as opposed to what's in the question),
const jsonParser = require("express").json();
const fetch = require('node-fetch');
module.exports = {
name: 'gql-transform',
schema: {
... // removed for brevity sakes
},
policy: () => {
return (req, res) => {
jsonParser(req, res, async () => {
try {
const response = await fetch(`${host}/graphql`, {...}) // removed config from fetch for brevity
res.send(response);
} catch (error) {
res.send({ error });
}
});
};
}
};
I am trying to build a small website. In that i using React for frontend, Nodejs for backend, and some third party api. Here my idea is, first to post the form data to nodejs. And from then i accepting that data in node and need to call an external api. For this purpose i am using axios. After receiving values from my api i have to send that value back to react application. And when i run my code in postman, the output is {}. I think that i am not getting values from my api but dont know how to resolve this. And i am new to these technologies. Someone pls help me to sort out this problem. Thanking you in advance. Here is my what i have tried so far.
const express = require('express');
const axios = require('axios');
const router = express.Router();
const request = require('request');
const app = express();
app.use(express.json());
app.use(express.urlencoded({ extended : false}));
router.get('/', (req, res) => {
res.send(" Express Homepage is running...");
});
async function callApi(emailid, pswd) {
return axios({
method:'post',
url:'http://51.X.X/api/login',
data: {
"email": `${emailid}`,
"password": `${pswd}`
},
headers: {'Content-Type': 'application/json' }
})};
callApi().then(function(response){
return response.data;
})
.catch(function(error){
console.log(error);
})
app.post('/api/login', (req, res) => {
let emailid = String(req.body.email);
let pswd = String(req.body.password);
const data = callApi(emailid, pswd);
if(data) {
res.send(data);
}else {
res.json({msg : " Response data not recieved.."})
}
});
use async/await syntax to handle asynchronous calls
app.post('/api/login', async (req, res) => {
let emailid = String(req.body.email);
let pswd = String(req.body.password);
const data = await callApi(emailid, pswd);
if(data) {
res.send(data);
}else {
res.json({msg : " Response data not recieved.."})
}
});
The problem is you are not waiting for async call to finish.
use async-await as mentioned in official doc https://www.npmjs.com/package/axios
function callAPI(){
const response = await axios({
method:'post',
url:'http://51.X.X/api/login',
data: {
"email": `${emailid}`,
"password": `${pswd}`
},
headers: {'Content-Type': 'application/json' }
})};
return response
}
app.post('/api/login', async (req, res) => {
let emailid = String(req.body.email);
let pswd = String(req.body.password);
//add try catch to catch exception
const data = await callApi(emailid, pswd);
if(data) {
//check for response from axios in official doc and send what data you
want to send
res.send(data);
}else {
res.json({msg : " Response data not recieved.."})
}
});
I have an API that returns some JSON from mongodb:
// In router.js
var api = require('api')
app.get('/lists', api.lists);
// In api.js
var db = require('db')
exports.lists = function(req, res) {
db.lists.find({}, function(err, lists) {
res.send(lists);
});
};
Is there a way to design the API so that I could also consume it from within my Node.js app? I'm trying to avoid having to duplicate any of the database code outside the API. I basically have a controller that can render the data server-side:
// In controller.js
var api = require('api')
exports.page = function(req, res) {
res.send(api.lists()); // This won't work
};
I found a hacky solution which was to pass a callback function to the api.lists(), but I have a feeling this is the "wrong" way to achieve this:
// In api.js
exports.lists = function(req, res, callback) {
db.lists.find({}, function(err, lists) {
if(callback){
callback(lists);
} else {
res.send(lists);
}
});
};
Any ideas?
I think the problem is that in your current code you are coupling your API to the response object. You can decouple them with something like this:
In router.js instead of using api.lists as the callback, define a function that will call api.lists with a callback that is wired to the response object. In this case api.list DOES NOT need to know about the response object but the function that we are creating does.
// In router.js
var api = require('api');
app.get('/lists', function(req, res) {
api.lists(function(err, lists) {
if(err) {
res.send('error page');
return;
}
res.send(lists);
});
});
In api.js we remove the reference to the response object. Now it will just call whatever callback it received with the appropriate data (err + lists). It's up to the callback to do whatever it pleases with the result.
// In api.js
var db = require('db')
exports.lists = function(callback) {
db.lists.find({}, function(err, lists) {
callback(err, lists);
});
};