Steps to reproduce
My environment:
Puppeteer version: 1.0.0
Platform / OS version: Linux / centos7.0
URLs (if applicable): http://1255.ijkuek.cn/
Node.js version: v9.4.0
What steps will reproduce the problem?
const puppeteer = require('puppeteer');
var args = process.argv.splice(2)
var url = args[0]
var ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0';
puppeteer.launch({
ignoreHTTPSErrors:true,
timeout: 1000,
args: ['--no-sandbox', '--disable-setuid-sandbox']}).then(async browser => {
const page = await browser.newPage();
await page.setExtraHTTPHeaders({
'upgrade-insecure-requests': '1'
});
page.setUserAgent(ua)
page.setDefaultNavigationTimeout(25000)
await page.setRequestInterception(true)
page.on('request', (request) => {
var type = request.resourceType()
if (type == 'image' || type == 'media')
request.abort();
else{
console.log("request: " + request.url())
request.continue();
}
});
page.on('response', (response) => {
console.log('response: ' + response.url())
if(type == 'document'){
response.text().then(function (textBody) {
console.log(textBody)
})
}
});
const response = await page.goto(url, {
waitUntil: 'networkidle2',
})
.catch(function(err){ if(err.toString().indexOf("Timeout")) {
browser.close();
console.log("Timeout!")
process.exit();
}})
browser.close();
});
What is the expected result?
the right redirect frame content
What happens instead?
the result is either timeout（to set timeout number larger useless ）or redirect to wrong url，finally，can't get the final content。but phantomjs can get it！
Related
Revising this question as I have made some progress and passport-twitter authentication is working mostly: VueJS app with Express/Knex/Postgresql backend.
The problem is that I am combining passport-twitter with my previous usage of passport-local using jwt. As a result, I have to generate a header token for the new Twitter user. Currently on logging in with Twitter, the header token is not being read. Without a valid token the app does crazy, time consuming things.
here is the header object from the initial Twitter login request, showing token as an undefined Object:
headers:
{ host: 'localhost:8000',
connection: 'keep-alive',
accept: 'application/json, text/plain, */*',
origin: 'http://localhost:8080',
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
token: '[object Object]',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
referer:
'http://localhost:8080/?user=%7B%22id%22%3A10,%22email%22%3Anull,%22token%22%3A%22eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6bnVsbCwiaWQiOjEwLCJleHAiOjE1NzYwNTU1MzYsImlhdCI6MTU3MDg2NzkzNn0.SUYL2KcnK8C9zYNLCcqGF99TjfIRxAcyg0Uc8WjQJD0%22%7D',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9',
'if-none-match': 'W/"79-a0G25cV4V6cfQoTIFr1X/nYU2Kg"' },
On refreshing the browser, the header token appears as expected and errors are gone.
headers:
{ host: 'localhost:8000',
connection: 'keep-alive',
accept: 'application/json, text/plain, */*',
origin: 'http://localhost:8080',
'user-agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36',
token:
'Token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6bnVsbCwiaWQiOjEwLCJleHAiOjE1NzYwNTU1MzYsImlhdCI6MTU3MDg2NzkzNn0.SUYL2KcnK8C9zYNLCcqGF99TjfIRxAcyg0Uc8WjQJD0',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-site',
referer:
'http://localhost:8080/?user=%7B%22id%22%3A10,%22email%22%3Anull,%22token%22%3A%22eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJlbWFpbCI6bnVsbCwiaWQiOjEwLCJleHAiOjE1NzYwNTU1MzYsImlhdCI6MTU3MDg2NzkzNn0.SUYL2KcnK8C9zYNLCcqGF99TjfIRxAcyg0Uc8WjQJD0%22%7D',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9',
'if-none-match': 'W/"79-a0G25cV4V6cfQoTIFr1X/nYU2Kg"' }
Here are the relevant files in Express:
authenticate.js is middleware, I am using to get the token from the header:
const jwt = require('express-jwt');
const getTokenFromHeaders = (req) => {
console.log('req from headers is: ', req)
const { headers: { token } } = req; // same as token = req.headers.token
if(token && token.split(' ')[0] === 'Token') {
return token.split(' ')[1];
}
return null;
};
const auth = {
required: jwt({
secret: 'secret',
userProperty: 'user',
getToken: getTokenFromHeaders,
}),
optional: jwt({
secret: 'secret',
userProperty: 'user',
getToken: getTokenFromHeaders,
credentialsRequired: false,
}),
};
module.exports = auth;
app.js:
const express = require('express')
const bodyParser = require('body-parser')
const path = require('path')
const dotenv = require('dotenv')
const Knex = require('knex')
const { Model } = require('objection')
const cors = require('cors');
const pathfinderUI = require('pathfinder-ui');
const session = require('express-session');
const passport = require('passport')
dotenv.config()
const app = express()
// objection setup
const knexfile = require('../knexfile')
const knex = Knex(knexfile[process.env.NODE_ENV])
Model.knex(knex)
//Configure our app
app.use('/pathfinder', function (req, res, next) {
pathfinderUI(app)
next()
}, pathfinderUI.router);
var corsOptions = {
origin: '*',
optionsSuccessStatus: 200 // some legacy browsers (IE11, various SmartTVs) choke on 204
}
app.use(cors(corsOptions));
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
var sess = {
secret: 'keyboard cat',
cookie: {}
}
if (app.get('env') === 'production') {
app.set('trust proxy', 1) // trust first proxy
sess.cookie.secure = true // serve secure cookies
}
app.use(session(sess))
app.use(passport.initialize());
app.use(passport.session());
require('./config/passport');
// api routes
// should we refactor this?
app.use("/api/auth", require("./routes/auth"))
app.use("/api/posts", require("./routes/posts"))
app.use("/api/acts", require("./routes/acts"))
app.use("/api/reports", require("./routes/reports"))
app.use("/api/groups", require("./routes/groups"))
app.use("/api/users", require("./routes/users"))
app.use("/api/memberships", require("./routes/memberships"))
app.use("/api/kudos", require("./routes/kudos"))
app.get('/', (req, res) => {
res.sendFile(path.join(__dirname, "index.html"))
})
module.exports = app
passport.js:
const passport = require('passport');
const LocalStrategy = require('passport-local');
const TwitterStrategy = require('passport-twitter').Strategy
const authHelpers = require('../auth/helpers');
var port = process.env.PORT || 8000;
const User = require('../models/User');
var trustProxy = false;
if (process.env.DYNO) {
// Apps on heroku are behind a trusted proxy
trustProxy = true;
}
passport.use(new LocalStrategy({
usernameField: 'user[email]',
passwordField: 'user[password]',
}, async (email, password, done) => {
const user = await User.query().where('email', email );
if (user && user.length === 1 && authHelpers.comparePass(password, user[0].password)) {
return done(null, user[0]);
} else {
return done(null, false, { errors: { 'email or password': 'is invalid' } });
}
}));
passport.use(new TwitterStrategy({
consumerKey: process.env.TWITTER_CONSUMER_KEY,
consumerSecret: process.env.TWITTER_SECRET_KEY,
callbackURL: process.env.TWITTER_CALLBACK_URL, //this will need to be dealt with
proxy: trustProxy
}, async function(token, tokenSecret, profile, done) {
let user = await User.query().findOne({twitter_id: profile.id_str})
console.log('hitting the passport twitter strategy')
if (user) {
// todo: update user with twitter profile
return done(null, user);
} else {
user = await authHelpers.createTwitterUser(profile);
console.log("The User from createTwitterUser is " + user);
return done(null, user);
}
}));
auth.js:
const express = require('express');
const User = require('../models/User');
const auth = require('../middlewares/authenticate');
const passport = require('passport');
const authHelpers = require('../auth/helpers')
let router = express.Router();
...
router.get('/twitter',
passport.authenticate('twitter')
);
router.get('/twitter/callback',
passport.authenticate('twitter', { failureRedirect: process.env.VUE_LOGIN_URL}),
function(req, res) {
// Successful authentication, redirect home.
const user = JSON.stringify(req.user);
console.log('hittingthe callback routesending stringified user: ', user);
res.redirect(process.env.VUE_HOME_URL + '?user=' + user);
});
module.exports = router;
helpers.js:
const bcrypt = require('bcryptjs');
const User = require('../models/User');
const jwt = require('jsonwebtoken');
...
async function createTwitterUser(profile) {
let newUser = await User.query()
.insert({
username: profile.screen_name,
bio: profile.description,
location: profile.location,
avatar: profile.profile_image_url_https,
twitter_id: profile.id_str //change to int
})
.returning('*');
console.log("new User is: ", newUser);
return newUser;
}
...
module.exports = {
comparePass,
createUser,
toAuthJSON
};
Hello i am creating an express service that prints to a pos printer at http request. The problem is that my solution does not print images on the first print but it does at the second and so on.
I am using https://github.com/song940/node-escpos
Here is my server.js
var print = require('./print')
var express = require('express')
var cors = require('cors')
var bodyParser = require('body-parser');
var app = express();
var corsOptions = {
origin: '*',
optionsSuccessStatus: 200 // some legacy browsers (IE11, various SmartTVs) choke on 204
}
app.get('/print', cors(corsOptions), (req, res) => {
print.demoPrint();
return res.send(req.body);
});
app.listen(3000, () => {
console.log('Express server is started')
});
Here is my print.js
const escpos = require('escpos');
const device = new escpos.USB(0x1504,0x003d);
const options = { encoding: "GB18030"};
const printer = new escpos.Printer(device);
const fs = require('fs')
const printSettings = JSON.parse(fs.readFileSync('printConfig.json', 'utf8'));
exports.demoPrint = () => {
device.open(function() {
printSettings.forEach((currentLine) => {
if(currentLine.printType === "text") {
console.log("PRINTING TEXT");
printer
.font(currentLine.font)
.align(currentLine.align)
.style('bu')
.size(currentLine.size, currentLine.size)
.text(currentLine.text)
}
else if(currentLine.printType === "image") {
escpos.Image.load(currentLine.path, (image) => {
console.log("PRINTING IMAGE");
printer
.align(currentLine.align)
.size(currentLine.size, currentLine.size)
.image(image)
});
}
else if(currentLine.printType === "barcode") {
console.log("PRINTING BARCODE");
printer
.align(currentLine.align)
.barcode(currentLine.text, 'CODE39', {
height: 64,
font: 'B'
});
}
});
printer
.text('\n')
.text('\n')
.cut()
.close();
});
};
Since this hasn't been answered yet I will provide the solution that worked for me. The problem was that the images were starting to load on the first time I submitted a request. By the second time I submitted a request the images were already loaded and were successfully printed.
I solved the problem by a adding a check that would not allow to print until the images were loaded.
Once I submit the form (either by form.submit() or element.click()), the page reloads but, is not redirected to the authenticated side of the site. It's simply the same page as the authentication failed page. The difference though is that, the new rendered page has the password removed from the password field.
var page = require('webpage').create();
page.onUrlChanged = function(url) {
console.log('Change to: ' + url);
};
page.onResourceError = function(resourceError) {
page.reason = resourceError.errorString;
page.reason_url = resourceError.url;
};
page.settings.userAgent = 'Mozilla/46.0.1 (X11; Linux x86_64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
page.viewportSize = { width: 1280, height: 1024 };
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.open('https://www.some site/default.asp',function(status) {
console.log("Status: " + status);
if(status === "success") {
page.evaluate(function() {
console.log("set values in a aform");
var x=document.getElementsByTagName("input");
x[8].value="blaa";
x[9].value="blaa";
x[10].value="blaa";
x[11].click();
});
window.setTimeout(function () {
phantom.exit();
}, 9000);
page.render('bsplogin1.png');
}
else{
console.log( "page not open!" );
phantom.exit( 0 );
}});
i'm very new of Express and server side in general, for my small react messaging app i'm using Express for the backend(a json file) and webpack dev server for frontend.
If i start just the Webpack server, the app show correctly, probelems come when i try to make the two works together and i think the problem is the proxy configuration of webpack-dev-server.
Here is my express server, json-api-server.js:
var express = require('express');
var serveStatic = require('serve-static');
var fs = require('fs');
var path = require('path');
var bodyParser = require('body-parser');
module.exports = (PORT) => {
const MESSAGES_FILE = path.join(__dirname, 'src/app/data/messages.json');
const app = express();
app.use(serveStatic(__dirname + '/build'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
// Additional middleware which will set headers that we need on each request.
app.use(function(req, res, next) {
// Set permissive CORS header - this allows this server to be used only as
// an API server in conjunction with something like webpack-dev-server.
res.setHeader('Access-Control-Allow-Origin', '*');
// Disable caching so we'll always get the latest comments.
res.setHeader('Cache-Control', 'no-cache');
next();
});
app.get('/messages', function(req, res) {
fs.readFile(MESSAGES_FILE, function(err, data) {
if (err) {
console.error(err);
process.exit(1);
}
res.json(JSON.parse(data));
});
});
app.post('/messages', function(req, res) {
fs.readFile(MESSAGES_FILE, function(err, data) {
if (err) {
console.error(err);
process.exit(1);
}
var messages = JSON.parse(data);
var newMessage = {
id: Date.now(),
body: req.body.body,
date: req.body.date,
from: req.body.from,
to: req.body.to
};
messages.push(newMessage);
fs.writeFile(MESSAGES_FILE, JSON.stringify(messages, null, 4), function(err) {
if (err) {
console.error(err);
process.exit(1);
}
res.json(messages);
});
});
});
app.listen(PORT, function (err) {
if (err) {
return console.log(err);
}
console.log('Listening at' + PORT );
});
}
This is webpack-server.js:
var webpack = require('webpack');
var WebpackDevServer = require('webpack-dev-server');
var config = require('./webpack.config');
module.exports = (PORT) => {
const backendPort = PORT - 1;
const server = new WebpackDevServer(webpack(config), {
publicPath: config.output.publicPath,
hot: true,
historyApiFallback: true,
proxy: {
'*' : {
target: 'http://localhost:' + backendPort
}
}
});
server.listen(PORT, 'localhost', function (err) {
if (err) {
return console.log(err);
}
console.log('Listening at ' + PORT);
});
}
And here is server.js:
var apiServer = require('./json-api-server');
var webpackServer = require('./webpack-server');
const PORT = process.env.PORT || 4001;
const PROD = process.env.NODE_ENV === 'production';
if (PROD) {
apiServer(PORT);
} else {
apiServer(PORT - 1);
webpackServer(PORT);
}
My filetree looks like:
--- /
----- server.js
----- webpack-server.js
----- json-api-server.js
----- src/
------- app/index.js
------- app/data/
-------------- messages.json
Both server start correctly and they don't give any errors in the terminal
I can reach localhost:4000/messages
I cannot reach localhost:4001. I got: "Can't get / "
Any helps? :)
Got it!
The * symbol here means: Use "*" to proxy all paths to the specified server.
proxy: {
'*' : {
target: 'http://localhost:' + backendPort
}
}
Which brings my index to be undefined.
So i have to point the key to the place where my json(or api) lives:
proxy: {
'/messages' : {
target: 'http://localhost:' + backendPort
}
}
As reference, this is also a valid solution:
proxy: {
'/messages': 'http://localhost:' + backendPort
}
I have a CasperJS test suite that needs to verify an image onload event. To test this I have a 1x1 pixel gif image, which I serve using PhantomJS webserver:
var fs = require('fs');
var webserver = require('webserver');
casper.test.begin('Image onload should be invoked', 1, {
setUp: function() {
var server = webserver.create();
this.server = server.listen(8080, function(request, response) {
if (request.url == '/') {
response.writeHead(200, { 'Content-Type': 'text/html' });
response.write('' +
'<!doctype html>\n' +
'<html>\n' +
'<head>\n' +
'<script type="text/javascript">\n' +
'window.onload = function() {\n' +
'var px = document.createElement("img");\n' +
'px.onload = function() {\n' +
'window._pxLoad = true;\n' +
'};\n' +
'px.src = "px.gif";\n' +
'};\n' +
'</script>\n' +
'</head>\n' +
'<body></body>\n' +
'</html>' +
'');
} else if (request.url.match(/px\.gif$/i)) {
response.writeHead(200, {
'Content-Type': 'image/gif',
'Cache-Control': 'no-cache'
});
var filePath = fs.workingDirectory + request.url.split('/').join(fs.separator).replace(/\d/gi, '');
response.write(fs.read(filePath));
}
response.close();
});
},
tearDown: function() {
this.server.close();
},
test: function(test) {
casper.start('http://localhost:8080', function() {
this.waitFor(function() {
return this.evaluate(function () {
return window._pxLoad !== undefined;
});
}, function then() {
var flag = this.evaluate(function () {
return window._pxLoad;
});
test.assertTruthy(flag, 'Image has been successfully loaded');
});
});
casper.run(function () {
test.done();
});
}
});
This test fails because window._pxLoad !== undefined did not evaluate within the 5000ms timeout. I even placed console.log statements inside image handler and they showed that my routing works, server do receive this /px.gif call, but looks like git image isn't served at all.
I tried to replace the call to local px.gif with the similar image from the Internet, this one, and test passed! So the problem is definitely relates to how local gif image is served by PhantomJS webserver.
Ok, looks like I've found an answer by myself. Well, sort of.
First of all, I couldn't make my PhantomJS webserver solution work. So I created a simple Node.js script which runs a webserver. It is spawned as a child process before running test suite:
img_server.js
var http = require('http');
var fs = require('fs');
var path = require('path');
var argv = process.argv;
var port = argv.length > 3 ? parseInt(argv[3], 10) || 8124;
http.createServer(function(req, res) {
var fileStream = fs.createReadStream(path.join(__dirname, 'px.gif'));
res.writeHead(200, {
'Content-Type': 'image/gif',
'Cache-Control': 'no-cache'
});
fileStream.pipe(res);
}).listen(port);
// This is important. Script should put something in the STDOUT when started.
// The CasperJS test suite will bind to this inside setUp hook to know when server is ready
console.log('Server running at http://127.0.0.1:' + port + '/');
Changes to CasperJS test suite:
var fs = require('fs');
var webserver = require('webserver');
var process = require('child_process');
var spawn = process.spawn;
casper.test.setUp(function(done) {
this.imgServerChild = spawn('node', ['img_server.js', '-p', '8180']);
// This where STDOUT from server script is used
this.imgServerChild.stdout.on("data", done);
});
casper.test.tearDown(function() {
this.imgServerChild.kill('SIGKILL');
});
// The rest of test suite
Of course I also changed the path to the image file inside the faked HTML output. Now image is served from different domain, which is totally fine for me. Now tests are working.