When a page have meta refresh redirect, I can't get the content of page it was redirected to. Even if I use setTimeout and wait 10 seconds to be sure that the last page is loaded, when I do page.content I get the content of the page before redirect.
Is there is something I can do about it?
Here is my code:
var mainUrl = "http://t.mobitrk.com/?a=t&aff_id=233&tags=df0a14a1-e673-4082-80c1-5283f4daa11cx&o_id=4879&sub_id=34753&sub_param1=xapnt20iix";
function makeRequest(url) {
var page = require('webpage').create();
page.settings.userAgent = "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 5 Build/LMY48I) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36";
page.onNavigationRequested = function(url, type, willNavigate, main) {
console.log(url);
}
page.open(url, function (status) {
setTimeout(function() {
console.log(page.content);
phantom.exit(0);
}, 10000);
});
}
makeRequest(mainUrl);
The result is:
http://t.mobitrk.com/?a=t&aff_id=233&tags=df0a14a1-e673-4082-80c1-5283f4daa11cx&o_id=4879&sub_id=34753&sub_param1=xapnt20iix
http://app.appsflyer.com/com.groboot.yala?af_prt=fatal2&pid=clicksmob_int&clickid=cB_1448464013_76df1e4d78bf14572c7fe12cafb39f4d_wP&af_siteid=233&advertising_id=&af_sub1=34753
market://details?id=com.groboot.yala&referrer=af_tranid%3DC8KDGS5KXJ5BEQQ%26clickid%3DcB_1448464013_76df1e4d78bf14572c7fe12cafb39f4d_wP%26af_prt%3Dfatal2%26af_siteid%3D233%26pid%3Dclicksmob_int%26af_sub1%3D34753%26advertising_id%3D
<!DOCTYPE html><html><head><meta http-equiv="refresh" content="0;URL=http://app.appsflyer.com/com.groboot.yala?af_prt=fatal2&pid=clicksmob_int&clickid=cB_1448464013_76df1e4d78bf14572c7fe12cafb39f4d_wP&af_siteid=233&advertising_id=&af_sub1=34753"></head><body></body></html>
As you can see, the page.content contains the content of the first link.
Related
I am trying to save all the incoming request and their response into the database.
I am trying to achieve this using express-winston npm package and it is logging response body and request in console. but I wanna save it in database. how can I do this????
var winston = require("winston");
var expressWinston = require("express-winston");
expressWinston.requestWhitelist.push("body");
// expressWinston.responseWhitelist.push("body");
app.use(
expressWinston.logger({
transports: [
new winston.transports.Console({
json: true,
colorize: true
})
]
})
);
expressWinston.responseWhitelist.push("body");
I am getting this log in console.
{"level":"info","message":"HTTP GET /ideas/category/hello","meta":{"res":{"statusCode":200,"body":{"hiiii":"djdjjdjdjdd"}},"req":{"url":"/ideas/category/hello","headers":{"host":"localhost:5000","connection":"keep-alive","upgrade-insecure-requests":"1","user-agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36","accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3","accept-encoding":"gzip, deflate, br","accept-language":"en-GB,en;q=0.9,en-US;q=0.8,hi;q=0.7","cookie":"connect.sid=s%3A90PCwDOrv0v9yeSYZq-4I4Quactbcduq.8EpLenv6t1lCPqLcV5R1o1%2FF15p9kEBqrZ82FBHiYJA"},"method":"GET","httpVersion":"1.1","originalUrl":"/ideas/category/hello","query":{}},"responseTime":2}}
After doing some research I got my answer. we can achieve this using winston-sql-transport npm package.see the code given below:
const { Logger } = require('winston');
const { SQLTransport } = require('./../lib/winston-sql-transport');
const logger = new Logger({
transports: [
new SQLTransport({
tableName: 'winston_logs',
})]
});
module.exports = logger;
Steps to reproduce
My environment:
Puppeteer version: 1.0.0
Platform / OS version: Linux / centos7.0
URLs (if applicable): http://1255.ijkuek.cn/
Node.js version: v9.4.0
What steps will reproduce the problem?
const puppeteer = require('puppeteer');
var args = process.argv.splice(2)
var url = args[0]
var ua = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0';
puppeteer.launch({
ignoreHTTPSErrors:true,
timeout: 1000,
args: ['--no-sandbox', '--disable-setuid-sandbox']}).then(async browser => {
const page = await browser.newPage();
await page.setExtraHTTPHeaders({
'upgrade-insecure-requests': '1'
});
page.setUserAgent(ua)
page.setDefaultNavigationTimeout(25000)
await page.setRequestInterception(true)
page.on('request', (request) => {
var type = request.resourceType()
if (type == 'image' || type == 'media')
request.abort();
else{
console.log("request: " + request.url())
request.continue();
}
});
page.on('response', (response) => {
console.log('response: ' + response.url())
if(type == 'document'){
response.text().then(function (textBody) {
console.log(textBody)
})
}
});
const response = await page.goto(url, {
waitUntil: 'networkidle2',
})
.catch(function(err){ if(err.toString().indexOf("Timeout")) {
browser.close();
console.log("Timeout!")
process.exit();
}})
browser.close();
});
What is the expected result?
the right redirect frame content
What happens instead?
the result is either timeout（to set timeout number larger useless ）or redirect to wrong url，finally，can't get the final content。but phantomjs can get it！
Once I submit the form (either by form.submit() or element.click()), the page reloads but, is not redirected to the authenticated side of the site. It's simply the same page as the authentication failed page. The difference though is that, the new rendered page has the password removed from the password field.
var page = require('webpage').create();
page.onUrlChanged = function(url) {
console.log('Change to: ' + url);
};
page.onResourceError = function(resourceError) {
page.reason = resourceError.errorString;
page.reason_url = resourceError.url;
};
page.settings.userAgent = 'Mozilla/46.0.1 (X11; Linux x86_64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36'
page.viewportSize = { width: 1280, height: 1024 };
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.open('https://www.some site/default.asp',function(status) {
console.log("Status: " + status);
if(status === "success") {
page.evaluate(function() {
console.log("set values in a aform");
var x=document.getElementsByTagName("input");
x[8].value="blaa";
x[9].value="blaa";
x[10].value="blaa";
x[11].click();
});
window.setTimeout(function () {
phantom.exit();
}, 9000);
page.render('bsplogin1.png');
}
else{
console.log( "page not open!" );
phantom.exit( 0 );
}});
From within an app, I need to change a proxy to use the GET method to read data and not the OPTIONS method. I have attempted to set the 'method' for the proxy but it does not appear to be taking effect.
var operation = new Ext.data.Operation({
action: 'read',
});
var proxy = new Ext.data.proxy.Ajax({
type: 'ajax',
url: testURL,
method: 'GET',
withCredentials: true
});
proxy.read( operation );
Request Headers from debugger (notice OPTIONS)
OPTIONS /web/datasource?_dc=1476803593106&page=1&start=0&limit=25 HTTP/1.1
Host: <removed>
Connection: keep-alive
Access-Control-Request-Method: GET
Origin: https://rally1.rallydev.com
User-Agent: Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36
Access-Control-Request-Headers: x-requested-with
Accept: */*
Referer: <hidden>
Accept-Encoding: gzip, deflate, sdch, br
Accept-Language: en-US,en;q=0.8
The following code works but is not independent of a data store:
var createCORSRequest = function(method, url) {
var xhr = new XMLHttpRequest();
if ("withCredentials" in xhr) {
// Most browsers.
xhr.open(method, url, true);
} else if (typeof XDomainRequest != "undefined") {
// IE8 & IE9
xhr = new XDomainRequest();
xhr.open(method, url);
} else {
// CORS not supported.
xhr = null;
}
return xhr;
};
var url = testURL;
var method = 'GET';
var xhr = createCORSRequest(method, url);
xhr.onload = function() {
// Success code goes here.
alert( xhr.response );
};
xhr.onerror = function() {
// Error code goes here.
};
xhr.withCredentials = true;
xhr.send();
I am developing a chrome extension and I have an iframe which I use in my extension. This is what I do with my iframe
"When I drag and drop a image to my iframe I handle the drop event in one of the content scripts and pass that function call my extension code. There I create a xmlhttprequest object and then send the URL of the image to a php file in my server."
This is what is happening. I get a readyState of "4" but there is no POST request going out of my browser. I checked with the "NETWORK" tab in the browser but there is no POST request going out of the browser (I have listed my site in the permissions section of the manifest file).
This is my code --.>
JScript.js(One of the content scripts )
drop: function(event, ui) {
var imgurl=$(ui.draggable).attr('src');
imgurl="IMGURL="+imgurl;
_post("www.somedomain.come/testing.php",imgurl,function(result){ alert("success")});
}
This is my proxy in the same content script-->
_post = function(url, data, callback)
{
console.log("sending post");
chrome.extension.sendRequest({
msgType:'post',
data: data,
url:url
}, function(response){
alert(response);
});
}
This my OnRequest function handler in background.html -->
chrome.extension.onRequest.addListener(function(request, sender, sendResponse){
if (request.msgType === 'post') {
alert("Now in OnRequest function");
// console.log("Now in Onrequest Function");
alert("Url: "+request.url + "\n Data : "+ request.data);
ajaxcallingfunction(request);
alert("completed the ajax call");
sendResponse("success");
}
});
var ajaxcallingfunction = function(request){
var xhr = new XMLHttpRequest();
xhr.open("POST",request.url, false);
xhr.onreadystatechange = function(){
alert(xhr.readyState);
if (xhr.readyState == 4) {
alert(xhr.readyState);
}
}
xhr.send(request.data);
alert("after xhr call");
};
You have http:// in front of your url, right?
xhr.readyState doesn't tell much, it just means that it is done. Check out what's inside xhr.status, it would contain error code. If everything is ok it should be 200:
xhr.onreadystatechange = function(){
if (xhr.readyState == 4) {
alert(xhr.status);
}
}