I have a problem :
2017-02-18T20:19:16 [DEBUG] WebPage - evaluateJavaScript "(function() { return (function () {\n var ev = document.createEvent(\"MouseEvents\");\n ev.initEvent(\"click\", true, true);\n\n document.querySelector(\"#jeje\").dispatchEvent(ev);\n })(); })()"
2017-02-18T20:19:16 [DEBUG] WebPage - updateLoadingProgress: 10
2017-02-18T20:19:16 [DEBUG] WebPage - evaluateJavaScript result QVariant(Invalid)
Code JS :
page.open(address, settings, function(status) {
// Check for page load success
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
if (status !== "success") {
console.log("==================Unable to access network==================");
} else {
// Wait for element to be visible
waitFor(function() {
// Check in the page if a specific element is now visible
return page.evaluate(function() {
return $("body").is(":visible");
});
}, function() {
console.log("==================The element should be visible now.==================");
page.evaluate(function() {
var ev = document.createEvent("MouseEvents");
ev.initEvent("click", true, true);
document.querySelector("#jeje").dispatchEvent(ev);
})
page.render(output);
phantom.exit();
});
}
})
});
I do not understand the problem. Could someone help me?
Thank you
Related
I'm trying to use $.getJSON inside PhantomJS but impossible to get the result of it. Any solution? I can not simply load or includeJs directly. The page has to be called from the same domain.
So I want to open a page and do the call from there.
Here is my current code which is not working:
var jqueryUrl = "https://code.jquery.com/jquery-latest.min.js";
page.open("http://www.example.com/", function(status) {
if (status === "success") {
page.includeJs(jqueryUrl, function() {
var result = page.evaluate(function() {
$.getJSON('http://www.example.com/someJson', function(data) {
return data;
});
});
console.log(result);
phantom.exit();
});
} else {
phantom.exit(1);
}
});
Thanks for any help!
You need to use page.onCallback with a combination with window.callPhantom because you are making an HTTP request in phantomjs context and the result needs to be returned only after the request is done.
I haven't tested exactly this code, but it should be something like this:
var jqueryUrl = "https://code.jquery.com/jquery-latest.min.js";
page.open("http://www.example.com/", function(status) {
if (status === "success") {
page.onCallback(function(data) {
// got the data!
console.log(data);
phantom.exit();
});
page.includeJs(jqueryUrl, function() {
page.evaluate(function() {
$.getJSON('http://www.example.com/someJson', window.callPhantom);
});
});
} else {
phantom.exit(1);
}
});
I'm creating an extenstion for google chrome that will perform checking if a stream on twitch.tv is online and will notify the user evey X minutes, I got that covered. What I'm looking for is a JScirpt code that will recognize if user is already on the streamers channel and will stop notifying him.
var username="$user";
setInterval(check,300000);
function check()
{
request("https://api.twitch.tv/kraken/streams/" + username, function() {
var json = JSON.parse(this.response);
if (json.stream == null)
{
chrome.browserAction.setIcon({ path: "offline.png" });
}
else
{
notify();
}
});
return 1;
}
function notify(){
var opt = {type: "basic",title: username + " is streaming!",message: "Click to join!",iconUrl: "start.png"};
chrome.notifications.create("", opt, function(notificationId)
{
setTimeout(function()
{
chrome.notifications.clear(notificationId, function(wasCleared) { console.log(wasCleared); });
}, 3000);
});
chrome.browserAction.setIcon({path:"online.png" });
}
chrome.browserAction.onClicked.addListener(function () {
chrome.tabs.create({ url: "http://www.twitch.tv/"+username });
});
function request(url, func, post)
{
var xhr = new XMLHttpRequest();
xhr.onload = func;
xhr.open(post == undefined ? 'GET' : 'POST', url, true);
xhr.send(post || '');
return 1;
}
check();
Use window.location.href to get the complete URL.
Use window.location.pathname to get URL leaving the host.
You can read more here.
I have a PhantomJs script in which I create a new wepage, inject jQuery into it and scrape a list of URL from it. After that I call a function passing the list of URL and create a new webpage for each one and try to recover certain information from it
var pageGlobal = require('webpage');
function createPage(){
var page = pageGlobal.create();
page.onAlert = function(msg) {
console.log(msg);
};
return page;
}
var page=createPage();
page.open('http://www.example.com/', function(status){
if ( status === "success" ) {
page.injectJs('jquery-1.6.1.min.js');
var urlList=page.evaluate(
function(){
var urlList=[];
window.console.log = function(msg) { alert(msg) };
$("td.row1>a").each(function(index, link) {
var link=$(link).attr('href');
urlList.push(link);
});
return urlList;
});
processUrlList(urlList);
}
});
function processUrlList(urlList){
for(i=0;i<urlList.length;i++){
var currentPage=createPage();
currentPage.open("http://www.example.com"+urlList[i], function(status){
if ( status === "success" ) {
if(currentPage.injectJs('jquery-1.6.1.min.js')===false){
console.log("Error en la inyeccion");
}
currentPage.evaluate(function() {
window.console.log = function(msg) { alert(msg) };
console.log("Evaluating");
$("showAdText").each(function(index, link) {
//Capture information about the entity in this URL
})
});
}
});
}
}
The problem is in the processUrlList function the injection of jQuery always fail returning false. Would it be a problem to create two or more page objects instead of reusing only one? What could be happening here?
I have a website with a login form. If a user is not logged and tries to access a internal page it will be redirected to the default page. For instance if I try to access
http://siteURL.PhantomPrint.aspx I will be redirected to http://siteURL/Default.aspx?ReturnUrl=PhantomPrint.aspx. And after login an automatic redirect will take place to the page.
After the redirect I want to render the page with Phantomjs and save it as pdf. The problem is that the rendering takes place before page load is finished and I can properly render the page only if I use timeouts. In this case, if the page loading takes longer than normal the resulted pdf is not the proper one.
Below you can find the java script code:
var page = require('webpage').create();
var index = 0,
page.onConsoleMessage = function (msg) {
console.log(msg);
};
var steps = [
function () {
//Load Login Page
page.open("http://siteURL.PhantomPrint.aspx", function () {
//Enter Credentials
page.evaluate(function () {
console.log("filling inputs");
var usernameInput = document.getElementById("txtUsername");
usernameInput.value = "user";
var passwordInput = document.getElementById("txtPassword");
passwordInput.value = "password";
var loginButton = document.getElementById("btnLogin");
loginButton.click();
console.log("login button was submitted");
});
});
},
function () {
// page.onLoadFinished = function () {
// Render the page to pdf
page.render('example.png');
phantom.exit();
console.log("rendering finished");
//});
}
];
interval = setInterval(function () {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 1000);
Any suggestions on how I can assure that rendering is done only after the redirected page is finishing loading are welcomed.
It looks like you want to process navigation steps. You would need to use page.onNavigationRequested to pick up if a page load/redirect was issued. This will be likely hard to maintain. You would also have to discard the idea of using a step array with setInterval.
Another possibility would be to specifically wait for some selector that is present in the target page using waitFor, but then again, this would make the use of setInterval impossible.
CasperJS is actually built on top of PhantomJS and uses steps to navigate the site. When you use any of the then* functions it will automatically pick up a page load and wait for page load finish until executing the callback.
var casper = require('casper').create();
casper.on("remote.message", function (msg) {
console.log(msg);
});
casper.start("http://siteURL/PhantomPrint.aspx", function () {
//Enter Credentials
this.evaluate(function () {
console.log("filling inputs");
var usernameInput = document.getElementById("txtUsername");
usernameInput.value = "user";
var passwordInput = document.getElementById("txtPassword");
passwordInput.value = "password";
});
this.click("#btnLogin");
this.echo("login button was submitted");
});
casper.then(function () {
this.capture('example.png');
});
casper.run();
This can be made even smaller by using casper.fillSelectors.
After more research I found a solution, see below code.
var loadInProgress = false;
page.onLoadStarted = function () {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function () {
loadInProgress = false;
console.log("load finished");
};
interval = setInterval(function () {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 100)
But I would like to know if there is no other solution which would not involve recursively function calling.
I am filling out and submitting a form using PhantomJS and then outputting the resulting page. The thing is, I have no idea if this thing is being submitted at all.
I print the resulting page, but it's the same as the original page. I don't know if this is because it redirects back or I didn't submit it or I need to wait longer or or or. In a real browser it sends a GET and receives a cookie, which it uses to send more GETS before eventually receiving the final result - flight data.
I copied this example How to submit a form using PhantomJS, using a diferent url and page.evaluate functions.
var page = new WebPage(), testindex = 0, loadInProgress = false;
page.onConsoleMessage = function(msg) {
console.log(msg);
};
page.onLoadStarted = function() {
loadInProgress = true;
console.log("load started");
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log("load finished");
};
var steps = [
function() {
//Load Login Page
page.open("http://www.klm.com/travel/dk_da/index.htm");
},
function() {
//Enter Credentials
page.evaluate(function() {
$("#ebt-origin-place").val("CPH");
$("#ebt-destination-place").val("CDG");
$("#ebt-departure-date").val("1/5/2013");
$("#ebt-return-date").val("10/5/2013");
});
},
function() {
//Login
page.evaluate(function() {
$('#ebt-flightsearch-submit').click() ;
# also tried:
# $('#ebt-flight-searchform').submit();
});
},
function() {
// Output content of page to stdout after form has been submitted
page.evaluate(function() {
console.log(document.querySelectorAll('html')[0].outerHTML);
});
}
];
interval = setInterval(function() {
if (!loadInProgress && typeof steps[testindex] == "function") {
console.log("step " + (testindex + 1));
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}, 50);
The site of interest is rather complicated to scrape. I logged the HTTP traffic from the US KLM site and got this:
GET /travel/us_en/apps/ebt/ebt_home.htm?name=on&ebt-origin-place=New+York+-+John+F.+Kennedy+International+%28JFK%29%2CNew+York&ebt-destination-place=Paris+-+Charles+De+Gaulle+Airport+%28CDG%29%2C+France&c%5B0%5D.os=JFK&c%5B0%5D.ost=airport&c%5B0%5D.ds=CDG&c%5B0%5D.dst=airport&c%5B1%5D.os=CDG&c%5B1%5D.ost=airport&c%5B1%5D.ds=JFK&inboundDestinationLocationType=airport&redirect=no&chdQty=0&infQty=0&c%5B0%5D.dd=2013-07-31&c%5B1%5D.dd=2013-08-14&c%5B1%5D.format=dd%2Fmm%2Fyyyy&flex=true&ebt-cabin-class=ECONOMY&adtQty=1&goToPage=&cffcc=ECONOMY&sc=false HTTP/1.1
Your injected values for the form elements are not what their server is looking for.
Inside page.evaluate(), you are sandboxed, but the sample code includes a hook to get sandboxed console activity onto the external console. For other debugging, you can also include object inspectors, etc., but they have to be injected into the page or part of the code passed into evaluate().