I have this beautiful code, all I want to make some pause between visits, so I add a 'setinterval', but this not works:
var page = require('webpage').create();
// the urls to navigate to
var urls = [
'http://blogger.com/',
'https://github.com/',
'http://reddit.com/'
];
var i = 0;
// the recursion function
var genericCallback = setInterval(function () {
return function (status) {
console.log("URL: " + urls[i]);
console.log("Status: " + status);
// exit if there was a problem with the navigation
if (!status || status === 'fail') phantom.exit();
i++;
if (status === "success") {
//-- YOUR STUFF HERE ----------------------
// do your stuff here... I'm taking a picture of the page
page.render('example' + i + '.png');
//-----------------------------------------
if (i < urls.length) {
// navigate to the next url and the callback is this function (recursion)
page.open(urls[i], genericCallback());
} else {
// try navigate to the next url (it is undefined because it is the last element) so the callback is exit
page.open(urls[i], function () {
phantom.exit();
});
}
}
};
},2000);
// start from the first url
page.open(urls[i], genericCallback());
the screenshot with the error I get:
maybe someone could help me and heal this code? Because I'm new to JS and to PhantomJS, any help will be apreciate.
I got this code from another stackoverflow answer here - Using Multiple page.open in Single Script
but I can't comment to the author , because I don't have 50 reputation
It should rather be something like this:
var page = require('webpage').create();
var urls = ['http://blogger.com/','https://github.com/','http://reddit.com/'];
var i = 0;
function OpenPage(){
setTimeout(function(){
page.open(urls[i], function(status) {
if (status == 'success') {
page.render('example' + i + '.png');
}
i++;
if(i <= urls.length - 1){
OpenPage();
}else{
phantom.exit();
}
});
},2000);
}
OpenPage();
Related
I have an app with Nuxt JS, and there is a route called posts that accepts parameters like so: .../posts/_id. When someone goes to /posts/put_news, they get post with name "Put News" and so on.
So, I wrote a validation method like so:
async validate({ params }) {
// await operations
const response = await axios.get('http://localhost:5000/listings_names')
var response_data = response.data
var str = (params.id).split('_').join(' ')
const arr2 = str.split(" ");
for (var i = 0; i < arr2.length; i++) {
arr2[i] = arr2[i].charAt(0).toUpperCase() + arr2[i].slice(1);
}
const str2 = arr2.join(" ");
var id_fix = str2
const obj = response_data.find(o => o.name == id_fix);
console.log(obj)
if (obj == undefined){
console.log('undefied, false')
return false
}
else{
return true;
}
},
The code does return false, but does nothing else. Once it returns "false" I expect nuxt to redirect the user to the error page, but it just stays on that page. I looked on the documentation, and it seems like the user should be automatically redirected to an error page, however nothing happens here. Also, my nuxt version is 2.15.8.
Thank you for the help
I fixed the issue, so all I had to do, was add redirect to validate function, and redirect to the error page, like so:
async validate({ params, redirect }) { //run-functions
if (obj == undefined) {
redirect('/not_found')
return false
} else {
return true
}
}
There is a problem. I have some urls. And there are a list of links on this urls, which I want to visit. Each of this links. There is no problem with looping throw urls, but problems with this links. Here is my code...
var urls = [];
var TEMPLATE = 'https://example.com/page/'
for (var i = 1; i > 0; i--) {
urls.push(TEMPLATE + i);
}
var page = require('webpage').create();
//Here is looping throw urls
function process(){
if (urls.length == 0){
phantom.exit();
} else{
url = urls.pop();
page = require('webpage').create();
page.open(url, onFinishedLoading);
}
}
function onFinishedLoading(status){
var links = page.evaluate(function() {
var arr = [];
//Here we are grab links inside urls
$('some.selector').each(function() {
arr.push( $('a', $(this)).attr("href"))
});
return arr;
});
//And this is just my tries to visit this links
link = links.pop();
//Just fine. Get the link
console.log(link);
parse(link);
setTimeout(function parse(link) {
page.open(link, function(status) {
var parsing = page.evaluate(function() {
return link + status;
});
//Don't work :(
console.log(parsing);
});
}, 1500);
page.release();
process();
// return links;
}
process();
Sorry for my silly question, i little know at phatom and JS.
Wish you can help me
I'm using PhantomJS to automate a page. What I do is:
do{
console.log(i);
i++;
page.open(url);
do { phantom.page.sendEvent('mousemove'); } while (page.loading);
if(page.injectJs('./Search.js') == false){
console.log("Search.js Failed")
}
var links = page.evaluate(function(json){
return search(json)
},json)
console.log(links);
} while(links == "")
So this leads me to opening the website repeated until what I'm looking for appears. But this also leads me to getting IP banned. What can I do to get around this?
Your IP is probably getting banned because the script generates too many requests to the website in very little time. So, you need to throttle requests, to apply a pause between them.
I would rewrite your script like this:
var page = require('webpage').create();
var url = "http://www.website.tld/";
var json = {"some" : "json"};
var i = 0;
var links;
// We abstract main code to a function so that we can call it
// again and again from itself
function getlinks (url, json) {
i++;
console.log(i);
page.open(url);
do { phantom.page.sendEvent('mousemove'); } while (page.loading);
if(page.injectJs('./Search.js') == false){
console.log("Search.js Failed")
}
var links = page.evaluate(function(json){
return search(json);
}, json);
if(links == "")
{
// No links scraped yet, so we wait for 3 seconds and try again
setTimeout(function(){
getlinks(url, json);
}, 3000)
}
else
{
console.log(links);
phantom.exit();
}
}
getlinks(url, json);
I'm testing out PhantomJS and trying to return all startups listed on angel.co. I decided to go with PhantomJS since I would need to paginate through the front page by clicking "Next" at the bottom. Right now this code does not return any results. I'm completely new to PhantomJS and have read through all the code examples so any guidance would be much appreciated.
var page = require('webpage').create();
page.open('https://angel.co/startups', function(status) {
if (status !== 'success') {
console.log('Unable to access network');
} else {
page.evaluate(function() {
var list = document.querySelectorAll('div.resume');
for (var i = 0; i < list.length; i++){
console.log((i + 1) + ":" + list[i].innerText);
}
});
}
phantom.exit();
});
By default, console messages evaluated on the page will not appear in your PhantomJS console.
When you execute code under page.evaluate(...), that code is being executed in the context of the page. So when you have console.log((i + 1) + ":" + list[i].innerText);, that is being logged in the headless browser itself, rather than in PhantomJS.
If you want all console messages to be passed along to PhantomJS itself, use the following after opening your page:
page.onConsoleMessage = function (msg) { console.log(msg); };
page.onConsoleMessage is triggered whenever you print to the console from within the page. With this callback, you're asking PhantomJS to echo the message to its own standard output stream.
For reference, your final code would look like (this prints successfully for me):
var page = require('webpage').create();
page.open('https://angel.co/startups', function(status) {
page.onConsoleMessage = function (msg) { console.log(msg); };
if (status !== 'success') {
console.log('Unable to access network');
} else {
page.evaluate(function() {
var list = document.querySelectorAll('div.resume');
for (var i = 0; i < list.length; i++){
console.log((i + 1) + ":" + list[i].innerText);
}
});
}
phantom.exit();
});
I'm trying to implement a form using the jQuery Form plugin. The form has three text fields and a file input and I am validating the form in the beforeSend callback. The problem is, whether the validation passes or not, the php script that handles the file upload gets loaded in the browser, which, obviously is not what I want to happen - I need to stay on the form's page.
You can take a look at the form and it's dependent files at http://www.eventidewebdesign.com/public/testUpload/. Indexing is on for that directory, so you can take a look at all of the related files. The form itself is on testUpload.php.
I'd appreciate it if someone could take a look at my code and help me figure out what's going on here.
Please write the following script instead of your, this will work.
<script>
$(document).ready( function() {
// Initialize and populate the datepicker
$('#sermonDate').datepicker();
var currentDate = new Date();
$("#sermonDate").datepicker('setDate',currentDate);
$("#sermonDate").datepicker('option',{ dateFormat: 'mm/dd/yy' });
/*
* Upload
*/
// Reset validation and progress elements
var formValid = true,
percentVal = '0%';
$('#uploadedFile, #sermonTitle, #speakerName, #sermonDate').removeClass('error');
$('#status, #required').empty().removeClass();
$('.statusBar').width(percentVal)
$('.percent').html(percentVal);
$('#frmSermonUpload').ajaxForm({
beforeSend: function() {
if (!ValidateUploadForm()) {
formValid = false;
console.log('validateuploadform returned false');
} else {
console.log('validateuploadform returned true');
formValid = true;
}
console.log('in beforeSend. formValid: ' + formValid);
if (!formValid) {
$('#uploadedFile').val('');
return false;
}
},
uploadProgress: function(event, position, total, percentComplete) {
console.log('in uploadProgress function. formValid: ' + formValid);
if (formValid) {
var percentVal = percentComplete + '%';
$('.statusBar').width(percentVal)
$('.percent').html(percentVal);
}
},
complete: function(xhr) {
console.log('in complete function. formValid: ' + formValid);
if (formValid) {
console.log('xhr.responseText: ' + xhr.responseText);
console.log('formValid: ' + formValid);
if (xhr.responseText === 'success') {
$('.statusBar').width('100%');
$('.percent').html('100%');
$('#status').html('Successfully uploaded the sermon.').addClass('successUpload');
// Clear the form
ClearForm();
} else if (xhr.responseText === 'fail') {
$('#status').html('There was a problem uploading the file. Try again.<br>If the problem persists, contact your system administrator.').addClass('errorUpload');
}
}
}
}); // End Upload Status Bar
});
function ValidateUploadForm() {
// Reset errors and clear message
$('#uploadedFile, #sermonTitle, #speakerName, #sermonDate').removeClass('error');
$('#required').empty();
var result = true;
title = $('#sermonTitle').val(),
speaker = $('#speakerName').val(),
date = $('#sermonDate').val(),
fileName = $('#uploadedFile').val();
extension = $('#uploadedFile').val().split('.').pop().toLowerCase();
//if (fileName !== '' && extension !== 'mp3') {
if ((fileName === '') || (extension !== 'mp3')) {
$('#uploadedFile').addClass('error');
$('#required').html('Only mp3 files are allowed!');
return false;
} else if (fileName === '') {
result = false;
} else if (title === '') {
$('#sermonTitle').addClass('error');
result = false;
} else if (speaker === '') {
$('#speakerName').addClass('error');
result = false;
} else if (date === '') {
$('#sermonDate').addClass('error');
result = false;
}
console.log('returning ' + result + ' from the validateuploadform function');
if (!result) { $('#required').html('All fields are required.'); }
return result;
}
function ClearForm() {
$('#uploadedFile, #sermonTitle, #sermonDate, #speakerName').val('').removeClass();
}
</script>