Cannot submit a form in phamtomjs - phantomjs

I'm new to PhantomJS. Following this example, I tried to create a script which logs in to avito.ru, then opens up an edit page, changes the price, submits the form, and as the server sends back a confirmation page, script should submit a new form for changes to take place.
So the cycle should be as follows:
Log in
GET .../items/edit/678347092
POST to .../items/edit/678347092
GET .../stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm
POST to .../stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm
My code is below:
var steps=[];
var testindex = 0;
var loadInProgress = false;
/*********SETTINGS*********************/
var webPage = require('webpage');
var page = webPage.create();
page.viewportSize = { width: 1366, height: 768 };
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
/*********SETTINGS END*****************/
console.log('All settings loaded, start with execution');
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
//Step 1 - Open Avito login page
function(){
console.log('Step 1 - opening Avito');
page.open("https://www.avito.ru/profile/login?next=%2Fprofile", function(status){
});
},
//Step 2 - Populate and submit the login form
function(){
console.log('Step 2');
console.log('URL: ' + page.url);
console.log('Populate and submit the login form');
page.evaluate(function(){
$('input[type="email"]').val('myemail');
$('input[type="password"]').val('mypassword');
$('.js-form-login').submit();
});
},
//Step 3 Profile page
function(){
console.log('Step 3');
console.log('URL: ' + page.url);
page.render('3-avito.png');
console.log("Opening edit page");
page.open("https://www.avito.ru/items/edit/678347092", function(status){
});
},
//Step 4 Edit the price
function(){
console.log('Step 4');
console.log('URL: ' + page.url);
page.render('4-edit-page.png');
console.log("Changing the price and submitting the form");
page.evaluate(function(){
document.querySelector('#item-edit__price').value = '21999';
document.forms[0].submit();
});
},
//Step 5 Confirm changes
function(){
console.log('Step 5');
console.log('URL: ' + page.url);
page.render('5-edit-confirm.png');
console.log("Final confirmation");
page.evaluate(function(){
document.forms[0].submit();
});
},
//Step 6 Render the result
function(){
console.log('Step 6');
console.log('URL: ' + page.url);
console.log("Rendering final page");
page.render('6-result.png');
}
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
steps[testindex]();
testindex++;
}
if (typeof steps[testindex] != "function") {
console.log("test complete!");
phantom.exit();
}
}
/**
* These listeners are very important in order to phantom work properly. Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() {
loadInProgress = true;
console.log('Loading started');
};
page.onLoadFinished = function() {
loadInProgress = false;
console.log('Loading finished');
};
page.onConsoleMessage = function(msg) {
console.log(msg);
};
I tried to paste the code inside page.evaluate to the Chrome's console, and it works fine. But as I can see in Fiddler, this script doesn't perform a first POST request (the form is not submitted at step 4). Also I have rendered images indicating that the price is changed at step 4, but since then all other images remain the same.
Below is the output of the script to the console:
All settings loaded, start with execution
Step 1 - opening Avito
Loading started
Loading finished
Step 2
URL: https://www.avito.ru/profile/login?next=/profile
Populate and submit the login form
Loading started
Loading finished
Step 3
URL: https://www.avito.ru/profile
Opening edit page
Loading started
Loading finished
Step 4
URL: https://www.avito.ru/items/edit/678347092
Changing the price and submitting the form
Step 5
URL: https://www.avito.ru/items/edit/678347092
Final confirmation
Step 6
URL: https://www.avito.ru/items/edit/678347092
Rendering final page
test complete!
Any ideas how to make this script work?
Update: the script sometimes fails to submit the first login form, sometimes the second form, and sometimes it submits the second form instead of the third at the step 5, rather than at the step 4.
I've used onResourceRequested to see the headers of the requests, and in the example below it doesn't submit the second form at the step 4, but submits the same form at the step 5. (So I think it is not the problem with the form validation). Here are some extracts:
Step 1 Open login page
Request (#1): {"headers":[{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157
Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":1,"method":"GET","time":"2015-11-
19T12:38:51.111Z","url":"https://www.avito.ru/profile/login?next=%2Fprofile"}
Loading started
...
Loading finished
Step 2
- POST login and password
Request (#28): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Content-Type","value":"multipart/form-data; boundary=----WebKitFormBoundaryrQFFuoCCfmZnvHH9"},
{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},{"name":"Referer","value":"https://www.avito.ru/profile/login?next=
%2Fprofile"},{"name":"Content-Length","value":"355"}],"id":28,"method":"POST","time":"2015-11-19T12:38:57.595Z","url":"https://www.avito.ru/profile/login"}
Loading started
Request (#29): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
{"name":"Referer","value":"https://www.avito.ru/profile/login?next=%2Fprofile"}],"id":29,"method":"GET","time":"2015-11-
19T12:38:58.360Z","url":"https://www.avito.ru/profile"}
...
Loading finished
Step 3 Open new page
Request (#72): {"headers":[{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157
Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}],"id":72,"method":"GET","time":"2015-11-
19T12:39:13.887Z","url":"https://www.avito.ru/items/edit/678347092"}
Loading started
...Loading finished
Step 4 Edit the price and submit changes
doesn't work
Step 5
- Confirm changes (instead it's performed submit of the form from the step 4)
Request (#107): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Content-Type","value":"application/x-www-form-urlencoded"},
{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},{"name":"Referer","value":"https://www.avito.ru/items/edit/678347092"},
{"name":"Content-Length","value":"8110"}],"id":107,"method":"POST","time":"2015-11-19T12:39:25.950Z","url":"https://www.avito.ru/items/edit/678347092"}
Loading started
Request (#108): {"headers":[{"name":"Origin","value":"https://www.avito.ru"},{"name":"User-Agent","value":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"},{"name":"Accept","value":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
{"name":"Referer","value":"https://www.avito.ru/items/edit/678347092"}],"id":108,"method":"GET","time":"2015-11-
19T12:39:26.709Z","url":"https://www.avito.ru/ulyanovsk/bytovaya_tehnika/stiralnaya_mashina_indesit_nws_7105_l_novaya_678347092/edit/confirm"}
Loading finished
Loading started
Loading finished
Step 6 - Screenshot

Related

Unable to login into PSN using Python requests module

I am trying to login into PSN https://www.playstation.com/en-in/sign-in-and-connect/ using python requests module and API got from the inspect element of browser. Below is the code
import requests
login_data = {
'password': "mypasswordhere",
'username': "myemailhere",
}
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36'
}
with requests.Session() as s1:
url = "https://auth.api.sonyentertainmentnetwork.com/2.0/oauth/token"
r = s1.post(url, data = login_data, headers = header)
print(r.text)
With this, I got below response from the server.
{"error":"invalid_client","error_description":"Bad client credentials","error_code":4102,"docs":"https://auth.api.sonyentertainmentnetwork.com/docs/","parameters":[]}
Can I know any alternative method to login into PSN network? Preferably using API model instead of selenium? My objective is to login into PSN network with my credentials and change password but seems got stuck in login page only...

Getting a 403 Forbidden when going to a URL with PhantomJS

If I go to the following web page in Chrome, it loads fine: https://www.cruisemapper.com/?poi=39
However, when I run the following PhantomJS script, which simply goes to the same URL and outputs the entire DOM string to the console, I get a 403 Forbidden message:
var page = require('webpage').create(),
url = 'https://www.cruisemapper.com/?poi=39';
page.open(url, function (status) {
if (status === 'success') {
console.log(page.evaluate(function () {
return document.documentElement.outerHTML;
}));
phantom.exit();
}
});
Here's the exact output to the console:
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>You don't have permission to access /
on this server.<br>
</p>
</body></html>
I thought that if I added some sort of user agent string, it might work. As such, I added the following above the console.log line:
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36';
But that didn't work. So then I tried the following instead:
page.customHeaders = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
};
But that didn't work either. Does anyone have any advice on how I can possibly hit up the URL above and not get a 403 Forbidden message? Thank you.
Your code works for me fine (I's suggest viewport size emulation though, see code). If you still get a 403, try changing your IP, it's possible that the site is on to you now (you probably visited that page lots of times).
var page = require('webpage').create(),
url = 'https://www.cruisemapper.com/?poi=39';
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36';
page.viewportSize = { width: 1440, height: 900 }; // <-- otherwise it's 400x300 by default
// It's good to watch for errors on the page
page.onError = function (msg, trace)
{
console.log(msg);
trace.forEach(function(item) {
console.log(' ', item.file, ':', item.line);
})
}
page.open(url, function (status) {
console.log(status);
page.render("page.png"); // Also useful to check if you get what you expect
if (status === 'success') {
console.log(page.evaluate(function () {
return document.documentElement.outerHTML;
}));
phantom.exit();
}
});

Using Selenium with Phantomjs in node not returning results

I have the following node route using selenium and chrome driver which is working correctly and returning expected html in the console:
app.get('/google', function (req, res) {
var driver = new webdriver
.Builder()
.forBrowser('chrome')
.build();
driver.get('https://www.google.com')
driver
.manage()
.window()
.setSize(1200, 1024);
driver.wait(webdriver.until.elementLocated({xpath: '//*[#id="lst-ib"]'}));
return driver
.findElement({xpath: '//*[#id="lst-ib"]'})
.sendKeys('stackoverflow' + webdriver.Key.RETURN)
.then((html) => {
return driver
.findElement({xpath: '//*[#id="rso"]/div[1]/div/div/div/div'})
.getAttribute("innerHTML")
})
.then((result) => {
console.log(result)
})
.then(() => {
res
.status(200)
.send('ok')
});
I have also installed the phantom js driver and tested that its working by returning the URL title - it works. When I use the above exact route and replace the chrome with phantomjs I get no results returned. There are no errors - just no print out in my console. The status and result are never sent to the browser so it doesn't appear to be stepping through promise chain.
Any suggestions?
The issue was that there was different html being rendered depending on the user agent. By forcing a user agent I was able to retrieve the results i needed.
Here is the code snippet replaced above to get this working.
.Builder()
// .forBrowser('phantomjs')
.withCapabilities(webdriver.Capabilities.phantomjs()
.set("phantomjs.page.settings.userAgent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"))
.build();

How to properly parse JSONP callback function in Meteor?

Does someone know how to parse JSONP callback in Meteor server methods?
I do
let response = HTTP.call('GET', AVIASALES_API_ENDPOINTS.getLocationFromIP, {
params: {
locale: 'en',
callback: 'useriata',
ip: clientIP
}
});
in response.content I’ve got
useriata({"iata":"MSQ","name":"Minsk","country_name":"Belarus"})
How to properly parse it?
It could help to know what you really try to accomplish? But here is a working example meteor actually doesn't do anything unusual with the requests.
Meteor.startup(function () {
var result = HTTP.call("GET", "https://api.github.com/legacy/repos/search/meteor", {
params: {},
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
}
});
console.log(result.data); // it's js object you can do result.data.repositories[0].name
console.log(JSON.stringify(result.data)); // json string
console.log(JSON.parse(JSON.stringify(result.data))) // if for some reason you need to parse it this way will work, but seems unnecessary
});
Update: The string you got back from the response wasn't valid JSON so you couldn't parse it used some regex to remove the invalid strings here is working example: http://meteorpad.com/pad/JCy5WkFsrtciG9PR5/Copy%20of%20Leaderboard

CasperJS fetchText() function echoing blank output

I'm trying to use fetchText() to print out the URL of a google search result to the terminal. Here's the image of what exactly I'm trying to print.
It's only prints out blank though! I don't see anything I'm doing wrong?
Code:
phantom.casperPath = "/usr/local/Cellar/casperjs/1.0.3/libexec/";
phantom.injectJs(phantom.casperPath + '/bootstrap.js');
var utils = require('utils');
var casper = require('casper').create();
casper.start('https://www.google.com/search?q=amazon+shoes');
casper.wait(3000, function () {
this.echo(this.fetchText('#rso > div:nth-child(1) > li:nth-child(1) > div > div > div > div.f.kv._TD > cite'));
}).run();
Google will change the page depending on the useragent string. So you need to set a string during creation (with example string)
var casper = require("casper").create({
pageSettings: {
userAgent: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36"
}
});
or with specific function
casper.userAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36");
Sometimes it is also necessary to set the viewport to something desktop-like, because PhantomJS' default viewport is 400x300 and Google might render a different site based on the viewport.