Downloading a file with CasperJS from POST attachment - phantomjs

I almost have this working, I just can't seem to download the file when it comes up. What am I doing wrong here? When clicking the button "Download Sales Report" a CSV should download, by my console.log() never even fires off.
var casper = require('casper').create();
casper.start('http://www.waynecountyauditor.org/Reports.aspx?ActiveTab=Sales')
.waitForText("Accept")
.thenClick('#ctl00_ContentPlaceHolder1_btnDisclaimerAccept')
.waitForText("View Sales")
.thenClick('#ctl00_ContentPlaceHolder1_WeeklySales_fvSalesReport_btnViewSales')
.waitForText("Download Sales Report")
.thenClick(x('//*[#id="ctl00_blSearchLinks"]/li[4]/a'))
.wait(1000)
.on('page.resource.received', function(resource) {
console.log('here');
if (resource.stage !== "end") {
return;
}
if (resource.url.indexOf('results.csv') > -1) {
this.download(resource.url, 'D:\Jobs\Currency\testing\ExportData.csv');
}
});
casper.run();

I finally figured it out. Vaviloff had 99% of what I needed. I was just missing 2 post variables. Thanks for your help Vaviloff!
// http://stackoverflow.com/questions/33903418/downloading-a-file-with-casperjs-from-post-attachment
var casper = require('casper').create({
verbose: true,
logLevel: 'debug',
pageSettings: {
loadImages: false // The WebPage instance used by Casper will
, loadPlugins: false // use these settings
, userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4'
}
});
var x = require('casper').selectXPath;
var utils = require('utils');
casper.on('remote.message', function(message) {
this.echo('LOG: ' + message);
});
casper.start()
.open('http://clintonoh.ddti.net/Reports.aspx?ActiveTab=Sales')
.waitForText("Accept")
.thenClick('#ctl00_ContentPlaceHolder1_btnDisclaimerAccept')
.waitForText("View Sales")
.thenClick('#ctl00_ContentPlaceHolder1_WeeklySales_fvSalesReport_btnViewSales')
.waitForText("Download Sales Report", function(){
// Adapted from: http://stackoverflow.com/questions/16144252/downloading-a-file-that-comes-as-an-attachment-in-a-post-request-response-in-pha
var res = this.evaluate(function() {
document.getElementById('__EVENTTARGET').value='ctl00$blSearchLinks' /* Was missing these 2 */
document.getElementById('__EVENTARGUMENT').value='4'
var res={};
f=document.getElementById("aspnetForm");
var previous_onsubmit = f.onsubmit;
f.onsubmit = function() {
//previous_onsubmit();
//iterate the form fields
var post={};
for(i=0; i<f.elements.length; i++) {
//console.log(f.elements[i].name + " = " + f.elements[i].value);
post[f.elements[i].name]=f.elements[i].value;
}
res.action = f.action;
res.post = post;
return false; //Stop form submission
}
// Trigger the click on the link.
var link = document.evaluate('//*[#id="ctl00_blSearchLinks"]/li[5]/a', document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
try {
var e = document.createEvent('MouseEvents');
e.initMouseEvent('click', true, true, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null);
link.dispatchEvent(e);
} catch(error){
console.log(error);
}
return res; //Return the form data to casper
});
//Start the download
casper.download(res.action, "./ExportData.csv", "POST", res.post);
//casper.capture("./image.png");
})
casper.run();

I finally got the answer after long time RD, we can use download node module for downloading the attachment as follows
const fs = require('fs');
const download = require('download');
download('http://unicorn.com/foo.pdf').then(data => {
fs.writeFileSync('dist/foo.pdf', data);
});`
Link to Download NPM Module

Related

Angular 5 HttpRequest reportProgress

I have a block of code that when executed in every other browser including IE11 works just fine, but for whatever reason when Edge runs this code I don't get a progress like the other browsers.
Edge logs this:
progress-- true
event.type == 0
A long pause and then writes:
progress: true
loaded: xxx
total: xxx
type: 1
Code:
Upload(file : File, id:number, endPoint:string) : void {
const formData = new FormData();
formData.append(file.name, file);
var auth = "Bearer " + this.Auth.GetAuth();
var autoThumb = this.stagedThumb == null ? 'true' : 'false';
const uploadRequest = new HttpRequest('POST', endPoint + `?id=${id}&thumb=${autoThumb}`, formData, {
reportProgress: true,
headers: new HttpHeaders({
'Authorization': auth
})
});
this.PerformUpload(uploadRequest);
}
PerformUpload(uploadRequest : HttpRequest<FormData>, progress: boolean = true) : void {
this.http.request(uploadRequest).subscribe(event =>
{
console.log("Http Event Message -- Progress: " + progress);
console.log(event);
console.log("End Event Message");
if (progress)
{
if (event.type == HttpEventType.UploadProgress)
{
this.progress = Math.round(100 * event.loaded / event.total);
}
else if (event.type == HttpEventType.Response)
{
this.message = event.body.toString();
setTimeout(() => {
this.ResetForm();
}, 2*1000);
}
}
});
}
In case you have not found the root cause yet, it seems this is an open issue for MS Edge.
See https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/12224510/

startRecording not working using RecordRTC with RTCMultiConnection

I am trying to record every new session/user added to RTCMultiConnection.
i am using the following demo url in application
https://rtcmulticonnection.herokuapp.com/demos/Audio+Video+TextChat+FileSharing.html
Now i have added the following cdn reference to the code.
https://cdn.webrtc-experiment.com/RecordRTC.js
and this is the code i am working with but connection.streams[event.streamid].startRecording(); is not working.
// ..................RTCMultiConnection Code.............
// ......................................................
var connection = new RTCMultiConnection();
var btnStopRec = document.getElementById("btnStopRecording");
connection.socketURL = 'https://rtcmulticonnection.herokuapp.com:443/';
connection.enableFileSharing = true;
connection.session = {
audio: true,
video: true,
data: true,
};
connection.sdpConstraints.mandatory = {
OfferToReceiveAudio: true,
OfferToReceiveVideo: true,
};
connection.onstream = function (event)
{
document.body.appendChild(event.mediaElement);
console.log("stream recording starts")
connection.streams[event.streamid].startRecording();
console.log("stream recording started")
}
I included all possible situations in a single snippet, below. Please take only the code that you need:
// global object that contains multiple recorders
var recorders = {};
// auto start recorder as soon as stream starts/begins
connection.onstream = function(event) {
document.body.appendChild(event.mediaElement);
recorders[event.streamid] = RecordRTC(event.stream, {
type: 'video'
});
recorders[event.streamid].startRecording();
};
// auto stop recorder as soon as stream stops/ends
connection.onstreamended = function(event) {
if (recorders[event.streamid]) {
recorders[event.streamid].stopRecording(function() {
var blob = recorders[event.streamid].getBlob();
var url = URL.createObjectURL(blob);
window.open(url);
delete recorders[streamid]; // clear
});
}
if (event.mediaElement.parentNode) {
event.mediaElement.parentNode.removeChild(event.mediaElement);
}
};
// stop single recorder
document.getElementById('manually-stop-single-recording').onclick = function() {
var streamid = prompt('Enter streamid');
recorders[streamid].stopRecording(function() {
var blob = recorders[streamid].getBlob();
var url = URL.createObjectURL(blob);
window.open(url);
delete recorders[streamid]; // clear
});
};
// stop all recorders
document.getElementById('manually-stop-all-recordings').onclick = function() {
Object.keys(recorders).forEach(function(streamid) {
recorders[streamid].stopRecording(function() {
var blob = recorders[streamid].getBlob();
var url = URL.createObjectURL(blob);
window.open(url);
delete recorders[streamid]; // clear
});
});
};
// record outside onstream event
// i.e. start recording anytime manually
document.getElementById('record-stream-outside-the-onstream-event').onclick = function() {
var streamid = prompt('Enter streamid');
var stream = connection.streamEvents[streamid].stream;
recorders[streamid] = RecordRTC(stream, {
type: 'video'
});
recorders[streamid].startRecording();
};

Why is my PhantomJS script outputting links from the first page and not the fourth?

I am trying to write out a phantomjs script that will print out all the 'http://librivox.org/' links on this web page:
https://librivox.org/reader/251?primary_key=251&search_category=reader&search_page=4&search_form=get_results
Here is my script:
var steps=[];
var testindex = 0;
var loadInProgress = false; //This is set to true when a page is still loading
var webPage = require('webpage');
var page = webPage.create();
var the_url = 'unknown';
page.onError = function(msg, trace) {
var msgStack = ['ERROR: ' + msg];
if (trace && trace.length) {
msgStack.push('TRACE:');
trace.forEach(function(t) {
msgStack.push(' -> ' + t.file + ': ' + t.line + (t.function ? ' (in function "' + t.function +'")' : ''));
});
}
console.error(msgStack.join('\n'));
phantom.exit(1);
};
page.settings.userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36';
page.settings.javascriptEnabled = true;
page.settings.loadImages = false;//Script is much faster with this field set to false
phantom.cookiesEnabled = true;
phantom.javascriptEnabled = true;
var system = require('system');
var args = system.args;
if (args.length === 1) {
console.log('usage: phantomjs --cookies-file=cookys.txt ./get-librivox-links-from-page.js');
} else {
args.forEach(function(arg, i) {
if ( i === 1 ) { the_url = arg; }
});
}
if ( the_url == 'unknown' ) { console.log('Please specify librivox url'); phantom.exit(); }
console.log( 'the_url is ' + the_url );
page.onConsoleMessage = function(msg) {
console.log(msg);
};
/**********DEFINE STEPS THAT FANTOM SHOULD DO***********************/
steps = [
function(url){
page.evaluate(function(url){
document.location.href = url;
},url);
},
function(){
page.evaluate(function(){
urls= [];
for (var i=document.links.length; i-->0;) {
if ( document.links[i].href.substring(0,20) == 'http://librivox.org/'.substring(0,20) ) {
console.log(document.links[i].href);
}
}
});
},
];
/**********END STEPS THAT FANTOM SHOULD DO***********************/
//Execute steps one by one
interval = setInterval(executeRequestsStepByStep,50);
function executeRequestsStepByStep(){
if (loadInProgress == false && typeof steps[testindex] == "function") {
if ( testindex == 0 ) {
steps[testindex](the_url);
} else {
steps[testindex]();
}
testindex++;
}
if (typeof steps[testindex] != "function") {
//We need to wait, after the steps is complete!
clearInterval(interval);interval=0;
setTimeout(function(){
setTimeout(phantom.exit,2000)
},3000);
}
}
/**
* These listeners are very important in order to phantom work properly.
* Using these listeners, we control loadInProgress marker which controls, weather a page is fully loaded.
* Without this, we will get content of the page, even a page is not fully loaded.
*/
page.onLoadStarted = function() { loadInProgress = true; };
page.onLoadFinished = function() { loadInProgress = false; };
page.onConsoleMessage = function(msg) { console.log(msg); };
I call the above script from a small shell script for convenience which looks like this:
$ cat run-get-librivox-links-from-page.sh
#!/bin/sh
script=/home/red/phantomjs/get-librivox-links-from-page.js
url=$1
if [ -z $url ]
then
echo "usage $0 <librivox url>"
exit 1
fi
/usr/bin/phantomjs --debug=false --cookies-file=cookys.txt \
$script $url
When I run the script like so:
$ ./run-get-librivox-links-from-page.sh "https://librivox.org/reader/251?primary_key=251&search_category=reader&search_page=4&search_form=get_results"
I get output that looks like I am output the links from search_page 1 instead of search_page 4:
the_url is https://librivox.org/reader/251?primary_key=251&search_category=reader&search_page=4&search_form=get_results
The page at https://librivox.org/reader/251?primary_key=251&search_category=reader&search_page=4&search_form=get_results displayed insecure content from http://archive.org/download/anythingycdo_mn_1302_librivox/anything_you_can_do_1302_thumb.jpg.
- above message repeated many times. remove for brevity. -
http://librivox.org/first-lensman-by-e-e-smith/
http://librivox.org/the-drums-of-jeopardy-by-harold-macgrath/
http://librivox.org/the-defiant-agents-by-andre-norton-2/
http://librivox.org/the-death-ship-by-william-clark-russell/
http://librivox.org/creatures-of-the-abyss-by-murray-leinster/
http://librivox.org/the-creature-from-beyond-infinity/
http://librivox.org/the-count-of-monte-cristo-by-alexandre-dumas/
http://librivox.org/the-cosmic-computer-by-h-beam-piper/
http://librivox.org/a-columbus-of-space-by-garrett-p-serviss/
http://librivox.org/the-colors-of-space-by-marion-zimmer-bradley-2/
http://librivox.org/the-colors-of-space-by-marion-zimmer-bradley/
http://librivox.org/the-city-at-worlds-end-by-edmond-hamilton/
http://librivox.org/citadel-of-fear-by-gertrude-barrows-bennett/
http://librivox.org/the-chessmen-of-mars-version-3-by-edgar-rice-burroughs/
http://librivox.org/the-bright-messenger-by-algernon-blackwood/
http://librivox.org/bat-wing-by-sax-rohmer/
http://librivox.org/at-the-earths-core-version-2-by-edgar-rice-burroughs/
http://librivox.org/astounding-stories-20-various/
http://librivox.org/astounding-stories-15-march-1931-by-ray-cummings/
http://librivox.org/astounding-stories-04-april-1930-by-ray-cummings/
http://librivox.org/astounding-stories-02-february-1930-by-various/
http://librivox.org/astounding-stories-01-january-1930-by/
http://librivox.org/anything-you-can-do-by-randall-garrett/

Parse multiple pages with phantomjs

I have made a code that parses all URL-s from a page. Next, I would like to get a href from every parsed URL <div class="holder"></div> and output it to a file and sepparate with a comma.
So far I have made this code. It is able to find all the URL-s need to be parsed and collects them to a comma sepparated file called output2.txt.
var resourceWait = 300,
maxRenderWait = 10000,
url = 'URL TO PARSE HREF-s FROM';
var page = require('webpage').create(),
count = 0,
forcedRenderTimeout,
renderTimeout;
page.viewportSize = { width: 1280, height : 1024 };
function doRender() {
var fs = require('fs');
var path = 'output2.txt';
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
fs.write(path,page.evaluate(function() {
return $('.urlDIV').find('a')
.map(function() {
return this.href;})
.get()
.join(',');
}), 'w');
phantom.exit()
});
}
page.onResourceRequested = function (req) {
count += 1;
clearTimeout(renderTimeout);
};
page.onResourceReceived = function (res) {
if (!res.stage || res.stage === 'end') {
count -= 1;
if (count === 0) {
renderTimeout = setTimeout(doRender, resourceWait);
}
}
};
page.open(url, function (status) {
if (status !== "success") {
phantom.exit();
} else {
forcedRenderTimeout = setTimeout(function () {
console.log(count);
doRender();
}, maxRenderWait);
}
});
Thanks in advance,
Martti

How to open a local pdf file using OpenURL in Titanium Appceleartor?

In Titanium Appcelerator, i tried to open a PDF file from local directory using OpenURL() method. It's not working properly. i tried on Android device.
My Code;
var myURL = "file:///storage/emulated/0/Android/data/com.test.testapp/cache/_tmp/sample.pdf";
Ti.Platform.openURL(myURL);
Try something like this:
try {
var f = Ti.Filesystem.getFile('your.pdf');
Ti.Android.currentActivity.startActivity(Ti.Android.createIntent({
action: Ti.Android.ACTION_VIEW,
type: 'application/pdf',
data: f.getNativePath()
}));
}
catch (err) {
var alertDialog = Titanium.UI.createAlertDialog({
title: 'No PDF Viewer',
message: 'We tried to open a PDF but failed. Do you want to search the marketplace for a PDF viewer?',
buttonNames: ['Yes','No'],
cancel: 1
});
alertDialog.show();
alertDialog.addEventListener('click', function(evt) {
if (evt.index == 0) {
Ti.Platform.openURL('http://search?q=pdf');
}
});
}
To open a remote PDF natively, you have to download it. Here is a solution which provides the user with options to preview or download the PDF.
var url = "http://www.polyu.edu.hk/iaee/files/pdf-sample.pdf";
var opts = {
cancel: 2,
options: ['Preview', 'Download', 'Cancel'],
selectedIndex: 2,
destructive: 0,
title: 'Open PDF'
};
var dialog = Ti.UI.createOptionDialog(opts);
dialog.addEventListener('click', function(e) {
if (e.index == 0) {
url = "https://docs.google.com/viewer?embedded=true&url=" + url;
var win = Ti.UI.createWindow();
var webView = Ti.UI.createWebView({url:url});
win.add(webView);
win.open();
} else if (e.index == 1) {
var filepath = url.split('/').pop();
var httpClient = Titanium.Network.createHTTPClient({
onload: function() {
var file = Titanium.Filesystem.getFile(Titanium.Filesystem.tempDirectory, filepath);
file.write(this.responseData);
try {
Ti.Android.currentActivity.startActivity(Ti.Android.createIntent({
action: Ti.Android.ACTION_VIEW,
type: 'application/pdf',
data: file.getNativePath()
}));
} catch (e) {
alert('No PDF reader found.');
}
}
});
httpClient.open('GET', url);
httpClient.send();
}
});
dialog.show();