infinity loading when try to scrap title article
i was trying to scrap title articles with cheerio in node js but i could not make it happen
infinity loading in google chrome and no content . here is my controller.js file that i trying to use cheerio to scrap title articles with cheerio and show handlesbars view engine
but view engie is working and everything is fine . but for scrap infinity loading in chrome with no error
classname that i use for this website is .c-entry-box--compact__title
maybe i am wrong . but i can not figure out
var express = require("express");
var router = express.Router();
var path = require("path");
var request = require("request");
var cheerio = require("cheerio");
var Comment = require("../models/Comment.js");
var Article = require("../models/Article.js");
router.get("/", function(req, res) {
res.redirect("/articles");
});
router.get("/scrape", function(req, res) {
request("http://www.theverge.com", function(error, response, html) {
var $ = cheerio.load(html);
var titlesArray = [];
$(".c-entry-box--compact__title").each(function(i, element) {
var result = {};
result.title = $(this)
.children("a")
.text();
result.link = $(this)
.children("a")
.attr("href");
if (result.title !== "" && result.link !== "") {
if (titlesArray.indexOf(result.title) == -1) {
titlesArray.push(result.title);
Article.count({ title: result.title }, function(err, test) {
if (test === 0) {
var entry = new Article(result);
entry.save(function(err, doc) {
if (err) {
console.log(err);
} else {
console.log(doc);
}
});
}
});
} else {
console.log("Article already exists.");
}
} else {
console.log("Not saved to DB, missing data");
}
});
res.redirect("/");
});
});
router.get("/articles", function(req, res) {
Article.find()
.sort({ _id: -1 })
.exec(function(err, doc) {
if (err) {
console.log(err);
} else {
var artcl = { article: doc };
res.render("index", artcl);
}
});
});
router.get("/articles-json", function(req, res) {
Article.find({}, function(err, doc) {
if (err) {
console.log(err);
} else {
res.json(doc);
}
});
});
router.get("/clearAll", function(req, res) {
Article.remove({}, function(err, doc) {
if (err) {
console.log(err);
} else {
console.log("removed all articles");
}
});
res.redirect("/articles-json");
});
router.get("/readArticle/:id", function(req, res) {
var articleId = req.params.id;
var hbsObj = {
article: [],
body: []
};
Article.findOne({ _id: articleId })
.populate("comment")
.exec(function(err, doc) {
if (err) {
console.log("Error: " + err);
} else {
hbsObj.article = doc;
var link = doc.link;
request(link, function(error, response, html) {
var $ = cheerio.load(html);
$(".l-col__main").each(function(i, element) {
hbsObj.body = $(this)
.children(".c-entry-content")
.children("p")
.text();
res.render("article", hbsObj);
return false;
});
});
}
});
});
router.post("/comment/:id", function(req, res) {
var user = req.body.name;
var content = req.body.comment;
var articleId = req.params.id;
var commentObj = {
name: user,
body: content
};
var newComment = new Comment(commentObj);
newComment.save(function(err, doc) {
if (err) {
console.log(err);
} else {
console.log(doc._id);
console.log(articleId);
Article.findOneAndUpdate(
{ _id: req.params.id },
{ $push: { comment: doc._id } },
{ new: true }
).exec(function(err, doc) {
if (err) {
console.log(err);
} else {
res.redirect("/readArticle/" + articleId);
}
});
}
});
});
module.exports = router;
The http://www.theverge.com Add content dynamically by scrolling.
This is example how to get the title by puppeteer
const puppeteer = require("puppeteer");
const getTitle = async () => {
try {
const browser = await puppeteer.launch({
headless: false,
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.goto('https://www.theverge.com');
await page.setViewport({
width:1920,
height:1080
});
// scroll down end of page
await page.evaluate(() => {
window.scrollTo(0, window.document.body.scrollHeight);
});
await page.waitForNavigation({ waitUntil: 'networkidle0' }), // (0 network connections for 500ms)
// get the title
titles = await page.evaluate(() => {
const textSelector = 'div.inline.pr-4.font-bold'
texts = Array.from(document.querySelectorAll(textSelector), row => row.innerText.trim() );
return texts;
});
await browser.close();
return Promise.resolve(titles);
} catch (error) {
return Promise.reject(error);
}
}
getTitle()
.then((titles) => {
console.log(titles); // first news search
})
This is result
[
'Is an upgraded M2 Ultra enough for a new Mac Pro and the Mac Studio?',
'Here’s the official trailer for Christopher Nolan’s next IMAX-filmed epic, Oppenheimer.',
'Sam Bankman-Fried’s ready to surrender himself to the US for extradition.',
'Who knew the thumb drive had such a contentious origin story?',
'But how many pebbles do you have in a jar?',
'This way for The Way of Water.',
'Netflix is taking Blockbuster behind the woodshed again.',
'I can’t escape the year-end wrap-ups.',
'The clock’s ticking if you want to get your gifts on time.',
'Want solar panels on your California home? Now might be the time.',
'Twitter Spaces has returned.',
'Apple’s facing another accusation of breaking labor laws.',
'Every game should have this feature.',
'Google’s working on simplifying smart home control on the wrist.',
'Apple could open up iOS, and the feds finally make a case against SBF.',
'I’m not the first, and I won’t be the last... but I do feel early.',
'For what it’s worth, Avatar: The Way of Water’s a good looking movie.',
'You may not want to upgrade to Apple’s new Home architecture.'
]
I have two functions and I am trying to call GetReceipt when the button (with uploadphoto function) has been clicked and right after the UploadPhoto function like getting data right after uploading a photo.
I tried putting this.GetReceipt() inside the UploadPhoto() function somehow it does not work but if i separate them into two different buttons both functions work perfectly.
Here is the code:
UploadPhoto=(pickedImagePath)=> {
var pickedImagePath = this.state.pickedImagePath;
console.log(pickedImagePath);
var APIURL="http://10.0.2.2/api/postform.php";
let uriParts = pickedImagePath.split('/');
let image = uriParts[uriParts.length - 1];
var ImageData= new FormData();
ImageData.append('photo', {
uri:pickedImagePath,
name: image,
type: 'image/jpeg'
})
fetch(APIURL,
{
headers:{
'Accept':'application/json',
'Content-Type':'multipart/form-data'
},
method:'POST',
body: ImageData
})
.then((response)=>
{
console.log('succ ')
console.log(response)
})
.catch((error)=>
{
console.log('err ')
console.log(error)
}
)
}
GetReceipt=()=> {
var title = this.state.title;
var amount = this.state.amount;
var transactionType = this.state.transactionType;
var note = this.state.note;
var date = this.state.date;
var APIURL="http://10.0.2.2/api/getform.php";
var headers2={
'Accept':'application/json',
'Content-Type':'application.json'
}
fetch(APIURL,
{
method:'POST',
headers: headers2,
})
.then((response)=>response.text())
.then((response)=>
{
// this.setState({array:response});
this.setState({title:response[3]});
this.setState({amount:response[0]});
this.setState({date:response[1]});
this.setState({note:response[2]});
// this.setState({results:response});
console.log("results:",response[2]);
console.log("merchant:",this.state.title);
console.log("results:",response);
})
.catch((error)=>
{
alert("Error"+error);
}
)
}
I'm creating an extenstion for google chrome that will perform checking if a stream on twitch.tv is online and will notify the user evey X minutes, I got that covered. What I'm looking for is a JScirpt code that will recognize if user is already on the streamers channel and will stop notifying him.
var username="$user";
setInterval(check,300000);
function check()
{
request("https://api.twitch.tv/kraken/streams/" + username, function() {
var json = JSON.parse(this.response);
if (json.stream == null)
{
chrome.browserAction.setIcon({ path: "offline.png" });
}
else
{
notify();
}
});
return 1;
}
function notify(){
var opt = {type: "basic",title: username + " is streaming!",message: "Click to join!",iconUrl: "start.png"};
chrome.notifications.create("", opt, function(notificationId)
{
setTimeout(function()
{
chrome.notifications.clear(notificationId, function(wasCleared) { console.log(wasCleared); });
}, 3000);
});
chrome.browserAction.setIcon({path:"online.png" });
}
chrome.browserAction.onClicked.addListener(function () {
chrome.tabs.create({ url: "http://www.twitch.tv/"+username });
});
function request(url, func, post)
{
var xhr = new XMLHttpRequest();
xhr.onload = func;
xhr.open(post == undefined ? 'GET' : 'POST', url, true);
xhr.send(post || '');
return 1;
}
check();
Use window.location.href to get the complete URL.
Use window.location.pathname to get URL leaving the host.
You can read more here.
I have a PhantomJs script in which I create a new wepage, inject jQuery into it and scrape a list of URL from it. After that I call a function passing the list of URL and create a new webpage for each one and try to recover certain information from it
var pageGlobal = require('webpage');
function createPage(){
var page = pageGlobal.create();
page.onAlert = function(msg) {
console.log(msg);
};
return page;
}
var page=createPage();
page.open('http://www.example.com/', function(status){
if ( status === "success" ) {
page.injectJs('jquery-1.6.1.min.js');
var urlList=page.evaluate(
function(){
var urlList=[];
window.console.log = function(msg) { alert(msg) };
$("td.row1>a").each(function(index, link) {
var link=$(link).attr('href');
urlList.push(link);
});
return urlList;
});
processUrlList(urlList);
}
});
function processUrlList(urlList){
for(i=0;i<urlList.length;i++){
var currentPage=createPage();
currentPage.open("http://www.example.com"+urlList[i], function(status){
if ( status === "success" ) {
if(currentPage.injectJs('jquery-1.6.1.min.js')===false){
console.log("Error en la inyeccion");
}
currentPage.evaluate(function() {
window.console.log = function(msg) { alert(msg) };
console.log("Evaluating");
$("showAdText").each(function(index, link) {
//Capture information about the entity in this URL
})
});
}
});
}
}
The problem is in the processUrlList function the injection of jQuery always fail returning false. Would it be a problem to create two or more page objects instead of reusing only one? What could be happening here?
I would like to learn phantomjs, but i can`t find good tutorial. I have 2 questions:
where is problem in following code (need to capture label of button and write to file):
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
page.open("http://vk.com", function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
page.evaluate(function() {
var str = $("#quick_login_button").text();
f = fs.open("ololo.txt", "w");
f.writeLine(str);
f.close();
console.log("done");
});
phantom.exit();
});
}
});
what tutorial in phantomjs you can advice to me? (not from official site)
Because execution is sandboxed, the web page has no access to the phantom objects.
var page = require('webpage').create();
var fs = require('fs');
page.onConsoleMessage = function(msg) {
phantom.outputEncoding = "utf-8";
console.log(msg);
};
page.open("http://vk.com", function(status) {
if ( status === "success" ) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
var str = page.evaluate(function() {
return $("#quick_login_button").text();
});
f = fs.open("ololo.txt", "w");
f.writeLine(str);
f.close();
console.log("done");
phantom.exit();
});
}
});
PhantomJS comes with a lot of included examples. Take a look here.