I'm trying to crawl a .aspx page, but it redirects me to a page which doesn't exist.
To solve this, I tried to set 'dont_merge_cookies': True and 'dont_redirect': True, and overwrite my start_requests, but now, it gives me an error "'Response' object has no attribute 'body_as_unicode'" and my response class type is 'scrapy.http.response.Response'.
Here's my code:
class Inon_Spider(BaseSpider):
name = 'Inon'
allowed_domains = ['www.shop.inonit.in']
start_urls = ['http://www.shop.inonit.in/Products/Inonit-Men-Jackets/QUIRK-BOX/Toy-Factory-Jacket---Soldiers/pid-1177471.aspx?Rfs=&pgctl=713619&cid=CU00049295']
#redirects to http://www.shop.inonit.in/Products/Inonit-Men-Jackets/QUIRK-BOX/Toy-Factory-Jacket---Soldiers/1177471
def start_requests(self):
start_urls = ['http://www.shop.inonit.in/Products/Inonit-Men-Jackets/QUIRK-BOX/Toy-Factory-Jacket---Soldiers/pid-1177471.aspx?Rfs=&pgctl=713619&cid=CU00049295']
for i in start_urls:
yield Request(i, meta = {
'dont_merge_cookies': True,
'dont_redirect': True,
'handle_httpstatus_list': [302]
},callback=self.parse)
def parse(self, response):
print "Response %s" %response.__class__
resp = TextResponse
item = DealspiderItem()
hxs = HtmlXPathSelector(resp)
title = hxs.select('//div[#class="aboutproduct"]/div[#class="container9"]/div[#class="ctl_aboutbrand"]/h1/text()').extract()
price = hxs.select('//span[#id="ctl00_ContentPlaceHolder1_Price_ctl00_spnWebPrice"]/span[#class="offer"]/span[#id="ctl00_ContentPlaceHolder1_Price_ctl00_lblOfferPrice"]/text()').extract()
prc = price[0].replace("Rs. ","")
description = []
item['price'] = prc
item['title'] = title
item['description'] = description
item['url'] = response.url
return item
Related
I am new to web-scraping.
I want to go to Webpage_A and follow all the links there.
Each of the link lead to a page where I can select some button and download the data in an Excel file.
I tried the below code. But I believe there is an error with
if link:
yield SeleniumRequest(
Instead of using "SeleniumRequest" to follow the links, what should I use?
If using pure Scrapy, I know I can use
yield response.follow(
Thank you
class testSpider(scrapy.Spider):
name = 'test_s'
def start_requests(self):
yield SeleniumRequest(
url='CONFIDENTIAL',
wait_time=15,
screenshot=True,
callback=self.parse
)
def parse(self, response):
tables_name = response.xpath("//div[#class='contain wrap:l']//li")
for t in tables_name:
name=t.xpath(".//a/span/text()").get()
link = t.xpath(".//a/#href").get()
if link:
yield SeleniumRequest(
meta={'table_name': name},
url= link,
wait_time=15,
screenshot=True,
callback=self.parse_table
)
def parse_table(self, response):
name = response.request.meta['table_name']
button_select=response.find_element_by_xpath("(//a[text()='Select All'])").click()
button_st_yr=response.find_element_by_xpath("//select[#name='ctl00$ContentPlaceHolder1$StartYearDropDownList'] /option[1]").click()
button_end_mth=response.find_element_by_xpath("//select[#name='ctl00$ContentPlaceHolder1$EndMonthDropDownList']/option[text()='Dec']").click()
button_download=response.find_element_by_xpath("//input[#id='ctl00_ContentPlaceHolder1_DownloadButton']").click()
yield{
'table_name': name
}
I'm new to Scrapy and I've a problem getting it to return more than the first row. This is the code:
import scrapy
class FarmtoolsSpider(scrapy.Spider):
name = 'farmtools'
allowed_domains = ['www.donedeal.ie']
start_urls = ['https://www.donedeal.ie/farmtools/']
def parse(self, response):
for row in response.xpath('//ul[#class="card-collection"]'):
yield {
'item_title': response.xpath('.//div[1]/p[#class="card__body-
title"]/text()').get(),
'item_county': response.xpath('.//ul[#class="card__body-
keyinfo"]/li[2]/text()').get(),
'item_price':
response.xpath('.//p[#class="card__price"]/span[1]/text()').get(),
'item_id': response.xpath('.//li[#class="card-
item"]/a/#href').get()
}
I would like it to have Title, County, Price, Id of each item in different rows. Actually if I run this code as it is it just gives me the first line.
I have tried getall but that just gives me blocks of each item.
Any help would be appreciated.
Here is working code which returns 30 rows:
class FarmtoolsSpider(scrapy.Spider):
name = 'farmtools'
allowed_domains = ['www.donedeal.ie']
start_urls = ['https://www.donedeal.ie/farmtools/']
def parse(self, response):
rows = response.xpath('//ul[#class="card-collection"]/li')
for row in rows:
yield {
'item_title': row.xpath('.//div[1]/p[#class="card__body-title"]/text()').get(),
'item_county': row.xpath('.//ul[#class="card__body-keyinfo"]/li[2]/text()').get(),
'item_price': row.xpath('.//p[#class="card__price"]/span[1]/text()').get(),
'item_id': row.xpath('.//li[#class="card-item"]/a/#href').get()
}
try row.xpath('.// ) instead of response.xpath
eg
for row in response.xpath('//ul[#class="card-collection"]'):
yield {'item_title': row.xpath('.//div[1]/p[#class="card__body-
title"]/text()').get(), etc...}
I have many other scripts with simlar basic code that work, but when I run this spider in cmd, and I open the .csv file to look at the "titles" saved, I get the xpath copied into excel. Any idea why?
import scrapy
class MovieSpider(scrapy.Spider):
name = 'movie'
allowed_domains = ['https://www.imdb.com/search/title?start=1']
start_urls = ['https://www.imdb.com/search/title?start=1/']
def parse(self, response):
titles = response.xpath('//*[#id="main"]/div/div/div[3]/div[1]/div[3]/h3/a')
pass
print(titles)
for title in titles:
yield {'Title': title}
--- Try Two Below:------
for subject in titles:
yield {
'Title': subject.xpath('.//h3[#class="lister-item-header"]/a/text()').extract_first(),
'Runtime': subject.xpath('.//p[#class="text-muted"]/span/text()').extract_first(),
'Description': subject.xpath('.//p[#class="text-muted"]/p/text()').extract_first(),
'Director': subject.xpath('.//*[#id="main"]/a/text()').extract_first(),
'Rating': subject.xpath('.//div[#class="inline-block ratings-imdb-rating"]/strong/text()').extract_first()
}
Use extract() or extract_first(), also use shorter and more capacious notation for xpath:
import scrapy
class MovieSpider(scrapy.Spider):
name = 'movie'
allowed_domains = ['https://www.imdb.com/search/title?start=1']
start_urls = ['https://www.imdb.com/search/title?start=1/']
def parse(self, response):
subjects = response.xpath('//div[#class="lister-item mode-advanced"]')
for subject in subjects:
yield {
'Title': subject.xpath('.//h3[#class="lister-item-header"]/a/text()').extract_first(),
'Rating': subject.xpath('.//div[#class="inline-block ratings-imdb-rating"]/strong/text()').extract_first(),
'Runtime': subject.xpath('.//span[#class="runtime"]/text()').extract_first(),
'Description': subject.xpath('.//p[#class="text-muted"]/text()').extract_first(),
'Directior': subject.xpath('.//p[contains(text(), "Director")]/a[1]/text()').extract_first(),
}
output:
I have managed to parse the list of advertisements, put some information in AdvertItem and load this item using AdvertLoader. But I could not figure out how I can get some extra information about each advertisement from item page details, put this additional information in the same AdvertItem object and then load the item with all information using AdvertLoader.
class AdvertLoader(ItemLoader):
default_input_processor = MapCompose(unicode.strip, remove_tags)
default_output_processor = Join()
class AdvertSpider(scrapy.Spider):
name = "adverts"
start_urls = [
"http://blablaadverts.com/",
]
adverts_list_xpath = '//table[#class="object-list-table"]/tbody/tr[#class="object-type-apartment"]'
advert_item_fields = {
'id': './#id',
'link': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/#href',
'status': 'normalize-space(.//td[contains(#class, "object-media")]/div/p/a/span[contains(#class, '
'"sold-overlay-list")]/span/text())',
'state': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'city': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'zone': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'address': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'rooms': './/td[contains(#class, "object-rooms")]/text()',
'area': 'normalize-space(.//td[contains(#class, "object-m2")]/text())',
'price': 'normalize-space(.//td[contains(#class, "object-price")]/p/text())',
}
advert_details_xpath = '//table[contains(#class, "object-data-meta")]/tbody/tr'
advert_item_details_fields = {
'floor': './/td/text()',
'built_in_year': './/td/text()',
'condition': './/td/text()',
'ownership': './/td/text()',
'energy_level': './/td/text()',
}
contact_name = '//div[contains(#class, "object-article-contact")]/p[#class="fn"]/text()'
next_page = '//li[contains(#class, "next")]/a/#href'
def parse(self, response):
selector = Selector(response)
for advert in selector.xpath(self.adverts_list_xpath):
loader = AdvertLoader(AdvertItem(), selector=advert)
for field, xpath in self.advert_item_fields.iteritems():
loader.add_xpath(field, xpath)
# This request is not working as I expect.
yield scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
yield loader.load_item()
next_page = response.xpath(self.next_page).extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield Request(next_page, callback=self.parse)
def parse_page2(self, response):
selector = Selector(response)
loader = response.meta['loader'] # type: AdvertLoader
loader.selector = selector
loader.add_xpath('contact_name', self.contact_name)
# yield loader.load_item()
Below code saves only information about each advertisement without extra details from second item details page.
Function parse_page2() is working if I run it separately from parse() function.
How can I collect all information and only then load my AdvertItem object in loader?
I am not sure I get you correctly or not.
But change this part of code
# This request is not working as I expect.
yield scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
yield loader.load_item()
to
# This request is not working as I expect.
scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
loader.load_item()
And then yield in this function when all information is available.
def parse_page2(self, response):
selector = Selector(response)
loader = response.meta['loader'] # type: AdvertLoader
loader.selector = selector
loader.add_xpath('contact_name', self.contact_name)
yield loader.load_item()
What is the correct way to nest Item data?
For example, I want the output of a product:
{
'price': price,
'title': title,
'meta': {
'url': url,
'added_on': added_on
}
I have scrapy.Item of:
class ProductItem(scrapy.Item):
url = scrapy.Field(output_processor=TakeFirst())
price = scrapy.Field(output_processor=TakeFirst())
title = scrapy.Field(output_processor=TakeFirst())
url = scrapy.Field(output_processor=TakeFirst())
added_on = scrapy.Field(output_processor=TakeFirst())
Now, the way I do it is just to reformat the whole item in the pipeline according to new item template:
class FormatedItem(scrapy.Item):
title = scrapy.Field()
price = scrapy.Field()
meta = scrapy.Field()
and in pipeline:
def process_item(self, item, spider):
formated_item = FormatedItem()
formated_item['title'] = item['title']
formated_item['price'] = item['price']
formated_item['meta'] = {
'url': item['url'],
'added_on': item['added_on']
}
return formated_item
Is this correct way to approach this or is there a more straight-forward way to approach this without breaking the philosophy of the framework?
UPDATE from comments: Looks like nested loaders is the updated approach. Another comment suggests this approach will cause errors during serialization.
Best way to approach this is by creating a main and a meta item class/loader.
from scrapy.item import Item, Field
from scrapy.contrib.loader import ItemLoader
from scrapy.contrib.loader.processor import TakeFirst
class MetaItem(Item):
url = Field()
added_on = Field()
class MainItem(Item):
price = Field()
title = Field()
meta = Field(serializer=MetaItem)
class MainItemLoader(ItemLoader):
default_item_class = MainItem
default_output_processor = TakeFirst()
class MetaItemLoader(ItemLoader):
default_item_class = MetaItem
default_output_processor = TakeFirst()
Sample usage:
from scrapy.spider import Spider
from qwerty.items import MainItemLoader, MetaItemLoader
from scrapy.selector import Selector
class DmozSpider(Spider):
name = "dmoz"
allowed_domains = ["example.com"]
start_urls = ["http://example.com"]
def parse(self, response):
mainloader = MainItemLoader(selector=Selector(response))
mainloader.add_value('title', 'test')
mainloader.add_value('price', 'price')
mainloader.add_value('meta', self.get_meta(response))
return mainloader.load_item()
def get_meta(self, response):
metaloader = MetaItemLoader(selector=Selector(response))
metaloader.add_value('url', response.url)
metaloader.add_value('added_on', 'now')
return metaloader.load_item()
After that, you can easily expand your items in the future by creating more "sub-items."
I think it would be more straightforward to construct the dictionary in the spider. Here are two different ways of doing it, both achieving the same result. The only possible dealbreaker here is that the processors apply on the item['meta'] field, not on the item['meta']['added_on'] and item['meta']['url'] fields.
def parse(self, response):
item = MyItem()
item['meta'] = {'added_on': response.css("a::text").extract()[0]}
item['meta']['url'] = response.xpath("//a/#href").extract()[0]
return item
Is there a specific reason for which you want to construct it that way instead of unpacking the meta field ?