Get some extra details from another page for each list item using Scrapy framework - scrapy

Question

I have managed to parse the list of advertisements, put some information in AdvertItem and load this item using AdvertLoader. But I could not figure out how I can get some extra information about each advertisement from item page details, put this additional information in the same AdvertItem object and then load the item with all information using AdvertLoader.
class AdvertLoader(ItemLoader):
default_input_processor = MapCompose(unicode.strip, remove_tags)
default_output_processor = Join()
class AdvertSpider(scrapy.Spider):
name = "adverts"
start_urls = [
"http://blablaadverts.com/",
]
adverts_list_xpath = '//table[#class="object-list-table"]/tbody/tr[#class="object-type-apartment"]'
advert_item_fields = {
'id': './#id',
'link': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/#href',
'status': 'normalize-space(.//td[contains(#class, "object-media")]/div/p/a/span[contains(#class, '
'"sold-overlay-list")]/span/text())',
'state': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'city': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'zone': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'address': './/td[#class="object-name"]/h2[contains(#class, "object-title")]/a/text()',
'rooms': './/td[contains(#class, "object-rooms")]/text()',
'area': 'normalize-space(.//td[contains(#class, "object-m2")]/text())',
'price': 'normalize-space(.//td[contains(#class, "object-price")]/p/text())',
}
advert_details_xpath = '//table[contains(#class, "object-data-meta")]/tbody/tr'
advert_item_details_fields = {
'floor': './/td/text()',
'built_in_year': './/td/text()',
'condition': './/td/text()',
'ownership': './/td/text()',
'energy_level': './/td/text()',
}
contact_name = '//div[contains(#class, "object-article-contact")]/p[#class="fn"]/text()'
next_page = '//li[contains(#class, "next")]/a/#href'
def parse(self, response):
selector = Selector(response)
for advert in selector.xpath(self.adverts_list_xpath):
loader = AdvertLoader(AdvertItem(), selector=advert)
for field, xpath in self.advert_item_fields.iteritems():
loader.add_xpath(field, xpath)
# This request is not working as I expect.
yield scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
yield loader.load_item()
next_page = response.xpath(self.next_page).extract_first()
if next_page is not None:
next_page = response.urljoin(next_page)
yield Request(next_page, callback=self.parse)
def parse_page2(self, response):
selector = Selector(response)
loader = response.meta['loader'] # type: AdvertLoader
loader.selector = selector
loader.add_xpath('contact_name', self.contact_name)
# yield loader.load_item()
Below code saves only information about each advertisement without extra details from second item details page.
Function parse_page2() is working if I run it separately from parse() function.
How can I collect all information and only then load my AdvertItem object in loader?

Answer 1

I am not sure I get you correctly or not.
But change this part of code
# This request is not working as I expect.
yield scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
yield loader.load_item()
to
# This request is not working as I expect.
scrapy.Request("http://blablaadverts.com/index.htmlnr=55&search_key=ca41231a29d2ab921aed02e864152c0e",
callback=self.parse_page2, meta={'loader': loader})
loader.load_item()
And then yield in this function when all information is available.
def parse_page2(self, response):
selector = Selector(response)
loader = response.meta['loader'] # type: AdvertLoader
loader.selector = selector
loader.add_xpath('contact_name', self.contact_name)
yield loader.load_item()