Scrapy - Extracting data from next page - scrapy

Question

I need your help folks, to page and extract elements. Here is my spider
import json
import scrapy
class YPSpider(scrapy.Spider):
name = 'yp'
start_urls = ['https://www.infobel.com/fr/france/business/50000/informatique_internet/']
def parse(self, response):
next_page_url = response.xpath('//ul[#class="pagination"]/li[#class="active"]/following-sibling::li[1]/a/#href').extract_first()
if next_page_url:
yield response.follow(next_page_url, callback=self.parse)
if response.meta.get('has_phone'):
item = response.meta['item']
response = json.loads(response.body)
item['phone'] = response['result']
yield item
else:
items = response.xpath('//*[contains(#class, "customer-box")]')
for item in items:
address_lines = item.xpath('.//span[contains(#class, "fa-map-marker")]/../span[#class="detail-text"]//text()').extract()
title = item.xpath('.//h2[#class="customer-item-name"]/a/text()').extract_first().strip()
address = address_lines[0].replace('\r', '').replace('\t', '').strip() if address_lines else ''
village = address_lines[1].replace('\r', '').replace('\t', '').strip() if len(address_lines) >= 1 else ''
phone = item.xpath('.//span[contains(#class, "icon-phone")]/../span[#class="detail-text"]/text()').extract()
item = {
'title': title,
'address': address,
'village': village,
'phone': phone,
}
if phone:
if phone[0].isnumeric():
item['phone'] = phone[0]
yield item
elif len(phone) >= 2:
yield scrapy.Request('https://www.infobel.com/fr/france/Search/Decrypt?encryptedString={}'.format(phone[1]), meta={'item': item, 'has_phone': True}
)
How can I make this crawl to go the page next and scrape elements from that page?
I appreciate your help
thank you in advance

Answer 1

You need to add this code to your parse method:
next_page_url = response.xpath('//ul[#class="pagination"]/li[#class="active"]/following-sibling::li[1]/a/#href').extract_first()
if next_page_url:
yield response.follow(next_page_url, callback=self.parse)
You need to read a bit about Python syntax before asking questions here:
def parse(self, response):
next_page_url = response.xpath('//ul[#class="pagination"]/li[#class="active"]/following-sibling::li[1]/a/#href').extract_first()
if next_page_url:
yield response.follow(next_page_url, callback=self.parse)
if response.meta.get('has_phone'):
item = response.meta['item']
response = json.loads(response.body)
item['phone'] = response['result']
yield item
else:
items = response.xpath('//*[contains(#class, "customer-box")]')
for item in items:
address_lines = item.xpath('.//span[contains(#class, "fa-map-marker")]/../span[#class="detail-text"]//text()').extract()
title = item.xpath('.//h2[#class="customer-item-name"]/a/text()').extract_first().strip()
address = address_lines[0].replace('\r', '').replace('\t', '').strip() if address_lines else ''
village = address_lines[1].replace('\r', '').replace('\t', '').strip() if len(address_lines) >= 1 else ''
phone = item.xpath('.//span[contains(#class, "icon-phone")]/../span[#class="detail-text"]/text()').extract()
item = {
'title': title,
'address': address,
'village': village,
'phone': phone,
}
if phone:
if phone[0].isnumeric():
item['phone'] = phone[0]
yield item
elif len(phone) >= 2:
yield scrapy.Request('https://www.infobel.com/fr/france/Search/Decrypt?encryptedString={}'.format(phone[1]), meta={'item': item, 'has_phone': True}
)