Scrapy CrawlSpider - Yielding multiple items from a single spider - scrapy

Question

i'm very new either with Scrapy or Python so my vocabulary might be inacurate
I'm trying to get two different items with my CrawlSpider, but i cannot find out how to do it. Currently only the first item "CourseItem" is crawled, the other one is not executed at all.
I believe it's probably the way im calling everything but can't figure exactly what. (callback=parse_course) ?
Each function is working properly if i make one spider per item.
I've tried to put everything in the same function but it doesn't help either.
spider.py
# -*- coding: utf-8 -*-
import scrapy
import re
import datetime
from scrapy.http import Request
from scrapy_spider.items import CourseItem, PronosticItem
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
class GenycourseSpider(scrapy.spiders.CrawlSpider):
name = 'genycourse'
allowed_domains = ['www.geny.com']
rules = (
Rule(
LinkExtractor(allow='/partants-pmu/*'), callback="parse_course"
),
)
def start_requests(self):
date = datetime.date(2016, 03, 12)
datefin = datetime.date(2016, 03, 12)
while date <= datefin:
url = 'http://www.geny.com/reunions-courses-pmu?date=%s' % date.isoformat()
date = date + datetime.timedelta(days=1)
yield Request(url)
def parse_course(self, response):
#Recupération des valeurs
date = response.request.url
reunion = response.xpath("//*[#id='yui-main']/div/div[2]/div[1]/text()").extract_first().strip()
course = response.xpath("//*[#id='yui-main']/div/div[2]/div[3]/span/strong/text()[1]").extract_first().strip()
type = response.xpath("//*[#id='yui-main']/div/div[2]/span").extract_first().strip()
hippodrome = response.xpath("//*[#id='navigation']/a[3]/text()").extract_first()
#Nettoyage du resultat
#On isole la date
date = date[33:43]
#On coupe au niveau de ( et on garde le numéro de la réunion
temp = re.split(r"\(", reunion)
reunion = temp [1]
reunion = re.sub(r'[R\)]', '', reunion)
#On identifie les courses et les stocke
type = re.findall(r'(Attel|Plat|Haies|Steeple-chase|Cross|Mont)', type)
plat = "Plat" in type
obstacle = "Haies" or "Steeple-chase" in type
#On met en place l'objet
courseinfo = CourseItem()
courseinfo['date'] = date
courseinfo['reunion'] = reunion
courseinfo['course'] = course
courseinfo['type'] = type
courseinfo['hippodrome'] = hippodrome
loopcheval = 1
numerocheval = 1
while numerocheval != None:
numerocheval = response.xpath("//*[#id='tableau_partants']/tbody/tr["+ str(loopcheval) +"]/td[1]/text()").extract_first()
if plat is True:
cotecheval = response.xpath("//*[#id='tableau_partants']/tbody/tr["+ str(loopcheval) +"]/td[12]/text()").extract_first().strip()
elif obstacle is True:
cotecheval = response.xpath("//*[#id='tableau_partants']/tbody/tr["+ str(loopcheval) +"]/td[11]/text()").extract_first().strip()
else:
cotecheval = response.xpath("//*[#id='tableau_partants']/tbody/tr["+ str(loopcheval) +"]/td[10]/text()").extract_first().strip()
if numerocheval is not None:
courseinfo['numero'] = numerocheval
courseinfo['cote'] = cotecheval
yield courseinfo
loopcheval = loopcheval + 1
def parse_prono(self, response):
date = response.request.url
reunion = response.xpath("//*[#id='yui-main']/div/div[2]/div[1]/text()").extract_first().strip()
course = response.xpath("//*[#id='yui-main']/div/div[2]/div[3]/span/strong/text()[1]").extract_first().strip()
date = date[33:43]
temp = re.split(r"\(", reunion)
reunion = temp [1]
reunion = re.sub(r'[R\)]', '', reunion)
pronoinfo = PronosticItem()
pronoinfo['date'] = date
pronoinfo['reunion'] = reunion
pronoinfo['course'] = course
ligne = 1
testpronostique = pronostiqueur = response.xpath("//*[#id='selectionsPresse']/table/tr[1]/td[1]/div/div[1]").extract_first()
if testpronostique is None:
pronoinfo['pronostiqueur'] = 'Pas de pronostic'
pronoinfo['premier'] = 0
pronoinfo['second'] = 0
yield pronoinfo
else:
while ligne <= 3:
while ligne <= 2:
colonne = 1
while colonne <= 3:
pronostiqueur = response.xpath("//*[#id='selectionsPresse']/table/tr["+ str(ligne) +"]/td["+ str(colonne) +"]/div/div[1]").extract_first()
pronostiqueur = re.findall(r'(Radio|AIP|Sud|Presse|Casaque|Courrier|che)', pronostiqueur)
pronostique = response.xpath("//*[#id='selectionsPresse']/table/tr["+ str(ligne) +"]/td["+ str(colonne) +"]/div/div[2]/text()").extract_first().strip()
pronostique = re.split(r" - ", pronostique)
premier = pronostique [0]
second = pronostique [1]
pronoinfo['pronostiqueur'] = pronostiqueur
pronoinfo['premier'] = premier
pronoinfo['second'] = second
yield pronoinfo
colonne = colonne + 1
if colonne == 4:
ligne = ligne + 1
if ligne == 3:
pronostiqueur = response.xpath("//*[#id='selectionsPresse']/table/tr[3]/td[1]/div/div[1]").extract_first()
pronostiqueur = re.findall(r'(Radio|AIP|Sud|Presse|Casaque|Courrier|che)', pronostiqueur)
pronostique = response.xpath("//*[#id='selectionsPresse']/table/tr[3]/td[1]/div/div[2]/text()").extract_first().strip()
pronostique = re.split(r" - ", pronostique)
premier = pronostique [0]
second = pronostique [1]
pronoinfo['pronostiqueur'] = pronostiqueur
pronoinfo['premier'] = premier
pronoinfo['second'] = second
yield pronoinfo
ligne = ligne + 1
items.py
class CourseItem(scrapy.Item):
date = scrapy.Field()
reunion = scrapy.Field()
course = scrapy.Field()
type = scrapy.Field()
hippodrome = scrapy.Field()
numero = scrapy.Field()
cote = scrapy.Field()
class PronosticItem(scrapy.Item):
date = scrapy.Field()
reunion = scrapy.Field()
course = scrapy.Field()
pronostiqueur = scrapy.Field()
premier = scrapy.Field()
second = scrapy.Field()
Im expecting to get both CourseItem() and PronosticItem() to get populated and sent to pipeplines and not just only CourseItem()
Thanks for your help !