Scrapy yield from a non call back method - scrapy

Question

I am trying to scrape a html file that has a json object with all the required test case data , but processing of the json happens in "find" and "parseTestCaseDetails" method where iteratively i get the test case details , which i finally parse in "findInterestedFields" , so my requirement is to yield the details of test cases to a json file from the the last method called in the hierarchy i.e findInterestedFields , is it possible to achieve this??
Thanks in advance!
import scrapy
import datetime
import json
import re
import collections
import time
import os
import js2xml
from scrapy.selector import HtmlXPathSelector
class AltiplanoAVInprogressTCSpiderTest(scrapy.Spider):
name = "AltiplanoAVInprogressTCSpiderTest"
buildNumber = []
FormatedBuildTime = []
keys = []
testcaseName=''
testcaseSuit=''
testcaseDoc=''
def start_requests(self):
print os.environ["jenkinsdomain"]
urls = [
os.environ["jenkinsdomain"] + "/job/InprogressFlakyAndBlockedTestCaseDetails/lastSuccessfulBuild/"
]
for url in urls:
print url
yield scrapy.Request(url=url, callback=self.parse, errback=self.parseerror)
def parseerror(self, failure):
print failure
def parse(self, response):
hxs = HtmlXPathSelector(response)
buildNumberString = hxs.select('normalize-space(//*[#id="main-panel"]/h1/text())').extract_first()
self.buildNumber = buildNumberString.split("#")[-1].split("(")[0].strip()
buildTimeUnformatd = buildNumberString.split("(")[-1].split(")")[0].strip().replace("PM", "").replace("AM", "")
buildTimeUnformatd = buildTimeUnformatd.strip()
t = time.strptime(buildTimeUnformatd, "%b %d, %Y %I:%M:%S")
self.FormatedBuildTime = time.strftime('%d %b %y %H:%M IST', t)
static_testDetailsUrl = os.environ[
"jenkinsdomain"] + "XX/Inprogress-ANV.html"
yield scrapy.Request(url=static_testDetailsUrl, callback=self.parseTestDetails, errback=self.parseerror)
def find(self, key, dictionary):
for k, v in dictionary.iteritems():
if k == key:
yield v
self.parseTestCaseDetails(v)
elif isinstance(v, dict):
for result in self.find(key, v):
yield result
elif isinstance(v, list):
for d in v:
for result in self.find(key, d):
yield result
def parseTestCaseDetails(self, testcases):
# get the list of test cases and again parse them one by one to get the name and other fields
for testcase in testcases:
self.findInterestedFields(testcase)
def findInterestedFields(self, dictionary):
jsonLoad = json.dumps(dictionary, indent=4)
loaded_json = json.loads(jsonLoad)
readabledatetime = datetime.datetime.now().strftime("%d %b %y %H:%M IST")
for k, v in loaded_json.iteritems():
if k == "name":
testcaseName = v
if k == "fullName":
testcaseSuit = v
if k == "doc":
testcaseDoc = v
yield {"name":"avInprogresstestcases",'avInprogresstestcases':{
'testcaseName':testcaseName
}}
def parseTestDetails(self, response):
data = response.xpath('//script[4]/text()').extract_first().strip()
jstree = js2xml.parse(data)
testDetailsInJson = js2xml.jsonlike.getall(jstree)
jsonLoad = json.dumps(testDetailsInJson[0], indent=4)
loaded_json = json.loads(jsonLoad)
list(self.find('tests', loaded_json))