scrapy sql or sqlite ... can't get required output - sql

Question

i'm trying to extract input fields from a website page and the page url (which contain these inputs ) and store them into a database ... ok
*** code works fine with no errors , but this isn't the desired output i want
spider code :
class MySpider(CrawlSpider):
name = 'isa_spider'
allowed_domains = ['testaspnet.vulnweb.com']
start_urls = ['http://testaspnet.vulnweb.com']
rules = (
Rule(SgmlLinkExtractor(allow=('/*' ) ),callback='parse_item'),)
def parse_item(self, response):
hxs = HtmlXPathSelector(response)
item=IsaItem()
item['response_fld']=response.url
res = hxs.select("//input[(#id or #name) and (#type = 'text' )]/#id ").extract()
item['text_input'] = res[0] if res else None # None is default value in case no field found
res = hxs.select("//input[(#id or #name) and (#type = 'password')]/#id").extract()
item['pass_input'] = res[0] if res else None # None is default value in case no field found
res = hxs.select("//input[(#id or #name) and (#type = 'file')]/#id").extract()
item['file_input'] = res[0] if res else None # None is default value in case no field found
return item
pipeline code
class SQLiteStorePipeline(object):
def __init__(self):
self.conn = sqlite3.connect('./project.db')
self.cur = self.conn.cursor()
def process_item(self, item, spider):
self.cur.execute("insert into inputs ( input_name) values(?)", (item['text_input'],))
self.cur.execute("insert into inputs ( input_name) values(?)", (item['pass_input'],))
self.cur.execute("insert into inputs ( input_name) values(?)", (item['file_input'],))
self.cur.execute("insert into links (link) values(?)", (item['response_fld'],))
self.conn.commit()
return item
database schema picture
required output picture
(sorry for not inserting images directly since my reputation is less than 10)

Answer 1

Haven't tested this:
class SQLiteStorePipeline(object):
def __init__(self):
self.conn = sqlite3.connect('./project.db')
self.cur = self.conn.cursor()
def process_item(self, item, spider):
cursor = self.cur
target_id = ? # determine target id
cursor.execute("insert into links (target, link) values(?, ?)", (target_id, item['response_fld'],))
link_id = cursor.lastrowid # find out just inserted link id
cursor.execute("insert into inputs (link_id, input_name, input_type) values(?, ?, ?)", (link_id, item['text_input'], 1))
cursor.execute("insert into inputs (link_id, input_name, input_type) values(?, ?, ?)", (link_id, item['pass_input'], 2))
cursor.execute("insert into inputs (link_id, input_name, input_type) values(?, ?, ?)", (link_id, item['file_input'], 3))
self.conn.commit()