What is Scrapy returning the same item instead of all items in the listing? - scrapy

I have created a spider which is meant to get me a historic view of lotto numbers. The script is working but it is returning the first item only and then duplicating it.
Here is my code which is returning the first row but then repeating it the number of times there is a row but it is supposed to return each row.
import scrapy
class LottoSpider(scrapy.Spider):
name = 'lottos'
start_urls = [
'https://www.lotteryextreme.com/powerball/results'
]
def parse(self, response):
for results in response.xpath('//*[contains(concat( " ", #class, " " ), concat( " ", "wyn", " " ))]'):
date = response.css('span.f20::text').get()
number1 = response.css('.results2 td::text')[0].extract()
number2 = response.css('.results2 td::text')[1].extract()
number3 = response.css('.results2 td::text')[2].extract()
number4 = response.css('.results2 td::text')[3].extract()
number5 = response.css('.results2 td::text')[4].extract()
powerball = response.css('.results2 td::text')[6].extract()
yield {
'date': date,
'number1': number1,
'number2': number2,
'number3': number3,
'number4': number4,
'number5': number5,
'powerball': powerball,
}

In the for loop you need to use results and not response.
import scrapy
class LottoSpider(scrapy.Spider):
name = 'lottos'
start_urls = ['https://www.lotteryextreme.com/powerball/results']
def parse(self, response):
for results in response.xpath('//*[contains(concat( " ", #class, " " ), concat( " ", "wyn", " " ))]'):
date = results.css('span.f20::text').get()
number1 = results.css('.results2 td::text')[0].get()
number2 = results.css('.results2 td::text')[1].get()
number3 = results.css('.results2 td::text')[2].get()
number4 = results.css('.results2 td::text')[3].get()
number5 = results.css('.results2 td::text')[4].get()
powerball = results.css('.results2 td::text')[6].get()
yield {
'date': date,
'number1': number1,
'number2': number2,
'number3': number3,
'number4': number4,
'number5': number5,
'powerball': powerball,
}

Related

My input function and len function does not return a correct result in some cases

I have a question I wrote this script to format an arabic text to a certain format. But I found a problem then when I type a longer sentence it adds more dots then there are letters. I will paste the code here and explain what the problem is.
import itertools
while True:
input_cloze_deletion = input("\nEnter: text pr 'exit'\n> ")
input_exit_check = input_cloze_deletion.strip().lower()
if input_exit_check == "exit":
break
# copy paste
interpunction_list = ["الله", ",", ".", ":", "?", "!", "'"]
# copy paste
interpunction_list = [ ",", ".", ":", "?", "!", "'", "-", "(", ")", "/", "الله", "اللّٰـه"]
text_replace_0 = input_cloze_deletion.replace(",", " ,")
text_replace_1 = text_replace_0.replace(".", " .")
text_replace_2 = text_replace_1.replace(":", " :")
text_replace_3 = text_replace_2.replace(";", " ;")
text_replace_4 = text_replace_3.replace("?", " ?")
text_replace_5 = text_replace_4.replace("!", " !")
text_replace_6 = text_replace_5.replace("'", " ' ")
text_replace_7 = text_replace_6.replace("-", " - ")
text_replace_8 = text_replace_7.replace("(", " ( ")
text_replace_9 = text_replace_8.replace(")", " ) ")
text_replace_10 = text_replace_9.replace("/", " / ")
text_replace_11 = text_replace_10.replace("الله", "اللّٰـه")
text_split_list = text_replace_11.split()
count_number = []
letter_count_list = []
index_list = itertools.cycle(range(1, 4))
for letter_count in text_split_list:
if letter_count in interpunction_list:
letter_count_list.append(letter_count)
elif "ـ" in letter_count:
letter_count = len(letter_count) - 1
count_number.append(letter_count)
print(letter_count)
else:
letter_count = len(letter_count)
count_number.append(letter_count)
print(letter_count)
for count in count_number:
letter_count_list.append(letter_count * ".")
zip_list = zip(text_split_list, letter_count_list)
zip_list_result = list(zip_list)
for word, count in zip_list_result:
if ((len(word)) == 2 or word == "a" or word == "و") and not word in interpunction_list :
print(f" {{{{c{(next(index_list))}::{word}::{count}}}}}", end="")
elif word and count in interpunction_list:
print(word, end = "")
else:
print(f" {{{{c{(next(index_list))}::{word}::{count}}}}}", end="")
when I type كتب عليـ ـنا و علـ ـي
the return is {{c1::كتب::...}} {{c2::عليـ::...}} {{c3::ـنا::...}} {{c1::و::..}} {{c2::علـ::..}} {{c3::ـي::..}}
but it should be
{{c1::كتب::...}} {{c2::عليـ::...}} {{c3::ـنا::..}} {{c1::و::.}} {{c2::علـ::..}} {{c3::ـي::.}}
I add a print function the print the len() results and the result is correct but it add an extra dot in some case.
But when I type just a single "و" it does a correct len() function but when I input a whole sentence it add an extra dot and I don't know why.
please help

Store Groovy SQL result as a variable?

I am very, very new to Groovy (like two days in). I am trying to take a SQL result which will return a single value, and store that value so that it can be put in the body of an email. However, my SQL result always returns NULL, but the other values in the email work just fine.
I know that the result will only return a single value, so how in the world can I get that value stored as a string?
def sql = getSqlInstance()
def user = "\'" + trmCurrentUser.username + "\'";
def name = "\'" + trmCurrentUser.displayname + "\'";
def email = "\'" + trmCurrentUser.email + "\'";
def comm = "\'" + trmComponent.id + "\'";
def rslt=[];
def cmpgn;
rslt[0] = sql.firstRow("select activity_id from DB.TABLE where communication_id=${comm}");
def cmpgn = rslt[0];
toAddress = trmCurrentUser.email;
fromAddress = "me#email.com";
trmUtilities.sendEmail (
//self explanatory
toAddress, fromAddress,
//body of email
user + " " + name + " " + email + " " + comm + " " + cmpgn[0],
//subject of email
" Action on " + trmComponent.componentType + " " + trmComponent.name
);
The resulting email I get is this (with the appropriate values other than the NULL):
'MYID' 'MY NAME' 'MYEMAIL' 'COMM' null

Trouble with break statement in While loop

I'm trying to write a program that collects information from the user, then outputs to a text file. The program collects input correctly, as the print statement I inserted tells me, but the break statement doesn't break the loop when the specified value is input, and the output file is never updated. Any suggestions you could give would really help. Thanks!
# 1. Define Greeting:
def greeting():
print("Welcome, user! Please enter the address data, or type 'quit'.")
# 2. Input
def inputData():
name = input("Enter the name: ")
address = input("Enter the address: ")
city = input("Enter the city: ")
state = input("Enter the state: ")
zipCode = input("Enter the zip: ")
phone = input("Enter the phone number: ")
addressData = str(name + ", " + address + ", " +
city + ", " + state + ", "+ zipCode +
", " + phone)
print(addressData)
return(name)
# 3. Write data to text file
def dataWrite():
f.write(addressData + "\n")
# 4. Display the goodbye message:
def goodbye():
print("Have a nice day!")
# 5. Define Main function:
def main():
greeting()
while name != "quit":
inputData()
if name == "quit":
break
else:
dataWrite()
f.close()
goodbye()
main()
def main():
greeting()
while name != "quit":
name = inputData()
if name == "quit":
break
else:
dataWrite()
f.close()
goodbye()
You are not getting the return value of inputData().

wxPython; passing a value to wx.TextCtrl from another class

I'd like to thank everyone in advance for taking the time to review this question and I'm sure a lot of people get hung up on this at first and I am also a bit new to OOP, I've primarily done vbscripts in the past, so this is a new frontier to me.
My problem is that I need to:
pass a value from one panel to another...
I'm sure its something simple, but my hair is getting grey over this one.
import wx
import win32com.client
class FinanceInfo(wx.Panel):
def __init__(self, parent):
wx.Panel.__init__(self, parent)
zbox = wx.BoxSizer(wx.VERTICAL)
self.Description = wx.TextCtrl(self, -1, style=wx.TE_READONLY|wx.TE_MULTILINE, size=(200,204))
zbox.Add(self.Description,0, wx.EXPAND,15)
self.SetSizer(zbox)
class Frame(wx.Frame):
def __init__(self, *args, **kwargs):
super(Frame, self).__init__(*args, **kwargs)
self.InitUI()
def InitUI(self):
panel = wx.Panel(self, -1)
box1 = wx.BoxSizer(wx.HORIZONTAL)
box2 = wx.BoxSizer(wx.HORIZONTAL)
box3 = wx.BoxSizer(wx.HORIZONTAL)
box4 = wx.BoxSizer(wx.HORIZONTAL)
box5 = wx.BoxSizer(wx.HORIZONTAL)
all_box = wx.BoxSizer(wx.VERTICAL)
overall = wx.BoxSizer(wx.HORIZONTAL)
nb = wx.Notebook(panel)
page2 = FinanceInfo(nb)
nb.AddPage(page2, "Finance Information")
first = wx.StaticText(panel, label="First Name: ")
last = wx.StaticText(panel, label="Last Name: ")
self.DATA = wx.ListBox(panel, style=wx.LB_SINGLE, size=(100,100))
self.Bind(wx.EVT_LISTBOX, self.OnSelection, id=self.DATA.GetId())
self.CLYa = wx.StaticText(panel, label="")
self.P2Da = wx.StaticText(panel, label="")
self.PLYa = wx.StaticText(panel, label="")
self.FN = wx.TextCtrl(panel, size=(75,-1))
self.LN = wx.TextCtrl(panel, size=(75,-1))
Search = wx.Button(panel, label="Search Patient")
self.Bind(wx.EVT_BUTTON, self.pulldata, id=Search.GetId())
Close = wx.Button(panel, label="Close Viewer")
self.Bind(wx.EVT_BUTTON, self.OnClose, id=Close.GetId())
box1.Add(first, 0, wx.ALL, 5)
box2.Add(last, 0, wx.ALL, 5)
box1.Add(self.FN, 1, wx.ALL, 5)
box2.Add(self.LN, 1, wx.ALL, 5)
box3.Add(self.DATA, 1 , wx.ALL, 5)
box4.Add(Search, 0, wx.ALL, 5)
box5.Add(Close, 0, wx.ALL, 5)
all_box.Add(box1, 0, wx.LEFT)
all_box.Add(box2, 0, wx.LEFT)
all_box.Add(wx.StaticLine(panel), 0, wx.ALL|wx.EXPAND, 5)
all_box.Add(box3, 0, wx.CENTER)
all_box.Add(box4, 0, wx.CENTER)
all_box.Add(box5, 0, wx.CENTER)
overall.Add(all_box,0,wx.EXPAND)
overall.Add(nb, 1, wx.EXPAND)
panel.SetSizer(overall)
self.SetSize((500, 275))
self.SetTitle("Maxident Historical Data Viewer")
self.Centre()
self.Show(True)
def OnClose(self, event):
quit()
def pulldata(self, event):
firstname = str(self.FN.GetValue())
lastname = str(self.LN.GetValue())
access = pullID(lastname.strip(), firstname.strip())
access.MoveFirst
dat = ""
while not access.EOF:
a = str(access.Fields("First Name").value)
b = str(access.Fields("Last Name").value)
PID = str(access.Fields("Patient Number").value)
name = str(a + " " + b + " :" + PID)
self.DATA.Insert(name, 0)
access.MoveNext()
def OnSelection(self, event):
x = str(self.DATA.GetStringSelection())
y = x.split(":")
PID = y[1]
pullfinancedata(PID)
def pullID(lastname, firstname):
DB = r"C:\Converted DBases\DATA.mdb"
engine = win32com.client.Dispatch("DAO.DBEngine.36")
db = engine.OpenDatabase(DB)
sql = "select [Patient Number], [First Name], [Last Name] from [tPatients] where [Last Name]='" + lastname.upper() + "' and [First Name]='" + firstname.upper() + "'"
access = db.OpenRecordset(sql)
return access
def pullfinancedata(PID):
DB = r"C:\Converted DBases\DATA.mdb"
engine = win32com.client.Dispatch("DAO.DBEngine.36")
db = engine.OpenDatabase(DB)
sql = "select * from [tPayment History] where [Patient Number]=" + PID
access = db.OpenRecordset(sql)
dat = ""
while not access.EOF:
PD = "Payment Date:\t" + str(access.Fields("Payment Date").value) + '\n'
PM = "Payment Method:\t" + str(access.Fields("Payment Method").value) + '\n'
PP = "Patient Payment:\t" + str(access.Fields("Patient Payment").value) + '\n'
IP = "Insurance Payment:\t" + str(access.Fields("Insurance Payment").value) + '\n'
dat = dat + PD + PM + PP + IP + "\n ------------------ \n"
access.MoveNext()
"""
THIS IS WHERE I NEED HELP!
"""
print dat
"""
I need this dat variable to be passed to the FinanceInfo class and
i've tried FinanceInfo.Description.SetValue(dat) but its not working
"""
def main():
ex = wx.App()
Frame(None)
ex.MainLoop()
if __name__ == '__main__':
main()
I would also be grateful for any other tips or tricks. Not sure if my BoxSizer's make any sense, I mostly got the examples around other places.
There are several ways to do this:
Keep a reference to each panel and pass them around willy nilly. Then you can do stuff like self.panelOne.MyTextCtrl.SetValue(self.otherText.GetValue())
Use wx.PostEvent to pass around the information
Use pubsub
There are probably other ways too, but I prefer the last one for this sort of thing. You can read a simple example here: http://www.blog.pythonlibrary.org/2010/06/27/wxpython-and-pubsub-a-simple-tutorial/
The answer from Mike is correct. For simple communication between a few classes I however prefer to just send a reference to the other class. It is a clean/transparent method that doesnt needs any additional library. See here an example
I hope you already solved ... but if not I revised your example using pubsub and adodbapi.
I used my mdb and adodbapi becouse logic using OpendDataBase and .movenext not works in my Python 2.7 installation .
enter code here
import wx
from wx.lib.pubsub import Publisher
import win32com.client
import adodbapi
adodbapi.adodbapi.verbose = True
# adds details to the sample printout
class FinanceInfo(wx.Panel):
def __init__(self, parent):
wx.Panel.__init__(self, parent)
Publisher().subscribe(self.showFrame, ("show.mainframe"))
zbox = wx.BoxSizer(wx.VERTICAL)
self.Description = wx.TextCtrl(self, -1, style=wx.TE_READONLY|wx.TE_MULTILINE, size=(200,204))
zbox.Add(self.Description,0, wx.EXPAND,15)
self.SetSizer(zbox)
def showFrame(self, msg):
"""
Shows the frame and shows the message sent in the
text control
"""
self.Description.SetValue(msg.data)
class Frame(wx.Frame):
def __init__(self, *args, **kwargs):
super(Frame, self).__init__(*args, **kwargs)
self.InitUI()
def InitUI(self):
#self.FinancePanelInfo = FinanceInfo(self)
panel = wx.Panel(self, -1)
box1 = wx.BoxSizer(wx.HORIZONTAL)
box2 = wx.BoxSizer(wx.HORIZONTAL)
box3 = wx.BoxSizer(wx.HORIZONTAL)
box4 = wx.BoxSizer(wx.HORIZONTAL)
box5 = wx.BoxSizer(wx.HORIZONTAL)
all_box = wx.BoxSizer(wx.VERTICAL)
overall = wx.BoxSizer(wx.HORIZONTAL)
nb = wx.Notebook(panel)
page2 = FinanceInfo(nb)
nb.AddPage(page2, "Finance Information")
first = wx.StaticText(panel, label="First Name: ")
last = wx.StaticText(panel, label="Last Name: ")
self.DATA = wx.ListBox(panel, style=wx.LB_SINGLE, size=(100,100))
self.Bind(wx.EVT_LISTBOX, self.OnSelection, id=self.DATA.GetId())
self.CLYa = wx.StaticText(panel, label="")
self.P2Da = wx.StaticText(panel, label="")
self.PLYa = wx.StaticText(panel, label="")
self.FN = wx.TextCtrl(panel, size=(75,-1))
self.LN = wx.TextCtrl(panel, size=(75,-1))
Search = wx.Button(panel, label="Search Patient")
self.Bind(wx.EVT_BUTTON, self.pulldata, id=Search.GetId())
Close = wx.Button(panel, label="Close Viewer")
self.Bind(wx.EVT_BUTTON, self.OnClose, id=Close.GetId())
box1.Add(first, 0, wx.ALL, 5)
box2.Add(last, 0, wx.ALL, 5)
box1.Add(self.FN, 1, wx.ALL, 5)
box2.Add(self.LN, 1, wx.ALL, 5)
box3.Add(self.DATA, 1 , wx.ALL, 5)
box4.Add(Search, 0, wx.ALL, 5)
box5.Add(Close, 0, wx.ALL, 5)
all_box.Add(box1, 0, wx.LEFT)
all_box.Add(box2, 0, wx.LEFT)
all_box.Add(wx.StaticLine(panel), 0, wx.ALL|wx.EXPAND, 5)
all_box.Add(box3, 0, wx.CENTER)
all_box.Add(box4, 0, wx.CENTER)
all_box.Add(box5, 0, wx.CENTER)
overall.Add(all_box,0,wx.EXPAND)
overall.Add(nb, 1, wx.EXPAND)
panel.SetSizer(overall)
self.SetSize((500, 275))
self.SetTitle("Maxident Historical Data Viewer")
self.Centre()
self.Show(True)
def OnClose(self, event):
quit()
def pulldata(self, event):
firstname = str(self.FN.GetValue())
lastname = str(self.LN.GetValue())
access = pullID(firstname.strip(), lastname.strip())
dat = ""
for rec in access:
a = str(rec[0])
b = str(rec[1])
#PID = str(access.Fields("Patient Number").value)
name = str(a + ":" + b)
print name
self.DATA.Insert(name, 0)
#access.MoveNext()
access.close
def OnSelection(self, event):
x = str(self.DATA.GetStringSelection())
y = x.split(":")
PID = y[0]
#PID = "Rossini Gianni"
dati = pullfinancedata(PID)
righe =""
for line in dati:
print line
riga = str(line)
righe = righe + riga + "\n"
Publisher().sendMessage(("show.mainframe"), righe)
def pullID(name, firstname):
#name = "ROSSINI GIANNI"
#DB = r"d:\coop&mie_doc\pdci\iscritti_2007.mdb"
# db = engine.OpenDatabase(DB)
# data_source = "D:\coop&mie_doc\pdci\iscritti_2007.mdb"
# mdw ="C:\Programmi\File comuni\System\System.mdw"
# DSN = "PROVIDER=Microsoft.Jet.OLEDB.4.0;DATA SOURCE=%s;Jet OLEDB:System Database=%s;" % (data_source, mdw)
SQL_statement = "select [name], [address] from [tessere_2008] where [name] LIKE '" + name.upper() + "%'"
#SQL_statement = "select [name], [address] from [tessere_2008] "
#access = db.OpenRecordset(sql)
# access = engine.Open(sql,conAccess,1,3)
_databasename = "d:\coop&mie_doc\pdci\iscritti_2007.mdb"
_table_name= 'tessere_2008'
_username = ''
_password = ''
_mdw = "C:\Programmi\File comuni\System\System.mdw"
constr = 'Provider=Microsoft.Jet.OLEDB.4.0;Data Source=%s;User Id=%s;Password=%s;Jet OLEDB:System Database=%s;' % (_databasename, _username, _password, _mdw)
#constr = 'Provider=Microsoft.Jet.OLEDB.4.0;Data Source=%s;Jet OLEDB:System Database=%s;' % (_databasename, _mdw)
conAccess = adodbapi.connect(constr)
accessdb = conAccess.cursor()
# accessdb = msaccess.AccessDb()
# connAccess = accessdb.connect("D:\coop&mie_doc\pdci\iscritti_2007.mdb", "Yram", "","C:\Programmi\File comuni\System\System.mdw")
print SQL_statement
accessdb.execute(SQL_statement)
print 'result rowcount shows as= %d. (Note: -1 means "not known")' \
% (accessdb.rowcount,)
# fields = access.getFields()
# print fields
# for item in access:
# print item
#get the results
access = accessdb.fetchmany(1)
#print them
for rec in access:
print rec
return accessdb
def pullfinancedata(PID):
print "pullfinancedata"
print PID
#DB = r"d:\coop&mie_doc\pdci\iscritti_2007.mdb"
#engine = win32com.client.Dispatch(r'ADODB.Recordset')
#db = engine.OpenDatabase(DB)
_databasename = "d:\coop&mie_doc\pdci\iscritti_2007.mdb"
_table_name= 'tessere_2008'
_username = ''
_password = ''
_mdw = "C:\Programmi\File comuni\System\System.mdw"
constr = 'Provider=Microsoft.Jet.OLEDB.4.0;Data Source=%s;User Id=%s;Password=%s;Jet OLEDB:System Database=%s;' % (_databasename, _username, _password, _mdw)
#constr = 'Provider=Microsoft.Jet.OLEDB.4.0;Data Source=%s;Jet OLEDB:System Database=%s;' % (_databasename, _mdw)
conAccess = adodbapi.connect(constr)
accessdb = conAccess.cursor()
#SQL_statement = "select [name], [address] from [tessere_2008] where [name] LIKE '" + name.upper() + "%'"
SQL_statement = "select * from [Tesseramento] where [anagra_iscritto]='" + PID + "'"
#access = db.OpenRecordset(sql)
print SQL_statement
accessdb.execute(SQL_statement)
print 'result rowcount shows as= %d. (Note: -1 means "not known")' \
% (accessdb.rowcount,)
dat = ""
#while not access.EOF:
# PD = "Payment Date:\t" + str(access.Fields("Payment Date").value) + '\n'
# PM = "Payment Method:\t" + str(access.Fields("Payment Method").value) + '\n'
# PP = "Patient Payment:\t" + str(access.Fields("Patient Payment").value) + '\n'
# IP = "Insurance Payment:\t" + str(access.Fields("Insurance Payment").value) + '\n'
# dat = dat + PD + PM + PP + IP + "\n ------------------ \n"
# access.MoveNext()
"""
THIS IS WHERE I NEED HELP!
"""
#get the results
access = accessdb.fetchmany(accessdb.rowcount)
#print them
#for rec in access:
# print rec
"""
I need this dat variable to be passed to the FinanceInfo class and
i've tried FinanceInfo.Description.SetValue(dat) but its not working
"""
return access
def main():
ex = wx.App()
Frame(None)
ex.MainLoop()
if __name__ == '__main__':
main()

groovy closure instantiate variables

is it possible to create a set of variables from a list of values using a closure??
the reason for asking this is to create some recursive functionality based on a list of (say) two three four or five parts
The code here of course doesn't work but any pointers would be helpful.then
def longthing = 'A for B with C in D on E'
//eg shopping for 30 mins with Fiona in Birmingham on Friday at 15:00
def breaks = [" on ", " in ", "with ", " for "]
def vary = ['when', 'place', 'with', 'event']
i = 0
line = place = with = event = ""
breaks.each{
shortline = longthing.split(breaks[i])
longthing= shortline[0]
//this is the line which obviously will not work
${vary[i]} = shortline[1]
rez[i] = shortline[1]
i++
}
return place + "; " + with + "; " + event
// looking for answer of D; C; B
EDIT>>
Yes I am trying to find a groovier way to clean up this, which i have to do after the each loop
len = rez[3].trim()
if(len.contains("all")){
len = "all"
} else if (len.contains(" ")){
len = len.substring(0, len.indexOf(" ")+2 )
}
len = len.replaceAll(" ", "")
with = rez[2].trim()
place = rez[1].trim()
when = rez[0].trim()
event = shortline[0]
and if I decide to add another item to the list (which I just did) I have to remember which [i] it is to extract it successfully
This is the worker part for then parsing dates/times to then use jChronic to convert natural text into Gregorian Calendar info so I can then set an event in a Google Calendar
How about:
def longthing = 'A for B with C in D on E'
def breaks = [" on ", " in ", "with ", " for "]
def vary = ['when', 'place', 'with', 'event']
rez = []
line = place = with = event = ""
breaks.eachWithIndex{ b, i ->
shortline = longthing.split(b)
longthing = shortline[0]
this[vary[i]] = shortline[1]
rez[i] = shortline[1]
}
return place + "; " + with + "; " + event
when you use a closure with a List and "each", groovy loops over the element in the List, putting the value in the list in the "it" variable. However, since you also want to keep track of the index, there is a groovy eachWithIndex that also passes in the index
http://groovy.codehaus.org/GDK+Extensions+to+Object
so something like
breaks.eachWithIndex {item, index ->
... code here ...
}