'NoneType' object has no attribute 'prettify' - selenium

def josaa_scrape():
"""
Sample usage: df = josaa_scrape("2018", "1")
df.info()
"""
with requests.Session() as s:
R = s.get(url, headers=headers)
data = {}
for key, value in params.items():
data.update({tag['name']: tag['value'] for tag in BeautifulSoup(R.content, 'lxml').select('input[name^=__]')})
data[key] = value
R = s.post(url, data=data)
table = BeautifulSoup(R.text, 'lxml').find(id = 'GridView1')
df = pd.read_html(table.prettify())[0]
df.dropna(inplace = True, how="all")
return df
It is throwing an error 'NoneType' object has no attribute 'prettify'. I have check all id's.

This actually means that BeautifulSoup(R.text, 'lxml').find(id = 'GridView1') returned nothing.
Fix that line so it will return a table element

Related

showing error " list indices must be integers or slices, not str "

showing error like "list indices must be integers or slices, not str"
Here MY APIView
class UploadViewSet(APIView):
parser_classes = (MultiPartParser, FormParser)
permission_classes = (AllowAny,)
def post(self, request, *args, **kwargs):
file = request.POST.get("content")
data = StringIO(file)
reader = csv.reader(data, delimiter=',')
for row in reader:
new_company = Company(
name=row['name'],
hr_name=row['hr_name'],
hr_email=row['hr_email'],
hr_verified=row['hr_verified'],
user_id=row['user_id'],
primary_phone=row['primary_phone'],
comments=row['comments'],
)
new_company.save()
return Response({"status": "success"},status.HTTP_201_CREATED)

Apppend and Delete Rows to Grid with GridTableBase

I am having trouble appending and deleting rows. My table changes a lot and must be rebuilt often so this has been a little tricky. All of my information comes from an SQL database. I am loading the results into a pandas DataFrame and then using it to populate the GridTableBase class. I am now trying to Append and Delete rows, but am having trouble overriding the class. I have been able to somewhat get it to work, but it behaves weird. For some reason, self.table.AppendRows(row) doesn't work and throws an error. The original was self.table.AppendRow(row), but AppendRow isn't a method. So I had to use a different method. I have to change a value in order to get the GridTableMessage to realize there has been a change, which is what I am doing here data.iloc[data.shape[0]-1,0] = str(val)
Ideally, I would add/delete the row from the table itself, but I can't figure out how to do that. I have derived most of my code from here https://github.com/wxWidgets/Phoenix/blob/master/demo/Grid_MegaExample.py but a lot of that will not work properly for me.
As of now, I can append a row, but for some reason, it appends 2 even though only one has been added to the DataFrame and GetNumberRows is returning the correct count. I assume it has something to do with the way I am accessing the table class. Can anyone provide some clarity?
def rowPopup(self, row, evt):
"""(row, evt) -> display a popup menu when a row label is right clicked"""
appendID = wx.Window.NewControlId()#wx.NewId()
deleteID = wx.Window.NewControlId()#wx.NewId()
x = self.GetRowSize(row)/2
if not self.GetSelectedRows():
self.SelectRow(row)
menu = wx.Menu()
xo, yo = evt.GetPosition()
menu.Append(appendID, "Append Row")
menu.Append(deleteID, "Delete Row(s)")
def append(event, self=self, row=row):#event, self=self, row=row
global data
#print("Append")
#self.table.AppendRows(row)
dlg = wx.TextEntryDialog(self,'Enter a new Key ID to insert into the ' + str("'") + data.columns[0] + str("'") + ' column.', 'Insert New Record')
dlg.SetValue("")
if dlg.ShowModal() == wx.ID_OK:
#print('You entered: %s\n' % dlg.GetValue())
val = dlg.GetValue()
#data[~pd.isnull(data).all(1)].fillna('')
#data['tables_id'].apply('(g)'.format)
data.loc[data.iloc[-1].name + 1,:] = ""
data.iloc[data.shape[0]-1,0] = str(val)
self.Reset()
#print(data)
#data = data.append(pd.Series(dtype='object'), ignore_index=True)
#self.data = DataTable(data)
#data[~pd.isnull(data).all(1)].fillna('')
#self.data = DataTable(data)
def delete(event, self=self, row=row):#event, self=self, row=row
global data
rows = self.GetSelectedRows()
data.drop(data.index[rows],inplace=True)
print (data)
self.Reset()
#self.table.DeleteRow(row)
#print(row)
#print(rows)
#EVT_MENU(self, appendID, append)
#EVT_MENU(self, deleteID, delete)
self.Bind(wx.EVT_MENU, append, id=appendID)
self.Bind(wx.EVT_MENU, delete, id=deleteID)
self.PopupMenu(menu, wx.Point(round(x), round(yo)))
menu.Destroy()
class DataTable(gridlib.GridTableBase):
def __init__(self, data):
gridlib.GridTableBase.__init__(self)
self.headerRows = 1
if data is None:
data = pd.DataFrame()
self.data = data
print("Instance")
#Store the row and col length to see if table has changed in size
self._rows = self.GetNumberRows()
self._cols = self.GetNumberCols()
self.odd=gridlib.GridCellAttr()
self.odd.SetBackgroundColour((217,217,217))
self.even=gridlib.GridCellAttr()
self.even.SetBackgroundColour((255,255,255))
def GetAttr(self, row, col, kind):
attr = [self.even, self.odd][row % 2]
attr.IncRef()
return attr
def GetNumberRows(self):
#print("# Rows:",len(self.data))
return len(self.data)# - 1
def GetTypeName(self, row, col):
#print(wx.grid.GRID_VALUE_STRING)
return wx.grid.GRID_VALUE_STRING
def GetNumberCols(self):
#print("# Cols:",len(self.data.columns)+ 1)
return len(self.data.columns) + 1
#return len(self.data.columns) #+ 1
def IsEmptyCell(self, row, col):
return False
def GetValue(self, row, col):
if col == 0:
try:
return self.data.index[row]
except:
print("Row,Col(",row,col,")","OOB")
return ""
else:
try:
return str(self.data.iloc[row, col - 1])
except:
print("Row,Col(",row,col,")","OOB")
return ""
def GetColLabelValue(self, col):
if col == 0:
if self.data.index.name is None:
return 'Index'
else:
return self.data.index.name
return self.data.columns[col - 1]
def ResetView(self, grid):
"""
(wxGrid) -> Reset the grid view. Call this to
update the grid if rows and columns have been added or deleted
"""
print('Old::' , self._rows, self._cols)
print('New::' , self.GetNumberRows(),self.GetNumberCols())
print(data)
grid.BeginBatch()
for current, new, delmsg, addmsg in [
(self._rows, self.GetNumberRows(), gridlib.GRIDTABLE_NOTIFY_ROWS_DELETED, gridlib.GRIDTABLE_NOTIFY_ROWS_APPENDED),
(self._cols, self.GetNumberCols(), gridlib.GRIDTABLE_NOTIFY_COLS_DELETED, gridlib.GRIDTABLE_NOTIFY_COLS_APPENDED),
]:
if new < current:
msg = gridlib.GridTableMessage(self,delmsg,new,current-new)
#grid.ProcessTableMessage(msg)
self.GetView().ProcessTableMessage(msg)
print("OvN:",self._rows,self.GetNumberRows())
return True
if new > current:
msg = gridlib.GridTableMessage(self,addmsg,new-current)
self.GetView().ProcessTableMessage(msg)
grid.ProcessTableMessage(msg)
#self.UpdateValues(grid)
msg = gridlib.GridTableMessage(self, gridlib.GRIDTABLE_REQUEST_VIEW_GET_VALUES)
grid.ProcessTableMessage(msg)
print("OvN:",self._rows,self.GetNumberRows())
grid.EndBatch()
self._rows = self.GetNumberRows()
self._cols = self.GetNumberCols()
# update the column rendering plugins
#self._updateColAttrs(grid)
# XXX
# Okay, this is really stupid, we need to "jiggle" the size
# to get the scrollbars to recalibrate when the underlying
# grid changes.
h,w = grid.GetSize()
grid.SetSize((h+1, w))
grid.SetSize((h, w))
grid.ForceRefresh()
def UpdateValues(self, grid):#self, grid
"""Update all displayed values"""
# This sends an event to the grid table to update all of the values
msg = gridlib.GridTableMessage(self, gridlib.GRIDTABLE_REQUEST_VIEW_GET_VALUES)
grid.table.ProcessTableMessage(msg)
class DataGrid(gridlib.Grid):
def __init__(self, parent, data, lc, tc): # data
gridlib.Grid.__init__(self, parent, - 1) #,colnames,-1 # data
self.lc = lc
self.tc = tc
self.table = DataTable(data)
self.SetTable(self.table, True)
self.Bind(gridlib.EVT_GRID_LABEL_RIGHT_CLICK, self.OnLabelRightClicked)
self.Bind(gridlib.EVT_GRID_CELL_RIGHT_CLICK, self.OnCellRightClick)
self.Bind(gridlib.EVT_GRID_CELL_CHANGED, self.onCellChanged) #wx.grid
def Reset(self):
"""reset the view based on the data in the table. Call
this when rows are added or destroyed"""
self.table.ResetView(self)
def OnCellRightClick(self, event):
print ("OnCellRightClick: (%d,%d)\n" % (event.GetRow(), event.GetCol()))
def OnLabelRightClicked(self, evt):
row, col = evt.GetRow(), evt.GetCol()
if row == -1: print("col")#self.colPopup(col, evt)
elif col == -1: self.rowPopup(row, evt)
def rowPopup(self, row, evt):
"""(row, evt) -> display a popup menu when a row label is right clicked"""
appendID = wx.Window.NewControlId()#wx.NewId()
deleteID = wx.Window.NewControlId()#wx.NewId()
x = self.GetRowSize(row)/2
if not self.GetSelectedRows():
self.SelectRow(row)
menu = wx.Menu()
xo, yo = evt.GetPosition()
menu.Append(appendID, "Append Row")
menu.Append(deleteID, "Delete Row(s)")
def append(event, self=self, row=row):#event, self=self, row=row
global data
#print("Append")
#self.table.AppendRows(row)
dlg = wx.TextEntryDialog(self,'Enter a new Key ID to insert into the ' + str("'") + data.columns[0] + str("'") + ' column.', 'Insert New Record')
dlg.SetValue("")
if dlg.ShowModal() == wx.ID_OK:
val = dlg.GetValue()
#data[~pd.isnull(data).all(1)].fillna('')
#data['tables_id'].apply('(g)'.format)
data.loc[data.iloc[-1].name + 1,:] = ""
data.iloc[data.shape[0]-1,0] = str(val)
self.Reset()
#print(data)
#self.data = DataTable(data)
def delete(event, self=self, row=row):#event, self=self, row=row
global data
rows = self.GetSelectedRows()
data.drop(data.index[rows],inplace=True)
print (data)
self.Reset()
self.Bind(wx.EVT_MENU, append, id=appendID)
self.Bind(wx.EVT_MENU, delete, id=deleteID)
self.PopupMenu(menu, wx.Point(round(x), round(yo)))
menu.Destroy()
class MainFrame(wx.Frame):
def __init__(self, parent, data): # (self, parent, data):
wx.Frame.__init__(self, parent, -1, "Varkey Foundation") #, size=(640,480))
#Create a panel
self.p = wx.Panel(self)
self.Maximize(True)
#Create blank dataframe
data = pd.DataFrame() #pd.DataFrame(np.random.randint(0,100,size=(200, 5)),columns=list('EFGHD')
#data.reset_index(drop=True, inplace=True)
self.data = DataTable(data)
self.nb = wx.Notebook(self.p)
self.p.SetBackgroundColour( wx.Colour( 0, 0, 0 ) ) # 38,38,38
self.nb.SetBackgroundColour(wx.Colour(58, 56, 56) )
#self.SetBackgroundColour( wx.Colour( 255, 255, 56 ) )
#create the page windows as children of the notebook
self.page1 = PageOne(self.nb)
self.page2 = PageTwo(self.nb)
self.page3 = PageThree(self.nb)
# add the pages to the notebook with the label to show on the tab
self.nb.AddPage(self.page1, "Data")
self.nb.AddPage(self.page2, "Analyze")
self.nb.AddPage(self.page3, "Change Log")
#CreateFonts
self.b_font = wx.Font(14,wx.ROMAN,wx.NORMAL,wx.BOLD, True)
self.lbl_font = wx.Font(14,wx.ROMAN,wx.NORMAL,wx.NORMAL, True)
self.cb_font = wx.Font(11,wx.SCRIPT,wx.ITALIC,wx.NORMAL, True)
self.h_font = wx.Font(18,wx.DECORATIVE,wx.ITALIC,wx.BOLD, True)
#Create username textcontrol <<<<<<<<<<<< Passed to grid class
self.tc_user =wx.TextCtrl(self.p,value='cmccall95',size = (130,25))
self.tc_password =wx.TextCtrl(self.p,value='Achilles95', style=wx.TE_PASSWORD | wx.TE_PROCESS_ENTER,size = (130,25))
self.tc_password.Bind(wx.EVT_TEXT_ENTER,self.onLogin)
self.tc_user.SetFont(self.cb_font)
self.tc_password.SetFont(self.cb_font)
#Create Change log lstCtrl <<<<<<<<<<<< Passed to grid class
self.lc_change = wx.ListCtrl(self.p,-1,style = wx.TE_MULTILINE | wx.LC_REPORT | wx.LC_VRULES)
self.lc_change.InsertColumn(0,"User ID")
self.lc_change.InsertColumn(1,"Status")
self.lc_change.InsertColumn(2,"Description")
self.lc_change.InsertColumn(3,"Date/Time")
#Set column widths
self.lc_change.SetColumnWidth(0, 75)
self.lc_change.SetColumnWidth(1, 75)
self.lc_change.SetColumnWidth(2, 450)
self.lc_change.SetColumnWidth(3, 125)
#Create the grid and continue layout
self.grid = DataGrid(self.page1, data, self.lc_change, self.tc_user)
#More layout code...
def onLoadNewData(self, event): #This is how I'm replacing the data in my table class
global data
self.Freeze()
if self.combo_table.GetValue():
#Connect to db
self.connect_mysql()
#Determine db table
self.getTable()
#Get new data
sql_query = "SELECT * FROM " + tbl
self.cursor.execute(sql_query)
temp = pd.read_sql(sql_query, con=self.db_con)
temp.reset_index(drop=True, inplace=True)
data = temp[~pd.isnull(temp).all(1)].fillna('')
#Create title #if data:
if not data.empty:
self.title.SetLabel(str(self.combo_table.GetValue()))
print(str(self.combo_table.GetValue()))
self.grid.Destroy()
self.grid = DataGrid(self.page1, data, self.lc_change, self.tc_user)
#self.grid.HideCol(0)
self.grid.AutoSizeColumns()
#Insert grid into existing sizer
self.p1_sizer.Insert(1,self.grid,1,wx.RIGHT| wx.LEFT|wx.EXPAND, 20)
self.p1_sizer.Layout()
#RESIZE
else:
print("Error:Dataframe is empty")
self.close_connection()
else:
print('CANT BE BLANK')
self.Thaw()
if __name__ == '__main__':
import sys
app = wx.App()
frame = MainFrame(None, sys.stdout) # (None, sys.stdout)
frame.Show(True)
app.MainLoop()

AttributeError: 'rare' is not a valid function for 'Series' object

I guess the problem is with the "apply" function, but what should I change?:
def value(val,spec_1,spec_2,spec_3,spec_4):
'''filtering Specific values'''
if 'DOOm1' in val or 'DOOm2' in val:
return spec_1
elif 'DOOm3' in val:
return spec_2
elif 'DOOm4' not in val:
return spec_3
else:
return spec_4
data['value'] = data.value.apply(value(val = data['value'],spec_1 ='Noted',spec_2= 'Noted',spec_3 = 'rare',spec_4 = 'DOOm4'))
data= data[data['value'] != 'rare']
I think you can use lambda function for pass values from value column like scalars:
data['value'] = data.value.apply(lambda x: value(x, spec_1 ='Noted',spec_2= 'Noted',spec_3 = 'rare',spec_4 = 'DOOm4'))

Odoo 9 context value missing in override method

in odoo9 I override the search_read method. The super method works ok. With the data returned I want to make a filter, the filter is on the context, the value was asigned on the click of the button comming from the view.
<button name="status_instalacion" string="Instalación" type="action" icon="fa-wrench fa-2x" context="{'stage_id' : 1, 'current_id': active_id}"/>
The problem occurs when I query the context in the search_read method. It exists but doesn't have the values I placed
context on click of button:
self._context
{u'lang': u'en_US', u'stage_id': 1, u'tz': False, u'uid': 1, u'current_id': 40, u'tipo_validacion': u'Sistemas Cr\xedticos', u'sistema_critico': u'AGUA'}
the stage_id is the value I want
context on read_search:
self._context
{u'lang': u'en_US', u'bin_size': True, u'tipo_validacion': u'Sistemas Cr\xedticos', u'tz': False, u'uid': 1,
u'active_test': False, u'sistema_critico': u'AGUA'}
as you can see the 'stage_id' value is missing
Tried also assigning the value to a property of the class, but the value never changes it is always the initial value.
from logging import getLogger
from openerp import api, fields, models
_logger = getLogger(__name__)
class MgmtsystemSistemasEquipos(models.Model):
""" Equipos."""
_name = 'mgmtsystem.sistemas.equipos'
dmy = 99 # ---> this value never changes
def dummy(self): # ---> tried calling a function. not work
return self.dmy
def set_dummy(self, id): # ----> set the value
self.dmy = id or self.dmy
codigo = fields.Char(
string=u'Código',
help=u"Código equipo",
required=True,
size=30)
name = fields.Char(
string=u'Nombre equipo',
required=True,
readonly=False,
index=True,
help="Nombre corto equipo",
size=30)
stage_id = fields.Many2one(
'mgmtsystem.action.stage',
'Fase',
default=_default_stage,
readonly=True)
#api.multi
def status_instalacion(self):
import pudb
pu.db
# save value to variable dmy to retrieve later
id = self._context.get('stage_id')
self.set_dummy(id)
#api.model
def search_read(
self, domain=None, fields=None, offset=0,
limit=None, order=None):
import pudb
pu.db
# here the variable allways has the original value (99)
current_stage_id = self.dmy
current_stage_id = self.dummy()
current_stage_id = getattr(self, dmy)
res = super(MgmtsystemSistemasEquipos, self).search_read(
domain, fields, offset, limit, order)
current_id = res[0]['id']
valid_protocols_ids = self._get_ids(
current_stage_id, current_id,
'mgmtsystem_equipos_protocolos',
'mgmtsystem_equipos_protocolos_rel',
'protocolo_id')
# # remove ids
res[0]['protocolos_ids'] = valid_protocols_ids
res[0]['informes_ids'] = valid_informes_ids
res[0]['anexos_ids'] = valid_anexos_ids
return res
# #api.multi
def _get_ids(self, current_stage_id, current_id, model, model_rel, field_rel):
import pudb
pu.db
# in this method the value of the variable is allways the original
current_stage_id = self.dummy()
sql = """ select a.id from
%s as a
join %s as b
on a.id = b.%s where b.equipo_id = %s
and a.stage_id = %s; """ % (model, model_rel, field_rel,
current_id, current_stage_id)
import psycopg2
try:
self.env.cr.execute(sql)
except psycopg2.ProgrammingError, ex:
message = 'Error trying to download data from server. \n {0} \n {1}'.format(ex.pgerror, sql)
_logger.info(message)
return False
rows = self.env.cr.fetchall()
list_of_ids = []
for row in rows:
list_of_ids.append(row[0])
return list_of_ids
I don't know Python very well, and thats the cause of my misunderstanding of how to read the value of the variable.
But then again, Why is the context modified in the search_read method?.
Thank you.
You should try following.
#api.model
def search_read(self, domain=None, fields=None, offset=0, limit=None, order=None):
import pudb
pu.db
# Here you need to get the value from the context.
current_stage_id = self._context.get('stage_id', getattr(self, dmy))
res = super(MgmtsystemSistemasEquipos, self).search_read(domain=domain, fields=fields, offset=offset, limit=limit, order=order)
current_id = res[0]['id']
valid_protocols_ids = self._get_ids(
current_stage_id, current_id,
'mgmtsystem_equipos_protocolos',
'mgmtsystem_equipos_protocolos_rel',
'protocolo_id')
# # remove ids
res[0]['protocolos_ids'] = valid_protocols_ids
res[0]['informes_ids'] = valid_informes_ids
res[0]['anexos_ids'] = valid_anexos_ids
return res
In your code those lines won't work just because there is no recordset available in self (it's correct behaviour search_read must have #api.model decorator).
# here the variable allways has the original value (99)
current_stage_id = self.dmy
current_stage_id = self.dummy()
current_stage_id = getattr(self, dmy)
So just remove those and lines and apply some other logic to get data.

python TypeError: expected string or buffer when parsing JSON from a file

I realize this problem has been answered for other folks but none of the threads are helping me solve it. I'm trying to parse a JSON structure and add all values in the sent_file when the keys match with the tweet_file. The error I'm getting
import sys
import json
def main():
sent_file = open(sys.argv[1])
tweet_file = open(sys.argv[2])
scores = {}
#tweet = {}
#tweet_text = {}
#hw()
#lines(sent_file)
#lines(tweet_file)
for line in sent_file:
term,score = line.split("\t")
scores[term] = int(score)
#print scores.items()
for tweets in tweet_file:
current_sent_value = 0
tweet = {} #this is a dict
#print type(tweets) str
tweet = json.loads(tweets)#[0] #this assignment changes tweet to a list. Why?
if 'text' in tweet:
tweet_text = {}
unicode_string = tweet['text']
encoded_string = unicode_string.encode('utf-8')
tweet_text = encoded_string.split()
for key in tweet_text:
for key in scores:
#print type(tweet_text) -- list
#print type(scores) --dict
if tweet_text.get(key) == scores.get(key): # get() does not work on a list. tweet_text is a list.
current_sent_value += scores(value)
print current_sent_value
if name == 'main':
main()
The error is here \assignment1\tweet_sentiment2.py", line 42, in main
if tweet_text.get(key) == scores.get(key): # get() does not work on a list. tweet_text is a list.
AttributeError: 'list' object has no attribute 'get'