I have a code that works for me about 70% of the time, the other times it throws the Error code 91 "Object Variable or With block Variable not set". If i click End and re-run it it will work fine.
The function is taking data that is entered into cells of an excel spreadsheet and populating text boxes, checking radio buttons, and selecting from drop-down lists on a webpage.
I can't post a link to the actual webpage for privacy issues but I'm hoping someone can help me why the error is coming up?
The line that shows the error is
drp.selectedIndex = Thisworkbook.sheets("sheet1").Range("L2").
Sub FillInternetForm()
Dim ie As Object
Set ie = CreateObject("InternetExplorer.Application")
'set references for copying to submitted reps sheet and deleting for next rep
Application.ScreenUpdating = False
'create new instance of IE.
ie.navigate "removed for privacy"
'go to web page listed inside quotes
ie.Visible = True
While ie.busy
DoEvents 'wait until IE is done loading page.
Wend
'select onboarding system CRIS or ENS
Set html = ie.document
Dim drp As HTMLFormElement
Set drp = html.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff45_1$ctl00 $DropDownChoice")
drp.selectedIndex = ThisWorkbook.Sheets("sheet1").Range("L2")
'set address nickname based on value
Set drp = html.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff32_1$ctl00 $Lookup")
drp.selectedIndex = ThisWorkbook.Sheets("sheet1").Range("m2")
'set market based on value
Set drp = html.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff6_1$ctl00$ DropDownChoice")
drp.selectedIndex = ThisWorkbook.Sheets("sheet1").Range("e2")
'check Not moved from another partner
ie.document.getElementById("ctl00_m_g_62853594_bb4b_4cec_8b5c_17fb6abb735e_ff46_1_ctl00_ctl01").Click
'input name and ssn based on excel sheet values
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff3_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("a2")
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff4_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("b2")
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff5_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("c2")
'input extra fields based on ICL value
'owner
ie.document.all("ctl00_m_g_62853594_bb4b_4cec_8b5c_17fb6abb735e_ff11_1_ctl00_ctl00_TextField").Value = ThisWorkbook.Sheets("sheet1").Range("j2")
'city
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff14_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("f2")
'state
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff15_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("g2")
'address
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff13_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("i2")
'phone
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff10_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("k2")
'zip
ie.document.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff16_1$ctl00$ctl00$TextField").Value = ThisWorkbook.Sheets("sheet1").Range("h2")
End Sub
While ie.busy is probably to blame. Use Loop Until ie.readyState = READYSTATE_COMPLETE instead.
I would cast .Range("L2") as an integer using CInt(.Range("L2")). You have a reference to the HTML Object library set. You should go ahead and add a reference to Microsoft Internet Controls. This way you'll get advantages of intellisense and internet constants. If you don't want to set the reference add Const READYSTATE_COMPLETE = 4 to the code and change the ie references back
Sub FillInternetForm()
Dim ie As InternetExplorer
Dim doc As HTMLDocument
Set ie = New InternetExplorer
'set references for copying to submitted reps sheet and deleting for next rep
Application.ScreenUpdating = False
'create new instance of IE.
ie.navigate "removed for privacy"
'go to web page listed inside quotes
ie.Visible = True
Do
DoEvents
Loop Until ie.readyState = READYSTATE_COMPLETE
With Sheets("sheet1")
'select onboarding system CRIS or ENS
Set doc = ie.document
Dim drp As HTMLFormElement
Set drp = HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff45_1$ctl00 $DropDownChoice")
drp.selectedIndex = CInt(.Range("L2"))
'set address nickname based on value
Set drp = HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff32_1$ctl00 $Lookup")
drp.selectedIndex = CInt(.Range("m2"))
'set market based on value
Set drp = HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff6_1$ctl00$ DropDownChoice")
drp.selectedIndex = CInt(.Range("e2"))
'check Not moved from another partner
doc.getElementById("ctl00_m_g_62853594_bb4b_4cec_8b5c_17fb6abb735e_ff46_1_ctl00_ctl01").Click
'input name and ssn based on excel sheet values
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff3_1$ctl00$ctl00$TextField").value = .Range("a2")
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff4_1$ctl00$ctl00$TextField").value = .Range("b2")
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff5_1$ctl00$ctl00$TextField").value = .Range("c2")
'input extra fields based on ICL value
'owner
doc.all("ctl00_m_g_62853594_bb4b_4cec_8b5c_17fb6abb735e_ff11_1_ctl00_ctl00_TextField").value = .Range("j2")
'city
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff14_1$ctl00$ctl00$TextField").value = .Range("f2")
'state
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff15_1$ctl00$ctl00$TextField").value = .Range("g2")
'address
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff13_1$ctl00$ctl00$TextField").value = .Range("i2")
'phone
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff10_1$ctl00$ctl00$TextField").value = .Range("k2")
'zip
doc.all("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff16_1$ctl00$ctl00$TextField").value = .Range("h2")
End With
End Sub
I agree with #Thomas Inzina in that it's probably because the document hasn't fully loaded yet.
But I've found with my past projects that the best way to handle these 30% failure rates is to just "wait a bit longer" before processing or referring to elements.
Try adding a DoEvents and a Sleep command before you selectedIndex call.
See if it makes a difference
At the top of your module (before any subs are declared)
Private Declare Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Then add the two lines before you start set the doc variable
Doevents
Sleep 2000 ' Sleep two seconds (2000 ms)
Set doc = ie.document
You can play with the sleep value once you get a number that works for you
EDIT - added solution
A more reliable way is to actually run a loop that has DoEvents/Sleep and an incrementing counter for numTries. It exits the loop only when the check for getElementById doesn't fail - or your maxTries counter is reached.
Pretty simple loop but let me know if you need an example
EDIT - EXAMPLE: A wait until loaded loop
Change this
Set drp = HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff45_1$ctl00 $DropDownChoice")
drp.selectedIndex = CInt(.Range("L2"))
To this:
Dim iTries As Integer
Dim iMaxTries As Integer ' it's better to turn this into a const at top of your sub
iMaxTries = 3
iTries = 0
While (iTries < iMaxTries) And IsNull(HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff45_1$ctl00 $DropDownChoice"))
iTries = iTries + 1
DoEvents
Sleep 750 ' try 3/4 second delays to start with
Wend
If iTries = iMaxTries Then
MsgBox "Did not load HTML element in " & iMaxTries & " tries"
Exit Sub
End If
' Should all be loaded and ready to process now
Set drp = HTML.getElementById("ctl00$m$g_62853594_bb4b_4cec_8b5c_17fb6abb735e$ff45_1$ctl00 $DropDownChoice")
drp.selectedIndex = CInt(.Range("L2"))
Related
I've written some code in vba to get the movie names from a torrent website using .createDocumentFromUrl() method. As I've never worked with this method and haven't found any remarkable information either on it to successfully go with, I get stuck. I have tried to create a script, though.
Here is my try:
Sub Fromurl_Method()
Const URL As String = "https://yts.am/browse-movies"
Dim hStart As HTMLDocument, hdoc As HTMLDocument
Set hStart = New HTMLDocument
Set hdoc = hStart.createDocumentFromUrl(URL, vbNullString)
Do While hdoc.readyState = "loading" Or hdoc.readyState = "interactive": DoEvents: Loop
Debug.Print hdoc.DocumentElement.innerHTML
End Sub
When I execute the above script, it fetches some html elements which are not from that website. I took a closer look into the elements I have parsed and noticed this line on the top This content cannot be displayed in a frame. It is behaving the same way with most of the sites. How can I make it successful? Thanks in advance.
Once again, my intention is to parse all the movie names from that site using .createDocumentFromUrl().
Sub Get_Info()
Dim Elems, e As Variant
Const READYSTATE_COMPLETE& = 4&
Dim ie As Object
Set ie = Nothing
DoEvents
Set ie = CreateObject("InternetExplorer.Application")
DoEvents
With ie
.Visible = false
.Navigate "https://yts.am/browse-movies"
While Not .readyState = READYSTATE_COMPLETE
DoEvents
Wend
End With
Dim i As Double
With ie.Document
Set Elems = .getElementsByTagName("a")
DoEvents
i = 2
For Each e In Elems
If e.getAttribute("class") = "browse-movie-title" Then
Range("A" & i).Value = e.innerText
i = i + 1
End If
Next e
End With
Set Elems = Nothing
Set e = Nothing
ie.Quit
Set ie = Nothing
End Sub
The code above will give you a list of all movies. Just modify the code to adapt it to your needs of getting the first one if you only need the first one.
I recently came across a new problem while working with my Excel VBA code in order to automate the input a value from a cell (order number) into Amazon's search box and searching for the order. I can't seem to locate the proper object to reference the search button on the homepage of the Amazon Seller home page in order to click and proceed. My code is this:
Option Explicit
Dim HTMLDoc As HTMLDocument
Dim MyBrowser As InternetExplorer
Sub MyAmazonSeller()
Dim MyHTML_Element As IHTMLElement
Dim MyURL As String
Dim oSignInLink As HTMLLinkElement
Dim oInputEmail As HTMLInputElement
Dim oInputPassword As HTMLInputElement
Dim oInputSigninButton As HTMLInputButtonElement
Dim oInputSearchOrder As HTMLInputElement
Dim oInputSearchButton As HTMLInputButtonElement
MyURL = "https://sellercentral.amazon.com/gp/homepage.html"
Set MyBrowser = New InternetExplorer
' Open the browser and navigate.
With MyBrowser
.Silent = True
.Navigate MyURL
.Visible = True
Do
DoEvents
Loop Until .ReadyState = READYSTATE_COMPLETE
End With
' Get the html document.
Set HTMLDoc = MyBrowser.Document
' See if you have the sign in link is because you are in the main
' page
Set oSignInLink = HTMLDoc.getElementById("signin-button-container")
If Not oSignInLink Is Nothing Then
oSignInLink.Click
Do
DoEvents
Loop Until MyBrowser.ReadyState = READYSTATE_COMPLETE
End If
' Get the email field and the next button
Set oInputEmail = HTMLDoc.getElementById("username")
Set oInputPassword = HTMLDoc.getElementById("password")
' Click the button and wait
oInputEmail.Value = "xxxxxxxxx#xxxxxx.net"
' Get the password field and the sign in button
Set oInputPassword = HTMLDoc.getElementById("password")
Set oInputSigninButton = HTMLDoc.getElementById("sign-in-button")
' Click the button and wait
oInputPassword.Value = "xxxxxxxx"
oInputSigninButton.Click
Do
DoEvents
Loop Until MyBrowser.ReadyState = READYSTATE_COMPLETE
Set oInputSearchOrder = HTMLDoc.getElementById("sc-search-field")
oInputSearchOrder.Value = "110-7706193-5695453"
Set oInputSearchButton = HTMLDoc.getElementByClassName("sc-search-button")
oInputSearchButton.Click
Do
DoEvents
Loop Until MyBrowser.ReadyState = READYSTATE_COMPLETE
Err_Clear:
If Err <> 0 Then
Err.Clear
Resume Next
End If
End Sub
The section right before the Err_Clear is the new snippet of code I've been trying to modify and work with. It seems that the search button does not have a proper ID, so therefore is more difficult to reference. I am getting an Error 91 every time the code gets to the second to last snippet.
Currently I have 2 pieces of code that work separately, but when used together they don't work properly.
The first code asks the user to input information which is stored. It then navigates to the correct webpage where it uses the stored user input information to navigate via filling and submitting a form. It arrives at the correct place.
The second code uses a specific URL via ie.navigate "insert url here" to navigate to the same place as the first code. It then scrapes URL data and stores it in a newly created sheet. It does this correctly.
When merging them I replace the navigation segment from the second code with the first code, but then it only stores the first 5 of 60 URLs as if it hadn't fully loaded the page before scraping data. It seems to skip the code directly after ie.document.forms(0).submit which is supposed to wait for the page to load before moving on to the scraping..
extra info: the button wasn't defined so I cannot just click it so I had to use ie.document.forms(0).submit
Summary of what I want the code to do:
request user input
store user input
open ie
navigate to page
enter user input into search field
select correct search category from listbox
submit form
'problem happens here
scrape url data
store url data in specific excel worksheet
The merged code:
Sub extractTablesData()
Dim ie As Object, obj As Object
Dim Var_input As String
Dim elemCollection As Object
Dim html As HTMLDocument
Dim Link As Object
Dim erow As Long
' create new sheet to store info
Application.DisplayAlerts = False
ThisWorkbook.Sheets("HL").Delete
ThisWorkbook.Sheets.Add.Name = "HL"
Application.DisplayAlerts = True
Set ie = CreateObject("InternetExplorer.Application")
Var_input = InputBox("Enter info")
With ie
.Visible = True
.navigate ("URL to the webpage")
While ie.readyState <> 4
DoEvents
Wend
'Input Term 1 into input box
ie.document.getElementById("trm1").Value = Var_input
'accessing the Field 1 ListBox
For Each obj In ie.document.all.Item("FIELD1").Options
If obj.Value = "value in listbox" Then
obj.Selected = True
End If
Next obj
' button undefined - using this to submit form
ie.document.forms(0).submit
'----------------------------------------------------------------
'seems to skip this part all together when merged
'Wait until IE is done loading page
Do While ie.readyState <> READYSTATE_COMPLETE
Application.StatusBar = "Trying to go to website…"
DoEvents
Loop
'----------------------------------------------------------------
Set html = ie.document
Set ElementCol = html.getElementsByTagName("a")
For Each Link In ElementCol
erow = Worksheets("HL").Cells(Rows.Count, 1).End(xlUp).Offset(1, 0).Row
Cells(erow, 1).Value = Link
Cells(erow, 1).Columns.AutoFit
Next
Application.StatusBar = “”
Application.ScreenUpdating = True
End With
End Sub
I've been stuck for quite some time on this and haven't found any solutions on my own so I'm reaching out. Any help will be greatly appreciated!
You mentioned you think the website might not be fully loaded. This is a common problem because of the more dynamic elements on a webpage. The easiest way to handle this is to insert the line:
Application.Wait Now + Timevalue("00:00:02")
This will force the code to pause for an additional 2 seconds. Insert this line below the code which waits for the page to load and this will give Internet Explorer a chance to catch back up. Depending on the website and the reliability of your connection to it I recommend adjusting this value anywhere up to about 5 seconds.
Most websites seem to require additional waiting like this, so handy code to remember when things don't work as expected. Hope this helps.
I solved this by using a completely different method. I used a query table with strings to go where I wanted.
Sub ExtractTableData()
Dim This_input As String
Const prefix As String = "Beginning of url"
Const postfix As String = "end of url"
Dim qt As QueryTable
Dim ws As Worksheet
Application.DisplayAlerts = False
ThisWorkbook.Sheets("HL").Delete
ThisWorkbook.Sheets.Add.Name = "HL"
Application.DisplayAlerts = True
This_input = InputBox("enter key info to go to specific url")
Set ws = ActiveSheet
Set qt = ws.QueryTables.Add( _
Connection:="URL;" & prefix & This_input & postfix, _
Destination:=Worksheets("HL").Range("A1"))
qt.RefreshOnFileOpen = True
qt.WebSelectionType = xlSpecifiedTables
'qt.webtables is key to getting the specific table on the page
qt.WebTables = 2
qt.Refresh BackgroundQuery:=False
End Sub
I am new to vba.
I am trying to use below code by David Zemens to fetch data from yelp
Option Explicit
Private Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Sub find()
'Uses late binding, or add reference to Microsoft HTML Object Library
' and change variable Types to use intellisense
Dim ie As Object 'InternetExplorer.Application
Dim html As Object 'HTMLDocument
Dim Listings As Object 'IHTMLElementCollection
Dim l As Object 'IHTMLElement
Dim r As Long
Set ie = CreateObject("InternetExplorer.Application")
With ie
.Visible = False
.Navigate "http://www.yelp.com/search?find_desc=boutique&find_loc=New+York%2C+NY&ns=1&ls=3387133dfc25cc99#start=10"
' Don't show window
'Wait until IE is done loading page
Do While .readyState <> 4
Application.StatusBar = "Downloading information, Please wait..."
DoEvents
Sleep 200
Loop
Set html = .Document
End With
Set Listings = html.getElementsByTagName("LI") ' ## returns the list
For Each l In Listings
'## make sure this list item looks like the listings Div Class:
' then, build the string to put in your cell
If InStr(1, l.innerHTML, "media-block clearfix media-block-large main-attributes") > 0 Then
Range("A1").Offset(r, 0).Value = l.innerText
r = r + 1
End If
Next
Set html = Nothing
Set ie = Nothing
End Sub
Problem is that it's not getting any data from the source.
Regards
There's a lot of work to be done.
Here's something that you can start with. Hopefully, you will be able to find the other pieces of information using the same logic. This will print business names in the immediate window. I've found the business names in meta tag description.
I've changed the sleep amount to 5 seconds. IE will be able to fully load and the rest of the code will be processed reliably. The initial 200 milliseconds gave results once every couple of runs. I guess this depends how fast your computer is so 5 seconds is pretty safe I guess.
Option Explicit
Declare Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Sub find()
'Uses late binding, or add reference to Microsoft HTML Object Library
' and change variable Types to use intellisense
Dim returnstring As String 'this is going to hold boutiques names
Dim ie As Object 'InternetExplorer.Application
Dim html As Object 'HTMLDocument
Dim meta As Object 'IHTMLElementCollection
Dim l As Object 'IHTMLElement
Dim r As Long
Set ie = CreateObject("InternetExplorer.Application")
With ie
.Visible = False
.Navigate "http://www.yelp.com/search?find_desc=boutique&find_loc=New+York%2C+NY&ns=1&ls=3387133dfc25cc99#start=10"
' Don't show window
'Wait until IE is done loading page
Do While .readyState <> 4
Application.StatusBar = "Downloading information, Please wait..."
DoEvents
Sleep 5000
Loop
Set html = .Document
End With
Set meta = html.GetElementsByTagName("META") ' ## returns attribures
Dim m As Object
For Each m In meta
If InStr(m.Content, "Reviews on Boutique in New York -") > 0 Then
returnstring = Replace(m.Content, "Reviews on Boutique in New York -", "")
End If
Next
Dim i As Integer
For i = 0 To UBound(Split(returnstring, ","))
Debug.Print (Split(returnstring, ",")(i))
Next
Set html = Nothing
Set ie = Nothing
End Sub
Myoutput:
I am trying to find a way to get the data from yelp.com
I have a spreadsheet on which there are several keywords and locations. I am looking to extract data from yelp listings based on these keywords and locations already in my spreadsheet.
I have created the following code, but it seems to get absurd data and not the exact information I am looking for.
I want to get business name, address and phone number, but all I am getting is nothing. If anyone here could help me solve this problem.
Sub find()
Dim ie As Object
Set ie = CreateObject("InternetExplorer.Application")
With ie
ie.Visible = False
ie.Navigate "http://www.yelp.com/search?find_desc=boutique&find_loc=New+York%2C+NY&ns=1&ls=3387133dfc25cc99#start=10"
' Don't show window
ie.Visible = False
'Wait until IE is done loading page
Do While ie.Busy
Application.StatusBar = "Downloading information, lease wait..."
DoEvents
Loop
' Make a string from IE content
Set mDoc = ie.Document
peopleData = mDoc.body.innerText
ActiveSheet.Cells(1, 1).Value = peopleData
End With
peopleData = "" 'Nothing
Set mDoc = Nothing
End Sub
If you right click in IE, and do View Source, it is apparent that the data served on the site is not part of the document's .Body.innerText property. I notice this is often the case with dynamically served data, and that approach is really too simple for most web-scraping.
I open it in Google Chrome and inspect the elements to get an idea of what I'm really looking for, and how to find it using a DOM/HTML parser; you will need to add a reference to Microsoft HTML Object Library.
I think you can get it to return a collection of the <DIV> tags, and then check those for the classname with an If statment inside the loop.
I made some revisions to my original answer, this should print each record in a new cell:
Option Explicit
Private Sub Sleep Lib "kernel32" (ByVal dwMilliseconds As Long)
Sub find()
'Uses late binding, or add reference to Microsoft HTML Object Library
' and change variable Types to use intellisense
Dim ie As Object 'InternetExplorer.Application
Dim html As Object 'HTMLDocument
Dim Listings As Object 'IHTMLElementCollection
Dim l As Object 'IHTMLElement
Dim r As Long
Set ie = CreateObject("InternetExplorer.Application")
With ie
.Visible = False
.Navigate "http://www.yelp.com/search?find_desc=boutique&find_loc=New+York%2C+NY&ns=1&ls=3387133dfc25cc99#start=10"
' Don't show window
'Wait until IE is done loading page
Do While .readyState <> 4
Application.StatusBar = "Downloading information, Please wait..."
DoEvents
Sleep 200
Loop
Set html = .Document
End With
Set Listings = html.getElementsByTagName("LI") ' ## returns the list
For Each l In Listings
'## make sure this list item looks like the listings Div Class:
' then, build the string to put in your cell
If InStr(1, l.innerHTML, "media-block clearfix media-block-large main-attributes") > 0 Then
Range("A1").Offset(r, 0).Value = l.innerText
r = r + 1
End If
Next
Set html = Nothing
Set ie = Nothing
End Sub