I've acquires an old VBScript that was used to retrived test score that I'm trying to convert to a VB.net Form app.
I'm stuck with this function
Function getit()
Dim xmlhttp
Dim pageNum
Dim objStream
Dim objDebugStream
Set objStream = CreateObject("ADODB.Stream")
objStream.Type = 1 'adTypeBinary
pageNum = 1
Do While pageNum > 0
Set xmlhttp=CreateObject("MSXML2.ServerXMLHTTP")
'strURL = DownloadDest
Wscript.Echo "Download-URL: " & strURL & "&page_num=" & pageNum
'For basic auth, use the line below together with user+pass variables above
xmlhttp.Open "GET", strURL & "&page_num=" & pageNum, false
xmlhttp.Send
Wscript.Echo "Download-Status: " & xmlhttp.Status & " " & xmlhttp.statusText
If xmlhttp.Status = 200 Then
If Left(LCase(xmlhttp.responseText),16) <> "no records found" Then
If objStream.State = 0 Then
objStream.Open
End If
objStream.Write xmlhttp.responseBody
If debugEachPage Then
Set objDebugStream = CreateObject("ADODB.Stream")
objDebugStream.Type = 1 'adTypeBinary
objDebugStream.Open
objDebugStream.Write xmlhttp.responseBody
objDebugStream.SaveToFile ".\sortest_aleks_" & classCode & "_page_" & pageNum & ".csv"
objDebugStream.Close
Set objDebugStream = Nothing
End If
Else
If pageNum = 1 Then
WScript.Echo "No Records Found for " & classCode
End If
pageNum = 0 ' Have to set this to exit loop
End If
Else
WScript.Echo "Response Status of " & xmlhttp.Status & " for " & classCode
End If
If pageNum <> 0 Then
pageNum = pageNum + 1
End If
Set xmlhttp=Nothing
Loop
If objStream.State <> 0 Then
objStream.SaveToFile LocalFile
objStream.Close
End If
Set objStream = Nothing
End Function
What I wrote looks like this
Private Sub GetALEKSData(ByVal strURL As String)
REM ======================================================================================================
' This Module will access the ALEKS Web Site and access the CofC foreign language scores for the terms indicated days
' The Comma Seperated Values (CSV) as then stored in the main form Text Box
'=========================================================================================================
Dim ALEKStr As System.IO.Stream = Nothing
Dim srRead As System.IO.StreamReader = Nothing
Try
'Create a WebReq for the URL
Dim WebReq As System.Net.WebRequest = System.Net.HttpWebRequest.Create(strURL)
'If required by the server, set the credentials.
WebReq.Credentials = CredentialCache.DefaultNetworkCredentials
'Get the Respponse.
Dim WebResp As System.Net.WebResponse = WebReq.GetResponse
' Display the status.
' If required by the server, set the credentials.
ALEKStr = WebResp.GetResponseStream
srRead = New System.IO.StreamReader(ALEKStr)
' read all the text
TextBox1.Text = srRead.ReadToEnd
Catch ex As Exception
TextBox1.Text = QQ REM Wipe Text box to indicate No DATA to Process
Finally
' Close Stream and StreamReader when done
srRead.Close()
ALEKStr.Close()
End Try
Debug.Print(TextBox1.Text)
REM Remove NO Data message
If InStr(TextBox1.Text, "No records match criteria.") > 0 Then TextBox1.Text = QQ
DataFileHasData = Len(TextBox1.Text) > 0
End Sub
It is returning with :Access denied: wrong3 HTTP header from
Not sure what I'm missing
Try this:
Private Sub GetALEKSData(ByVal strURL As String)
REM ======================================================================================================
' This Module will access the ALEKS Web Site and access the CofC foreign language scores for the terms indicated days
' The Comma Seperated Values (CSV) as then stored in the main form Text Box
'=========================================================================================================
Using wc As New System.Net.WebClient()
Try
wc.Credentials = CredentialCache.DefaultNetworkCredentials
TextBox1.Text = wc.DownloadString(strURL)
Catch
TextBox1.Text = QQ
End Try
End Using
Debug.Print(TextBox1.Text)
If TextBox1.Text.Contains("No records match criteria.") Then TextBox1.Text = QQ
DataFileHasData = Not String.IsNullorWhiteSpace(TextBox1.Text)
End Sub
And if that doesn't work, the error message says, "Access Denied", so the problem is probably this line:
wc.Credentials = CredentialCache.DefaultNetworkCredentials
If that still doesn't help, install fiddler and compare the HTTP requests sent by the old vbscript to the new VB.Net code. You'll be able to see exactly what you're missing.
Setting the UserAgent fixed the issue
Private Sub GetWEBData(ByVal strURL As String)
REM ======================================================================================================
' This Module will access the WEB Web Site and access the CofC foreign language scores for the terms indicated days
' The Comma Seperated Values (CSV) as then stored in the main form Text Box
'=========================================================================================================
'Clear existing data
Try
'Create a WebReq for the URL
Dim WebReq As HttpWebRequest = CType(WebRequest.Create(strURL), HttpWebRequest)
'If required by the server, set the credentials.
WebReq.Credentials = CredentialCache.DefaultNetworkCredentials
WebReq.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
'Get the Respponse.
'Dim WebResp As System.Net.WebResponse = WebReq.GetResponse
Dim WebResp As HttpWebResponse = CType(WebReq.GetResponse(), HttpWebResponse)
' Display the status.
' Console.WriteLine(WebResp.StatusDescription)
' Open the stream using a StreamReader for easy access.
Dim WEBtream As Stream = WebResp.GetResponseStream()
' Open the stream using a StreamReader for easy access.
Dim srRead As New StreamReader(WEBtream)
' Read the content.
Dim responseFromServer As String = srRead.ReadToEnd()
' Display the content.
TextBox1.Text = responseFromServer
TextBox1.Refresh()
'Console.WriteLine(responseFromServer)
' Cleanup the streams and the response.
srRead.Close()
WEBtream.Close()
WebResp.Close()
Catch ex As Exception
MsgBox("WEB DATA READ ERROR OCCURED", MsgBoxStyle.Critical, "Program Error")
End Try
End Sub
Related
it has been some days since I implemented in several PCs my Google Drive upload programme. It is developed in VB.NET console app and I can say it works out pretty good. Except that in some PCs it continues throwing the taskcanceled exception. Basically the programme works at night, where the internet traffic in the pcs should be likely to 0 and it continues trying to upload to a google folder until it succesfully do it OR the time (of an hour usually) runs out.
The Exception looks like this
Ex: System.AggregateException: One or more errors occured. ---> System.Threading.Tasks.TaskCanceledException: A task was canceled.
--- Fine della traccia dello stack dell'eccezione interna ---
in System.Threading.Tasks.Task.ThrowIfExceptional(Boolean includeTaskCanceledExceptions)
in System.Threading.Tasks.Task`1.GetResultCore(Boolean waitCompletionNotification)
in System.Threading.Tasks.Task`1.get_Result()
in Google.Apis.Upload.ResumableUpload.Upload() in C:\Apiary\2021-09-08.15-52-39\Src\Support\Google.Apis\Upload\ResumableUpload.cs:riga 388
in TeraDriveServ.Service1.UploadFile(String FilePath) in C:\Sviluppo\TeraDriveServ\Service1.vb:row 291
---> (Internal exception 0) System.Threading.Tasks.TaskCanceledException: A Task was canceled.<---
This is the code I use
Private Sub UploadFile(FilePath As String)
Dim pathDb As String = System.Configuration.ConfigurationManager.AppSettings("DB")
If Service.ApplicationName <> "Google Drive VB Dot Net" Then CreateService()
Dim mail As String = ""
Dim destinatari As String = ""
Dim TheFile As New Google.Apis.Drive.v2.Data.File()
Dim nome As String = Path.GetFileName(FilePath)
Dim infofile As New IO.FileInfo(FilePath)
' I get some data from the file
Dim dataultimaModifica As DateTime = infofile.LastWriteTime
Dim nomefileeffettivo As String = infofile.Name
Dim estensione = Path.GetExtension(FilePath)
'Getting the file folder
Dim folderId As String = System.Configuration.ConfigurationManager.AppSettings("CARTELLADRIVE")
TheFile.Parents = New List(Of ParentReference) From {New ParentReference() With {.Id = folderId}}
TheFile.Title = nome
TheFile.Description = ""
Dim esiste As Boolean = False
' A query to check if the file is uploaded
Using con As System.Data.SQLite.SQLiteConnection = New System.Data.SQLite.SQLiteConnection("data source=" & pathDb & "databaseFile.db3")
Using com As System.Data.SQLite.SQLiteCommand = New System.Data.SQLite.SQLiteCommand(con)
con.Open()
com.CommandText = "Select ID FROM CARICATI WHERE NOME = '" & nome & "' AND DATA='" & dataultimaModifica & "'"
Using reader As System.Data.SQLite.SQLiteDataReader = com.ExecuteReader()
While reader.Read()
esiste = True
End While
con.Close()
End Using
End Using
End Using
If esiste = True Then
' already loaded
Return
Else
Select Case estensione
Case ".bak"
TheFile.MimeType = "application/octet-stream"
Case ".zip"
TheFile.MimeType = "application/zip"
'Case ".jpeg", ".jpg"
' TheFile.MimeType = "image/ jpeg"
'Case ".txt"
' TheFile.MimeType = "text/plain"
'Case ".pdf"
' TheFile.MimeType = "application/pdf"
'Case ".xls"
' TheFile.MimeType = "application/ vnd.ms - excel"
'Case ".doc"
' TheFile.MimeType = "application/msword"
'Case ".docx"
' TheFile.MimeType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
'Case ".xlsx"
' TheFile.MimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
'Case ".odt"
' TheFile.MimeType = "application/ vnd.oasis.opendocument.text"
Case Else
' I dont like the file extension
Return
End Select
Dim fatto As Integer = 0
' I upload the file details in the DB
Using con As System.Data.SQLite.SQLiteConnection = New System.Data.SQLite.SQLiteConnection("data source=" & pathDb & "databaseFile.db3")
Using com As System.Data.SQLite.SQLiteCommand = New System.Data.SQLite.SQLiteCommand(con)
con.Open()
com.CommandText = "INSERT INTO CARICATI (NOME,DATA) VALUES('" & nome & "','" & dataultimaModifica & "') "
com.ExecuteNonQuery()
fatto = con.LastInsertRowId
' UPLOADED
con.Close()
End Using
End Using
End If
Try
Dim ByteArray As Byte() = System.IO.File.ReadAllBytes(FilePath)
Dim Stream As New System.IO.MemoryStream(ByteArray)
Dim UploadRequest As Google.Apis.Drive.v2.FilesResource.InsertMediaUpload = Service.Files.Insert(TheFile, Stream, TheFile.MimeType)
logger.Debug("Oggetto creato")
' UploadRequest.ContentStream.WriteTimeout = 600000
'If contaUp = 0 Then
' UploadRequest.Service.HttpClient.Timeout = Service.HttpClient.Timeout
'End If
'contaUp = 1
logger.Debug("Starting upload: " & DateTime.Now)
' Here is where the exception points at, the Google Upload() Method
UploadRequest.Upload()
logger.Debug("Upload completed: " & DateTime.Now)
Dim file As File = Nothing
file = UploadRequest.ResponseBody
If file Is Nothing Then
logger.Debug("Upload failed")
' deletes the data from db to try again
AnnulloCaricamento(nome, dataultimaModifica, pathDb)
Else
' I send some stuff to a server
logger.Debug("Cerco di fare POST")
Dim testoSucc As String = System.Configuration.ConfigurationManager.AppSettings("TESTOS")
POST(nomefileeffettivo, dataultimaModifica)
eseguito = True
End If
Catch ex As Exception
' deletes the data from db to try again
AnnulloCaricamento(nome, dataultimaModifica, pathDb) ''05/10/2021
End Try
End Sub
Thanks everyone
My code downloads files in loop but after the last file downloads it keeps downloading files that aren't there. Website shows redirect and 404 error.
I'm new with visual basic so I'm asking for help here.
My.Computer.Network.DownloadFile(strFullUrlDownload, strFullSavePath, False, 1000)
404 error
redirect
Public Class Form1
Private Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click
Dim strMainUrl As String = "http://jixxer.com/123/"
Dim dt As DateTime = DateTime.Now
Dim dtDate As String = dt.ToString("yyyy-MM-dd")
Dim strSlash As String = "/"
Dim strPdf As String = "pdf"
Dim strDot As String = "."
Dim strPage As String = "page"
Dim strPageNbr As String = 1
Dim intCounter As Integer = 1
Dim strPageCounter As String = String.Format("{0:000}", intCounter)
Dim strSavePath As String = "D:\dls\title1\"
Dim strFullSavePath As String = strSavePath & strPageCounter & strDot & strPdf
Dim strFullUrlDownload As String = strMainUrl & dtDate & strSlash & strPdf & strSlash & strPage & strPageNbr & strDot & strPdf
Do Until strPageCounter = 200
' Downloads the resource with the specified URI to a local file.
My.Computer.Network.DownloadFile(strFullUrlDownload, strFullSavePath, False, 1000)
intCounter = intCounter + 1
strPageNbr = strPageNbr + 1
strPageCounter = String.Format("{0:000}", intCounter)
strFullSavePath = strSavePath & strPageCounter & strDot & strPdf
strFullUrlDownload = strMainUrl & dtDate & strSlash & strPdf & strSlash & strPage & strPageNbr & strDot & strPdf
Loop
End Sub
End Class
Try
'TRY to download the file using https first...
My.Computer.Network.DownloadFile(New Uri("https://" & ServerAddress & WebLogoPath & Convert.ToString(RowArray(0)) & ".png"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) & "\" & AppDataFolder & PCLogoPath & Convert.ToString(RowArray(0)) & ".png", "", "", False, 500, True)
Catch ex_https As Exception
'Unable to locate file or write file
'If the operation timed out...
If (ex_https.Message = "The operation has timed out") Then
'Re-TRY to download the file using http instead, as a time out error may indicate that HTTPS is not supported.
Try
My.Computer.Network.DownloadFile(New Uri("http://" & ServerAddress & WebLogoPath & Convert.ToString(RowArray(0)) & ".png"), Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) & "\" & AppDataFolder & PCLogoPath & Convert.ToString(RowArray(0)) & ".png", "", "", False, 500, True)
Catch ex_http As Exception
'Most likely, the file doesn't exist on the server. Either way, we cannot obtain the file so we need to perform the same action,
'which is handled outside of this Try block.
End Try
Else
'This is most likely a 404 error. Either way, we cannot obtain the file (and the connection is not timing out) - so
'we need to perform the same action, which is handled outside of this Try block.
End If
End Try
I just put the counter at 200 to test and make sure it works. But I know I need a way to quit on error but not sure how to code it yet. Appreciate any help.
If you don't know how many documents are stored in that remote directory, you have to handle the exception when a page is not found.
It's always possible to receive WebExceptions when a resource is requested from a site, so you should handle this case anyway.
I suggest to use the WebClient class directly instead of Network.DownloadFile(), which may be handy if you want to show a predefined UI of the progress (when it's possible), but using WebClient directly, lets you perform the download asynchrounously if you need it to, using the async/await pattern and the WebClient.DownloadFileTaskAsync() method.
Another suggestion: use a method to download those files, so you can call it from anywhere in your code. You can use a class or a module to store your methods, so you don't clutter your UI and you can also easily reuse these classes or modules in different projects, just including in a project the file that contains them.
Your code could be modified as follow (synchronous version):
You need to pass to the DownloadPdfPages method the remote base address: http://jixxer.com/123, the Path where the files are store (filesPath).
The third and fourth parameters are optional:
- If you don't specify a resourceName, Date.Now.ToString("yyyy-MM-dd") is assumed,
- If you don't specify a startPage, it will default to 1, converted in page1.pdf (the example here asks to start from page 3).
Note: I'm using String Interpolation here: $"page{startPage + pageCount}.pdf".
If your VB.Net version doesn't support it, use String.Format() instead.
Private Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click
Dim numberOfPages = DownloadPdfPages("http://jixxer.com/123", "D:\dls\title1", "", 3)
If numberOfPages > 0 Then
MessageBox.Show($"Download completed. Number of pages: {numberOfPages}")
Else
MessageBox.Show("Download failed")
End If
End Sub
Private Function DownloadPdfPages(baseAddress As String, filesPath As String, Optional resourceName As String = "", Optional startPage As Integer = 1) As Integer
If String.IsNullOrEmpty(resourceName) Then resourceName = Date.Now.ToString("yyyy-MM-dd")
Dim resourceAddr = Path.Combine(baseAddress, resourceName, "pdf")
Dim pageCount = 0
Dim client = New WebClient()
Try
Do
Dim documentName = $"page{startPage + pageCount}.pdf"
Dim resourceUri = New Uri(Path.Combine(resourceAddr, documentName), UriKind.Absolute)
Dim fileName = Path.Combine(filesPath, documentName)
client.DownloadFile(resourceUri, fileName)
pageCount += 1
Loop
Catch ex As WebException
If ex.Response IsNot Nothing Then
Dim statusCode = DirectCast(ex.Response, HttpWebResponse).StatusCode
If statusCode = HttpStatusCode.NotFound Then
Return pageCount
End If
ElseIf ex.Status = WebExceptionStatus.ProtocolError AndAlso ex.Message.Contains("404") Then
Return pageCount
Else
' Log and/or ...
Throw
End If
Return 0
Finally
client.Dispose()
End Try
End Function
Asynchronous version, using the WebClient.DownloadFileTaskAsync() method.
Just a few changes ae necessary, note the Async keyword added to both the Button.Click handler and the DownloadPdfPagesAsync() method.
The Await keyword is then used to wait for a method to complete, without blocking the UI:
Private Async Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click
Dim numberOfPages = Await DownloadPdfPagesAsync("http://jixxer.com/123", "D:\dls\title1", "", 3)
If numberOfPages > 0 Then
MessageBox.Show($"Download completed. Number of pages: {numberOfPages}")
Else
MessageBox.Show("Download failed")
End If
End Sub
Private Async Function DownloadPdfPagesAsync(baseAddress As String, filesPath As String, Optional resourceName As String = "", Optional startPage As Integer = 1) As Task(Of Integer)
If String.IsNullOrEmpty(resourceName) Then resourceName = Date.Now.ToString("yyyy-MM-dd")
Dim resourceAddr = Path.Combine(baseAddress, resourceName, "pdf")
Dim pageCount = 0
Dim client = New WebClient()
Try
Do
Dim documentName = $"page{startPage + pageCount}.pdf"
Dim resourceUri = New Uri(Path.Combine(resourceAddr, documentName), UriKind.Absolute)
Dim fileName = Path.Combine(filesPath, documentName)
Await client.DownloadFileTaskAsync(resourceUri, fileName)
pageCount += 1
Loop
Catch ex As WebException
If ex.Response IsNot Nothing Then
Dim statusCode = DirectCast(ex.Response, HttpWebResponse).StatusCode
If statusCode = HttpStatusCode.NotFound Then
Return pageCount
End If
ElseIf ex.Status = WebExceptionStatus.ProtocolError AndAlso ex.Message.Contains("404") Then
Return pageCount
Else
' Log and/or ...
Throw
End If
Return 0
Finally
client.Dispose()
End Try
End Function
Recently installed new version of Neo4j on Windows 7 Prof PC. Able to create nodes using API batch inserts. Cypher queries from web interface work but now fail from VB.NET code at the line after the comment 'retrieve results of query, which will be in JSon. This ran okay on the previous Neo4j version (2.2.x)
Public Shared Function DBQuery(URI As String, PostString As String) As DataView
'runs query and returns JSon results as a dataview
'Uses POST method to access Neo4j Server API
Dim S As String = ""
Dim HttpWReq As System.Net.HttpWebRequest = System.Net.HttpWebRequest.Create(URI)
HttpWReq.Method = "POST"
HttpWReq.ContentType = "application/json"
HttpWReq.Accept = "application/json"
Dim B1() As Byte = System.Text.Encoding.Unicode.GetBytes(PostString, 0, Len(PostString))
'POST query
'http://blog.micic.ch/net/using-neo4j-graph-db-with-c-net
HttpWReq.Connection = "Open"
HttpWReq.ContentLength = B1.Length
Dim newStream As IO.Stream = HttpWReq.GetRequestStream()
'this method closes stream before calling getResponse
Using newStream
newStream.Write(B1, 0, B1.Length)
End Using
'retrieve results of query, which will be in JSon
Dim HttpWResp As System.Net.HttpWebResponse = CType(HttpWReq.GetResponse(), System.Net.HttpWebResponse)
HttpWReq.KeepAlive = False
HttpWReq.Timeout = 15000000
Dim E As System.Text.Encoding = System.Text.Encoding.GetEncoding(HttpWResp.CharacterSet)
Dim SR As IO.StreamReader = New IO.StreamReader(HttpWResp.GetResponseStream, encoding:=E)
S = SR.ReadToEnd 'JSon result
Return JSonToDV(S)
End Function
Documentation for v2.3.0 indicates the need for a different conf file setting, but this is not working. The documentation is at http://neo4j.com/docs/2.3.0-M01/server-configuration.html . The neo4j-server.properties file originally had no entry for org.neo4j.server.database.location=data/graph.db. Adding the suggested line (org.neo4j.server.database.location="C:/Data/Neo4j/UMLS/graph.db") and then the database failed to start. Would appreciate suggested solutions.
The problem was not with Neo4j 2.3.0 but with the VB.NET code. The corrected code, which works is:
Public Shared Function DBQuery(URI As String, PostString As String, method As EnumLib.WebServiceMethod) As DataView
'Used for individual API calls; see BulkUpload for other method
'Uses POST method to access Neo4j Server API
Dim ID As Long = 0
Dim HttpWReq As System.Net.HttpWebRequest = System.Net.HttpWebRequest.Create(URI)
Select Case method
Case EnumLib.WebServiceMethod.POST
HttpWReq.Method = "POST"
Case EnumLib.WebServiceMethod.GET
HttpWReq.Method = "GET"
End Select
HttpWReq.ContentType = "application/json"
HttpWReq.Accept = "application/json"
Dim B1() As Byte = System.Text.Encoding.UTF8.GetBytes(PostString, 0, Len(PostString))
'http://blog.micic.ch/net/using-neo4j-graph-db-with-c-net
HttpWReq.Connection = "Open"
Dim S As String = ""
Try
HttpWReq.ContentLength = B1.Length
Dim newStream As IO.Stream = HttpWReq.GetRequestStream()
'this method closes stream before calling getResponse
Using newStream
newStream.Write(B1, 0, B1.Length)
End Using
Dim HttpWResp As System.Net.HttpWebResponse = CType(HttpWReq.GetResponse(), System.Net.HttpWebResponse)
Dim E As System.Text.Encoding = System.Text.Encoding.GetEncoding(HttpWResp.CharacterSet)
Dim SR As IO.StreamReader = New IO.StreamReader(HttpWResp.GetResponseStream, encoding:=E)
S = SR.ReadToEnd
Catch ex As System.Net.WebException
MsgBox("Message: " & vbLf & ex.Message)
Dim RS As IO.StreamReader = New IO.StreamReader(ex.Response.GetResponseStream)
Dim SS As String = RS.ReadToEnd
PostReturnString = "WebException Error: " & ex.Message & vbLf & vbLf & ex.Status & vbLf & vbLf & SS
' MsgBox("Status: " & vbLf & ex.Status & vbLf & vbLf & SS)
End Try
Return JSonToDV(S)
End Function
My program has been using:
Dim DLLink1 As String
DLLink1 = Trim(TextBox2.Text)
Dim DownloadDirectory1 As String
DownloadDirectory1 = Trim(TextBox4.Text)
Try
Button3.Enabled = False
' My.Computer.Network.DownloadFile(DLLink1, (DownloadDirectory1 + "/UpdatedClient.zip"))
Dim HttpReq As HttpWebRequest = DirectCast(WebRequest.Create(DLLink1), HttpWebRequest)
Using HttpResponse As HttpWebResponse = DirectCast(HttpReq.GetResponse(), HttpWebResponse)
Using Reader As New BinaryReader(HttpResponse.GetResponseStream())
Dim RdByte As Byte() = Reader.ReadBytes(1 * 1024 * 1024 * 10)
Using FStream As New FileStream(DownloadDirectory1 + "/UpdatedClient.zip", FileMode.Create)
FStream.Write(RdByte, 0, RdByte.Length)
End Using
End Using
End Using
Finally
MsgBox("Finished Download.")
Button3.Enabled = True
Label4.Visible = True
I tried this previously, and it didn't work at all:
My.Computer.Network.DownloadFile(DLLink1, (DownloadDirectory1 + "/UpdatedClient.zip"))
The website requires you to be logged in, so I made a spare account for the program:
WebBrowser1.Navigate("http://www.mpgh.net/forum/admincp/")
Timer1.Start()
Button2.Enabled = False
Then
WebBrowser1.Document.GetElementById("vb_login_username").SetAttribute("value", "AutoUpdaterAccount")
WebBrowser1.Document.GetElementById("vb_login_password").SetAttribute("value", "password")
Dim allelements As HtmlElementCollection = WebBrowser1.Document.All
For Each webpageelement As HtmlElement In allelements
If webpageelement.GetAttribute("type") = "submit" Then
webpageelement.InvokeMember("click")
Timer1.Stop()
Label5.Text = "Authorized."
Button2.Enabled = True
So now you're logged into the account, on the website, but when the code above to download runs, it downloads a zip, but it's corrupted. So I opened it with notepad++ and this is what I get (Does this mean it didn't login for the download, and it only logged in with the webbrowser and they aren't linked? Or something? Like My firefox logins aren't linked with chrome?:
The code is huge, it's like a HTML coding. Here is the link to a online notepad I put it on:
http://shrib.com/nCOucdfL
Another thing, a webbrowser can't be showing on the program, it can be on the outside not showing, like I did with the login. They also can't click the save button like on a normal web browser when a window pops up, I want it to download automatically to where they set it using a button which sets the directory as DownloadDirectory1
It must be your lucky day because today I woke up and decided that I would like to help you with your cause. I first tried to get the download to work with the web browser control but unfortunately I am sure this is not possible without extending the web browser control and we don't want to do that today.
As I mentioned in the comments, the only way I really know that this is possible (without user interaction) is to log in via the HttpWebRequest method. It's pretty tricky stuff. Definitely not for beginners.
Now I must admit that this isn't the cleanest, most "proper" and user-friendly code around, so if anyone wants to suggest a better way to do things, I am all ears.
I suggest you test this first before you incorporate it into your existing app. Just create a new vb.net app and replace all of the code in Form1 with the code below. You will have to update the usernamehere and passwordhere strings with your real username and password. Also, the file is saving to C:\file.rar by default so you can change this path if you want. This code completely removes the need for the web browser control (unless you are using it for something else) so most likely you can remove that from your real application once you incorporate this properly:
Imports System.Net
Imports System.IO
Imports System.Text
Public Class Form1
Private Const gsUserAgent As String = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:35.0) Gecko/20100101 Firefox/35.0"
Const sUsername As String = "usernamehere"
Const sPassword As String = "passwordhere"
Const sMainURL As String = "http://www.mpgh.net/"
Const sCheckLoginURL As String = "http://www.mpgh.net/forum/login.php?do=login"
Const sDownloadURL As String = "http://www.mpgh.net/forum/attachment.php?attachmentid=266579&d=1417312178"
Const sCookieLoggedInMessage As String = "mpgh_imloggedin=yes"
Dim oCookieCollection As CookieCollection = Nothing
Dim sSaveFile As String = "c:\file.rar"
Private Sub Form1_Load(sender As Object, e As EventArgs) Handles MyBase.Load
StartScrape()
End Sub
Private Sub StartScrape()
Try
Dim bContinue As Boolean = True
Dim sPostData(15) As String
sPostData(0) = UrlEncode("vb_login_username")
sPostData(1) = UrlEncode(sUsername)
sPostData(2) = UrlEncode("vb_login_password")
sPostData(3) = UrlEncode(sPassword)
sPostData(4) = UrlEncode("vb_login_password_hint")
sPostData(5) = UrlEncode("Password")
sPostData(6) = UrlEncode("s")
sPostData(7) = UrlEncode("")
sPostData(8) = UrlEncode("securitytoken")
sPostData(9) = UrlEncode("guest")
sPostData(10) = UrlEncode("do")
sPostData(11) = UrlEncode("login")
sPostData(12) = UrlEncode("vb_login_md5password")
sPostData(13) = UrlEncode("")
sPostData(14) = UrlEncode("vb_login_md5password_utf")
sPostData(15) = UrlEncode("")
If GetMethod(sMainURL) = True Then
If SetMethod(sCheckLoginURL, sPostData, sMainURL) = True Then
' Login successful
If DownloadMethod(sDownloadURL, sMainURL) = True Then
MessageBox.Show("File downloaded successfully")
Else
MessageBox.Show("Error downloading file")
End If
End If
End If
Catch ex As Exception
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
End Try
End Sub
Private Function GetMethod(ByVal sPage As String) As Boolean
Dim req As HttpWebRequest
Dim resp As HttpWebResponse
Dim stw As StreamReader
Dim bReturn As Boolean = True
Try
req = HttpWebRequest.Create(sPage)
req.Method = "GET"
req.AllowAutoRedirect = False
req.UserAgent = gsUserAgent
req.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
req.Headers.Add("Accept-Language", "en-us,en;q=0.5")
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")
req.Headers.Add("Keep-Alive", "300")
req.KeepAlive = True
resp = req.GetResponse ' Get the response from the server
If req.HaveResponse Then
' Save the cookie info if applicable
SaveCookies(resp.Headers("Set-Cookie"))
resp = req.GetResponse ' Get the response from the server
stw = New StreamReader(resp.GetResponseStream)
stw.ReadToEnd() ' Read the response from the server, but we do not save it
Else
MessageBox.Show("No response received from host " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End If
Catch exc As WebException
MessageBox.Show("Network Error: " & exc.Message.ToString & " Status Code: " & exc.Status.ToString & " from " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function SetMethod(ByVal sPage As String, ByVal sPostData() As String, sReferer As String) As Boolean
Dim bReturn As Boolean = False
Dim req As HttpWebRequest
Dim resp As HttpWebResponse
Dim str As StreamWriter
Dim sPostDataValue As String = ""
Try
req = HttpWebRequest.Create(sPage)
req.Method = "POST"
req.UserAgent = gsUserAgent
req.Accept = "application/x-shockwave-flash,text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5"
req.Headers.Add("Accept-Language", "en-us,en;q=0.5")
req.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7")
req.Referer = sReferer
req.ContentType = "application/x-www-form-urlencoded"
req.Headers.Add("Pragma", "no-cache")
req.Headers.Add("Keep-Alive", "300")
If oCookieCollection IsNot Nothing Then
' Pass cookie info from the login page
req.CookieContainer = SetCookieContainer(sPage)
End If
str = New StreamWriter(req.GetRequestStream)
If sPostData.Count Mod 2 = 0 Then
' There is an even number of post names and values
For i As Int32 = 0 To sPostData.Count - 1 Step 2
' Put the post data together into one string
sPostDataValue &= sPostData(i) & "=" & sPostData(i + 1) & "&"
Next i
sPostDataValue = sPostDataValue.Substring(0, sPostDataValue.Length - 1) ' This will remove the extra "&" at the end that was added from the for loop above
' Post the data to the server
str.Write(sPostDataValue)
str.Close()
' Get the response
resp = req.GetResponse
If req.HaveResponse Then
If resp.Headers("Set-Cookie").IndexOf(sCookieLoggedInMessage) > -1 Then
' Save the cookie info
SaveCookies(resp.Headers("Set-Cookie"))
bReturn = True
Else
MessageBox.Show("The email or password you entered are incorrect." & vbCrLf & vbCrLf & "Please try again.", "Unable to log in", MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
bReturn = False
End If
Else
' This should probably never happen.. but if it does, we give a message
MessageBox.Show("The email or password you entered are incorrect." & vbCrLf & vbCrLf & "Please try again.", "Unable to log in", MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
bReturn = False
End If
Else
' Did not specify the correct amount of parameters so we cannot continue
MessageBox.Show("POST error. Did not supply the correct amount of post data for " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End If
Catch ex As Exception
MessageBox.Show("POST error. " & ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function DownloadMethod(ByVal sPage As String, sReferer As String) As Boolean
Dim req As HttpWebRequest
Dim bReturn As Boolean = False
Try
req = HttpWebRequest.Create(sPage)
req.Method = "GET"
req.AllowAutoRedirect = False
req.UserAgent = gsUserAgent
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
req.Headers.Add("Accept-Language", "en-US,en;q=0.5")
req.Headers.Add("Accept-Encoding", "gzip, deflate")
req.Headers.Add("Keep-Alive", "300")
req.KeepAlive = True
If oCookieCollection IsNot Nothing Then
' Set cookie info so that we continue to be logged in
req.CookieContainer = SetCookieContainer(sPage)
End If
' Save file to disk
Using oResponse As System.Net.WebResponse = CType(req.GetResponse, System.Net.WebResponse)
Using responseStream As IO.Stream = oResponse.GetResponseStream
Using fs As New IO.FileStream(sSaveFile, FileMode.Create, FileAccess.Write)
Dim buffer(2047) As Byte
Dim read As Integer
Do
read = responseStream.Read(buffer, 0, buffer.Length)
fs.Write(buffer, 0, read)
Loop Until read = 0
responseStream.Close()
fs.Flush()
fs.Close()
End Using
responseStream.Close()
End Using
oResponse.Close()
End Using
bReturn = True
Catch exc As WebException
MessageBox.Show("Network Error: " & exc.Message.ToString & " Status Code: " & exc.Status.ToString & " from " & sPage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
bReturn = False
End Try
Return bReturn
End Function
Private Function SetCookieContainer(sPage As String) As System.Net.CookieContainer
Dim oCookieContainerObject As New System.Net.CookieContainer
Dim oCookie As System.Net.Cookie
For c As Int32 = 0 To oCookieCollection.Count - 1
If IsDate(oCookieCollection(c).Value) = True Then
' Fix dates as they seem to cause errors/problems
oCookieCollection(c).Value = Format(CDate(oCookieCollection(c).Value), "dd-MMM-yyyy hh:mm:ss")
End If
oCookie = New System.Net.Cookie
oCookie.Name = oCookieCollection(c).Name
oCookie.Value = oCookieCollection(c).Value
oCookie.Domain = New Uri(sPage).Host
oCookie.Secure = False
oCookieContainerObject.Add(oCookie)
Next
Return oCookieContainerObject
End Function
Private Sub SaveCookies(sCookieString As String)
Dim sCookieStrings() As String = sCookieString.Trim.Replace(" HttpOnly,", "").Replace(" HttpOnly", "").Replace(" domain=.mpgh.net,", "").Split(";".ToCharArray())
oCookieCollection = New CookieCollection
For Each sCookie As String In sCookieStrings
If sCookie.Trim <> "" Then
Dim sName As String = sCookie.Trim().Split("=".ToCharArray())(0)
Dim sValue As String = sCookie.Trim().Split("=".ToCharArray())(1)
oCookieCollection.Add(New Cookie(sName, sValue))
End If
Next
End Sub
Private Function UrlEncode(ByRef URLText As String) As String
Dim AscCode As Integer
Dim EncText As String = ""
Dim bStr() As Byte = Encoding.ASCII.GetBytes(URLText)
Try
For i As Long = 0 To UBound(bStr)
AscCode = bStr(i)
Select Case AscCode
Case 48 To 57, 65 To 90, 97 To 122, 46, 95
EncText = EncText & Chr(AscCode)
Case 32
EncText = EncText & "+"
Case Else
If AscCode < 16 Then
EncText = EncText & "%0" & Hex(AscCode)
Else
EncText = EncText & "%" & Hex(AscCode)
End If
End Select
Next i
Erase bStr
Catch ex As WebException
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
End Try
Return EncText
End Function
End Class
I have an Excel macro that has been in use for years which posts to a database using an XMLHttp call. The code is digitally signed.
Recently the site which is being posted to has enabled ADFS. Now instead of getting xml back I get the contents of the ADFS authentication form. There is no prompt for credentials in it since authentication already occurred. It I open the url from a web browser it goes through as expected with existing credentials used and the page loaded.
I tried setting the trusted setting for the url and allowed external content but that didn't matter.
Have I missed something?
The html I get back looks like...
<html><head><title>Working...</title></head><body><form method="POST" name="hiddenform" action="https://isvcci.jttest.com:444/"><input type="hidden" name="wa" value="wsignin1.0" />
...
<noscript><p>Script is disabled. Click Submit to continue.</p><input type="submit" value="Submit" /></noscript></form><script language="javascript">window.setTimeout('document.forms[0].submit()', 0);</script></body></html>
This is the vba:
Sub PostXml(strType As String, strAddress As String, objXml As MSXML2.DOMDocument60)
Dim objHttp As MSXML2.XMLHTTP60, objXmlResponse As MSXML2.DOMDocument60, objNode As MSXML2.IXMLDOMNode
Dim strText As String
Set objHttp = New MSXML2.XMLHTTP60
objHttp.Open "POST", strAddress, False
objHttp.setRequestHeader "Content-Type", "text/xml; charset=utf-8"
objHttp.send objXml
Set objXmlResponse = objHttp.responseXML
rem responseXML is always empty but responseText has the adfs page <------
Set objNode = objXmlResponse.SelectSingleNode("root/errorMessage")
If objNode Is Nothing Then
MsgBox "Error: Unable to retrieve expected response from the server." + vbCrLf + "The opportunity may not have been updated."
Else
... code for success goes here
End If
End Sub
Thanks for any assistance!
XMLHttp wouldn't work over adfs so I used an InternetExplorer control instead. It's a hassle to get the resulting xml back though using a page which sets a form value would probably be simpler. The resulting xml gets returned formatted like what you see in a web browser. I use a simple regex to remove dashes outside of tags.
I'm not that experienced with vba and excel so there might be better ways to code this but it works.
Sub PostXml(strType As String, strAddress As String, objXml As MSXML2.DOMDocument60)
Dim objHttp As MSXML2.XMLHTTP60, objXmlResponse As MSXML2.DOMDocument60, objNode As MSXML2.IXMLDOMNode
Dim objDoc As MSHTML.HTMLDocument
Dim strText As String, strHeaders As String, strPostData As String
Dim MyBrowser As InternetExplorer
Dim PostData() As Byte
Dim expr As VBScript_RegExp_55.RegExp
Dim colMatch As VBScript_RegExp_55.MatchCollection
Dim vbsMatch As VBScript_RegExp_55.Match
Dim sMatchString As String
' XMLHttp doesn't work with ADFS so browser was used
Set MyBrowser = New InternetExplorer
strHeaders = "Content-Type: text/xml; charset=utf-8" & vbCrLf
PostData = StrConv(objXml.XML, vbFromUnicode)
MyBrowser.Visible = False
MyBrowser.navigate strAddress, 0, "", PostData, strHeaders
Do While MyBrowser.Busy Or MyBrowser.readyState <> 4
Loop
Set objDoc = MyBrowser.Document
strText = objDoc.body.innerText
Set expr = New VBScript_RegExp_55.RegExp
expr.Pattern = "(?:\s| |^)(-)(?=\s|\r|\n|$)"
expr.IgnoreCase = True
expr.MultiLine = True
expr.Global = True
strText = expr.Replace(strText, "")
Set objXmlResponse = New MSXML2.DOMDocument60
Set objNode = Nothing
If objXmlResponse.LoadXML(strText) Then
Set objNode = objXmlResponse.SelectSingleNode("root/errorMessage")
'Else
'MsgBox "Invalid XML " & objXmlResponse.parseError.ErrorCode & "," & objXmlResponse.parseError.reason
End If
MyBrowser.Quit
Set MyBrowser = Nothing
Rem MsgBox "response =" & vbCrLf & objXmlResponse.XML
If objNode Is Nothing Then
MsgBox "Error: Unable to retrieve expected response from the server."
Else
strText = objNode.Text
If strText > "" Then
MsgBox strText, vbOKOnly, "Error"
Else
' it worked, read the xml here
End If
End If
End Sub