Now i have:
Dim i As Integer
Dim web As New WebClient With {.Proxy = Nothing}
'http://1.2.3.4/sql.db
Dim attkstring As String = "/sql.db"
i = 0
Dim shellx As Integer = 0
Dim sUrl As String = shells.Items(shellx) & attkstring
For Each item In shells.Items
Try
If web.OpenRead(sUrl) = True Then
End If
Catch ex As Exception
End Try
shellsloadedtext.Items.Add(sUrl)
shellx += 1
Next
But in my ListBox write all (ip+url), I need to write the IP + the URL just in case work, and now writes them even if there are no
Ex: real ip: 7.7.7.7 false ip: 1.1.1.1
I write in the ip list 7.7.7.7 1.1.1.1
And my ListBox shellsloadedtext wirte 7.7.7.7/sql.db and 1.1.1.1/sql.db
Help me
Try this:
Imports System.Net
Function UrlExists(destination As String) As Boolean
Dim url As New Uri(destination)
Dim request As WebRequest = WebRequest.Create(url)
Try
Using response As WebResponse = request.GetResponse
Return True
End Using
Catch ex As Exception
Return False
End Try
End Function
Related
I want to check if URL has valid page (not 404, just 200).
Code I've tried:
Dim request As HttpWebRequest = DirectCast(WebRequest.Create(myurl), HttpWebRequest)
request.KeepAlive = True
Dim response As HttpWebResponse = DirectCast(request.GetResponse(), HttpWebResponse)
If response.StatusCode = HttpStatusCode.OK Then
MsgBox("OK")
End If
Yet every URL that I enter leaves OK response, even if I enter http://mywebsite.com/blahblah.
It's not same on all websites (works fine with example.com), but it doesn't work on my website. Why?
In my browser I see 404 page, but code says it's OK.
Edit: Just to mention that my website has Cloudflare on.
Try this out and see if it work's for you... As mentioned above in my comment's:
A 404 just means the page isn't found on the server, status will return ok even if a page isn't found and the server was reached and responded
Public Class WebPage
Public Property PageSource As String = String.Empty
Public Property Status As HttpStatusCode = HttpStatusCode.NotFound
Public Property WebError As String = String.Empty
End Class
Public Shared Function GetWebPage(ByVal Website As String) As WebPage
Dim web As New WebPage() With {.Status = HttpStatusCode.OK}
Try
Using source As New System.Net.WebClient()
web.PageSource = source.DownloadString(Website)
End Using
Return web
Catch exweb As WebException
If exweb.Status = WebExceptionStatus.ProtocolError AndAlso exweb.Message.Contains("404") Then
web.Status = HttpStatusCode.NotFound
Else
web.Status = HttpStatusCode.BadRequest
End If
web.WebError = exweb.Message
Catch ex As Exception
web.Status = HttpStatusCode.NotFound
web.WebError = ex.Message
End Try
Return web
End Function
Usage Example
Dim webObj As WebPage = GetWebPage("THESITE")
If Not String.IsNullOrEmpty(webObj.WebError) Then
MessageBox.Show(webObj.WebError)
ElseIf webObj.Status = HttpStatusCode.OK Then
MessageBox.Show("OK")
End If
This will do what you want.
Function URLExists(url As String) As Boolean
Dim Request As Object
Dim ff As Integer
Dim rc As Variant
On Error GoTo EndNow
Set Request = CreateObject("WinHttp.WinHttpRequest.5.1")
With Request
.Open "GET", url, False
.send
rc = .StatusText
End With
Set Request = Nothing
If rc = "OK" Then URLExists = True
Exit Function
EndNow:
End Function
I have a program that runs in the background looping to check if a page on the site has been changed. It works once and shows the message box but if I change it again it won't do anything.
Imports System.Net
Imports System.String
Imports System.IO
Module Main
Sub Main()
While 1 = 1
Dim client As WebClient = New WebClient()
Dim reply As String = client.DownloadString("http://noahcristinotesting.dx.am/file.txt")
If reply.Contains("MsgBox") Then
Dim Array() As String = reply.Split(":")
MessageBox.Show(Array(2), Array(1))
Dim request As System.Net.FtpWebRequest = DirectCast(System.Net.WebRequest.Create("ftp://noahcristinotesting.dx.am/noahcristinotesting.dx.am/file.txt"), System.Net.FtpWebRequest)
request.Credentials = New System.Net.NetworkCredential("username", "password")
request.Method = System.Net.WebRequestMethods.Ftp.UploadFile
Dim path As String = "C:\test.txt"
Dim createText As String = "completed"
File.WriteAllText(path, createText)
Dim fileftp() As Byte = System.IO.File.ReadAllBytes("C:\test.txt")
Dim strz As System.IO.Stream = request.GetRequestStream()
strz.Write(fileftp, 0, fileftp.Length)
strz.Close()
strz.Dispose()
End If
End While
End Sub
End Module
Not sure at this moment what is causing it to crash when run outside of the IDE, but try trapping exceptions that are being thrown in the loop. I imagine there's an exception happening, cratering your app. The below catch block is by no means production ready, normally you want to catch specific exceptions in order to handle them effectively, but this is a cheap way to see if an exception is being thrown and what it is at runtime.
Sub Main()
Try
While 1 = 1
Dim client As WebClient = New WebClient()
Dim reply As String = client.DownloadString("http://noahcristinotesting.dx.am/file.txt")
If reply.Contains("MsgBox") Then
Dim Array() As String = reply.Split(":")
MessageBox.Show(Array(2), Array(1))
Dim request As System.Net.FtpWebRequest = DirectCast(System.Net.WebRequest.Create("ftp://noahcristinotesting.dx.am/noahcristinotesting.dx.am/file.txt"), System.Net.FtpWebRequest)
request.Credentials = New System.Net.NetworkCredential("username", "password")
request.Method = System.Net.WebRequestMethods.Ftp.UploadFile
Dim path As String = "C:\test.txt"
Dim createText As String = "completed"
File.WriteAllText(path, createText)
Dim fileftp() As Byte = System.IO.File.ReadAllBytes("C:\test.txt")
Dim strz As System.IO.Stream = request.GetRequestStream()
strz.Write(fileftp, 0, fileftp.Length)
strz.Close()
strz.Dispose()
End If
End While
Catch ex As Exception
MsgBox(ex.ToString)
End Try
End Sub
Alternatively, you could check your event viewer in windows to see if a .net application exception is being logged. Event Viewer > Windows Logs > Application
My application is a web scraper (for the most part) that stores information in a database. I have 2 classes so far:
clsSpyder - This essentially rolls-up the scraper processes
clsDB - This does any database processes
My test program looks over all the URLs, scrapes, pushes into the database. It is pretty simple sequentially, but I would like to have say N number of threads running those processes (scrape and store). My sequential code is this:
Private Sub Button4_Click(sender As Object, e As EventArgs) Handles Button4.Click
'Grab List
Dim tDS As New DataSet
Dim tDB As New clsTermsDB
Dim tSpyder As New clsAGDSpyder
Dim sResult As New TermsRuns
'Grab a list of all URLS
tDS = tDB.GetTermsList(1)
Try
For Each Row As DataRow In tDS.Tables(0).Rows
rtbList.AppendText(Row("url_toBeCollected") & vbCrLf)
sResult = tSpyder.SpiderPage(Row("url_toBeCollected"))
'If nothing is found, do not store
If sResult.html <> "" And sResult.text <> "" Then
tDB.InsertScrape(Now(), sResult.html, sResult.text, Row("url_uid"), 1)
End If
Next
Exit Sub
Catch ex As Exception
MessageBox.Show(ex.Message)
End Try
End Sub
With that in mind and noting that I am passing variables to the SpiderPage and InsertScrape methods.. How could I implement threading? It's gotta be simple, but I feel like I have been googling and trying things for days without success :(
*** ADDED: SpiderPage method:
Public Function SpiderPage(PageURL As String) As TermsRuns
Dim webget As New HtmlWeb
Dim node As HtmlNode
Dim doc As New HtmlDocument
Dim docNOHTML As HtmlDocument
Dim uri As New Uri(PageURL)
Dim wc As HttpWebRequest = DirectCast(WebRequest.Create(uri.AbsoluteUri), HttpWebRequest)
Dim wcStream As Stream
wc.AllowAutoRedirect = True
wc.MaximumAutomaticRedirections = 3
'Set Headers
wc.UserAgent = "Mozilla/5.0 (Macintosh; I; Intel Mac OS X 11_7_9; de-LI; rv:1.9b4) Gecko/2012010317 Firefox/10.0a4"
wc.Headers.Add("REMOTE_ADDR", "66.83.101.5")
wc.Headers.Add("HTTP_REFERER", "66.83.101.5")
'Set HTMLAgility Kit Useragent Spoofing (not needed, I don't think)
webget.UserAgent = "Mozilla/5.0 (Macintosh; I; Intel Mac OS X 11_7_9; de-LI; rv:1.9b4) Gecko/2012010317 Firefox/10.0a4"
'Certification STuff
wc.UseDefaultCredentials = True
wc.Proxy.Credentials = System.Net.CredentialCache.DefaultCredentials
ServicePointManager.ServerCertificateValidationCallback = AddressOf AcceptAllCertifications
'Create Cookie Jar
Dim CookieJar As New CookieContainer
wc.CookieContainer = CookieJar
'Keep Alive Settings
wc.KeepAlive = True
wc.Timeout = &H7530
'Read the web page
Dim wr As HttpWebResponse = Nothing
Try
wcStream = wc.GetResponse.GetResponseStream
doc.Load(wcStream)
'Remove HTML from the document
docNOHTML = RemoveUnWantedTags(doc)
'Grab only the content inside the <body> tag
node = docNOHTML.DocumentNode.SelectSingleNode("//body")
'Output
SpiderPage = New TermsRuns
SpiderPage.html = node.InnerHtml
SpiderPage.text = node.InnerText
Return SpiderPage
Catch ex As Exception
'Something goes here when scraping returns an error
SpiderPage = New TermsRuns
SpiderPage.html = ""
SpiderPage.text = ""
End Try
End Function
*** Added InsertScrape:
Public Function InsertScrape(scrape_ts As DateTime, scrape_html As String, scrape_text As String, url_id As Integer, tas_id As Integer) As Boolean
Dim myCommand As MySqlClient.MySqlCommand
Dim dt As New DataTable
'Create ds/dt for fill
Dim ds As New DataSet
Dim dtbl As New DataTable
Try
'Set Connection String
myConn.ConnectionString = myConnectionString
'Push Command to Client Object
myCommand = New MySqlClient.MySqlCommand
myCommand.Connection = myConn
myCommand.CommandText = "spInsertScrape"
myCommand.CommandType = CommandType.StoredProcedure
myCommand.Parameters.AddWithValue("#scrape_ts", scrape_ts)
myCommand.Parameters("#scrape_ts").Direction = ParameterDirection.Input
myCommand.Parameters.AddWithValue("#scrape_html", scrape_html)
myCommand.Parameters("#scrape_html").Direction = ParameterDirection.Input
myCommand.Parameters.AddWithValue("#scrape_text", scrape_text)
myCommand.Parameters("#scrape_text").Direction = ParameterDirection.Input
myCommand.Parameters.AddWithValue("#url_id", url_id)
myCommand.Parameters("#url_id").Direction = ParameterDirection.Input
myCommand.Parameters.AddWithValue("#tas_id", tas_id)
myCommand.Parameters("#tas_id").Direction = ParameterDirection.Input
'Open Connection
myConn.Open()
myCommand.ExecuteNonQuery()
'Close Connection
myConn.Close()
InsertScrape = True
Catch ex As Exception
'Put Message Here
InsertScrape = False
MessageBox.Show(ex.Message)
End Try
End Function
thanks in advance.
Now i have:
Dim i As Integer
Dim web As New WebClient With {.Proxy = Nothing}
'http://1.2.3.4/sql.db
Dim attkstring As String = "/sql.db"
i = 0
Dim shellx As Integer = 0
Dim sUrl As String = shells.Items(shellx) & attkstring
For Each item In shells.Items
Try
If web.OpenRead(sUrl) = True Then
End If
Catch ex As Exception
End Try
shellsloadedtext.Items.Add(sUrl)
shellx += 1
Next
But in my ListBox write all (ip+url), I need to write the IP + the URL just in case work, and now writes them even if there are no
Ex: real ip: 7.7.7.7 false ip: 1.1.1.1
I write in the ip list 7.7.7.7 1.1.1.1
And my ListBox shellsloadedtext wirte 7.7.7.7/sql.db and 1.1.1.1/sql.db
Help me
Does this help?
Dim sUrl As String = ips.Items(urlchk) & urlstring
web.DownloadString(sUrl)
yourListBox.Items.Add(sUrl)
Reference:
HTTP GET with .NET WebClient
I'm using vb.net 2005, I've got the following code running a thread to download a file. However, the process fails sometimes when trying to read the local copy of the file. I think I may need to unlock the local file somehow but I'm not sure how to do this. Can someone take a look and advise me ?
Dim BP1Ended As Boolean = False
Private Sub BackgroundProcess1()
BP1Ended = False
mPadFileStatus = DownloadFile(mstrPadUrl, mLocalFile)
BP1Ended = True
End Sub
'---'
Dim t As System.Threading.Thread
t = New System.Threading.Thread(AddressOf BackgroundProcess1)
t.Start()
Dim ProcessStartTime As Date = Now()
Do While ProcessStartTime.AddMinutes(1) >= Date.Now
Application.DoEvents()
If BP1Ended = True Then
Exit Do
End If
Loop
t.Abort()
PadFileStatus = mPadFileStatus
If BP1Ended = False Then
Application.DoEvents()
AddConsoleMsg("Downloading file.... Aborted", True)
End If
'---'
Public Function DownloadFile(ByVal pstrRequestedFile As String, ByVal pstrDestinationFile As String, Optional ByVal TimeOut As Integer = 120) As DownloadStatus
Dim input As IO.Stream
Dim Req As System.Net.HttpWebRequest = Nothing
Dim Response As System.Net.HttpWebResponse
Try
Req = System.Net.HttpWebRequest.Create(pstrRequestedFile)
Catch ex As Exception
Return DownloadStatus.UnknownError
End Try
Req.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)"
Req.Timeout = TimeOut * 1000 '120 * 1000 '1 second = 1 000 milliseconds
Try
Response = Req.GetResponse
input = Response.GetResponseStream
Dim streamreader As New StreamReader(input, System.Text.Encoding.GetEncoding("windows-1252")) 'System.Text.Encoding.UTF8)'
Dim s_response As String = streamreader.ReadToEnd()
streamreader.Close()
Dim filestream As New FileStream(pstrDestinationFile, FileMode.Create)
Dim streamwriter As New StreamWriter(filestream, System.Text.Encoding.GetEncoding("windows-1252")) ' System.Text.Encoding.UTF8)'
streamwriter.Write(s_response)
streamwriter.Flush()
streamwriter.Close()
Dim length As Long = 1000000 * 100
Dim pos As Long = 0
If Response.ContentLength > 0 Then
length = Response.ContentLength
End If
If length > 0 Then
Return DownloadStatus.OK
End If
input.Close()
Catch ew As System.Net.WebException
If ew.Status = WebExceptionStatus.NameResolutionFailure Or ew.Status = WebExceptionStatus.ProtocolError Then
Return DownloadStatus.FileNotFound
ElseIf ew.Status <> WebExceptionStatus.Success Then
Return DownloadStatus.UnknownError
End If
'Dim errorRespone As HttpWebResponse = CType(ew.Response, HttpWebResponse)
'If errorRespone.StatusCode = HttpStatusCode.NotFound Then '404
' Return DownloadStatus.FileNotFound
'Else
' Return DownloadStatus.UnknownError
'End If'
Catch ex As Exception 'Don't know'
Return DownloadStatus.UnknownError
End Try
End Function
'---'
Dim OpenFile As FileStream
'OpenFile = New FileStream(pstrPadFile, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)
'FAILS HERE
OpenFile = New FileStream(pstrPadFile, FileMode.Open, FileAccess.Read, FileShare.Read)
You don't close the file stream in case of exception which might lead to the lock. Make sure you always dispose disposable resources by wrapping them in Using statements:
Using filestream As FileStream = New FileStream(pstrDestinationFile, FileMode.Create)
Using streamwriter As StreamWriter = New StreamWriter(filestream, Encoding.GetEncoding("windows-1252"))
streamwriter.Write(s_response)
End Using
End Using
Same stands true for the response and response streams. They should be properly disposed.
Also you may consider using the WebClient class which has methods like DownloadFile and DownloadData which could make your life much easier.