I want to escape the \/\/ from url to get thumb with Uri.UnescapeDataString("\/\/") on this url from the HTML source using request:
https:\/\/scontent.fhel5-1.fna.fbcdn.net\/v\/t15.5256-10\/fr\/cp0\/e15\/q65\/48734886_2061090874011522_801576553975644160_n.jpg?_nc_cat=107&_nc_sid=08861d&_nc_ohc=4jeujL2uPDwAX_FBn97&_nc_ht=scontent.fhel5-1.fna&oh=672481858f1d7a2bd2655da89b57bacd&oe=5F63E18C
So my thumb will be decoded and those sign replaced so it can be parsed as string in show my the picturebox imgCover.ImageLocation = _thumbURL.ToString()
My code:
Private Sub getNFO()
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls
Dim apiURL As String = "https://m.facebook.com/watch/?v="
Dim _videoID As String = txtURL.Text
Try
Dim the_request As System.Net.HttpWebRequest = System.Net.HttpWebRequest.Create(apiURL + _videoID)
the_request.UserAgent = "Mozilla/5.0 (iPhone; CPU iPhone OS 13_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Mobile/15E148 Safari/604.1"
the_request.Timeout = 5000
Dim the_response As System.Net.HttpWebResponse = the_request.GetResponse
Dim stream_reader As System.IO.StreamReader = New System.IO.StreamReader(the_response.GetResponseStream())
Dim yt_source As String = stream_reader.ReadToEnd
If String.IsNullOrEmpty(yt_source) Then
MessageBox.Show("Facebook ID Request Error")
Else
Dim r1 As Regex = New Regex("(?<=description"":"").*?([^[]+?)(?="")")
Dim m1 = r1.Match(yt_source)
_titleURL = m1.Value
Dim r2 As Regex = New Regex("(?<=thumbnailUrl"":"").*?([^[]+?)(?="")")
Dim m2 = r2.Match(yt_source)
_thumbURL = m2.Value
End If
lbl_Title.Text = _titleURL.ToString()
imgCover.ImageLocation = _thumbURL.ToString()
Catch ex As Exception
MessageBox.Show(ex.Message)
End Try
End Sub
Finally make it bymyself :)
Dim decodedThumbURL = _thumbURL.Replace("\", String.Empty)
Related
It's like 5h I'm testing backconnect proxy I have which rotate on each request, but when I try to connect with proxy credentials (username , pass) it always connect and receive response from stream even if I type wrong pass, which makes me sick.
My code:
Public Shared Sub TestProxyConn()
Try
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12
Dim _proxyIP As String = Form1.txtRotatingIP.Text
Dim _proxyPort As Integer = Convert.ToInt32(Form1.txtRotatingPort.Text)
Dim _proxyUser As String = Form1.txtRotatingUser.Text
Dim _proxyPass As String = Form1.txtRotatingPass.Text
Dim myProxy As IWebProxy = New WebProxy("http://" & _proxyIP & ":" & _proxyPort)
Dim _url As String = "https://www.whatsmyua.info/"
Dim the_request As HttpWebRequest = WebRequest.Create(_url)
If Form1.chAuthMethod.Checked = True Then
the_request.UseDefaultCredentials = False
myProxy.Credentials = New NetworkCredential(_proxyUser, _proxyPass)
End If
the_request.Proxy = myProxy
the_request.Timeout = 10000
the_request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"
Dim the_response As System.Net.HttpWebResponse = the_request.GetResponse
Dim stream_reader As New System.IO.StreamReader(the_response.GetResponseStream())
Dim _source As String = stream_reader.ReadToEnd
If _source.Contains("my user agent") Then
MessageBox.Show("Connected!")
End If
Catch ex As System.Net.WebException
MessageBox.Show("Connection failed!")
End Try
End Sub
I use MS Visual Studio 2017 in Windows 10 and tried following httpclient by VB.net in order to get html of, for example, yahoo.com. I could get Header OK status 200 while its contents consists of several funny letters and finally resulsted in empty when converting to string.
Would you please advise how to receive contents by means of httpclient?
I tried variety sets of header which are not a cause of the problem. I assume the cause exists in how to convert the contents or I may need to access again the site after receiving a header in order to receive the content???.
Dim url As String = "https://yahoo.com"
Dim httpclienthandler0 = New HttpClientHandler()
httpclienthandler0.UseCookies = True
httpclienthandler0.SslProtocols = SecurityProtocolType.Tls Or SecurityProtocolType.Tls11 Or SecurityProtocolType.Tls12 Or SecurityProtocolType.Tls13
httpclienthandler0.ServerCertificateCustomValidationCallback = AddressOf OnRemoteCertificateValidationCallback
Dim httpclient0 As New HttpClient(httpclienthandler0)
httpclient0.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
httpclient0.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate, br")
httpclient0.DefaultRequestHeaders.Add("Accept-Language", "en-us;q=0.7,en;q=0.3")
httpclient0.DefaultRequestHeaders.Add("Cache-Control", "max-age=0")
httpclient0.DefaultRequestHeaders.Add("Connection", "keep-alive")
httpclient0.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1")
httpclient0.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36")
httpclient0.Timeout = TimeSpan.FromSeconds(10.0)
Try
Dim res As HttpResponseMessage = Await httpclient0.GetAsync(New Uri(url))
res.EnsureSuccessStatusCode()
Dim responseBody As String = Await res.Content.ReadAsStringAsync()
Dim enc As System.Text.Encoding = System.Text.Encoding.GetEncoding("utf-8")
Dim html As String = String.Empty
Using stream = Await res.Content.ReadAsStreamAsync()
Using reader = DirectCast(New IO.StreamReader(stream, enc, True), IO.TextReader)
html = Await reader.ReadToEndAsync()
End Using
End Using
Catch ex As HttpRequestException
Dim ex1 As Exception = ex
While (ex1 IsNot Nothing)
Console.WriteLine("Ex Message: {0} ", ex1.Message)
ex1 = ex1.InnerException
End While
Catch ex As TaskCanceledException
Dim ex2 As Exception = ex
Console.WriteLine(vbCr + "Timeout!)
Console.WriteLine("Ex MEssage: {0} ", ex2.Message)
End Try
Following codes derived from above are specific code that gets empty contents.
Dim responseBody As String = Await res.Content.ReadAsStringAsync()
and I tried next code instead of responseBody.
Using stream = Await res.Content.ReadAsStreamAsync()
Using reader = DirectCast(New IO.StreamReader(stream, enc, True), IO.TextReader)
html = Await reader.ReadToEndAsync()
End Using
End Using
However both code resulted in empty.
Thank you indeed for your time to read and answer.
I want to take the Flight schedule from the following address:
http://fo-apac.ttinteractive.com/Zenith/FrontOffice/(S(nves1yv4xxoia40cmotixof1))/USBangla/en-GB/BookingEngine/SearchFlights?__cnv=tShqK&json={"BookingPathArguments":null,"OriginDestinations":[{"IsOpen":false,"DataIdOrigin":6337,"DataIdDestination":6707,"DateTime":"2016-04-27T00:00:00.000"}],"TravelerTypes":[{"DataId":1,"TravelerCount":1},{"DataId":2,"TravelerCount":0},{"DataId":3,"TravelerCount":0}],"Currency":{"Code":"BDT"},"PromoCode":null,"DisplayRealAvailability":false,"Visibility":0,"ExtendedSearchDayCount":3}
If you paste the address to browser address bar and use firebug (or any fiddler), you will see this page sends 3 jquery ajax calls to bring the schedule. The following ajax POST actually fetch the schedule.
http://fo-apac.ttinteractive.com/Zenith/FrontOffice/(S(nves1yv4xxoia40cmotixof1))/USBangla/en-GB/FlexibleFlightStaticAjax/FlexibleFlightListLoadSelectedDays?__cnv=mxw0s
PostData : SaleConditionAccepted=false&ExtendedSearchDayCount=3&DoNotCheckCache=false&AlreadyLoggedIn=false&TempDataGuid=nves1yv4xxoia40cmotixof1&CurrencyCode=BDT&FareBasisDataId=&Travelers[0][DataId]=1&Travelers[0][Count]=1&UserSelections[0][SelectedDate]=2016-04-27T00:00:00&UserSelections[0][ReferenceDate]=2016-04-27T00:00:00&UserSelections[0][DataIdOrigin]=6337&UserSelections[0][DataIdDestination]=6707&UserSelections[0][GenericClassDataId]=&UserSelections[0][SelectedSegments]=&JsonPrepareBookingRequest=&PromoCode=
I am sending the request with httpwebrequest, but for unknown reason, I miss the session. I used CookieContainer to keep the cookies. I used the following function to send httprequest:
Public Function GetPostWP(ByVal Url As String, ByVal CkCont As CookieContainer, Optional ByVal PostData As String = "", Optional ByVal refSite As String = "") As String
Dim pStr As String = ""
Try
Dim Http As HttpWebRequest = WebRequest.Create(Url)
If refSite <> "" Then Http.Referer = refSite
Http.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate")
Http.CookieContainer = CkCont 'Initial CkCont is Nothing
Http.KeepAlive = True
Http.AllowAutoRedirect = True
'Http.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
Http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
Http.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
If PostData <> "" Then
Http.Method = "POST"
Http.ContentLength = PostData.Length
Http.ContentType = "application/x-www-form-urlencoded"
Dim PostStream As StreamWriter = New StreamWriter(Http.GetRequestStream())
PostStream.Write(PostData)
PostStream.Close()
End If
Using WebResponse As HttpWebResponse = Http.GetResponse()
Dim responseStream As Stream = WebResponse.GetResponseStream()
If (WebResponse.ContentEncoding.ToLower().Contains("gzip")) Then
responseStream = New GZipStream(responseStream, CompressionMode.Decompress)
ElseIf (WebResponse.ContentEncoding.ToLower().Contains("deflate")) Then
responseStream = New DeflateStream(responseStream, CompressionMode.Decompress)
End If
Dim reader As StreamReader = New StreamReader(responseStream, Encoding.Default)
pStr = reader.ReadToEnd()
responseStream.Close()
End Using
tmpCky = CkCont 'tmpCky is a Public CookieContainer Variable to hold cookies for future use.
GetPostWP = pStr
Catch ex As Exception
GetPostWP = "Error : " & ex.Message
End Try
End Function
Dim Cky As New CookieContainer
Dim Txt as String = GetPostWP(PostAddress, Cky, PostData, RefAdd)
Cky = tmpCky
Can anyone analyze the page, please?
I am trying to code some self bot to do some stuff for me on my self hosted wordpress blogs, so I managed to login successfully using webrequests had some problems but solved them.
But now I'm trying to go to permalinks page for example to get some data but when Im trying to do it using the login cookie it just redirects me back to login page like I'm not using the login cookie.
So basically this is the login function:
cleanurl = TextBox3.Text
logincookie = New CookieContainer
Dim url As String = cleanurl & "/wp-login.php"
Dim postreq As HttpWebRequest = DirectCast(HttpWebRequest.Create(url), HttpWebRequest)
Dim cookies As New CookieContainer
postreq.CookieContainer = cookies
postreq.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b6pre) Gecko/20100903 Firefox/4.0b6pre"
postreq.KeepAlive = True
postreq.Timeout = 120000
postreq.Method = "POST"
postreq.Referer = url
postreq.AllowAutoRedirect = False
postreq.ContentType = "application/x-www-form-urlencoded"
Dim postData As String = "log=" & TextBox1.Text & "&pwd=" & TextBox2.Text & "&wp-submit=Log+In&redirect_to=" & cleanurl & "/wp-admin" & "&testcookie=1"
Dim encoding As New UTF8Encoding
Dim byteData As Byte() = encoding.GetBytes(postData)
postreq.ContentLength = byteData.Length
Dim postreqstream As Stream = postreq.GetRequestStream()
postreqstream.Write(byteData, 0, byteData.Length)
postreqstream.Close()
Dim postresponse As HttpWebResponse
postresponse = DirectCast(postreq.GetResponse, HttpWebResponse)
'/ The Following is set next request with authentication cookie
Dim nextreq As HttpWebRequest = DirectCast(HttpWebRequest.Create(cleanurl), HttpWebRequest)
nextreq.ContentType = "application/x-www-form-urlencoded"
nextreq.Method = "GET"
nextreq.CookieContainer = New CookieContainer
nextreq.CookieContainer.Add(postresponse.Cookies)
nextreq.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b6pre) Gecko/20100903 Firefox/4.0b6pre"
nextreq.KeepAlive = True
Dim nextresponse As HttpWebResponse
nextresponse = DirectCast(nextreq.GetResponse, HttpWebResponse)
logincookie = nextreq.CookieContainer
logincookie.Add(nextresponse.Cookies)
Dim postreqreader As New StreamReader(nextresponse.GetResponseStream())
Dim thepage As String = postreqreader.ReadToEnd
nextresponse.Close()
RichTextBox1.Text = thepage
WebBrowser1.DocumentText = thepage
Refresh()
If thepage.Contains("ERROR") Then
MsgBox("Error logging in!")
Else
MsgBox("Lets Start Blogging!")
End If
It works perfect, gets me to the URL homepage and shows me that Im logged in.
but when Im launching this code to get to the permalinks edit page it just returns the login page source code means it wasn't able to login.
Dim nextreq As HttpWebRequest = DirectCast(HttpWebRequest.Create(cleanurl & "/wp-admin/options-permalink.php"), HttpWebRequest)
nextreq.ContentType = "application/x-www-form-urlencoded"
nextreq.Method = "GET"
nextreq.CookieContainer = logincookie
nextreq.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:2.0b6pre) Gecko/20100903 Firefox/4.0b6pre"
nextreq.KeepAlive = True
Dim nextresponse As HttpWebResponse = DirectCast(nextreq.GetResponse, HttpWebResponse)
nextreq.CookieContainer.Add(nextresponse.Cookies)
Dim postreqreader As New StreamReader(nextresponse.GetResponseStream())
Dim thepage As String = postreqreader.ReadToEnd
nextresponse.Close()
RichTextBox1.Text = thepage
WebBrowser1.DocumentText = thepage
Refresh()
Here is an alternative. Try the Wordpress API (xmlrpc.php) to access and modify data in your Wordpress installation. I have used CookComputing.XmlRpc to achieve this.
Here is a sample on how to create a new WordPress post:
Imports CookComputing.XmlRpc
Public Sub Add_New_Post_Sample()
Dim NewPostContent As New NewPostContent
NewPostContent.post_type = "post"
NewPostContent.post_status = "draft"
NewPostContent.post_title = "MyTitle"
NewPostContent.post_author = "1"
NewPostContent.post_excerpt = ""
NewPostContent.post_content = "MyContent"
NewPostContent.post_date_gmt = "2015-01-21 00:00:00"
NewPostContent.post_name = "my-unique-url"
NewPostContent.post_password = ""
NewPostContent.comment_status = "closed"
NewPostContent.ping_status = "closed"
NewPostContent.sticky = False
NewPostContent.post_thumbnail = 0
NewPostContent.post_parent = 0
NewPostContent.Mycustom_fields = New String() {""}
Dim API = DirectCast(XmlRpcProxyGen.Create(GetType(IgetCatList)), IgetCatList)
Dim clientprotocal = DirectCast(API, XmlRpcClientProtocol)
clientprotocal.Url = "http://myurl.com/xmlrpc.php"
Dim result As String = API.NewPost(1, "admin", "password", NewPostContent)
Console.WriteLine(result)
End Sub
Public Structure NewPostContent
Public post_type As String
Public post_status As String
Public post_title As String
Public post_author As Integer
Public post_excerpt As String
Public post_content As String
Public post_date_gmt As DateTime
Public post_name As String
Public post_password As String
Public comment_status As String
Public ping_status As String
Public sticky As Boolean
Public post_thumbnail As Integer
Public post_parent As Integer
Public Mycustom_fields As String()
End Structure
I think there is a problem with cookiecontainer (httpwebrequest)
there are 2 main functions in my class, first getting a new form (a hidden token in form tag), and it should set the cookies (which not set) and then second function (doLogin) should login the form. why cookie is not set...?
here is my codes: (you can test my code with "http://wetcatdesign.com/wiki/" as wikiURL)
Public Class wiki_submitter
Dim CookieJar As New CookieContainer
Public wikiURL As String
Private Function cutStr(ByVal Str As String, ByVal startStr As String, _
ByVal finishStr As String, Optional ByVal startPos As Integer = 1) As String
Dim start As Integer = InStr(startPos, Str, startStr) + Len(startStr)
Dim finish As Integer = InStr(start + 1, Str, finishStr)
cutStr = Mid(Str, start, finish - start)
End Function
Public Function GetNewForm()
Try
Dim req As HttpWebRequest = HttpWebRequest.Create(wikiURL & "index.php?title=Special:UserLogin&returnto=Main_Page")
Dim res As HttpWebResponse
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"
res = req.GetResponse()
'Setting cookies
req.CookieContainer = CookieJar
'getting HTML result
Dim sr As StreamReader = New StreamReader(res.GetResponseStream())
Dim HTML = sr.ReadToEnd
sr.Close()
Dim wpLoginToken As String = cutStr(HTML, "<input type=""hidden"" name=""wpLoginToken"" value=""", """") ' finding wpLoginToken parameter
GetNewForm = wpLoginToken
Catch ex As Exception
MsgBox(ex.Message)
End Try
End Function
Public Function doLogin(ByVal username As String, ByVal pass As String)
Dim req As HttpWebRequest = HttpWebRequest.Create(wikiURL & "index.php?title=Special:UserLogin&action=submitlogin&type=login")
Dim res As HttpWebResponse
Dim HTML As String
'-------Setting up headers------------
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"
req.Referer = wikiURL & "index.php?title=Special:UserLogin&returnto=Main_Page"
req.ContentType = "application/x-www-form-urlencoded" 'Form content type
req.Method = "POST" 'data will be send in POST method
req.CookieContainer = CookieJar 'Setting cookies
'-------------------------------------
Dim sw As StreamWriter = New StreamWriter(req.GetRequestStream)
Dim poststring = "wpLoginToken=" & GetNewForm() & "&wpLoginattempt=Log in&wpName=" & username & "&wpPassword=" & pass
Try
sw.Write(poststring)
Catch ex As Exception
MsgBox(ex.Message)
Finally
sw.Close()
End Try
res = req.GetResponse()
Dim sr As StreamReader = New StreamReader(res.GetResponseStream())
HTML = sr.ReadToEnd 'HTML as result
sr.Close()
doLogin = HTML 'returns HTML result
End Function
End Class
What happens if you set it before res = req.GetResponse() ?
Firstly, you need to set cookies before sending the request and secondly you need to extract cookies from the response when you expect them. Here's one way of doing it
Public Class wiki_submitter
Dim CookieJar As New CookieContainer
Public wikiURL As String
Private Function cutStr(ByVal Str As String, ByVal startStr As String, _
ByVal finishStr As String, Optional ByVal startPos As Integer = 1) As String
Dim start As Integer = InStr(startPos, Str, startStr) + Len(startStr)
Dim finish As Integer = InStr(start + 1, Str, finishStr)
cutStr = Mid(Str, start, finish - start)
End Function
Public Function GetNewForm()
Try
Dim req As HttpWebRequest = HttpWebRequest.Create(wikiURL & "index.php?title=Special:UserLogin&returnto=Main_Page")
Dim res As HttpWebResponse
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"
res = req.GetResponse()
'Setting cookies
'req.CookieContainer = CookieJar
SaveIncomingCookies(res, wikiURL)
'getting HTML result
Dim sr As StreamReader = New StreamReader(res.GetResponseStream())
Dim HTML = sr.ReadToEnd
sr.Close()
Dim wpLoginToken As String = cutStr(HTML, "<input type=""hidden"" name=""wpLoginToken"" value=""", """") ' finding wpLoginToken parameter
GetNewForm = wpLoginToken
Catch ex As Exception
MsgBox(ex.Message)
End Try
End Function
Public Function doLogin(ByVal username As String, ByVal pass As String)
Dim req As HttpWebRequest = HttpWebRequest.Create(wikiURL & "index.php?title=Special:UserLogin&action=submitlogin&type=login")
Dim res As HttpWebResponse
Dim HTML As String
'-------Setting up headers------------
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"
req.Referer = wikiURL & "index.php?title=Special:UserLogin&returnto=Main_Page"
req.ContentType = "application/x-www-form-urlencoded" 'Form content type
req.Method = "POST" 'data will be send in POST method
'req.CookieContainer = CookieJar 'Setting cookies
'-------------------------------------
Dim sw As StreamWriter = New StreamWriter(req.GetRequestStream)
Dim poststring = "wpLoginToken=" & GetNewForm() & "&wpLoginattempt=Log in&wpName=" & username & "&wpPassword=" & pass
Try
req.CookieContainer = CookieJar
sw.Write(poststring)
Catch ex As Exception
MsgBox(ex.Message)
Finally
sw.Close()
End Try
res = req.GetResponse()
SaveIncomingCookies(res, wikiURL)
Dim sr As StreamReader = New StreamReader(res.GetResponseStream())
HTML = sr.ReadToEnd 'HTML as result
sr.Close()
doLogin = HTML 'returns HTML result
End Function
Private Function SaveIncomingCookies(ByRef response As HttpWebResponse, ByRef Uri As String)
If response.Headers("Set-Cookie") <> Nothing Then
CookieJar.SetCookies(New Uri("http://wetcatdesign.com"), response.Headers("Set-Cookie"))
End If
End Function
End Class