I am trying to download
"https://www.google.com/search?sclient=psy-ab&biw=1472&bih=740&espv=2&tbm=vid&btnG=Search&q=%25%25%25#q=iran&tbm=nws";
by the following code:
string url = "https://www.google.com/search?sclient=psy-ab&biw=1472&bih=740&espv=2&tbm=vid&btnG=Search&q=%25%25%25#q=iran&tbm=nws";
try
{
string htmlPage = "";
//http request preparing
CookieContainer CC = new CookieContainer();
HttpWebRequest objRequest = (HttpWebRequest)WebRequest.Create(url);
objRequest.Timeout = 60000;
objRequest.Proxy = null;
objRequest.UseDefaultCredentials = true;
objRequest.KeepAlive = false; //THIS DOES THE TRICK
objRequest.ProtocolVersion = HttpVersion.Version10; // THIS DOES THE TRICK
objRequest.CookieContainer = CC;
//http request sending
using (HttpWebResponse objResponse = (HttpWebResponse)objRequest.GetResponse())
{
using (Stream strm = objResponse.GetResponseStream())
{
using (StreamReader objStreamReader = new StreamReader(strm))
{
htmlPage = objStreamReader.ReadToEnd();
}
}
};
if (htmlPage.Contains("No results found for") || htmlPage.Contains("(without quotes):") || htmlPage.Contains("Make sure all words are spelled correctly."))
{
return dtResult;
}
else
{
Regex objEgEx = new Regex(#"[\r\n][ ]+\.[\r\n][ ]+");
htmlPage = objEgEx.Replace(htmlPage, string.Empty);
int startIndex = htmlPage.IndexOf("<div class =\"g\">");
if (startIndex == -1)
{ Console.Write("problem in parsing"); }
but HttpWebRequest download the first page of the google instead of the url I hd saved for it which is the address of the video search service of Google results' page.
what should I change so that it download the url I want?
You are downloading the page, not the query. Due that the search of google doesn't load a new page but updates a page. Maybe have a look into google search api
Related
I am trying to attach files (screenshots) to an Azure DevOps work item via a C# desktop app. I have managed to attach files, but they're not valid image files, which leads me to believe that I'm doing something wrong in uploading them.
From the documentation DevOps Create Attachment below is the section on the Request body of the API call, which is rather vague.
From a GitHub discussion this answer seems to suggest that I just upload the binary content directly, which is what I'm doing.
My code is as follows
var img = File.ReadAllBytes(fname);
string query = #"/_apis/wit/attachments?fileName=" + fname + #"&api-version=6.0"
string response = AzureUtils.AttachFile(query, img, "POST", false, "application/octet-stream");
Is it correct that I literally pass in the byte array which is read from the file (variable img) as the body?
Why is it not a valid file when I look at it in DevOps?
The code for AttachFile is
public static string AttachFile(string query, byte[] data = null, string method = "GET",
bool dontUseBaseURL = false, string contentType = "application/json-patch+json")
{
try
{
HttpWebRequest request = WebRequest.Create(query) as HttpWebRequest;
request.ContentType = contentType;
request.Method = method;
request.Proxy.Credentials = CredentialCache.DefaultCredentials;
request.Headers.Add("Authorization", "Basic " +
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{1}", ["AzurePAT"]))));
if (data != null)
{
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(data);
}
}
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
string result = string.Empty;
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
result = reader.ReadToEnd();
}
request = null;
response = null;
return result;
}
In MVC I could generate .xsl or .pdf file with no issues with File(), but with the web Api nothing is happening when the action is fired! This is what I have tried so far.
I have tried a couple of solutions in here including this one Web API and report viewer
but nothing has worked for me.
public HttpResponseMessage Export(ExportVolunteerSearchFilter searchModel)
{
if (searchModel.Equals(null))
{
return Request.CreateResponse(HttpStatusCode.BadRequest);
}
var volunteers = _volunteersService.ExportAllVolunteersData(searchModel);
ReportViewer ReportViewer1 = new ReportViewer();
ReportViewer1.SizeToReportContent = true;
ReportViewer1.LocalReport.ReportPath =
System.Web.HttpContext.Current.Server.MapPath("~/Reports/VolunteersReport.rdlc");
ReportViewer1.LocalReport.EnableExternalImages = true;
ReportViewer1.LocalReport.DataSources.Clear();
ReportDataSource _rsource = new ReportDataSource("DataSet1", volunteers);
ReportViewer1.LocalReport.DataSources.Add(_rsource);
ReportViewer1.LocalReport.Refresh();
Warning[] warnings;
string[] streamIds;
string mimeType = string.Empty;
string encoding = string.Empty;
string extension = string.Empty;
string fileName = "reportVolunteer";
HttpResponseMessage response = new HttpResponseMessage(HttpStatusCode.OK);
var stream = new FileStream(System.Web.HttpContext.Current.Server.MapPath("~/Reports/VolunteersReport.rdlc"), FileMode.Open);
response.Content = new StreamContent(stream);
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
response.Content.Headers.ContentDisposition.FileName = fileName;
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/xls");
return response;
}
I have done it as:-
response.Content = new PushStreamContent(
async (outstream) =>
{
await getDataMethod(outstream)
},
new MediaTypeHeadrerValue(mediaType:"application/xls"));
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = $"test.xls"
};
return response;
I am provided a webservice url similar to:
http://ericdev35:7280/persons/persons/
and a username and password.
I want to make a post call on this web service from WPF application.
The data to be sent to the service is the first name and last name of a person in the format:
"fname=Abc&lname=Xyz"
How can I make a call for this in C#?
Here is the code that I have tried:
HttpWebRequest httpWebRequest = (HttpWebRequest) WebRequest.Create("http://ericdev35:7280/persons/persons/");
httpWebRequest.Method = "POST";
httpWebRequest.ContentType = "application/json";
httpWebRequest.Credentials = new NetworkCredential(username, password);
string data = "fname=Abc&lname=Xyz";
StreamWriter writer = new StreamWriter(httpWebRequest.GetRequestStream());
writer.Write(data);
writer.Close();
This does not give me any error but I cannot see the data that I have posted. Is there anything that needs to be corrected?
Is the Content Type correct?
This Method posts json.
After that it gets the response and deserialize the Json Object.
private static string PostJson<T1>(string p_url, string p_json, string p_method, out T1 p_target)
{
var httpWebRequest = (HttpWebRequest)WebRequest.Create(p_url);
httpWebRequest.UseDefaultCredentials = true;
httpWebRequest.ContentType = "application/json";
httpWebRequest.Method = p_method;
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
streamWriter.Write(p_json);
streamWriter.Flush();
streamWriter.Close();
}
HttpWebResponse httpResponse;
try
{
httpResponse = (HttpWebResponse)httpWebRequest.GetResponse();
}
catch (WebException ex)
{
httpResponse = ex.Response as HttpWebResponse;
}
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
var a_result = streamReader.ReadToEnd();
//If you dont need a Json object delete anything behind here
try
{
p_target = JsonConvert.DeserializeObject<T1>(a_result);
}
catch { p_target = default(T1); }
return a_result;
}
}
I have applicaiton which is the combination of MVC 4 + Web Api + SQL server.
I am trying to download the doc file to MVC but i have tried the below step.
I have Web API where i have written the below code. when i send the rowid it has the value stored in the DB as varbinary. file format can be any thing like .doc,pdf etc ... but however I am looking for the first doc or PDF file format.
When I call the Web api it will create the PDF file and download it , but the file is completely corrupted.
[ResponseType(typeof(MandateExceptionDO))]
[HttpGet]
[Route("api/DealingMandate/GetExceptionDoc/{rowId}")]
public HttpResponseMessage GetExceptionDoc(int rowId)
{
IDealingMandates repository = new DealingMandatesRepository();
List<MandateExceptionDO> mandateexceptiondoc =new List<MandateExceptionDO>();
mandateexceptiondoc = repository.GetExceptionDoc(rowId);
HttpResponseMessage response = new HttpResponseMessage(HttpStatusCode.OK);
//response.Content = new ByteArrayContent(mandateexceptiondoc[0].Content);
//response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("multipart/form-data");
//byte[] fileBytes = System.IO.File.ReadAllBytes(mandateexceptiondoc[0].Content);
response.Content = new ByteArrayContent(mandateexceptiondoc[0].Content);
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment");
response.Content.Headers.ContentDisposition.FileName = "testing.pdf";
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/pdf");
//return Ok(mandateexceptiondoc);
return response;
}
I am able to fix this issue on the web api i made the byte as string as show below
String doc = Convert.ToBase64String(customermandate.Content);
and for the MVC side i converted back to byte from the string
var doc = restClient.Execute(request);
var response = doc.Content.Substring(1, doc.Content.Length - 2).Replace(#"\/", "/");
byte[] docBytes = Convert.FromBase64String(response);
if (doc.Content != null && doc.Content.Length > 0 && !string.IsNullOrEmpty(doc.Content))
{
Response.Clear();
Response.ContentType = "application/pdf";
Response.AddHeader("content-disposition", "attachment; filename=" + FileName);
Response.BinaryWrite(docBytes);
Response.End();
}
I have an assignment, I want to download the data from some specific web site, I checked the topic of httpwebrequest and httpwebresponse but still I m not able to understand how we can fill the text boxes on that web site and press the login button to go inside the web site, login into the web site is the first step.
I want to use VB.net for this task, if anybody can help us.
Thanks In Advance.
I had written an article on this using ASP.NET MVC, it should help you : Introduction to Web Scraping with HttpWebRequest using ASP.NET MVC 3
Web Scraping means parsing those pages to extract pieces of information in a structured way. It also refers to creating a programmatic interface, an API, that interacts with a site through an HTML interface meant for humans.
Please dont think that your program will type in username and password in to thier respective textboxes and then the program will press the login button for you to go inside the website.
Web Scraping does not happen in this manner.
You need to study how the website POST the Login username and password and then using your code you need to do the same thing to get the cookie. At each step, rather on each page you ll need to see how the website works using firebug or chrome dev tool and then send the POST or GET data accordingly to get what you want.
Below is what I had written to scrape data from my WordPress website, I have added comments wherever applicable to make the code easier to read.
Below are the constant that you need to define, ‘UserName’ and ‘Pwd’ are the login details to my WordPress account, ‘Url’ stand for the login page url and ‘ProfileUrl’ is the address of the page where the profile details are shown.
const string Url = "http://yassershaikh.com/wp-login.php";
const string UserName = "guest";
const string Pwd = ".netrocks!!"; // n this not my real pwd :P
const string ProfileUrl = "http://yassershaikh.com/wp-admin/profile.php";
public ActionResult Index()
{
string postData = Crawler.PreparePostData(UserName, Pwd, Url);
byte[] data = Crawler.GetEncodedData(postData);
string cookieValue = Crawler.GetCookie(Url, data);
var model = Crawler.GetUserProfile(ProfileUrl, cookieValue);
return View(model);
}
I had created a static class called “Crawler”, here’s the code for it.
// preparing post data
public static string PreparePostData(string userName, string pwd, string url)
{
var postData = new StringBuilder();
postData.Append("log=" + userName);
postData.Append("&");
postData.Append("pwd=" + pwd);
postData.Append("&");
postData.Append("wp-submit=Log+In");
postData.Append("&");
postData.Append("redirect_to=" + url);
postData.Append("&");
postData.Append("testcookie=1");
return postData.ToString();
}
public static byte[] GetEncodedData(string postData)
{
var encoding = new ASCIIEncoding();
byte[] data = encoding.GetBytes(postData);
return data;
}
public static string GetCookie(string url, byte[] data)
{
var webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.Method = "POST";
webRequest.ContentType = "application/x-www-form-urlencoded";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2";
webRequest.AllowAutoRedirect = false;
Stream requestStream = webRequest.GetRequestStream();
requestStream.Write(data, 0, data.Length);
requestStream.Close();
var webResponse = (HttpWebResponse)webRequest.GetResponse();
string cookievalue = string.Empty;
if (webResponse.Headers != null && webResponse.Headers["Set-Cookie"] != null)
{
cookievalue = webResponse.Headers["Set-Cookie"];
// Modify CookieValue
cookievalue = GenerateActualCookieValue(cookievalue);
}
return cookievalue;
}
public static string GenerateActualCookieValue(string cookievalue)
{
var seperators = new char[] { ';', ',' };
var oldCookieValues = cookievalue.Split(seperators);
string newCookie = oldCookieValues[2] + ";" + oldCookieValues[0] + ";" + oldCookieValues[8] + ";" + "wp-settings-time-2=1345705901";
return newCookie;
}
public static List<string> GetUserProfile(string profileUrl, string cookieValue)
{
var webRequest = (HttpWebRequest)WebRequest.Create(profileUrl);
webRequest.Method = "GET";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2";
webRequest.AllowAutoRedirect = false;
webRequest.Headers.Add("Cookie", cookieValue);
var responseCsv = (HttpWebResponse)webRequest.GetResponse();
Stream response = responseCsv.GetResponseStream();
var htmlDocument = new HtmlDocument();
htmlDocument.Load(response);
var responseList = new List<string>();
// reading all input tags in the page
var inputs = htmlDocument.DocumentNode.Descendants("input");
foreach (var input in inputs)
{
if (input.Attributes != null)
{
if (input.Attributes["id"] != null && input.Attributes["value"] != null)
{
responseList.Add(input.Attributes["id"].Value + " = " + input.Attributes["value"].Value);
}
}
}
return responseList;
}
Hope this helps.