I am sending some requests to an API in a batch in a .Net core 3.1 project, response time is very quick but Is there anything further I can do? In my actual scenario, I will be sending 700 requests with 5 requests at a time.
static async Task ThreadRequets()
{
List<string> userIds1 = new List<string>() { "1", "2", "3", "4","2757","2756" };
Stopwatch watch = new Stopwatch();
watch.Start();
foreach (var batch in BuildChunksWithLinqAndYield(userIds1, 5))
{
var tasks = batch.Select(id => getUser(id));
var users = await Task.WhenAll(tasks);
}
watch.Stop();
}
static IEnumerable<IEnumerable<T>> BuildChunksWithLinqAndYield<T>(List<T> fullList, int batchSize)
{
int total = 0;
while (total < fullList.Count)
{
yield return fullList.Skip(total).Take(batchSize);
total += batchSize;
}
}
static async Task<string> getUser(string userID)
{
using (var restClient = new RestClient($"https://gorest.co.in/public/v2/users/{userID}"))
{
var request = new RestRequest();
request.AddHeader("Authorization", "Bearer mytoken");
RestResponse response = await restClient.GetAsync(request);
JObject result = JObject.Parse(response.Content);
string name = result["name"].ToString();
return name;
}
}
Related
I have following code segment it works for small file. But if the file is larger then application is loading for long and recieves No 'Access-Control-Allow-Origin' header is present on the requested resource.
[HttpPost]
[ScopeAuthorize(Constants.ClaimScopeSGCanManageAll, Constants.ClaimScopeUserCanManage)]
[DisableRequestSizeLimit, RequestFormLimits(MultipartBodyLengthLimit = int.MaxValue, ValueLengthLimit = int.MaxValue)]
public async Task<IActionResult> UploadFile()
{
if (!Request.Form.Files.Any())
{
throw new Common.Exceptions.ValidationException("Empty file");
}
IFormFile formFile = Request.Form.Files[0];
var csvDatas = new List<PatientsCSVItem>();
using (var reader = new StreamReader(formFile.OpenReadStream()))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(';');
//process csv rows
}
}
PatientCsvLog executionLog = _patientCsvManager.AddOrUpdatePatientsByCsvData(csvDatas, _userManager.GetLoggedUserId(User));
if (executionLog == null)
{
throw new ArgumentNullException(nameof(executionLog));
}
var response = new
{
NumberRecordImported = executionLog.NumberRecordImported,
NumberRecordUpdated = executionLog.NumberRecordUpdated,
NumberRecordDiscarded = executionLog.NumberRecordDiscarded,
DiscardedRecordList = executionLog.DiscardedRecordList
};
return Ok(response);
}
How to download multiple files from s3 buckets. I could not find any better option on SO.
Here is my code for single file download. Given list of Urls, I am looping to download multiple files.
public async Task Download(string url, Stream output)
{
var s3Uri = new AmazonS3Uri(url);
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to stream is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}
Download files given s3 urls
var list = new List<Stream>();
foreach(var url in urls)
{
var stream = new MemoryStream();
await Download(url,ms);
list.Add(stream);
}
Is there any better option to download multiple files at once from S3?
I finally decided to implement my own version
public class StreamWrapper
{
public string Url { get; set; }
public Stream Content { get; set; }
public string FileName { get; set; }
}
public async Task Download(IList<StreamWrapper> inout, int maxConcurrentDownloads)
{
if (maxConcurrentDownloads <= 0)
{
maxConcurrentDownloads = 20;
}
if (!inout.HasAny())
return;
var tasks = new List<Task>();
for (int i = 0; i < inout.Count; i++)
{
StreamWrapper wrapper = inout[i];
AmazonS3Uri s3Uri = null;
if (AmazonS3Uri.TryParseAmazonS3Uri(wrapper.Url, out s3Uri))
{
tasks.Add(GetObject(s3Uri, wrapper.Content));
}
if (tasks.Count == maxConcurrentDownloads || i == inout.Count - 1)
{
await Task.WhenAll(tasks);
tasks.Clear();
}
}
}
private async Task GetObject(AmazonS3Uri s3Uri, Stream output)
{
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to local file system is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}
Am trying to create a an azure function that is triggered in a Logic Apps,
The functions purpose is to web crawl certain web sites, take the desired information, compare that with a a SQL Server database in Azure, compare if we already have that information if not add it.
My issue is that when ever i run it I get the Server 500 error, I assume its accessing the database that cause. Help?
public static async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req, ILogger log
)
{
log.LogInformation("C# HTTP trigger function processed a request.");
string RequestBody = await new StreamReader(req.Body).ReadToEndAsync();
{
return await CrawlBlog(0, RequestBody);
}
}
private static async Task<IActionResult> CrawlBlog(int Picker, string req)
{
int BlogPicker = Picker;
string TheResult = req;
//Get the url we want to test
var Url = "";
if (BlogPicker == 0)
{
Url = "*********";
}
else if (BlogPicker == 1)
{
Url = "*********";
}
/*
else if (BlogPicker == 2)
{
Url = "https://azure.microsoft.com/en-in/blog/?utm_source=devglan";
}
*/
else
{
TheResult = "False we got a wrong pick";
return (ActionResult)new OkObjectResult
( new {TheResult });
}
var httpClient = new HttpClient();
var html = await httpClient.GetStringAsync(Url);
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
//a list to add all availabel blogs we found
var Blog = new List<BlogStats>();
switch (BlogPicker)
{
case 0:
{
var divs =
htmlDocument.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("class", "").Equals("home_blog_sec_text")).ToList();
foreach (var divo in divs)
{
var Blogo = new BlogStats
{
Summary = divo.Descendants("p").FirstOrDefault().InnerText,
Link = divo.Descendants("a").FirstOrDefault().ChildAttributes("href").FirstOrDefault().Value,
Title = divo.Descendants("a").FirstOrDefault().InnerText
};
Blog.Add(Blogo);
}
break;
}
case 1:
{
var divs =
htmlDocument.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("class", "").Equals("post_header_title two_third last")).ToList();
foreach (var divo in divs)
{
//string TheSummary = "we goofed";
var ThePs = divo.Descendants("p").ToList();
var Blogo = new BlogStats
{
Summary = ThePs[1].GetDirectInnerText(),
Link = divo.Descendants("a").LastOrDefault().ChildAttributes("href").FirstOrDefault().Value,
Title = divo.Descendants("a").FirstOrDefault().InnerText
};
Blog.Add(Blogo);
}
break;
}
}
TheResult = await SqlCheck(Blog[0].Title, Blog[0].Summary, Blog[0].Link); //error 500
return (ActionResult)new OkObjectResult
(
new
{
TheResult
}
);
}
public static async Task<string> SqlCheck(string Tit, string Sumy, string Lin)
{
SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder();
builder.DataSource = "flygon.database.windows.net";
builder.UserID = "*****";
builder.Password = "********";
builder.InitialCatalog = "torkoal";
System.Data.DataSet ds = new System.Data.DataSet();
SqlConnection connection = new SqlConnection(builder.ConnectionString);
connection.Open();
SqlCommand CheckCommand = new SqlCommand("SELECT * FROM TableBoto WHERE Link = #id3 ", connection);
CheckCommand.Parameters.AddWithValue("#id3", Lin);
SqlDataAdapter dataAdapter = new SqlDataAdapter(CheckCommand);
dataAdapter.Fill(ds);
int i = ds.Tables[0].Rows.Count;
if (i > 0)
{
return $" We got a Duplicates in title : {Tit}";
}
try
{
{
string query = $"insert into TableBoto(Title,Summary,Link) values('{Tit}','{Sumy}','{Lin}');";
SqlCommand command = new SqlCommand(query, connection);
SqlDataReader reader = await command.ExecuteReaderAsync();
reader.Close();
}
}
catch (SqlException)
{
// Console.WriteLine(e.ToString());
}
connection.Close();
return $" Success Ign +{Tit} + Ign {Sumy}+ Ign {Lin} Ign Success SQL ";
}
}
500 HTTP status code is a generic code which means that the server was not able to process the request due to some issues, First step would be to add some exception handling to your function and see if the failure occurs and where it occurs.
On Side note, you should not use HTTP client in the way used in the code, you should not new it up every time your function executes, this client should be static in nature. Refer Manage connections in Azure Functions
I'm trying to send list of objects from MVC to WEBAPI using below methods. API is able to able receive the list from controller but, value of each item in the list is either empty/null on API side.
Can anyone please help me to fix this?
Controller Method:
private List<FCM.Models.Facility> GetFacilityDetails()
{
var url = "http://localhost:64664/";
var facilies = new List<Facility>();
facilies.Add( new Facility{ FCLT_ID = 100, FCLT_NM = "Facility 100" });
facilies.Add( new Facility{ FCLT_ID = 200, FCLT_NM = "Facility 200" });
facilies.Add( new Facility{ FCLT_ID = 300, FCLT_NM = "Facility 300" });
var json = JsonConvert.SerializeObject(facilies);
var _client = new RestClient(url);
var request = new RestRequest("api/facility/details", Method.GET) { RequestFormat = DataFormat.Json };
facilies.ForEach(fclt =>
request.AddParameter("facilites", fclt, ParameterType.GetOrPost));
var response = _client.Execute<List<FCM.Models.Facility>>(request);
if (response.Data == null)
{
throw new Exception(response.ErrorMessage);
}
return response.Data;
}
WebAPI method:
[Route("api/facility/details")]
public IEnumerable<Facility> GetFullAddress([FromUri] IEnumerable<Facility> facilities)
{
return null;
}
Like the comment suggested you maybe want to issue a POST request instead, but if you would like to send an array with a GETrequest you could do it like this (with System.Net.Http.HttpClient):
Add a Format method to you Facility class:
public class Facility
{
public int FCLT_ID { get; set; }
public string FCLT_NM { get; set; }
public string Format(int index)
{
return $"[{index}].FCLT_ID={FCLT_ID}&[{index}].FCLT_NM={FCLT_NM}";
}
}
Define a class which can format the array values:
public class FacilityList : List<Facility>
{
public string Format()
{
var builder = new StringBuilder();
for (var i = 0; i < Count; i++)
{
builder.Append(this[i].Format(i));
if(i != Count -1)
{
builder.Append("&");
}
}
return builder.ToString();
}
}
And then issue the request:
var client = new HttpClient()
{
BaseAddress = new Uri("http://localhost:64664/"),
DefaultRequestHeaders = {Accept = {new MediaTypeWithQualityHeaderValue("application/json")}}
};
var facilities = new FacilityList
{
new Facility {FCLT_ID = 100, FCLT_NM = "Facility 100"},
new Facility {FCLT_ID = 200, FCLT_NM = "Facility 200"},
new Facility {FCLT_ID = 300, FCLT_NM = "Facility 300"}
};
var format = facilities.Format();
var response = client.GetAsync("api/facility/details?" + format).GetAwaiter().GetResult();
var result = JsonConvert.DeserializeObject<IEnumerable<Facility>>(response.Content.ReadAsStringAsync().GetAwaiter().GetResult());
This will bind to your controller action:
[Route("api/facility/details")]
public IHttpActionResult Get([FromUri] IEnumerable<Facility> facilities)
{
// Do stuff..
return Ok(facilities);
}
I want to create a registrant for a webinar using GoToWebinar API's. I came across the code at gotowebinar api php
I provided my username and password to get the oAuth object. This worked perfectly fine as described.
Now I want to do something like this:
I have a Registration page. When user fills in the required details, selects the 'register to webinar' option and clicks on 'Submit', I want to enrol him for that webinar using CreateRegistrant API. The problem is, I am not able to get the oAuth object without providing username and password. Is there a way to pass this programatically and create oAuth object?
I store my API key, UserID and password in my WebConfig then read them into a Login Object for use when I do authorization. Here's how I do it in C#:
public class Login
{
public string UserId
{ get { return System.Configuration.ConfigurationManager.AppSettings["GTWUserId"]; } }
public string Password
{ get { return System.Configuration.ConfigurationManager.AppSettings["GTWPassword"]; } }
public string APIKey
{ get { return System.Configuration.ConfigurationManager.AppSettings["GTWAPIKey"]; } }
}
public string DoAuthorize()
{
Login lg = new Login();
string sError = "";
// first we need to create the uri for the web request
string uri = String.Format("https://api.citrixonline.com/oauth/access_token?grant_type=password&user_id={0}&password={1}&client_id={2}",
lg.UserId, lg.Password, lg.APIKey);
// then the request to login is created and sent. From the response
// we need to store at least the access token and the organizer key
// to use for further calls
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(uri);
request.Accept = "application/json";
request.ContentType = "application/json";
try
{
var response = request.GetResponse();
//the following lines duplicate the response stream so we can read it for
//deserialization and also re-read it and write it out.
using (MemoryStream ms = new MemoryStream())
{
var stream = response.GetResponseStream();
stream.CopyTo(ms);
ms.Position = 0;
stream.Close();
DataContractJsonSerializer ser = new DataContractJsonSerializer(typeof(ResponseDirectLogin));
var deserialized = (ResponseDirectLogin)ser.ReadObject(ms);
auth.OauthToken = deserialized.AccessToken;
auth.OrganizerKey = deserialized.OrganizerKey;
}
}
catch (WebException e)
{
using (var sr = new StreamReader(e.Response.GetResponseStream()))
sError = sr.ReadToEnd();
sError = String.Concat(sError, "/n", uri);
}
return sError;
}
public class Auth {
public string OauthToken { get; set; }
public string OrganizerKey { get; set; }
}
public static Auth auth = new Auth(); // This is actually in a BaseControlelr inherited by our MVC Home Controller.
public string DoRegister(string WebinarKey)
{
// Here we authorize if we haven't alerady
if (auth.OauthToken == null)
{
sMessage = DoAuthorize();
}
// first we need to create the uri for the web request
// OrganizerKey is your authorization key for the webinar organizer
string uri = String.Format(#"https://api.citrixonline.com/G2W/rest/organizers/{0}/webinars/{1}/registrants",
OrganizerKey, WebinarKey);
//then create and serialize the registrant object
// This is for when you have questions on your webinar, you can omit them if you don't have any
List<questions> q = GetQuestionKeys(Key, OrganizerKey);
List<response> responses_ = new List<response>();
foreach (var question in q)
{
response res1 = new response();
res1.questionKey = question.questionKey;
// determine which question and set the response
if (question.question == "question")
{
res1.responseText = "response";
responses_.Add(res1);
}
}
var registrant = new Registrant
{
firstName = FirstName,
lastName = LastName,
email = EmailAddress,
responses = responses_.ToArray()
};
JavaScriptSerializer ser = new JavaScriptSerializer();
string json = ser.Serialize(registrant);
// then the request to create a registrant is created and sent
// N.B. we need to include the access token to the headers to access
// the user's account and data
try {
WebClient client = new WebClient();
client.Headers = new WebHeaderCollection();
client.Headers.Add("Accept", "application/vnd.citrix.g2wapi-v1.1+json");
client.Headers.Add("Content-type", "application/json");
client.Headers.Add("Authorization", string.Format("OAuth oauth_token={0}", OAuthToken));
try
{
string resp = client.UploadString(uri, "POST", json);
var ok = ser.Deserialize<ResponseCreateRegistrantOk>(resp);
}
catch (WebException e)
{
//if there is an error, e.g. the registrant exists already
// we need an alternative deserialization
Stream s = new MemoryStream();
using (Stream response = e.Response.GetResponseStream())
{
byte[] buffer = new byte[1024];
int byteCount;
do
{
byteCount = response.Read(buffer, 0, buffer.Length);
s.Write(buffer, 0, byteCount);
} while (byteCount > 0);
}
s.Seek(0, SeekOrigin.Begin);
string content = new StreamReader(s, Encoding.UTF8).ReadToEnd();
s.Seek(0, SeekOrigin.Begin);
using (var err = new StreamReader(s))
{
var sb = new StringBuilder("Registration Error\n");
if (content.IndexOf("int_err_code") > -1)
{
var dupe = ser.Deserialize<ResponseCreateRegistrantDuplicate>(err.ReadToEnd());
sb.AppendFormat(String.Format("Error Code: {0}<br />", dupe.ErrorCode));
sb.AppendFormat(String.Format("Message: {0}<br />", dupe.Message));
}
else
{
var dupe = ser.Deserialize<ResponseCreateRegistrantDuplicate>(err.ReadToEnd());
sb.AppendFormat(String.Format("Description: {0}<br />", dupe.Description));
//sb.AppendFormat(String.Format("Incident: {0}<br />", dupe.Incident));
//sb.AppendFormat(String.Format("Registrant key: {0}<br />", dupe.RegistrantKey));
sb.AppendFormat(String.Format("Join Url: {0}<br />", dupe.JoinUrl));
}
sMessage = sb.ToString();
}
}
} catch (Exception exc) {
exc.Data.Add("stringInfo", "inside");
return "";
}
return sMessage;
}