Net core api to upload 1GB size csv file - asp.net-core

I have following code segment it works for small file. But if the file is larger then application is loading for long and recieves No 'Access-Control-Allow-Origin' header is present on the requested resource.
[HttpPost]
[ScopeAuthorize(Constants.ClaimScopeSGCanManageAll, Constants.ClaimScopeUserCanManage)]
[DisableRequestSizeLimit, RequestFormLimits(MultipartBodyLengthLimit = int.MaxValue, ValueLengthLimit = int.MaxValue)]
public async Task<IActionResult> UploadFile()
{
if (!Request.Form.Files.Any())
{
throw new Common.Exceptions.ValidationException("Empty file");
}
IFormFile formFile = Request.Form.Files[0];
var csvDatas = new List<PatientsCSVItem>();
using (var reader = new StreamReader(formFile.OpenReadStream()))
{
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var values = line.Split(';');
//process csv rows
}
}
PatientCsvLog executionLog = _patientCsvManager.AddOrUpdatePatientsByCsvData(csvDatas, _userManager.GetLoggedUserId(User));
if (executionLog == null)
{
throw new ArgumentNullException(nameof(executionLog));
}
var response = new
{
NumberRecordImported = executionLog.NumberRecordImported,
NumberRecordUpdated = executionLog.NumberRecordUpdated,
NumberRecordDiscarded = executionLog.NumberRecordDiscarded,
DiscardedRecordList = executionLog.DiscardedRecordList
};
return Ok(response);
}

Related

Retreive the file name and size uploaded via postman in asp.net core

I have tried the following code ..
When I try out the following code I get 404 content not found. Also my controller doesn't get hit when I try to debug the code ..
public async Task<IActionResult> Download(string filename)
{
if (filename == null)
return Content("filename not present");
var path = Path.Combine(Directory.GetCurrentDirectory(),"wwwroot", filename);
var memory = new MemoryStream();
using (var stream = new FileStream(path, FileMode.Open))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
return File(memory, GetContentType(path), Path.GetFileName(path));
}
Upload file with size and filename as response:
public async Task<IActionResult> OnPostUploadAsync(IFormFile file)
{
long size = file.Length;//in bytes
if (file.Length > 0)
{
var name = Path.GetRandomFileName();
//var fileOriginName = file.FileName;
var path = Path.Combine(System.IO.Directory.GetCurrentDirectory(), "wwwroot", name);
using (var stream = System.IO.File.Create(path))
{
await file.CopyToAsync(stream);
}
return Ok(new { size = size, filename = name });
}
else {
return Ok(new { size = 0, filename = ""});
}
}
===============================================
I have an api like below and I had a file test.txt in wwwroot folder.
public async Task<IActionResult> Download(string filename)
{
if (filename == null)
return Content("filename not present");
var path = Path.Combine(System.IO.Directory.GetCurrentDirectory(), "wwwroot", filename);
var memory = new MemoryStream();
using (var stream = new FileStream(path, FileMode.Open))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
var contentType = "APPLICATION/octet-stream";
return File(memory, contentType, Path.GetFileName(path));
}
Then when I call localhost:port/home/download?filename=test then I'll get exception that file can't find. And when I call localhost:port/home/Download?filename=test.txt it succeeds.
My API is in an asp.net core MVC project and I think you need to check the contentType.

ASP .Net Core file upload - getting form data when [DisableFormValueModelBinding] attribute is in place

I went ahead and implemented an ASP .Net Core file upload controller per the documentation and it requires using a [DisableFormValueModelBinding] attribute for streaming large files. I got that working fine. Unfortunately, when using that attribute it seems to block my JSON properties coming in from the form.
Is there any way to get both the file and the form data here? Here is my controller code (the request.form calls are where I am having issues):
[Route("{caseNbr:int}/Document")]
[ResponseType(typeof(CaseDocumentModel))]
[DisableFormValueModelBinding]
[HttpPost]
public async Task<IActionResult> PostDocument(int caseNbr)
{
string errorTrackingFileName = string.Empty;
try
{
UserSessionModel userSessionModel = SessionExtensions.CurrentUserSession;
if (!MultipartRequestHelper.IsMultipartContentType(Request.ContentType))
{
return BadRequest("Bad Request");
}
var boundary = MultipartRequestHelper.GetBoundary(MediaTypeHeaderValue.Parse(Request.ContentType), _defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, HttpContext.Request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out var contentDisposition);
if (hasContentDispositionHeader)
{
if (!MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
{
return BadRequest("Bad Request");
}
var fileName = WebUtility.HtmlEncode(contentDisposition.FileName.Value);
errorTrackingFileName = fileName;
var trustedFileNameForFileStorage = fileName; //Path.GetRandomFileName();
var streamedFileContent = await FileHelpers.ProcessStreamedFile(section, contentDisposition, ModelState, _permittedExtensions, _fileSizeLimit);
if (!ModelState.IsValid)
{
return BadRequest("Bad Request");
}
using (var targetStream = System.IO.File.Create(Path.Combine(_tempFilePath, trustedFileNameForFileStorage)))
{
await targetStream.WriteAsync(streamedFileContent);
**//This is where I am having trouble:**
string descrip = HttpContext.Request.Form["Description"].ToString();
string docType = HttpContext.Request.Form["DocType"].ToString() ?? "Document";
bool isGeneralFileUpload = false;
if (string.IsNullOrWhiteSpace(Request.Form["GeneralFileUpload"]) == false && AppHelper.IsBool(Request.Form["GeneralFileUpload"]))
isGeneralFileUpload = bool.Parse(Request.Form["GeneralFileUpload"]);
int transcriptionJobId = 0;
if (string.IsNullOrWhiteSpace(Request.Form["TranscriptionJobId"]) == false && AppHelper.IsNumeric(Request.Form["TranscriptionJobId"]))
transcriptionJobId = int.Parse(Request.Form["TranscriptionJobId"]);
CaseDocumentModel createdCaseDocumentModel;
if (docType.Equals("Dictation"))
createdCaseDocumentModel = DictationRepository.ProcessDictationFile(userSessionModel.DBID, caseNbr, _tempFilePath, fileName, userSessionModel);
else if (isGeneralFileUpload)
createdCaseDocumentModel = DashboardAdjusterRepository.CreateGeneralFileUploadDocument(_tempFilePath, fileName, userSessionModel, docType, descrip);
else if (docType.Equals("Transcription"))
createdCaseDocumentModel = TranscriptionRepository.UploadTranscriptionFile(userSessionModel.DBID, _tempFilePath, fileName, userSessionModel, transcriptionJobId);
else
createdCaseDocumentModel = CaseRepository.CreateCaseDocumentRecord(userSessionModel.DBID, caseNbr, descrip, docType, _tempFilePath, fileName, userSessionModel);
return Ok(createdCaseDocumentModel);
}
}
// Drain any remaining section body that hasn't been consumed and
// read the headers for the next section.
section = await reader.ReadNextSectionAsync();
}
}
catch (Exception ex)
{
AppHelper.WriteErrorLog("CaseController PostDocument failed due to " + ex.Message + " case number was " + caseNbr + " file name was " + errorTrackingFileName);
return BadRequest("Bad Request");
}
return BadRequest("Bad Request");
}
Here is a sample call with Postman:
Screen shot of Postman

How to download multiple files at once from S3 using C# AWS SDK

How to download multiple files from s3 buckets. I could not find any better option on SO.
Here is my code for single file download. Given list of Urls, I am looping to download multiple files.
public async Task Download(string url, Stream output)
{
var s3Uri = new AmazonS3Uri(url);
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to stream is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}
Download files given s3 urls
var list = new List<Stream>();
foreach(var url in urls)
{
var stream = new MemoryStream();
await Download(url,ms);
list.Add(stream);
}
Is there any better option to download multiple files at once from S3?
I finally decided to implement my own version
public class StreamWrapper
{
public string Url { get; set; }
public Stream Content { get; set; }
public string FileName { get; set; }
}
public async Task Download(IList<StreamWrapper> inout, int maxConcurrentDownloads)
{
if (maxConcurrentDownloads <= 0)
{
maxConcurrentDownloads = 20;
}
if (!inout.HasAny())
return;
var tasks = new List<Task>();
for (int i = 0; i < inout.Count; i++)
{
StreamWrapper wrapper = inout[i];
AmazonS3Uri s3Uri = null;
if (AmazonS3Uri.TryParseAmazonS3Uri(wrapper.Url, out s3Uri))
{
tasks.Add(GetObject(s3Uri, wrapper.Content));
}
if (tasks.Count == maxConcurrentDownloads || i == inout.Count - 1)
{
await Task.WhenAll(tasks);
tasks.Clear();
}
}
}
private async Task GetObject(AmazonS3Uri s3Uri, Stream output)
{
GetObjectRequest getObjectRequest = new GetObjectRequest
{
BucketName = s3Uri.Bucket,
Key = System.Net.WebUtility.UrlDecode(s3Uri.Key)
};
using (var s3Client = new AmazonS3Client(s3Uri.Region))
{
// dispose the underline stream when writing to local file system is done
using (var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest).ConfigureAwait(false))
{
using (var responseStream = getObjectResponse.ResponseStream)
{
await responseStream.CopyToAsync(output);
}
}
}
output.Seek(0L, SeekOrigin.Begin);
}

Win Phone 8.1 : OutOfMemory Exception while downloading large files (30 MB)

I am creating a universal Windows Phone 8.1 App. I am trying to download the file and view it into launcher. I works for small file less than 15 MB files. But when file size is more than 15 MB, I got the out of memory exception.
async private Task<object> GetMailAttachments(string attachNotify)
{
try
{
cmdBarMailItem.IsEnabled = false;
if (await Device.IsNetworkAvailable())
{
cts = new CancellationTokenSource();
// Ignore SSL Certificate which is untrusted,expired and has invalid hostname.
var filter = new HttpBaseProtocolFilter() { AllowUI = false };
filter.IgnorableServerCertificateErrors.Add(Windows.Security.Cryptography.Certificates.ChainValidationResult.Untrusted);
filter.IgnorableServerCertificateErrors.Add(Windows.Security.Cryptography.Certificates.ChainValidationResult.Expired);
filter.IgnorableServerCertificateErrors.Add(Windows.Security.Cryptography.Certificates.ChainValidationResult.InvalidName);
// Start calling the soap service #userGetAttachmentByIndex
using (var client = new System.Net.Http.HttpClient(new WinRtHttpClientHandler(filter)))
{
//Prepare parameters which is to be post via soap envelope.
List<KeyValuePair<string, string>> parameter = new List<KeyValuePair<string, string>>();
parameter.Add(new KeyValuePair<string, string>("sessionId", GlobalInfo.SessionID));
parameter.Add(new KeyValuePair<string, string>("attachmentIndex", attachNotify.Split('|')[1].ToString()));
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("text/xml"));
client.DefaultRequestHeaders.Add("SOAPAction", "userGetAttachmentByIndex");
var postContent = new StringContent(StringHelper.ConstructSoapRequest(parameter, "userGetAttachmentByIndex"), Encoding.UTF8, "text/xml");
// Getting response from soap service
var response = await client.PostAsync(new Uri(AppEnv.ServiceEndPoint), postContent, cts.Token);
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
string soapResponse = await response.Content.ReadAsStringAsync();
var soap = XDocument.Parse(soapResponse);
XNamespace ns = "http://service.webservice.cryoserver.ci";
var base64BinaryStr = soap.Descendants(ns + "userGetAttachmentByIndexResponse").First().Descendants(ns + "return").First().Descendants(ns + "attachmentType").First().Descendants(ns + "binaryData").First().Descendants(ns + "base64Binary").First().Value;
await saveStringToLocalFile(base64BinaryStr);
var file = await Windows.Storage.ApplicationData.Current.LocalFolder.GetFileAsync("myTest.pdf");
bool x = await Windows.System.Launcher.LaunchFileAsync(file);
return x;
}
}
}
cmdBarMailItem.IsEnabled = true;
}
catch (TaskCanceledException)
{
PopupRetrieve.IsOpen = false;
ProgressBar.IsVisible = false;
cmdBarMailItem.IsEnabled = true;
}
catch(Exception ex)
{
cmdBarMailItem.IsEnabled = true;
ProgressBar.IsVisible = false;
MessageBox.Show(AlertType.Connectivity);
}
return null;
}
async Task saveStringToLocalFile(string content)
{
try
{
// saves the string 'content' to a file 'filename' in the app's local storage folder
// byte[] fileBytes = System.Text.Encoding.UTF8.GetBytes(content.ToCharArray());
byte[] byteArray = Convert.FromBase64String(content);
// create a file with the given filename in the local folder; replace any existing file with the same name
StorageFile file = await Windows.Storage.ApplicationData.Current.LocalFolder.CreateFileAsync("myTest.pdf", CreationCollisionOption.ReplaceExisting);
// write the char array created from the content string into the file
using (var stream = await file.OpenStreamForWriteAsync())
{
stream.Write(byteArray, 0, byteArray.Length);
stream.Flush();
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
I got the exception on executing the line
string soapResponse = await response.Content.ReadAsStringAsync();
Anybody have an idea why the exception occurs ? What could be possible solution to fix it.
Any help would be appriciable. :)

Error while uploading file method in Client Object Model Sharepoint 2010

Error while uploading file method in Client Object Model + Sharepoint 2010. Once the file got uploaded. After that though the code compiles with no error
I get the error while executing
"{"Value does not fall within the expected range."}
{System.Collections.Generic.SynchronizedReadOnlyCollection}
I have a method which takes care of functionality to upload files
///////////////////////////////////////////////////////////////////////////////////////////
public void Upload_Click(string documentPath, byte[] documentStream)
{
String sharePointSite = "http://cvgwinbasd003:28838/sites/test04";
String documentLibraryUrl = sharePointSite +"/"+ documentPath.Replace('\\','/');
////////////////////////////////////////////////////////////////////
//Get Document List
List documentsList = clientContext.Web.Lists.GetByTitle("Doc1");
var fileCreationInformation = new FileCreationInformation();
//Assign to content byte[] i.e. documentStream
fileCreationInformation.Content = documentStream;
//Allow owerwrite of document
fileCreationInformation.Overwrite = true;
//Upload URL
fileCreationInformation.Url = documentLibraryUrl;
Microsoft.SharePoint.Client.File uploadFile = documentsList.RootFolder.Files.Add(
fileCreationInformation);
//uploadFile.ListItemAllFields.Update();
clientContext.ExecuteQuery();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
In the MVC 3.0 application in the controller I have defined the following method to invoke the upload method.
//////////////////////////////////////////////////////////////////////////////////////////////////
public ActionResult ProcessSubmit(IEnumerable<HttpPostedFileBase> attachments)
{
System.IO.Stream uploadFileStream=null;
byte[] uploadFileBytes;
int fileLength=0;
foreach (HttpPostedFileBase fileUpload in attachments)
{
uploadFileStream = fileUpload.InputStream;
fileLength=fileUpload.ContentLength;
}
uploadFileBytes= new byte[fileLength];
uploadFileStream.Read(uploadFileBytes, 0, fileLength);
using (DocManagementService.DocMgmtClient doc = new DocMgmtClient())
{
doc.Upload_Click("Doc1/Doc2/Doc2.1/", uploadFileBytes);
}
return RedirectToAction("SyncUploadResult");
}
//////////////////////////////////////////////////////////////////////////////////////////////////
Please help me to locate the error
I think your documentLibraryUrl needs to be relative. This is working for me with Sharepoint 2013
[HttpPost]
[ValidateAntiForgeryToken]
[SharePointContextFilter]
public ActionResult Upload()
{
if (Request.Files.Count > 0)
{
HttpPostedFileBase file = Request.Files[0];
if (file != null && file.ContentLength > 0)
{
var fileName = Path.GetFileName(file.FileName);
var spContext = SharePointContextProvider.Current.GetSharePointContext(HttpContext);
using (var clientContext = spContext.CreateUserClientContextForSPHost())
{
if (clientContext != null)
{
FileCreationInformation newFile = new FileCreationInformation();
using (MemoryStream ms = new MemoryStream())
{
file.InputStream.CopyTo(ms);
byte[] array = ms.GetBuffer();
newFile.Content = array;
}
List docs = clientContext.Web.Lists.GetByTitle("Documents");
Folder folder = docs.RootFolder;
clientContext.Load(folder);
clientContext.ExecuteQuery();
newFile.Url = docs.RootFolder.ServerRelativeUrl + "/" + fileName;
Microsoft.SharePoint.Client.File uploadFile = docs.RootFolder.Files.Add(newFile);
clientContext.Load(uploadFile);
clientContext.ExecuteQuery();
//Set the metadata
Microsoft.SharePoint.Client.ListItem item = uploadFile.ListItemAllFields;
string docTitle = string.Empty;
item["Title"] = docTitle;
item.Update();
clientContext.ExecuteQuery();
}
}
}
}
return RedirectToAction("Index", new { SPHostUrl = SharePointContext.GetSPHostUrl(HttpContext.Request).AbsoluteUri });
}