Display PDF from azure blob in browsers using Microsoft Azure Storage SDK for Node.js and JavaScript for Browsers - azure-storage

I am trying to use Microsoft Azure Storage SDK for Node.js and JavaScript for Browsers (https://github.com/Azure/azure-storage-node) to display PDF contents stored in Azure blob in browsers. So far I couldn't find any examples on how to do it.
I tried to follow the suggestion from https://github.com/Azure/azure-storage-node/issues/440, But couldn't make it work. I am using Azure function.
module.exports = async function (context, req) {
let accessToken = await getAccessToken();
let container = req.params.container;
let filename = req.params.filename;
let tokenCredential = new azure.TokenCredential(accessToken);
let storageAccountName = process.env.StorageAccountName;
let blobService = azure.createBlobServiceWithTokenCredential(`https://${storageAccountName}.blob.core.windows.net/`, tokenCredential);
return new Promise((resolve, reject) => {
let readStream = blobService.createReadStream(container, filename, function (error, result, response) {
if (error) {
context.log(error);
context.log(response);
context.res = {
status: 400,
body: response
};
resolve(context.res);
}
});
let body = '';
readStream.on('data', (chunk) => {
body += chunk;
});
readStream.on('end', () => {
context.res = {
headers: {
'Content-Type': "application/pdf"
},
body: body
};
resolve(context.res);
});
});
};
But I got "Couldn't open PDF" error message in the browser or timeout error.

For downloading blob in browser environment, using URL with SAS is recommended, and in the framework you are using, would an accessible URL pointing to PDF be enough?
Please follow example:
Download Blob
BlobService provides interfaces for downloading a blob into browser memory. Because of browser's sandbox limitation, we cannot save the downloaded data trunks into disk until we get all the data trunks of a blob into browser memory. The browser's memory size is also limited especially for downloading huge blobs, so it's recommended to download a blob in browser with SAS Token authorized link directly.
Shared access signatures (SAS) are a secure way to provide granular access to blobs and containers without providing your storage account name or keys. Shared access signatures are often used to provide limited access to your data, such as allowing a mobile app to access blobs.

Related

putObject upload broken files on S3 only when its by API

I have a problem when i try to upload a file on S3 through my API
I use the "putObject" method, and the thing that surprise me it's it work when i run my serverless application locally, with serverless-offline, i can push the all file on s3 and i can open it
But when i deploy my application on API Gateway, if i use the API Gateway route, the file is lightweight than original, and i can't open it, it tell me that the file is corrupted
If anyone has an idea, it could really help me
Thanks
My putObject method looks like this
const bucketName = _.get(getBucket,'bucketName');
const extension = _.get(data,'media.filename').split('.').pop();
const keyName = _.get(data,'keyName') + '.' + extension;
const content = _.get(data,'media.content')
let params = {
Bucket: bucketName,
Key: keyName,
ContentType: _.get(data,'media.contentType'),
Body: content,
ACL: 'public-read'
};
return new Promise((resolve, reject) => {
s3.putObject(params, function(err, data) {
err
?
reject(console.log(err)) :
resolve(
response(200,"Object Added")
);
});
});
Uploading files via ApiGateway is not a good idea, I'd strongly advise using resigned URL https://docs.aws.amazon.com/AmazonS3/latest/dev/PresignedUrlUploadObject.html
If you want to upload through ApiGateway, than provided information is not enough. Try to log the received lambda event, that should help

How to Upload a csv file lager than 10MB on S3 using Lambda /API Gateway

Hello I am new here on AWS i was trying to upload a csv file on my bucket s3 but when the file is larger than 10mb it is returing "{"message":"Request Entity Too Large"}" I am using postman to do this. Below is the current code I created but in the future I will add some validation to change the name of the file that being uploaded into my format. Is there any way to do this with this kind of code or if you have any suggestion that can help me with the issue I have encountered?
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const bucket = process.env.UploadBucket;
const prefix = "csv-files/";
const filename = "file.csv";
exports.handler = (event, context, callback) => {
let data = event.body;
let buff = new Buffer(data, 'base64');
let text = buff.toString('ascii');
console.log(text);
let textFileSplit = text.split('?');
//get filename split
let getfilename = textFileSplit[0].split('"');
console.log(textFileSplit[0]);
console.log(textFileSplit[1]);
// //remove lower number on csv
let csvFileSplit = textFileSplit[1].split('--')
const params = {
Bucket: bucket,
Key: prefix + getfilename[3],
Body: csvFileSplit[0]
};
s3.upload(params, function (err, data) {
if (err) {
console.log('error uploading');
callback(err);
}
console.log("Uploaded")
callback(null, "Success")
});
}
For scenarios like this one, we normally use a different approach.
Instead of sending the file to lambda through API Gateway, you send the file directly to S3. This will make your solution more robust and cost you less because you don't need to transfer the data to API Gateway and you don't need to process the entire file inside the lambda.
The question is: How do you do this in a secure way, without opening your S3 Bucket to everyone on the internet and uploading anything to it? You use s3 signed urls. Signed Urls are a feature of S3 that allows you to bake in the url the correct permissions to upload an object to a secured bucket.
In summary the process will be:
Frontend sends a request to API Gateway;
API Gateway forward the request to a Lambda Function;
The Lambda Function generate a signed Url with the permissions to upload the object to a specific s3 bucket;
API Gateway sends back the response from Lambda Function to the Frontend. Frontend upload the file to the signed Url.
To generate the signed url you will need to use the normal aws-sdk in your lambda function. There you will call the method getSignedUrl (signature depends on your language). You can find more information about signed urls here.

How to send multiple images in a Expressjs api get request with sendFIle()

I'm looking for away to send multiple images in one GET request from an Expressjs server through an api.
I want to create an image gallery of each users uploaded images in a MEAN stack. When images are uploaded using multer, the image information is saved to mongodb, including the userid of whoever uploaded it.
When on angularjs, I want user to have access to any of the images they have previously uploaded. Currently I'm sending one file on a GET request based on user id. Is there anyway of sending multiple files in one json. I'm currently using Expressjs's res.sendFile, but haven't found any info about sending multiple back yet.
https://expressjs.com/en/api.html#res.sendFile
Here is my current get request:
exports.getUpload = function(req, res) {
Upload.find({createdby: req.params.Id}).exec(function(err, upload) {
errorhandle.errorconsole(err, 'file found');
console.log(upload[0]);
var options = {
root: '/usr/src/app/server/public/uploads/images'
};
var name = "" + upload[0].storedname +"";
console.log(name);
res.sendFile(name, options,function(err) {
errorhandle.errorconsole(err, 'file sent');
});
});
};
You can't with res.sendFile. In fact I don't think you can at all. Maybe with HTTP/2 Server Push
, but I'm not sure.
What you can do is send a JSON response with a link to all the images:
exports.getUpload = async (req, res) => {
const uploads = await Upload.find({ createdby: req.params.Id }).exec()
const response = uploads.map(image => {name: `https://example.com/uploads/images/${image.storedname}`})
res.json(response)
}
Note error handling omitted.

getaddrinfo ENOTFOUND API Google Cloud

I'm trying to execute API.AI tutorial for building a weather bot for Google Assistant (the one here: https://dialogflow.com/docs/getting-started/basic-fulfillment-conversation)
I made everything successfully, created the bot within API, created the Fulfillments, installed NodeJS on my pc, connected Google Cloud Platform, etc.
Then I created the index.js file by copying it exactly how it's stated on API.ai tutorial with my API key from World Weather Organisation (see below).
But when I use the bot, it doesn't work. On the Google Cloud Platform the error is always the same:
Error: getaddrinfo ENOTFOUND api.worldweatheronline.com
api.worldweatheronline.com:80
at errnoException (dns.js:28)
at GetAddrInfoReqWrap.onlookup (dns.js:76)
No matter how often I do it I get the same error. So I don't actually reach the API. I tried to see if anything changed from WWO side (URL, etc.) but apparently no. I updated NodeJS and still same issue. I refreshed the Google Cloud platform completely and didn't help.
That one I really can't debug. Could anyone help?
Here's the code from API.ai:
'use strict';
const http = require('http');
const host = 'api.worldweatheronline.com';
const wwoApiKey = '[YOUR_API_KEY]';
exports.weatherWebhook = (req, res) => {
// Get the city and date from the request
let city = req.body.result.parameters['geo-city']; // city is a required param
// Get the date for the weather forecast (if present)
let date = '';
if (req.body.result.parameters['date']) {
date = req.body.result.parameters['date'];
console.log('Date: ' + date);
}
// Call the weather API
callWeatherApi(city, date).then((output) => {
// Return the results of the weather API to Dialogflow
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ 'speech': output, 'displayText': output }));
}).catch((error) => {
// If there is an error let the user know
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ 'speech': error, 'displayText': error }));
});
};
function callWeatherApi (city, date) {
return new Promise((resolve, reject) => {
// Create the path for the HTTP request to get the weather
let path = '/premium/v1/weather.ashx?format=json&num_of_days=1' +
'&q=' + encodeURIComponent(city) + '&key=' + wwoApiKey + '&date=' + date;
console.log('API Request: ' + host + path);
// Make the HTTP request to get the weather
http.get({host: host, path: path}, (res) => {
let body = ''; // var to store the response chunks
res.on('data', (d) => { body += d; }); // store each response chunk
res.on('end', () => {
// After all the data has been received parse the JSON for desired data
let response = JSON.parse(body);
let forecast = response['data']['weather'][0];
let location = response['data']['request'][0];
let conditions = response['data']['current_condition'][0];
let currentConditions = conditions['weatherDesc'][0]['value'];
// Create response
let output = `Current conditions in the ${location['type']}
${location['query']} are ${currentConditions} with a projected high of
${forecast['maxtempC']}°C or ${forecast['maxtempF']}°F and a low of
${forecast['mintempC']}°C or ${forecast['mintempF']}°F on
${forecast['date']}.`;
// Resolve the promise with the output text
console.log(output);
resolve(output);
});
res.on('error', (error) => {
reject(error);
});
});
});
}
Oh boy, in fact the reason was most stupid ever. I didn't enable "billing" on Google Cloud Platform and that's why it blocked everything (even though I'm using a free test of the API). They just wanted my credit card number. It works now
I had the same issue trying to hit my db. Billing wasn't the fix as I had billing enabled already.
For me it was knexfile.js setup for MySql - specifically the connection object. In that object, you should replace the host key with socketPath; and prepend /cloudsql/ to the value. Here's an example:
connection: {
// host: process.env.APP_DB_HOST, // The problem
socketPath: `/cloudsql/${process.env.APP_DB_HOST}`, // The fix
database: process.env.APP_DB_NAME,
user: process.env.APP_DB_USR,
password: process.env.APP_DB_PWD
}
Where process.env.APP_DB_HOST is your Instance connection name.
PS: I imagine that even if you're not using Knex, the host or server parameter of a typical DB connectionstring will have to be called socketPath when connecting to Google Cloud SQL.

Correct code to upload local file to S3 proxy of API Gateway

I created an API function to work with S3. I imported the template swagger. After deployment, I tested with a Node.js project by the npm module aws-api-gateway-client.
It works well with: get bucket lists, get bucket info, get one item, put a bucket, put a plain text object, however I am blocked with put a binary file.
firstly, I ensure ACL is allowed with all permissions on S3. secondly, binary support also added
image/gif
application/octet-stream
The code snippet is as below. The behaviors are:
1) after invokeAPI, the callback function is never hit, after sometime, the Node.js project did not respond. no any error message. The file size (such as an image) is very small.
2) with only two times, the uploading seemed to work, but the result file size is bigger (around 2M bigger) than the original file, so the file is corrupt.
Could you help me out? Thank you!
var filepathname = './items/';
var filename = 'image1.png';
fs.stat(filepathname+filename, function (err, stats) {
var fileSize = stats.size ;
fs.readFile(filepathname+filename,'binary',function(err,data){
var len = data.length;
console.log('file len' + len);
var pathTemplate = '/my-test-bucket/' +filename ;
var method = 'PUT';
var params = {
folder: '',
item:''
};
var additionalParams = {
headers: {
'Content-Type': 'application/octet-stream',
//'Content-Type': 'image/gif',
'Content-Length': len
}
};
var result1 = apigClient.invokeApi(params,pathTemplate,method,additionalParams,data)
.then(function(result){
//never hit :(
console.log(result);
}).catch( function(result){
//never hit :(
console.log(result);
});;
});
});
We encountered the same problem. API Gateway is meant for limited data (10MB as of now), limits shown here,
http://docs.aws.amazon.com/apigateway/latest/developerguide/limits.html
Self Signed URL to S3:
Create an S3 self signed URL for POST from the lambda or the endpoint where you are trying to post.
How do I put object to amazon s3 using presigned url?
Now POST the image directly to S3.
Presigned POST:
Apart from posting the image if you want to post additional properties, you can post it in multi-form format as well.
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#createPresignedPost-property
If you want to process the file after delivering to S3, you can create a trigger from S3 upon creation and process with your Lambda or anypoint that need to process.
Hope it helps.