Expressjs Multer Error Handling with Array - express

I am using multer's array method as a middleware for my post route and I'm trying to figure out how I can send the error callback from the fileFilter function in the multer options setup as a flash message when the group of files that a user is uploading contains at least one file with the wrong format. My current fileFilter setup achieves this, but instead of sending the user to a blank page with the File Selected is not supported message, I was looking for a way to communicate that to the route using multer as middleware.
Here is my multer setup:
var upload = multer({
storage: multerS3({
s3: s3,
bucket: options.Bucket,
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: options.ACL,
key: function(req, file, cb){
var fileNameFormatted = file.originalname.replace(/\s+/g, '-').toLowerCase();
cb(null, req.user.organizationId + '/' + uploadDate + '/' + fileNameFormatted);
}
}),
fileFilter: function(req, file, cb){
if(!file.originalname.match(/\.(jpg|jpeg|png|gif|csv|xls|xlsb|xlsm|xlsx)$/)){
return cb('File Selected is not supported');
}
cb(null, true);
}
});
Here is my route with upload.array('fileUpload', 5) fileUpload is the name of my file input field and 5 is the multer file length limiting feature.
.post(upload.array('fileUpload', 5), function(req, res) {
//Configure Uploaded S3 File Path strings based on environment for use in DB
var uploadedFiles = req.files;
var s3FilePath = [];
for (var prop in uploadedFiles) {
console.log(uploadedFiles[prop].key);
if (app.get('env') === 'production' || app.get('env') === 'staging') {
s3FilePath = 'https://files.test-site.com/' + uploadedFiles[prop].key;
} else {
s3FilePath.push(uploadedFiles[prop].location);
}
}
models.Blog.update({
blogId: req.body.blogId,
blogDate: req.body.blogDate,
}, {
where: {
userId: req.user.userId,
blogId: req.body.blogId
}
}).then(function(blog) {
console.log('This is the blog ' + blog);
var files = _.map(s3FilePath, function(file) {
console.log(file);
return {
fileName: file,
blogId: req.body.blogId
};
});
return models.BlogFile.bulkCreate(files);
}).then(function() {
res.redirect('/app');
}).catch(function(err) {
res.send(err);
console.log('File Post Error ' + err);
});
});

Related

Unexpected end of multipart data nodejs multer s3

iam trying to upload image in s3 this is my code
const upload = require('../services/file_upload');
const singleUpload = upload.single('image');
module.exports.uploadImage = (req,res) => {
singleUpload(req, res, function (err) {
if (err) {
console.log(err);
return res.status(401).send({ errors: [{ title: 'File Upload Error', detail: err}] });
}
console.log(res);
return res.json({ 'imageUrl': req.file.location });
});
}
FileUpload.js
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true)
} else {
cb(new Error('Invalid Mime Type, only JPEG and PNG'), false);
}
}
const upload = multer({
fileFilter,
storage: multerS3({
s3,
bucket: 'image-bucket',
acl: 'public-read',
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, {fieldName: 'TESTING_META_DATA!'});
},
key: function (req, file, cb) {
cb(null,"category_"+Date.now().toString()+".png")
}
})
})
module.exports = upload;
I tried to test api with postmanin serverless local it is giving this error
Error: Unexpected end of multipart data
at D:\Flutter\aws\mishpix_web\node_modules\dicer\lib\Dicer.js:62:28
at process._tickCallback (internal/process/next_tick.js:61:11) storageErrors: []
After deploying online. i tried the api. the file is uploaded in server but its a broken
Are you using aws-serverless-express. aws-serverless-express will forward original request to Buffer as utf8 encoding. So multipart data will be lost or error. I am not sure why.
So, I change aws-serverless-express to aws-serverless-express-binary and everything worked.
yarn add aws-serverless-express-binary
Hope this help!

Multer isn't passing in express put route

I'm trying to upload an image to the file system with Multer. Please take a look at the relevant data in my route:
const
..
multer = require('multer'),
..;
const storage = multer.diskStorage({
destination: function (req, file, callback) {
callback(null, './uploads');
},
filename: function (req, file, callback) {
callback(null, req.params.id + file.originalname);
}
}),
upload = multer({storage: storage}).single('profilePic');
router.put(
'/:id',
middleware.isLoggedIn,
(req, res, next) => {
User
.findByIdAndUpdate(
req.params.id, req.body.user,
(err, updatedUser) => {
if (err) {
return req.flash('error', err.message);
}
upload(req, res, (err) => {
if (err) {
eval(locus);
return req.flash('error', err.message);
}
updatedUser = req.body.user;
eval(locus);
//redirect show page
res.redirect('/dashboard/profile/' + req.params.id + '/edit');
});
});
});
module.exports = router;
When I look at updatedUser the first thing I see is
{ profilePic: 'data:image/jpeg;base64,....} what am I doing wrong? It's not even updating the page now that I have the upload function in here. What I really want to do is get the destination to work on s3 but I need to get this to save first.
So, this is a the most basic example of uploading an image using multer:
var express = require('express')
var multer = require('multer')
var app = express()
var storage = multer.diskStorage({
// define where the file should be uploaded, else it will be uploaded to the system temp dir
destination: function (req, file, cb) {
// ./uploads should be created beforehand
cb(null, './uploads')
},
// define "filename", else a random name will be used for the uploaded file
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + file.originalname)
}
})
var upload = multer({ storage: storage })
// pic is the name of image field in the form
app.put('/profile', upload.single('pic'), function (req, res, next) {
console.log(req.file)
res.send('Uploaded')
})
app.listen(3000)
And here is an example curl command to upload an image from the file system to the above app:
curl -X PUT -F 'pic=#/projects/eg/foto.png' localhost:3000/profile
Make sure the example works fine, to ensure you understand how multer handles file uploads, and that the issue is not with multer.
That said and done, User.findByIdAndUpdate seems to be storing the image data as a base64 encoded string somewhere; I have no idea what User.findByIdAndUpdate connects to. It is beyond the domain of multer.
Someone on our Gitter channel (https://gitter.im/expressjs/express) might be able to suggest something. Join us there.

Lambda File Write to S3

For the past six months I have been downloading the NASA APOD and saving to an S3 bucket using a Lambda function. Up until 12/23/2016 all was working as expected. Now when I check my bucket, the images are there but size 0 bytes. I have included my code below. Does anyone know if there has been a change? Thanks!
var AWS = require("aws-sdk");
var https = require('https');
var http = require('http');
var fs = require('fs');
// Incoming Handler
// <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
exports.handler = (event, context, callback) => {
GetAPOD();
};
// <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
// Functions
// <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
function GetAPOD() {
var nasa_api_key = 'MY KEY GOES HERE'
, nasa_api_path = '/planetary/apod?api_key=' + nasa_api_key;
var options = {
host: 'api.nasa.gov',
port: 443,
path: nasa_api_path,
method: 'GET'
};
// Connect to the NASA API and get the APOD.
var req = https.request(options, function (res) {
console.log('Open connection to NASA.');
res.setEncoding('utf-8');
var responseString = '';
res.on('data', function (data) {
responseString = data;
});
res.on('end', function () {
console.log('API Response: ' + responseString);
var responseObject = JSON.parse(responseString)
, image_date = responseObject['date']
, image_url = responseObject['url']
, image_hdurl = responseObject['hdurl']
, media_type = responseObject['media_type'];
if (media_type == 'image') {
var image_name = image_date + '.jpg';
var s3 = new AWS.S3();
var s3Bucket = new AWS.S3( { params: {Bucket: 'nasa-apod'} } );
// Check to see if the image already exists in the S3 bucket.
// If not we will upload the image to S3.
var head_data = {Key: image_name};
s3Bucket.headObject(head_data, function(err, output_head_data) {
if (output_head_data) {
console.log("Image exists on S3.");
}
else {
console.log("Image does not exists on S3.");
// Image has not been uploaded to S3, open a stream and download the image to the /tmp folder.
var file = fs.createWriteStream("/tmp/" + image_name);
var request = http.get(image_url, function(response) {
console.log("Opening file stream.");
// Pipe the data into the file stream and save to disk.
response.pipe(file);
response.on('end', function () {
// File is written to disk, we are going to check that it exists.
var fileName = "/tmp/" + image_name;
fs.exists(fileName, function(exists) {
if (exists) {
console.log("File exits in /tmp folder.");
// Get the stats for the image, will need this for the ContentLength
fs.stat(fileName, function(error, stats) {
if (error) {
console.log("Stat Error: " + error);
}
else {
console.log("Opening file stream.");
var image_stream = fs.createReadStream(fileName);
// Begin the upload process to S3.
var param_data = {Key: image_name, Body: image_stream, ContentType: "image/jpeg", ContentLength: stats.size, ACL: "public-read"};
s3Bucket.putObject(param_data, function(err, output_data) {
if (err) {
console.log('Error uploading data to S3: ' + err);
}
else {
console.log('Image successfully uploaded.');
}
});
}
});
}
else {
console.log('File does not exist in the /tmp folder.');
}
});
});
});
}
});
}
else {
console.log("Media Type: " + media_type);
}
});
});
req.on('error', function (e) {
console.error('HTTP error: ' + e.message);
});
req.end();
}
// <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
Found out that NASA APOD API is now using https and not http for images. I had to adjust my code to use https for the image path.

Express file upload with multer and gridfs (corrupted file ?)

I'm trying to upload a file (an image), the upload is fine, the file is stored in Mongo and have the same content type and same size as the original file, then when I try to download it, the file is corrupted but keeps the same content type (if I upload a pdf, it is recognized as a pdf, if it is a png, it is also recognized, but I can't open them).
I don't understand what is wrong with this, it is pretty simple and standard.
For the upload from the client, I use angular ng-upload, for the download, it is a simple GET request to the route defined in the code.
EDIT :
The file is well uploaded on the server, so the problem is when I try to read it from GridFS.
The file that is downloaded is bigger than the one uploaded ! So the file isn't as the original and is corrupted, why ?
Here is my code.
//BACKEND
//ROUTES
var multer = require('multer');
var upload = multer({ dest: './tmp/'});
router.post('/:id/logo',upload.single('file'), uploadFile);
router.get('/:id/logo', getFile);
//Controller
var Grid = require('gridfs-stream');
var mongoose = require('mongoose');
var fs = require('fs');
uploadLogo = function(req, res) {
var gfs = Grid(mongoose.connection.db, mongoose.mongo);
var writeStream = gfs.createWriteStream();
fs.createReadStream(req.file.path).pipe(writeStream);
writeStream.on('close', function(file) {
res.status(200).send({fileId: file._id});
});
writeStream.on('error', function(e) {
res.status(500).send("Could not upload file");
});
}
getFile = function(req, res) {
var gfs = Grid(mongoose.connection.db, mongoose.mongo);
// Check if the file exist
gfs.findOne({ _id: req.params.id}, function(err, file) {
if(err) {
res.status(404).end();
} else if(!file){
res.status(404).end();
} else {
var readstream = gfs.createReadStream({
_id: file._id
});
res.set('Content-Type', file.contentType);
readstream.on('error', function (err) {
res.send(500, err);
});
readstream.on('open', function () {
readstream.pipe(res);
});
}
});
}
//FRONT END
$scope.newFileUpload = function(file) {
$scope.upload(file);
}
$scope.upload = function(file) {
if (file && !angular.isUndefined(file.name)) {
Upload.upload({
url: 'api/' + $scope.myId + '/logo',
fields: {
'type': 'logo'
},
file: file
}).success(function(data, status, headers, config) {
$scope.imageId = data.fileId;
}).error(function(data, status, headers, config) {
console.log('file upload error status: ' + status);
});
}
};
//THE HTML
<div class="drop-box"
ngf-drop
ngf-select
ng-model="imageLogo"
ngf-drag-over-class="dragover"
ngf-allow-dir="true"
accept="image/*"
ngf-pattern="'image/*'"
ngf-change="newFileUpload(imageLogo)"
ngf-multiple="false"
ngf-resize="{width: 200, height: 50}">
Drop logo image here or click to upload</div>
<input type="file" nv-file-select uploader="uploader"/>

Connect-busboy - Amazon S3 - Validation and return

I am trying to directly upload files from the browser to Amazon S3 using connect-busboy with the following code (https://github.com/thaume/s3-streaming)
route - index.js:
var awsUpload = require('../services/aws-streaming');
// Index route
// ===========
exports.index = function(req, res){
if (req.method === 'POST') {
return awsUpload(req, function(err, url) {
res.send(JSON.stringify(url));
});
}
res.writeHead(200, { Connection: 'close' });
res.end('<html><head></head><body>\
<form method="POST" enctype="multipart/form-data">\
<input type="file" name="filefield"><br />\
<input type="submit">\
</form>\
</body></html>');
};
with my modified version of aws-streaming.js
// Initialize aws client
// =====================
var config = require('../config/' + 'development');
var Knox = require('knox');
var moment = require('moment');
var crypto = require('crypto');
// Create the knox client with your aws settings
Knox.aws = Knox.createClient({
key: config.aws.AWS_ACCESS_KEY_ID,
secret: config.aws.AWS_SECRET_ACCESS_KEY,
bucket: config.aws.S3_BUCKET_NAME,
region: 'eu-west-1'
});
// S3 upload service - stream buffers to S3
// ========================================
var s3UploadService = function(req, next) {
req.files = {};
req.busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
if (!filename) {
// If filename is not truthy it means there's no file
return;
}
//////////////// CHECK FOR MIMETYE /////////////////////
// If file is not "text/plain" - return //
if (mimetype != "text/plain") {
console.log('true!')
return; // A JSON array with an error "Wrong file type"!
}
// Create the initial array containing the stream's chunks
file.fileRead = [];
file.on('data', function(chunk) {
// Push chunks into the fileRead array
this.fileRead.push(chunk);
});
file.on('error', function(err) {
console.log('Error while buffering the stream: ', err);
});
file.on('end', function() {
// Concat the chunks into a Buffer
var finalBuffer = Buffer.concat(this.fileRead);
req.files[fieldname] = {
buffer: finalBuffer,
size: finalBuffer.length,
filename: filename,
mimetype: mimetype
};
// Generate date based folder prefix
var datePrefix = moment().format('YYYY[/]MM');
var key = crypto.randomBytes(10).toString('hex');
var hashFilename = key + '-' + filename;
var pathToArtwork = '/artworks/' + datePrefix + '/' + hashFilename;
var headers = {
'Content-Length': req.files[fieldname].size,
'Content-Type': req.files[fieldname].mimetype,
'x-amz-acl': 'public-read'
};
Knox.aws.putBuffer( req.files[fieldname].buffer, pathToArtwork, headers, function(err, response){
if (err) {
console.error('error streaming image: ', new Date(), err);
return next(err);
}
if (response.statusCode !== 200) {
console.error('error streaming image: ', new Date(), err);
return next(err);
}
console.log('Amazon response statusCode: ', response.statusCode);
console.log('Your file was uploaded');
next();
});
});
});
req.busboy.on('error', function(err) {
console.error('Error while parsing the form: ', err);
next(err);
});
req.busboy.on('finish', function() {
console.log('Done parsing the form!');
// When everythin's done, render the view
next(null, 'http://www.google.com');
});
// Start the parsing
req.pipe(req.busboy);
};
module.exports = s3UploadService;
What I would like to do is to validate the mimetype and return a json array with the error message, ending the parsing of the form and not upload the file. Have added code to aws-streaming, but it will not return even if validates to true. What have I done wrong?
Also the code runs the callback when it is finished parsing the form, but I would like it to be run when the file is actually uploaded. How can I achieve this, comment out the next() in 'finish' event and move it to the Knox.aws.putBuffer?
Im using Express 4