How do I serve a file from S3 through Meteor Iron Router - amazon-s3

My question is very similar to this one which describes how to serve a local file using Iron Router. I need to do the same, but instead of reading the file synchronously from disk, I need to get the file from S3 which is an asynchronous call.
The problem appears to be the fact that the action method has returned before the asynchronous s3.getObject completes giving me the following error.
Error: Can't render headers after they are sent to the client.
I'm assuming that Iron Router is generating the response for me when it realizes that I haven't handled the response in my action method, but I'm stumped about how to tell it to wait for my asynchronous call to finish.
Here is my code.
Router.map(function () {
this.route('resumeDownload', {
where: 'server',
path: '/resume/:_id',
action: function () {
var response = this.response;
var candidate = Candidates.findOne(this.params._id);
if (!candidate || !candidate.resumeS3Key) {
// this works fine because the method hasn't returned yet.
response.writeHead(404);
return response.end();
}
var s3 = new AWS.S3();
s3.getObject({Bucket: 'myBucket', Key: candidate.resumeS3Key}, function (err, data) {
if (err) {
// this will cause the error to be displayed
response.writeHead(500);
return response.end();
}
// this will also cause the error to be displayed
response.writeHead(200, {'Content-Type': data.ContentType});
response.end(data.Body);
});
}
});
});

I was able to solve this one myself. I needed to use a future in my action method.
Here is the working code.
Router.map(function () {
this.route('resumeDownload', {
where: 'server',
path: '/resume/:_id',
action: function () {
var response = this.response,
candidate = Candidates.findOne(this.params._id);
if (!candidate || !candidate.resumeS3Key) {
response.writeHead(404);
return response.end();
}
var Future = Npm.require('fibers/future'),
s3 = new AWS.S3(),
futureGetObject = Future.wrap(s3.getObject.bind(s3)),
data = futureGetObject({Bucket: 'myBucket', Key: candidate.resumeS3Key}).wait();
response.writeHead(200, {'Content-Type': data.ContentType});
response.end(data.Body);
}
});
});

Related

Netlify server functions unable to handle multipart/form-data

I’m working on a Netlify Function where we take form data for a job application (including a file upload) and pass the data on to a third-party API for use in their system. I was following along with this handy post (thanks!) —
https://www.netlify.com/blog/2021/07/29/how-to-process-multipart-form-data-with-a-netlify-function/
— but seem to have run into a situation where the data in the file is not handled properly (for example, PDFs turn up with blank content, though ASCII metadata appears to be at least partly intact), at least when using the Netlify CLI; I have yet to try on a deploy preview. Writing to a local directory confirms that the issue isn’t with the third party API. Is there something I’m missing when working with these files? Example code below (note that I’ve also attempted to work with the Buffer data, with identical results).
Fetch function to call the Netlify Function:
const data = new FormData(form);
fetch('/.netlify/functions/apply', {
method: 'POST',
body: data,
}).then(res => {
if (!res.ok && res.status !== 406) {
throw new Error('oh no');
}
return res.json();
}).then(data => {
if (Array.isArray(data.missingRequiredFields) && data.missingRequiredFields.length > 0) {
console.log(data);
showMissingFields(data.missingRequiredFields);
} else {
showConfirmationMessage(data.message);
}
}).catch(err => {
showWarningMessage('Something went wrong; please try again.');
}).finally(() => {
submitButton.removeAttribute('disabled');
});
And here’s our Netlify Function:
const Busboy = require("busboy");
const FormData = require("form-data");
const fetch = require("node-fetch");
function parseMultipartForm(event) {
// source: https://www.netlify.com/blog/2021/07/29/how-to-process-multipart-form-data-with-a-netlify-function/
return new Promise(resolve => {
const fields = {};
const busboy = new Busboy({
// uses request headers to extract the form boundary value
headers: event.headers,
});
// before parsing anything, we need to set up some handlers.
// whenever busboy comes across a file ...
const f = [];
busboy.on("file", (fieldname, filestream, filename, transferEncoding, mimeType) => {
// ... we take a look at the file's data ...
filestream.on("data", data => {
fields[fieldname] = {
filename,
type: mimeType,
content: data,
transferEncoding,
};
});
});
// whenever busboy comes across a normal field ...
busboy.on("field", (fieldName, value) => {
// ... we write its value into `fields`.
fields[fieldName] = value;
});
// once busboy is finished, we resolve the promise with the resulted fields.
busboy.on("finish", () => {
resolve(fields);
});
// now that all handlers are set up, we can finally start processing our request!
busboy.write(event.body);
});
}
/** ***************************************************************************
* Serverless function
**************************************************************************** */
exports.handler = async function(event, context) {
// parse the incoming multipart/form-data data into fields object
const fields = await parseMultipartForm(event);
// create new formdata object to be send to Lever
const form = new FormData();
for (const [key, value] of Object.entries(fields)) {
if (key === "resume") {
// append "resume" with the file buffer and add the file name
form.append("resume", value.content, { filename: value.filename });
} else {
form.append(key, value);
}
}
};
Any guidance you might have would be greatly appreciated.

Managing Gcal Response + express and header issue

I'm new to node and banging my head against a wall on what should be a simple node+express+googlecal+pug issue
node/express route accepts requests and calls controller
controller ensures validation of auth and then...
executes a successful gcal function...console.log has the data i need
trying to directly (in controller function) returns "Cannot set headers after they are sent to the client"....why is a call to Gcal API forcing a response back to client?
Trying to make it more micro via individual calls to each function results in same result
What am I missing here?
getcalendars: async function(oAuth2Client, res) {
const calendar = google.calendar({ version: "v3", auth: oAuth2Client });
cal = await calendar.calendarList.list(
{},
(err, result) => {
//console.log("HEADERS SENT1?: "+res.headersSent);
if (err) {
console.log('The API returned an error: ' + err);
return;
}
console.log(JSON.stringify(result));
message2 = JSON.stringify(result)
res.render('schedules', {message2: message2})
return
});
},
EDIT: Calling function
router.route('/dashboard/schedules')
.get(async function(req, res) {
if (req.session.loggedin) {
//x = gcalController.getcalendars(req, res);
token = await gcalController.gettoken(req, res);
isAuth = await gcalController.calauth(token);
listcalendars = await gcalController.getcalendars(isAuth,res);
} else {
res.redirect("/?error=failedAuthentication")
//res.send('Please login to view this page!');
}
});
Can't set headers already sent happens when you're sending a response more than once. Usually you can terminate the function by returning your res.send() call.
It looks like the express middleware that created the res object is sending a response by the time your res.render() gets pulled out of the microtask queue.
Can you show the full code? It seems that this is probably originating in the scope where getcalendars is called.

Axios interceptors don't send data to API in production Heroku app

This is part 2 of me debugging my application in production
In part 1, I managed to at least see what was causing my problem and managed to solve that.
When I send a request to my API which is hosted on Heroku using axios interceptor, every single request object looks like this in the API
{ 'object Object': '' }
Before sending out data to the API, I console.log() the transformRequest in axios and I can see that the data I am sending is actually there.
Note: I have tested this process simply using
axios.<HTTP_METHOD>('my/path', myData)
// ACTUAL EXAMPLE
await axios.post(
`${process.env.VUE_APP_BASE_URL}/auth/login`,
userToLogin
);
and everything works and I get data back from the server.
While that is great and all, I would like to abstract my request implementation into a separate class like I did below.
Does anyone know why the interceptor is causing this issue? Am I misusing it?
request.ts
import axios from "axios";
import { Message } from "element-ui";
import logger from "#/plugins/logger";
import { UsersModule } from "#/store/modules/users";
const DEBUG = process.env.NODE_ENV === "development";
const service = axios.create({
baseURL: process.env.VUE_APP_BASE_URL,
timeout: 5000,
transformRequest: [function (data) {
console.log('data', data)
return data;
}],
});
service.interceptors.request.use(
config => {
if (DEBUG) {
logger.request({
method: config.method,
url: config.url
});
}
return config;
},
error => {
return Promise.reject(error);
}
);
service.interceptors.response.use(
response => {
console.log('axios interception response', response)
return response.data;
},
error => {
const { response } = error;
console.error('axios interception error', error)
if (DEBUG) {
logger.error(response.data.message, response);
}
Message({
message: `Error: ${response.data.message}`,
type: "error",
duration: 5 * 1000
});
return Promise.reject({ ...error });
}
);
export default service;
Login.vue
/**
* Sign user in
*/
async onClickLogin() {
const userToLogin = {
username: this.loginForm.username,
password: this.loginForm.password
};
try {
const res = await UsersModule.LOGIN_USER(userToLogin);
console.log("res", res);
this.onClickLoginSuccess();
} catch (error) {
throw new Error(error);
}
}
UsersModule (VUEX Store)
#Action({ rawError: true })
async [LOGIN_USER](params: UserSubmitLogin) {
const response: any = await login(params);
console.log('response in VUEX', response)
if (typeof response !== "undefined") {
const { accessToken, username, name, uid } = response;
setToken(accessToken);
this.SET_UID(uid);
this.SET_TOKEN(accessToken);
this.SET_USERNAME(username);
this.SET_NAME(name);
}
}
users api class
export const login = async (data: UserSubmitLogin) => {
return await request({
url: "/auth/login",
method: "post",
data
});
};
I'm not sure what you're trying to do with transformRequest but that probably isn't what you want.
A quote from the documentation, https://github.com/axios/axios#request-config:
The last function in the array must return a string or an instance of Buffer, ArrayBuffer, FormData or Stream
If you just return a normal JavaScript object instead it will be mangled in the way you've observed.
transformRequest is responsible for taking the data value and converting it into something that can actually be sent over the wire. The default implementation does quite a lot of work manipulating the data and setting relevant headers, in particular Content-Type. See:
https://github.com/axios/axios/blob/885ada6d9b87801a57fe1d19f57304c315703079/lib/defaults.js#L31
If you specify your own transformRequest then you are replacing that default, so none of that stuff will happen automatically.
Without knowing what you're trying to do it's difficult to advise further but you should probably use a request interceptor rather than transformRequest for whatever it is you're trying to do.

axios.post is returning error when used with redux-saga

I recently converted my redux-thunk middleware code to use redux-saga and it was working all these days fine and all of a sudden it is throwing an error. Not sure why!!
My Spring Boot REST Client is returning the proper response and no errors in the log. And if i make the same request using swagger i am getting the response back as expected so there is nothing wrong on the server side.
I have the following code
const LOGIN_URL = 'http://localhost:8888/api/a/login';
export function* loginUserAsync(action) {
console.log('.loginUserAsync() : action:', action);
yield put({ type: LoginConstants.LOGIN_USER_IN_PROGRESS });
const postParams = {
username: action.props.username,
password: action.props.password
};
const headerParams = {
headers: {
'Content-Type': 'application/json',
//'Content-Type': 'x-www-form-urlencoded'
}
};
console.log('headerParams', headerParams);
console.log('postParams', postParams);
try {
console.log('Before making async post call using axios');
const response = yield call(axios.post, LOGIN_URL, postParams, headerParams);
let token;
console.log('response', response);
if (response.headers) {
token = response.headers['x-auth-token'];
AsyncStorage.setItem('jwt', token);
}
// Login Succeeded fire Login Success Action
yield put({
type: LoginConstants.LOGIN_USER_SUCCESS,
token,
account: response.data
});
const navigatorUID = Store.getState().navigation.currentNavigatorUID;
Store.dispatch(NavigationActions.push(navigatorUID, Router.getRoute('home')));
} catch (error) {
// Login Failed fire Login Failure Action
console.log('loginUserAync() : error:[' + JSON.stringify(error) + ']');
yield put({
type: LoginConstants.LOGIN_USER_FAILURE,
error: error.data
});
}
}
export function* loginUser() {
console.log('.loginUser() :');
yield takeEvery(LoginConstants.LOGIN_USER, loginUserAsync);
}
In the console i am seeing the following:
I have no idea why it stopped working all of a sudden.
Thanks
Sateesh
For some reason localhost and 127.0.0.1 are not being recognized and i have to use the actual IP Address.
I had that Issue when i tried to run it in my mac book. It always worked with localhost in Ubuntu.

Sails JS with Redis for caching

As I said in my previous questions, I am trying to learn how to use sails.js, what I'm trying to do now is to cache the response of an api to redis. I have searched on how to do this, but I can't make it to work. Without caching, I call the api through ajax.
Any thoughts on how I will be able to do it using my controller? How can I call the api using the controller in sails.js and cache the response using redis?
You can use https://github.com/mranney/node_redis
Steps:
Add to package.json
"redis": "^0.12.1"
Run
npm install
Create a service module /api/services/CachedLookup.js
var redis = require("redis"),
client = redis.createClient();
module.exports = {
rcGet: function (key, cb) {
client.get(key, function (err, value) {
return cb(value);
});
},
fetchApi1: function (cb) {
var key = 'KEY'
CachedLookup.rcGet(key, function (cachedValue) {
if (cachedValue)
return cb(cachedValue)
else {//fetch the api and cache the result
var request = require('request');
request.post({
url: URL,
form: {}
}, function (error, response, body) {
if(error) {
//handle error
}
else {
client.set(key, response);
return cb(response)
}
});
}
});
}
}
Inside the controller
CachedLookup.fetchApi1(function (apiResponse) {
res.view({
apiResponse: apiResponse
});
});