Lambda Edge Origin Request 502 - amazon-s3

I am getting in the invocation of lambda edge in origin request for Cloudfront, this error, I am trying to change the metatags of a Single Page Application done to React:
Validation error: The Lambda function returned an invalid entry in the headers object, the header must have a value field, header: cache-control has one or more entries which doesn't have a value field.
And this is my lambda edge function
const path = require('path');
const https = require('https');
const zlib = require('zlib');
let originalUri;
const downloadContent = (url, callback) => {
https.get(url, (res) => {
let response;
let body = '';
if (res.headers['content-encoding'] === 'gzip') {
response = res.pipe(zlib.createGunzip());
} else {
response = res;
}
response.on('data', (chunk) => {
body += chunk;
});
response.on('end', () => {
callback(true, body, res.headers);
});
}).on('error', (e) => callback(false, e));
};
const fetchMetaData = (url, callback) => {
downloadContent(url, (isOk, result, headers) => {
if (!isOk) {
console.log('Error fetching meta data:', result);
callback(false);
} else {
const metaData = JSON.parse(result);
let metaTags = '';
console.log('metaData whate:', metaData);
if (metaData) {
if (metaData.title) {
metaTags += '<title>' + metaData.title + '</title>';
metaTags += '<meta property=\"og:title\" content=\"' + metaData.title + '\" />';
}
if (metaData.description) {
metaTags += '<meta name=\"description\" content=\"' + metaData.description + '\" />';
metaTags += '<meta property=\"og:description\" content=\"' + metaData.description + '\" />';
}
if (metaData.images) {
for (let i = 0; i < metaData.images.length; i++) {
const image = metaData.images[i];
metaTags += '<meta property=\"og:image\" content=\"' + image + '\" />';
}
}
}
metaTags += '<meta property=\"og:url\" content=\"' + originalUri + '" />';
metaTags += '<meta property=\"og:type\" content=\"website\" />';
callback(true, metaTags, headers);
}
});
};
const fetchIndexHtmlAndCreateCloudFrontResponse = (url, metaTags, metaHeaders, callback) => {
downloadContent(url, (isOk, result, headers) => {
if (!isOk) {
callback(false);
} else {
var finalBody = result.replace(/(<title>title<\/title>)/gi, metaTags);
const responseHeaders = {
'content-type': [{key:'Content-Type', value: 'text/html'}],
'content-encoding' : [{key:'Content-Encoding', value: 'gzip'}],
'accept-ranges': [{key:'Accept-Ranges', value: 'bytes'}]
};
let eTag = '';
if (metaHeaders) {
const metaEtag = metaHeaders['etag'];
if (metaEtag) {
eTag = metaEtag.replace(/"/g, '');
}
}
if (headers) {
const lastModified = headers['last-modified'];
const cacheControl = headers['cache-control'];
const contentETag = headers['etag'];
if (lastModified) {
responseHeaders['last-modified'] = [{key:'Last-Modified', value: lastModified}]
}
if (lastModified) {
responseHeaders['cache-control'] = [{key:'Cache-Control', value: cacheControl}]
}
if (contentETag) {
eTag += contentETag.replace(/"/g, '');;
}
}
if (eTag !== '') {
responseHeaders['etag'] = [{key:'ETag', value: eTag}]
}
const newResponse = {
status: '200',
statusDescription: 'OK',
headers: responseHeaders,
body: finalBody,
};
callback(true, newResponse);
}
});
};
exports.handler = (event, context, callback) => {
const { request, response, config } = event.Records[0].cf;
originalUri = request.uri;
const parsedPath = path.parse(originalUri);
if (parsedPath.ext === '' && parsedPath.dir === '/collections') {
console.log('parsedPath:', parsedPath.base);
request.uri = '/index.html';
let metaUrl = 'https://a2mfja.execute-api.eu-west-1.amazonaws.com/testenv/metatags';
fetchMetaData(metaUrl, (isOk, metaTags, metaHeaders) => {
if (!isOk) {
return callback(null, request); // Return same request so CloudFront can process as usual.
} else {
const contentUrl = 'https://d3lyhnumbmrole.cloudfront.net/index.html';
fetchIndexHtmlAndCreateCloudFrontResponse(contentUrl, metaTags, metaHeaders, (isOk, newResponse) => {
if (!isOk) {
return callback(null, request);
} else {
//newResponse.headers = request.headers;
return callback(null, newResponse);
}
});
}
});
} else {
return callback(null, request);
}
};

The Error was here, cacheControl was undefined in the headers:
if (lastModified) {
responseHeaders['cache-control'] = [{key:'Cache-Control', value: cacheControl}]
}

Related

vue method for loop wait for function complete

In this vue component, I have a method containing a for loop, calling another method. The second method does a request to the appserver. I need the first function waiting for the second to continue the for-loop. I've tried several async await options but doesn't understand how to implement it.
methods: {
selectFiles(files) {
this.progressInfos = [];
this.selectedFiles = files;
},
uploadFiles() {
this.message = "";
//var result = 0;
for (let i = 0; i < this.selectedFiles.length; i++) {
console.log(i)
//result = await this.upload(i, this.selectedFiles[i]);
this.upload(i, this.selectedFiles[i]);
}
},
upload(idx, file) {
this.progressInfos[idx] = { percentage: 0, fileName: file.name };
//console.log("FinDocuNum:" + financialDocument.finDocId)
FinancialDocumentDataService.upload(1, file, (event) => {
this.progressInfos[idx].percentage = Math.round(100 * event.loaded / event.total);
}).then((response) => {
let prevMessage = this.message ? this.message + "\n" : "";
this.message = prevMessage + response.status;
return 1;
}).catch(() => {
this.progressInfos[idx].percentage = 0;
this.message = "Could not upload the file:" + file.name;
return 0;
});
}
}
The upload function must be async and return a promise like this:
async upload(file) {
return new Promise((resolve, reject) => {
axios({url: url, data: file, method: 'POST'})
.then(resp => {
resolve(resp)
})
.catch(err => {
reject(err)
})
})
},

Unknown field found while parsing JSONEachRow format Click house

I'm getting this error
I'm inserting the data into the Clickhouse and i get this error because some of fields are null.
Unknown error field: Code: 117. DB::Exception: Unknown field found while parsing JSONEachRow format: SharesStats: (at row 1): While executing JSONEachRowRowInputFormat. (INCORRECT_DATA) (version 21.11.6.7 (official build))
const Clickhouse = require('#apla/clickhouse');
const { default: axios } = require('axios');
const stockCode = require('./stockCode');
const clickhouse = new Clickhouse({
host: 'localhost',
port: 8123,
username: 'default',
password: '',
dataObjects: true,
queryOptions: {
database: 'stocks'
}
});
const indexStream = clickhouse.query(
'INSERT INTO test2',
{
format: 'JSONEachRow'
},
(err) => {
if (err) {
return console.error(err);
}
}
);
const fetchDataFundamentals = async (stock) => {
try {
const { data: response } = await axios.get(
`https://eodhistoricaldata.com/api/fundamentals/${stock}?api_token=${apiKey}&fmt=json`
);
const nestedFields = [
'SharesStats',
'Technicals',
'SplitsDividends',
'outstandingShares',
'Earnings',
'Financials'
];
const data = {
...response.General,
...response.Highlights,
...response.Valuation,
...response.Holders,
...response.InsiderTransactions,
SharesStats: response.SharesStats,
Technicals: response.Technicals,
SplitsDividends: response.SplitsDividends,
outstandingShares: response.outstandingShares,
Earnings: response.Earnings,
Financials: response.Financials
};
Object.keys(data).forEach((key) => {
if (nestedFields.includes(key)) {
data[key] = Object.keys(data[key]).map((nestedKey) => {
if (!data[key][nestedKey]) {
data[key][nestedKey] = 0;
} else if (typeof data[key][nestedKey] === 'object') {
data[key][nestedKey] = JSON.stringify(data[key][nestedKey]);
}
return data[key][nestedKey];
});
} else if (!data[key]) {
delete data[key];
} else if (typeof data[key] === 'object') {
data[key] = JSON.stringify(data[key]);
}
});
console.log(data);
indexStream.write(data);
indexStream.end();
} catch (error) {
console.error(error.message);
}
};
// Promise.all(stockCode.map((code) => fetchDataFundamentals(code)))
// .then(() => {
// indexStream.end();
// console.log('Fundametals data inserted');
// })
// .catch((err) => {
// console.log(err.message);
// });
try {
fetchDataFundamentals('1010.SR');
} catch (error) {
console.error(error.message);
}

Hapi Js File Upload Struggling

today i trying to get some file upload with Hapi Js, i follow all Google Result with similarity of code.
this the code :
server.route({
method: "POST",
path: `${PUBLIC_URL}${THEME_URL}/create`,
handler: async (request: any, reply: ResponseToolkit) => {
console.log(request.payload.file, 'payload')
return reply.response(request.payload)
},
options: {
payload: {
output: 'stream',
allow: 'multipart/form-data',
parse: false,
}
}
})
with thats code i cant get request.payload my file or data, this is my request with postman:
post file with postman
enter image description here
i got undifined at request.payload.file
if i turn payload :{parse:true} i get unsuported media types
thanks for attention
If you are using the below version then you must be using the following syntax
#hapi/hapi: 18.x.x +
payload: {
parse: true,
multipart: {
output: 'stream'
},
maxBytes: 1000 * 1000 * 5, // 5 Mb
}
Also, you can also try using Joi to validate your payload.
{
method: 'POST',
path: '/upload',
options: {
payload: {
maxBytes: 209715200,
output: 'stream',
parse: true,
allow: 'multipart/form-data',
multipart: true // <-- this fixed the media type error
},
handler: async (req, reply) => {
try {
// await importData(req.payload)
// return reply.response("IMPORT SUCCESSFULLY")
const data = await req.payload;
// let final = await importFile(data)
// return reply.response("final", final)
if (data.file) {
let name = await data.file.hapi.filename;
console.log("FIlename: " + name);
let path = await __dirname + "/uploads/" + name;
let file = await fs.createWriteStream(path);
await data.file.pipe(file);
await data.file.on('end', async function (err) {
// var ret = {
// filename: data.file.hapi.filename,
// headers: data.file.hapi.headers
// }
if (typeof require !== 'undefined')
XLSX = require('xlsx');
const workbook = await XLSX.readFile(path);
var sheetName = workbook.SheetNames;
console.log("row======>>>>");
await sheetName.forEach(async () => {
let xlData = await XLSX.utils.sheet_to_json(workbook.Sheets[sheetName[0]]);
console.log("xlData", xlData);
for (let i = 0; i < xlData.length; i++) {
console.log("if condition", xlData[i].phone)
const userCheck = await getUserIdService({ where: { phone: xlData[i].phone } });
console.log("userCheck", userCheck.data)
console.log("test", !(userCheck.data === null));
if (!(userCheck.data === null)) {
console.log("finally ", userCheck.data?.phone)
await uploadUpdateService(xlData[i], { where: { phone: userCheck.data?.phone } });
// return finalUpdate
// return reply.response("updated")
}
else if (!xlData[i].customerID) {
await uploadCreate(xlData[i]);
// return finalCreate
}
}
})
})
}
} catch (err) {
console.log('Err----------------------' + err);
// error handling
return reply.response(Boom.badRequest(err.message, err))
// return reply.response(Boom.badRequest(err.message, err));
}
}
}
}

Add and Retrieve Audio

I have added and retrived image in MongoDB using Node. Can I use the same code with some adjustment? Suggest me.
upload.ts
var multer = require("multer");
export let UPLOAD_PATH = "uploads";
const storage = multer.diskStorage({
destination: function(req, file, cb) {
req;
file;
cb(null, UPLOAD_PATH);
},
filename: function(req, file, cb) {
req;
cb(null, file.fieldname + "-" + Date.now() + ".jpg");
}
});
export const upload = multer({ storage: storage }).single("avatar");
image.controller.ts
Upload
this._model.findOne(
{ ["user"]: new mongoose.Types.ObjectId(user._id) },
img => {
upload(req, res, err => {
if (err) {
res.status(500).json(null);
} else {
// Create a new image model and fill the properties
let newImage = new Image();
newImage.filename = req.file.filename;
newImage.originalName = req.file.originalname;
newImage.desc = req.body.desc;
newImage.url =
req.protocol + "://" + req.get("host") + "/images/" + newImage._id;
newImage.user = user._id;
newImage.save(err => {
if (err) {
res.status(400).json(null);
} else {
res.status(201).json(img);
}
});
}
});
}
);
Retrive
getImage = (req, res) => {
const user = this.getUser(req, res);
this._model.findOne({ ['user']: new mongoose.Types.ObjectId(user._id) }, (err, image) => {
if (err) {
res.status(500).json(null);
}
else if (image == null) {
res.status(200).json(image);
} else {
// stream the image back by loading the file
res.setHeader('Content-Type', 'image/jpeg');
fs.createReadStream(path.join(UPLOAD_PATH, image.filename)).pipe(res);
}
})
};
Is it is possible to use same code with some modification to add and retrieve audio files using Node, Express in Mongo?

AWS S3 Signature Does Not Match

I am stuck on s3.getSignedUrl and getting the error 'Signature Does Not Match, Signature calculated does not match...'. I have checked the credentials, and so many other things, but am not making any progress. I have my access key and secret saved in a credentials file.
The URL I'm getting back is https://postcard-photo-repo-dev.s3.amazonaws.com/Screenshot7.png?AWSAccessKeyId=AKIAJWHTSREEZUZGGO3A&Expires=1534619652&Signature=MkyVjARuo3PaO6lAYV6Li%2FAaR9E%3D
upload.js file:
function getSignedRequest(file) {
const xhr = new XMLHttpRequest();
xhr.open('GET', `/api/feed/sign-s3?file-name=${file.name}&file-type=${file.type}`);
xhr.onreadystatechange = () => {
if(xhr.readyState === 4){
if(xhr.status === 200){
const response = JSON.parse(xhr.responseText);
uploadFile(file, response.signedRequest, response.url);
} else {
alert('Could not get signed URL');
}
}
};
xhr.send();
}
function uploadFile(file, signedRequest, url){
const xhr = new XMLHttpRequest();
xhr.open('PUT', signedRequest);
xhr.onreadystatechange = () => {
if(xhr.readyState === 4) {
if(xhr.status === 200) {
$('#preview').src = url;
$('#avatar-url').value = url;
} else {
alert('Could not upload file');
}
}
};
xhr.send(file);
}
routes file:
router.get('/sign-s3', (req, res) => {
const s3 = new aws.S3();
const fileName = req.query['file-name'];
const fileType = req.query['file-type'];
const s3Params = {
Bucket: S3_BUCKET,
Key: fileName,
Expires: 600,
// ACL: 'public-read',
// ContentType: fileType
};
s3.getSignedUrl('putObject', s3Params, (err, data) => {
if(err) {
console.log(err);
return res.end();
}
console.log(data);
const returnData = {
signedRequest: data,
url: `https://${S3_BUCKET}.s3.amazonaws.com/${fileName}`
};
res.write(JSON.stringify(returnData));
res.end();
});
});