In this vue component, I have a method containing a for loop, calling another method. The second method does a request to the appserver. I need the first function waiting for the second to continue the for-loop. I've tried several async await options but doesn't understand how to implement it.
methods: {
selectFiles(files) {
this.progressInfos = [];
this.selectedFiles = files;
},
uploadFiles() {
this.message = "";
//var result = 0;
for (let i = 0; i < this.selectedFiles.length; i++) {
console.log(i)
//result = await this.upload(i, this.selectedFiles[i]);
this.upload(i, this.selectedFiles[i]);
}
},
upload(idx, file) {
this.progressInfos[idx] = { percentage: 0, fileName: file.name };
//console.log("FinDocuNum:" + financialDocument.finDocId)
FinancialDocumentDataService.upload(1, file, (event) => {
this.progressInfos[idx].percentage = Math.round(100 * event.loaded / event.total);
}).then((response) => {
let prevMessage = this.message ? this.message + "\n" : "";
this.message = prevMessage + response.status;
return 1;
}).catch(() => {
this.progressInfos[idx].percentage = 0;
this.message = "Could not upload the file:" + file.name;
return 0;
});
}
}
The upload function must be async and return a promise like this:
async upload(file) {
return new Promise((resolve, reject) => {
axios({url: url, data: file, method: 'POST'})
.then(resp => {
resolve(resp)
})
.catch(err => {
reject(err)
})
})
},
Related
When a csv file is uploaded on my s3 bucket, my lambda will be triggered to insert my data into DynamoDB.
I need a stream because the file is too large to be downloaded as full object.
const batchWrite = async (clientDynamoDB, itemsToProcess) => {
const ri = {};
ri[TABLE_DYNAMO] = itemsToProcess.map((itm) => toPutRequest(itm));
const params = { RequestItems: ri };
await clientDynamoDB.batchWriteItem(params).promise();
};
function runStreamPromiseAsync(stream, clientDynamoDB) {
return new Promise((resolve, reject) => {
const sizeChunk = 25;
let itemsToProcess = [];
stream
.pipe(fastCsv.parse({headers: Object.keys(schemaGeData), trim: true}))
.on("data", (row) => {
stream.pause();
itemsToProcess.push(row);
if (itemsToProcess.length === sizeChunk) {
batchWrite(clientDynamoDB, itemsToProcess).finally(() => {
stream.resume();
});
itemsToProcess = [];
}
})
.on("error", (err) => {
console.log(err);
reject("Error");
})
.on("end", () => {
stream.pause();
console.log("end");
batchWrite(clientDynamoDB, itemsToProcess).finally(() => {
resolve("OK");
});
});
});
}
module.exports.main = async (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const object = event.Records[0].s3;
const bucket = object.bucket.name;
const file = object.object.key;
const agent = new https.Agent({
keepAlive: true
});
const client = new AWS.DynamoDB({
httpOptions: {
agent
}
});
try {
//get Stream csv data
const stream = s3
.getObject({
Bucket: bucket,
Key: file
})
.createReadStream()
.on('error', (e) => {
console.log(e);
});
await runStreamPromiseAsync(stream, client);
} catch (e) {
console.log(e);
}
};
When my file is 1000 lines everything is inserted but when I have 5000 lines, my function insert only around 3000 lines and this number is random... Sometimes more sometimes less..
So I'd like to understand what am I missing here ?
I also read this article but to be honest even if you pause the second stream, the first one is still running.. So if someone have any ideas on how to do this, it would be greatly appreciated !
Thanks
I found out why It was not fully processed, it's because the callback of batchWriteItem can return unprocess Items. So I change the function batchWrite and also the runPromiseStreamAsync a little bit because i might not have all the items processed from itemsToProcess.
Anyway here is the full code :
const batchWrite = (client, itemsToProcess) => {
const ri = {};
ri[TABLE_DYNAMO] = itemsToProcess.map((itm) => toPutRequest(itm));
const items = { RequestItems: ri };
const processItemsCallback = function(err, data) {
return new Promise((resolve, reject) => {
if(!data || data.length === 0){
return resolve();
}
if(err){
return reject(err);
}
let params = {};
params.RequestItems = data.UnprocessedItems;
return client.batchWriteItem(params, processItemsCallback);
});
};
return client.batchWriteItem(items, processItemsCallback );
};
function runStreamPromiseAsync(stream, clientDynamoDB) {
return new Promise((resolve, reject) => {
const sizeChunk = 25;
let itemsToProcess = [];
let arrayPromise = [];
stream
.pipe(fastCsv.parse({headers: Object.keys(schemaGeData), trim: true}))
.on("error", (err) => {
console.log(err);
reject("Error");
})
.on('data', data => {
itemsToProcess.push(data);
if(itemsToProcess.length === sizeChunk){
arrayPromise.push(batchWrite(clientDynamoDB, itemsToProcess));
itemsToProcess = [];
}
})
.on('end', () => {
if(itemsToProcess.length !== 0){
arrayPromise.push(batchWrite(clientDynamoDB, itemsToProcess));
}
resolve(Promise.all(arrayPromise).catch(e => {
reject(e)
}));
});
});
}
I'm building a webapp in Nuxt.js and it's growing quite a bit.
I have a page which does two things: one when i'm creating a task and one when managing that task.
This page has a lot of methods, divided for when i create the task and when i manage the task.
How can i split these modules in two files and then import then only when I need them?
These methods need also to access the component's state and other function Nuxt imports such as axios.
async create() {
if (this.isSondaggioReady()) {
try {
await this.getImagesPath()
const o = { ...this.sondaggio }
delete o.id
o.questions = o.questions.map((question) => {
delete question.garbageCollector
if (question.type !== 'checkbox' && question.type !== 'radio') {
delete question.answers
delete question.hasAltro
} else {
question.answers = question.answers.map((answer) => {
delete answer._id
delete answer.file
delete answer.error
if (answer.type !== 'image') delete answer.caption
return answer
})
}
if (question.hasAltro) {
question.answers.push({
type: 'altro',
value: ''
})
}
return question
})
console.log('TO SEND', JSON.stringify(o, null, 2))
this.$store.commit('temp/showBottomLoader', {
show: true,
message: 'Crezione del sondaggio in corso'
})
const { data } = await this.$axios.post('/sondaggi/admin/create', o)
this.sondaggio.id = data
const s = {
_id: data,
author: this.$auth.user.email.slice(0, -13),
title: this.sondaggio.title
}
this.$store.commit('temp/pushHome', { key: 'sondaggi', attr: 'data', data: [...this.$store.state.temp.home.sondaggi.data, s] })
this.$store.dispatch('temp/showToast', 'Sondaggio creato correttamente')
this.$router.replace('/')
} catch (e) {
console.log(e)
this.$store.dispatch('temp/showToast', this.$getErrorMessage(e))
} finally {
this.$store.commit('temp/showBottomLoader', {
show: false,
message: 'Crezione del sondaggio in corso'
})
}
}
},
Here there's an example of what a method does. It calls an async functions which relies on HTTP axios requests:
async getImagesPath() {
this.sondaggio.questions.forEach((question, i) => {
question.answers.forEach((answer, j) => {
if (answer.file instanceof File || answer.value.includes('data:image')) {
this.uploadingImages.push({
coords: [i, j],
percentage: 0,
file: answer.file || answer.value
})
}
})
})
const requests = []
this.uploadingImages.forEach((img) => {
const temp = new FormData()
temp.append('img', img.file)
const req = this.$axios.post('/sondaggi/admin/images/add/' + this.sondaggio.title.replace(/\s+/g, ''), temp, {
onUploadProgress: function (progressEvent) {
img.percentage = Math.round(((progressEvent.loaded * 100) / progressEvent.total) * 90 / 100)
},
onDownloadProgress: function (progressEvent) {
img.percentage = 90 + Math.round(((progressEvent.loaded * 100) / progressEvent.total) * 10 / 100)
},
headers: { 'Content-Type': 'multipart/form-data' }
})
.catch((err) => {
console.log(err)
img.percentage = 150
})
requests.push(req)
})
try {
const response = await Promise.all(requests)
response.forEach(({ data }, i) => {
this.sondaggio.questions[this.uploadingImages[i].coords[0]].answers[this.uploadingImages[i].coords[1]].value = data[0]
})
this.$set(this.sondaggio, 'hasImages', this.uploadingImages.length > 0)
this.uploadingImages = []
await Promise.resolve()
} catch (e) {
console.log('handling gloval err', e)
await Promise.reject(e)
}
},
As you can see axios requests modify the component's state
I am developing a mobile app, which uses React-Native, Redux and React-Native-Contacts.
In an action creator, I am trying to call a function (from React-Native-Contacts), which has callbacks (rather than promises). Following is my code.
I want the action "acContactImport" to wait for the helper function "fContactImport" to finish before proceeding to the "dispatch" statement. It does not wait for it. How can I make it wait for it?
// Action Creator:
import Contacts from 'react-native-contacts';
import { PermissionsAndroid } from 'react-native';
export const acContactImport = (userID) => {
return async (dispatch) => {
let contactList = [];
try {
// The following statement should be executed asynchronously.
// However, it is not. How can I make it asynchronous?
contactList = await fContactImport();
dispatch({
type: CONTACT_IMPORT,
payload: { contactList: contactList },
});
}
catch (err) {
console.log("acContactImport - catch: ", err);
}
};
};
// Helper Function 1
export const fContactImport = async () => {
let contactList = [];
if (Platform.OS === "android") {
PermissionsAndroid.request(PermissionsAndroid.PERMISSIONS.READ_CONTACTS, {
title: "Contacts",
message: "This app would like to view your contacts."
})
.then(() => {
contactList = _loadContacts();
});
} else {
contactList = _loadContacts();
}
}
// Helper Function 2
export const _loadContacts = () => {
Contacts.getAll((err, data2) => {
if (err) {
return [];
}
else {
let candidates = [];
for (let i = 0; i < data2.length; i++) {
let candidateObject = { name: candidateRecord.givenName + " " + candidateRecord.middleName + " " + candidateRecord.familyName };
candidates.push(candidateObject);
}
return contactList;
}
});
}
Apparently, the problem is NOT in the action creator, but in the helper functions, which do not support the new style of promises (async/wait). So, here is the solution...
// Action Creator:
// No changes are required. Original code is good.
// Helper Function 1
export const fContactImport = async () => {
let contactList = [];
if (Platform.OS === "android") {
try {
const granted = await PermissionsAndroid.request(
PermissionsAndroid.PERMISSIONS.READ_CONTACTS,
{
title: "Contacts",
message: "This app would like to view your contacts."
},
);
if (granted === PermissionsAndroid.RESULTS.GRANTED) {
contactList = await _loadContacts();
} else {
console.log('Contacts access denied');
}
} catch (err) {
console.warn(err);
}
} else {
contactList = await _loadContacts();
}
return contactList;
}
// Helper Function 2
export const _loadContacts = () => {
return new Promise((resolve, reject) => {
Contacts.getAll((err, data2) => {
if (err) {
reject([]);
}
else {
let candidates = [];
for (let i = 0; i < data2.length; i++) {
let candidateObject = { name: candidateRecord.givenName + " " + candidateRecord.middleName + " " + candidateRecord.familyName };
candidates.push(candidateObject);
}
resolve(candidates);
}
});
});
}
I tried to stress test my api in ExpressJS and to handler multi request I used Promise.all and then Async.queue with concurrency option.
Promise:
export const myapi = async (args1, args2) => {
console.log('args:', args1, args2);
let testing_queue = [];
testing_queue.push(new Promise(async (resolve, reject) => {
let result = await doAComplexQuery(args1, args2); // SELECT... JOIN...
if (!result || result.length <= 0)
reject(new Error('Cannot find anything!'));
resolve(result);
}
));
return await Bluebird.map(testing_queue, async item => {
return item;
}, {concurrency: 4}); };
Async.queue: (https://www.npmjs.com/package/async)
export const myapi = async (args1, args2) => {
console.log('args:', args1, args2);
let testing_queue = Async.queue(function (task, callback) {
console.log('task', task);
callback();
}, 4);
testing_queue.push(async function () {
let result = await doAComplexQuery(args1, args2); // SELECT... JOIN...
if (!result || result.length <= 0)
throw new Error('Cannot find anything!');
return result;
}
);};
And try to make request as much as possible:
const response = async function () {
return await Axios.post('http://localhost:3000/my-api', {
"args1": "0a0759eb",
"args2": "b9142db8"
}, {}
).then(result => {
return result.data;
}).catch(error => {
console.log(error.message);
});
};
for (var i = 0; i < 10000; i++) {
response();
}
And Run. The #1 way returns many ResourceTimeout or Socket hang up responses. Meanwhile, the #2 returns success response for all requests and runs even faster.
So is the Async.queue better in this case?
I think it could help the speed if you raise the concurrency limit on your promise.map.
I'm doing a network scanner in react-native,
The problem is my api request are working but not in the for loop,
if I just made a for loop with 30loop its ok, but if I got more than 30loops
nothing is working.
if the IP to find is 192.168.1.79
and I set num_two to 60 and num_one to 1, this work good.
But if I set num_two to 50 and num_one to 1, this isn't working anymore.
My searching function:
searchInNetwork(net) {
return new Promise(async (resolve, reject) => {
for (let num_one = 1 ; num_one < 255 ; num_one++) {
let reseau = net + String(num_one);
for (let num_two = 50 ; num_two < 255 ; num_two++) {
let url = 'http://' + reseau + '.' + String(num_two) + ':3000/connect';
await ApiService.getRequest(url, 100)
.then((data) => {
console.log("SUCCESS");
ApiService.url = 'http://' + reseau + '.' + String(num_two) + ':3000';
ApiService.connected = true;
resolve();
})
` .catch((error) => {
if (url === "http://192.168.1.79:3000/connect")
console.log(error);
})
}
}
});
}
My ApiService class:
class ApiService {
static url = 'http://0.0.0.0:3000';
static connected = false;
static getUrl() {
return (this.url);
}
static getRequest(route, timeout) {
return new Promise(async (resolve, reject) => {
fetch(route)
.then((response) => response.json())
.then((responseJson) => {
ApiService.connected = true;
resolve(responseJson);
})
.catch((error) => {
ApiService.connected = false;
reject(error);
});
if (timeout) {
const error = new Error("Timeout net");
setTimeout(reject, timeout, error);
}
});
};
};
module.exports = ApiService;