rxjs, call next on generator only after previous operation completes - iterator

I'm uploading a file by creating slices of file using Blob.slice() in a generator function
export function* chunkFile(file: File, chunkSize: number) {
let chunkStart = 0;
const _chunkEnd = chunkStart + chunkSize;
let chunkEnd = _chunkEnd > file.size ? file.size : _chunkEnd;
while (chunkStart < file.size) {
yield <ChunkType>{
chunk: file.slice(chunkStart, chunkEnd),
start: chunkStart,
end: chunkEnd
};
chunkStart = chunkEnd;
const _chunkEndIn = chunkStart + chunkSize;
chunkEnd = _chunkEndIn > file.size ? file.size : _chunkEndIn;
}
}
and I'm uploading file like this
Observable.from(chunckFile(file,chunkSize)).concatMap(uploadRoutine).subscribe();
But all chunks are created at same time.
what I need is create new chunck (call next on generator) only when current chunk upload completes.

Found Solution by my own
export function rxIterable<T, R>(source: Iterator<T>, consumer: (value: T) => Observable<R>) {
const first = source.next();
if (first.done) {
return empty<R>();
}
return consumer(first.value).pipe(
expand(() => {
const next = source.next();
if (next.done) {
return empty<R>();
}
return consumer(next.value);
}),
finalize(() => source.return())
);
}

Related

React-native-fs : How to use readDir recursively using .map()?

I tried to get all the files and directories available in a folder using react-native-fs.
I created a function to get all the files and directories recursively in a folder, I call this function this way :
const data = await scanDir(path);
I first tried using the .map() function but my function return only some elements :
async function scanDir(pathOfDirToScan, data = {directory: [], files: []}) {
const readedFilesAndDir = await FS.readDir(pathOfDirToScan);
Object.keys(readedFilesAndDir).map(async key => {
if (readedFilesAndDir[key].isDirectory()) {
const directoryPath = pathOfDirToScan + '/' + readedFilesAndDir[key].name;
data.directory.push(directoryPath);
data = await scanDir(directoryPath, data);
} else {
data.files.push(pathOfDirToScan + '/' + readedFilesAndDir[key].name);
}
});
return data;
}
It seems my function return the data after the first time map is executed, but the function continue after that.
I then tried with a for loop and it works as intended :
async function scanDir(pathOfDirToScan, data = {directory: [], files: []}) {
const readedFilesAndDir = await FS.readDir(pathOfDirToScan);
for (let i = 0; i < readedFilesAndDir.length; i++) {
if (readedFilesAndDir[i].isDirectory()) {
const directoryPath = pathOfDirToScan + '/' + readedFilesAndDir[i].name;
data.directory.push(directoryPath);
data = await scanDir(directoryPath, data);
} else {
data.files.push(pathOfDirToScan + '/' + readedFilesAndDir[i].name);
}
}
return data;
}
What should I do to make the function properly works using .map() ?
The FS.readDir(dirpath) returns an array of objects as per docs. Object.keys(obj) is not required for iteration in that case, just readedFilesAndDir.map() will do your task.
Copy and pasted your own code with some corrections. Hope, it helps:
async function scanDir(pathOfDirToScan, data = {directory: [], files: []}) {
const readedFilesAndDir = await FS.readDir(pathOfDirToScan);
readedFilesAndDir.map(async eachItem=> {
if (eachItem.isDirectory()) {
const directoryPath = pathOfDirToScan + '/' + eachItem.name;
data.directory.push(directoryPath);
data = await scanDir(directoryPath, data);
} else {
data.files.push(pathOfDirToScan + '/' + eachItem.name);
}
});
return data;
}

How to write unit test component with FileReader.addEventListener in angular 8?

I use angular 8 and i want to test my component with FileReader.
I can not test a FileReader in my processFile function.
Maybe my work is badly written? Can you help me please to understand.
IF I understand correctly, I have to test a class (Filereader) in a process function
my component
processFile(imageInput: any) {
const file: File = imageInput.files[0];
const reader = new FileReader();
let size: number = 2097152
if (file) {
if (file.size <= size) {
this.sharingDataService.setUploadIsOk(true)
reader.addEventListener('progress', (event:any) =>{
this.progressValue = this.progressBar(event)
if (event.lengthComputable) {
// console.log(event.loaded+ " / " + event.total)
}
})
reader.addEventListener('loadstart', (event:any) =>{
this.progressValue =0;
this.textDuringUploadBefore = "No"
this.textDuringUploadAfter = ''
// console.log('start');
})
reader.addEventListener('loadend', (event:any) =>{
// console.log('end');
})
reader.addEventListener('load', (event: any) => {
console.log(event);
this.selectedFile = new ImageSnippet(event.target.result, file);
this.fileName = this.selectedFile.file.name;
this.fileNameExt =this.fileName.split('.').pop();
this.displayAddPhoto = false;
this.selectedFile.status = 'ok';
this.getLoadCallBack(file)
// this.ng2ImgMax.resizeImage(file, 900,600).subscribe(
// result =>{
// // console.log('compress', );
// this.textDuringUploadAfter= "Yes!!!"
// this.textDuringUploadBefore= ''
// this.fileForm.patchValue({
// image: new File([result], result.name)
// });
// this.imageIsLoad = true
// this.sharingDataService.setUploadIsOk(false)
// }
// )
// this.imageOutput.emit(this.fileForm)
});
reader.readAsDataURL(file);
} else {
const msg ="This picture is too big."
+ '<br/>' + "Please upload an image of less than 2MB."
// this.sharedFunctionService.openDialogAlert(msg, 'home')
this.errorService.openDialogError(msg)
this.imageIsLoad = false
this.sharingDataService.setUploadIsOk(false)
}
}
}
getLoadCallBack(file:File){
this.ng2ImgMax.resizeImage(file, 900,600).subscribe(
result =>{
// console.log('compress', );
this.textDuringUploadAfter= "Yes"
this.textDuringUploadBefore= ''
this.fileForm.patchValue({
image: new File([result], result.name)
});
console.log(this.fileForm);
this.imageIsLoad = true
this.sharingDataService.setUploadIsOk(false)
}
)
this.imageOutput.emit(this.fileForm)
}
my spec.ts
it('processFile', () => {
// const mockEvt = { target: { files: [fileInput] } };
// const mockReader: FileReader = jasmine.createSpyObj('FileReader', ['readAsDataURL', 'onload']);
// spyOn(window as any, 'FileReader').and.returnValue(mockReader);
// spyOn(component, 'getLoadCallBack').and.callThrough();
const file = new File([''], 'test-file.jpg', { lastModified: null, type: 'image/jpeg' });
const fileInput = { files: [file] };
const eventListener = jasmine.createSpy();
spyOn(window as any, "FileReader").and.returnValue({
addEventListener: eventListener
})
component.processFile(fileInput);
i have got an error
TypeError: reader.readAsDataURL is not a function
how to test my processFile function?
I trie many way but no sucess

Extendscript Photoshop: Is there a way to save out jpeg with specific KB file sizes for web?

I'm trying to utilize the following. what this does is saves out Jpeg accordingly to less then a a certain size set. I was curious if anyone knew a way to specifically direct a filename to get a filesize that is equal to the following or as close as possible.
Filename_160x600.png = Filename_160x600.jpg 39kb
Filename_300x600.png = Filename_300x600.jpg 59kb
Filename_1500x513.png = Filename_1500x513.jpg 150kb
saveJPG(
{
path: activeDocument.path,
maxSize: 50 //size in kbs
})
function saveJPG(_data)
{
if (_data.path == undefined) return false;
_data.name = _data.name == undefined ? activeDocument.name : _data.name;
_data.quality == undefined && _data.quality = 75;
if (!new Folder(_data.path).exists)
{
alert("Output path doesn't exist!"); //you can add a function to create a path if needed
return false
}
var options = new ExportOptionsSaveForWeb(),
jpgFile = new File(_data.path + '/' + getName(_data.name) + '.jpg');
options.format = SaveDocumentType.JPEG;
options.quality = _data.quality;
activeDocument.exportDocument(jpgFile, ExportType.SAVEFORWEB, options);
if (_data.maxSize != undefined)
{
var ms = _data.maxSize * 1000;
if (jpgFile.length > ms)
{
if (!jpgFile.remove())
{
alert('Save file is locked, please make sure it\'s not opened anywhere');
return
};
saveJPG(
{
path: _data.path,
name: _data.name,
maxSize: _data.maxSize,
quality: _data.quality - 2
});
}
};

Import SQL dump within Node environment

I'd like a npm script to create/configure/etc. and finally import a SQL dump. The entire creation, configuring, etc. is all working, however, I cannot get the import to work. The data never is inserted. Here's what I have (nevermind the nested callback as they'll be turned into promises):
connection.query(`DROP DATABASE IF EXISTS ${config.database};`, err => {
connection.query(`CREATE DATABASE IF NOT EXISTS ${config.database};`, err => {
connection.query('use DATABASENAME', err => {
const sqlDumpPath = path.join(__dirname, 'sql-dump/sql-dump.sql');
connection.query(`SOURCE ${sqlDumpPath}`, err => {
connection.end(err => resolve());
});
})
});
});
I also tried the following with Sequelize (ORM):
return new Promise(resolve => {
const sqlDumpPath = path.join(__dirname, 'sql-dump/sql-dump.sql');
fs.readFile('./sql/dump.sql', 'utf-8', (err, data) => {
sequelize
.query(data)
.then(resolve)
.catch(console.error);
});
});
Here's how I set up my initial Sequelized import using the migrations framework. There is plenty of going on here but in short I:
find the latest sql-dump in the migrations folder
read the file using fs
split the text into queries
check if its a valid query and if so apply some cleaning that my data required (see related post)
push an array full of queries - I start with making sure that the database is clean by calling the this.down first
run everything as a promise (as suggested here) using the mapSeries (not the map)
Using sequelize-cli you can in your shell create a migration by writing:
sequelize migration:create
And you will automatically have the file where you enter the code below. In order to execute the migration you simply write:
sequelize db:migrate
"use strict";
const promise = require("bluebird");
const fs = require("fs");
const path = require("path");
const assert = require("assert");
const db = require("../api/models"); // To be able to run raw queries
const debug = require("debug")("my_new_api");
// I needed this in order to get some encoding issues straight
const Aring = new RegExp(String.fromCharCode(65533) +
"\\" + String.fromCharCode(46) + "{1,3}", "g");
const Auml = new RegExp(String.fromCharCode(65533) +
String.fromCharCode(44) + "{1,3}", "g");
const Ouml = new RegExp(String.fromCharCode(65533) +
String.fromCharCode(45) + "{1,3}", "g");
module.exports = {
up: function (queryInterface, Sequelize) {
// The following section allows me to have multiple sql-files and only use the last dump
var last_sql;
for (let fn of fs.readdirSync(__dirname)){
if (fn.match(/\.sql$/)){
fn = path.join(__dirname, fn);
var stats = fs.statSync(fn);
if (typeof last_sql === "undefined" ||
last_sql.stats.mtime < stats.mtime){
last_sql = {
filename: fn,
stats: stats
};
}
}
}
assert(typeof last_sql !== "undefined", "Could not find any valid sql files in " + __dirname);
// Split file into queries
var queries = fs.readFileSync(last_sql.filename).toString().split(/;\n/);
var actions = [{
query: "Running the down section",
exec: this.down
}]; // Clean database by calling the down first
for (let i in queries){
// Skip empty queries and the character set information in the 40101 section
// as this would most likely require a multi-query set-up
if (queries[i].trim().length == 0 ||
queries[i].match(new RegExp("/\\*!40101 .+ \\*/"))){
continue;
}
// The manual fixing of encoding
let clean_query = queries[i]
.replace(Aring, "Å")
.replace(Ouml, "Ö")
.replace(Auml, "Ä");
actions.push({
query: clean_query.substring(0, 200), // We save a short section of the query only for debugging purposes
exec: () => db.sequelize.query(clean_query)
});
}
// The Series is important as the order isn't retained with just map
return promise.mapSeries(actions, function(item) {
debug(item.query);
return item.exec();
}, { concurrency: 1 });
},
down: function (queryInterface, Sequelize) {
var tables_2_drop = [
"items",
"users",
"usertypes"
];
var actions = [];
for (let tbl of tables_2_drop){
actions.push({
// The created should be created_at
exec: () => db.sequelize.query("DROP TABLE IF EXISTS `" + tbl +"`")
});
}
return promise.map(actions, function(item) {
return item.exec();
}, { concurrency: 1 });/**/
}
};
Based loosely on Max Gordon's answer, here's my code to run a MySQL Dump file from NodeJs/Sequelize:
"use strict";
const fs = require("fs");
const path = require("path");
/**
* Start off with a MySQL Dump file, import that, and then migrate to the latest version.
*
* #param dbName {string} the name of the database
* #param mysqlDumpFile {string} The full path to the file to import as a starting point
*/
module.exports.migrateFromFile = function(dbName, mysqlDumpFile) {
let sequelize = createSequelize(dbName);
console.log("Importing from " + mysqlDumpFile + "...");
let queries = fs.readFileSync(mysqlDumpFile, {encoding: "UTF-8"}).split(";\n");
console.log("Importing dump file...");
// Setup the DB to import data in bulk.
let promise = sequelize.query("set FOREIGN_KEY_CHECKS=0"
).then(() => {
return sequelize.query("set UNIQUE_CHECKS=0");
}).then(() => {
return sequelize.query("set SQL_MODE='NO_AUTO_VALUE_ON_ZERO'");
}).then(() => {
return sequelize.query("set SQL_NOTES=0");
});
console.time("Importing mysql dump");
for (let query of queries) {
query = query.trim();
if (query.length !== 0 && !query.match(/\/\*/)) {
promise = promise.then(() => {
console.log("Executing: " + query.substring(0, 100));
return sequelize.query(query, {raw: true});
})
}
}
return promise.then(() => {
console.timeEnd("Importing mysql dump");
console.log("Migrating the rest of the way...");
console.time("Migrating after importing mysql dump");
return exports.migrateUp(dbName); // Run the rest of your migrations
}).then(() => {
console.timeEnd("Migrating after importing mysql dump");
});
};

How to yield in map?

I am using react native to build my application and I am trying to convert my variables to camelCase using the code below
export default function formatToCamelCase(inputObject: {[key: string]: any}) {
let snakeKeys = Object.keys(inputObject);
let newObject = {};
for (let key of snakeKeys) {
if (typeof inputObject[key] !== 'object') {
newObject[parseToCamelCase(key)] = inputObject[key];
} else if (inputObject[key] !== null && !Array.isArray(inputObject[key])) {
newObject[parseToCamelCase(key)] = formatToCamelCase(inputObject[key]);
} else {
newObject[parseToCamelCase(key)] = inputObject[key];
}
}
return newObject;
}
export function formatArrayToCamelCase(inputArray: Array<{[key: string]: any}>) {
return inputArray.map((object) => {
return formatToCamelCase(object);
});
}
And I am trying to call the formatArrayToCamelCase method in the function here (in a separate file):
import formatToCamelCase, {formatArrayToCamelCase} from '../helpers/formatToCamelCase';
export function* fetchWeighInWeighOutEvent(action: FetchWeighInWeighOutEventAction): any {
...
Object.values(locations).map((location) => {
let timeslotsArray = location.timeslots;
if (timeslotsArray.length > 0) {
//Problem here - shows that yield is a reserved word
let camelCaseTimeslots = yield call(formatArrayToCamelCase, timeslotsArray);
}
});
I was unable to put the yield call in the .map, when trying to run the code, I get the following error:
SyntaxError: yield is a reserve word
How can I overcome this issue so that I can call the formatArrayToCamelCase function and convert my array accordingly?