In my worker I am converting a base64 string I get from the request to a blob with some function. However, when I try to PUT the blob into my bucket, I get "Network Connection Lost" error. I can successfully PUT just the base64 string or any other string but not a blob. Here is my worker:
// Function to convert b64 to blob (working fine)
function b64toBlob(b64Data, contentType, sliceSize=512) {
const byteCharacters = atob(b64Data);
const byteArrays = [];
for (let offset = 0; offset < byteCharacters.length; offset += sliceSize) {
const slice = byteCharacters.slice(offset, offset + sliceSize);
const byteNumbers = new Array(slice.length);
for (let i = 0; i < slice.length; i++) {
byteNumbers[i] = slice.charCodeAt(i);
}
const byteArray = new Uint8Array(byteNumbers);
byteArrays.push(byteArray);
}
const blob = new Blob(byteArrays, {type: contentType});
return blob;
}
export default {
async fetch(request, env) {
const url = new URL(request.url);
const key = url.pathname.slice(1);
switch (request.method) {
case 'PUT':
const contentType = 'application/pdf';
const b64Data = request.body;
const blob = b64toBlob(b64Data, contentType);
try {
await env.qa_sub_agreements_bucket.put(key, blob, { // Failing here
httpMetadata: request.headers,
})
return new Response(blob) // Successfully returns the blob when above PUT is commented out
} catch (e) {
console.error(e.message, e.stack); // Logs out "Error: Network Connection Lost"
}
Hard to say definitively because the Worker posted doesn't appear to be totally complete. An eagle-eyed coworker spotted that it looks like the problem may be that you're invoking atob on a ReadableStream and likely that conversion is what's throwing the exception.
Related
Im trying to use exactInput() function for UniV3 interface but when trying to execute the code the transactions fails https://goerli.etherscan.io/tx/0xb0d5e4b491610b9db8d98cc938008ba2a4e1a06e67b05ed87ac6c0ca3ad61dab
I know eth send shows 0 in this one but even especifying amount it fails, I dont know what to change..
I have checked many codes out there and cant see the mistake, please could someone give me some advice?
const {abi: V3SwapRouterABI} = require('#uniswap/v3-periphery/artifacts/contracts/interfaces/ISwapRouter.sol/ISwapRouter.json')
const { ethers } = require("ethers")
require("dotenv").config()
const INFURA_URL_TESTNET = process.env.INFURA_URL_TESTNET
const PRIVATE_KEY = process.env.PRIVATE_KEY
const WALLET_ADDRESS = process.env.WALLET_ADDRESS
// now you can call sendTransaction
const wethToken= "0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6"
const Uni= "0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984"
const UniswapRouter="0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45"
const UniV3Contract = new ethers.Contract(
UniswapRouter,
V3SwapRouterABI
)
const provider = new ethers.providers.JsonRpcProvider(INFURA_URL_TESTNET)
const wallet = new ethers.Wallet(PRIVATE_KEY)
const signer = wallet.connect(provider)
const FEE_SIZE = 3
function encodePath(path, fees) {
if (path.length != fees.length + 1) {
throw new Error('path/fee lengths do not match')
}
let encoded = '0x'
for (let i = 0; i < fees.length; i++) {
// 20 byte encoding of the address
encoded += path[i].slice(2)
// 3 byte encoding of the fee
encoded += fees[i].toString(16).padStart(2 * FEE_SIZE, '0')
}
// encode the final token
encoded += path[path.length - 1].slice(2)
return encoded.toLowerCase()
}
async function getToken() {
const path = encodePath([wethToken, Uni], [3000])
const deadline = Math.floor(Date.now()/1000) + (60*10)
const params = {
path: path,
recipient: WALLET_ADDRESS,
deadline: deadline,
amountIn: ethers.utils.parseEther('0.01'),
amountOutMinimum: 0
}
const encodedData = UniV3Contract.interface.encodeFunctionData("exactInput", [params])
const txArg = {
to: UniswapRouter,
from: WALLET_ADDRESS,
data: encodedData,
gasLimit: ethers.utils.hexlify(1000000)
}
const tx = await signer.sendTransaction(txArg)
console.log('tx: ', tx)
const receipt = tx.wait()
console.log('receipt: ', receipt)
}
module.exports = { getToken
You will need to remove the Deadline.. The new router 0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45 moved deadline to the multi-call function (since the router is designed to be multi-call)
I am processing an audio buffer with an OfflineAudioContext with the following node layout:
[AudioBufferSourceNode] -> [AnalyserNode] -> [OfflineAudioContext]
This works very good on Chrome (106.0.5249.119) but on Safari 16 (17614.1.25.9.10, 17614) each time I run the analysis takes longer and longer. Both running on macOS.
What's curious is that I must quit Safari to "reset" the processing time.
I guess there's a memory leak?
Is there anything that I'm doing wrong in the JavaScript code that would cause Safari to not garbage collect?
async function processFrequencyData(
audioBuffer,
options
) {
const {
fps,
numberOfSamples,
maxDecibels,
minDecibels,
smoothingTimeConstant,
} = options;
const frameFrequencies = [];
const oc = new OfflineAudioContext({
length: audioBuffer.length,
sampleRate: audioBuffer.sampleRate,
numberOfChannels: audioBuffer.numberOfChannels,
});
const lengthInMillis = 1000 * (audioBuffer.length / audioBuffer.sampleRate);
const source = new AudioBufferSourceNode(oc);
source.buffer = audioBuffer;
const az = new AnalyserNode(oc, {
fftSize: numberOfSamples * 2,
smoothingTimeConstant,
minDecibels,
maxDecibels,
});
source.connect(az).connect(oc.destination);
const msPerFrame = 1000 / fps;
let currentFrame = 0;
function process() {
const frequencies = new Uint8Array(az.frequencyBinCount);
az.getByteFrequencyData(frequencies);
// const times = new number[](az.frequencyBinCount);
// az.getByteTimeDomainData(times);
frameFrequencies[currentFrame] = frequencies;
const nextTime = (currentFrame + 1) * msPerFrame;
if (nextTime < lengthInMillis) {
currentFrame++;
const nextTimeSeconds = (currentFrame * msPerFrame) / 1000;
oc.suspend(nextTimeSeconds).then(process);
}
oc.resume();
}
oc.suspend(0).then(process);
source.start(0);
await oc.startRendering();
return frameFrequencies;
}
const buttonsDiv = document.createElement('div');
document.body.appendChild(buttonsDiv);
const initButton = document.createElement('button');
initButton.onclick = init;
initButton.innerHTML = 'Load audio'
buttonsDiv.appendChild(initButton);
const processButton = document.createElement('button');
processButton.disabled = true;
processButton.innerHTML = 'Process'
buttonsDiv.appendChild(processButton);
const resultElement = document.createElement('pre');
document.body.appendChild(resultElement)
async function init() {
initButton.disabled = true;
resultElement.innerText += 'Loading audio... ';
const audioContext = new AudioContext();
const arrayBuffer = await fetch('https://gist.githubusercontent.com/marcusstenbeck/da36a5fc2eeeba14ae9f984a580db1da/raw/84c53582d3936ac78625a31029022c8fdb734b2a/base64audio.txt').then(r => r.text()).then(fetch).then(r => r.arrayBuffer())
resultElement.innerText += 'finished.';
resultElement.innerText += '\nDecoding audio... ';
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
resultElement.innerText += 'finished.';
processButton.onclick = async () => {
processButton.disabled = true;
resultElement.innerText += '\nStart processing... ';
const t0 = Date.now();
await processFrequencyData(audioBuffer, {
fps: 30,
numberOfSamples: 2 ** 13,
maxDecibels: -25,
minDecibels: -70,
smoothingTimeConstant: 0.2,
});
resultElement.innerText += `finished in ${Date.now() - t0} ms`;
processButton.disabled = false;
};
processButton.disabled = false;
}
I guess this is really a bug in Safari. I'm able to reproduce it by rendering an OfflineAudioContext without any nodes. As soon as I use suspend()/resume() every invocation takes a little longer.
I'm only speculating here but I think it's possible that there is some internal mechanism which tries to prevent the rapid back and forth between the audio thread and the main thread. It almost feels like one of those login forms which takes a bit longer to validate the password every time you try.
Anyway I think you can avoid using suspend()/resume() for your particular use case. It should be possible to create an OfflineAudioContext for each of the slices instead. In order to get the same effect you would only render the particular slice with each OfflineAudioContext.
const currentTime = 0;
while (currentTime < duration) {
const offlineAudioContext = new OfflineAudioContext({
length: LENGTH_OF_ONE_SLICE,
sampleRate
});
const audioBufferSourceNode = new AudioBufferSourceNode(
offlineAudioContext,
{
buffer
}
);
const analyserNode = new AnalyserNode(offlineAudioContext);
audioBufferSourceNode.start(0, currentTime);
audioBufferSourceNode
.connect(analyserNode)
.connect(offlineAudioContext.destination);
await offlineAudioContext.startRendering();
const frequencies = new Uint8Array(analyserNode.frequencyBinCount);
analyserNode.getByteFrequencyData(frequencies);
// do something with the frequencies ...
currentTime += LENGTH_OF_ONE_SLICE * sampleRate;
}
I think the only thing missing would be the smoothing since each of those slices will have it's own AnalyserNode.
I'm trying to use a CloudFlare worker to manage my backend ESI fragments but i get an error:
Uncaught (in promise) TypeError: Body has already been used. It can only be used once. Use tee() first if you need to read it twice.
Uncaught (in response) TypeError: Body has already been used. It can only be used once. Use tee() first if you need to read it twice.
I don't find where the body has already been used
The process is:
get a response with the parts
Transform the body by replacing parts fragments with sub Backend calls (streamTransformBody function)
return the response
addEventListener("fetch", event => {
event.respondWith(handleRequest(event.request))
});
const esiHeaders = {
"user-agent": "cloudflare"
}
async function handleRequest(request) {
// get cookies from the request
if(cookie = request.headers.get("Cookie")) {
esiHeaders["Cookie"] = cookie
console.log(cookie)
}
// Clone the request so that it's no longer immutable
newRequest = new Request(request)
// remove cookie from request
newRequest.headers.delete('Cookie')
// Add header to get <esi>
newRequest.headers.set("Surrogate-Capability", "abc=ESI/1.0")
console.log(newRequest.url);
const response = await fetch(newRequest);
let contentType = response.headers.get('content-type')
if (!contentType || !contentType.startsWith("text/")) {
return response
}
// Clone the response so that it's no longer immutable
const newResponse = new Response(response.body, response);
let { readable, writable } = new TransformStream()
streamTransformBody(newResponse.body, writable)
newResponse.headers.append('x-workers-hello', 'Hello from
Cloudflare Workers');
return newResponse;
}
async function streamTransformBody(readable, writable) {
const startTag = "<".charCodeAt(0);
const endTag = ">".charCodeAt(0);
let reader = readable.getReader();
let writer = writable.getWriter();
let templateChunks = null;
while (true) {
let { done, value } = await reader.read();
if (done) break;
while (value.byteLength > 0) {
if (templateChunks) {
let end = value.indexOf(endTag);
if (end === -1) {
templateChunks.push(value);
break;
} else {
templateChunks.push(value.subarray(0, end));
await writer.write(await translate(templateChunks));
templateChunks = null;
value = value.subarray(end + 1);
}
}
let start = value.indexOf(startTag);
if (start === -1) {
await writer.write(value);
break;
} else {
await writer.write(value.subarray(0, start));
value = value.subarray(start + 1);
templateChunks = [];
}
}
}
await writer.close();
}
async function translate(chunks) {
const decoder = new TextDecoder();
let templateKey = chunks.reduce(
(accumulator, chunk) =>
accumulator + decoder.decode(chunk, { stream: true }),
""
);
templateKey += decoder.decode();
return handleTemplate(new TextEncoder(), templateKey);
}
async function handleTemplate(encoder, templateKey) {
const linkRegex = /(esi:include.*src="(.*?)".*\/)/gm
let result = linkRegex.exec(templateKey);
let esi
if (!result) {
return encoder.encode(`<${templateKey}>`);
}
if (result[2]) {
esi = await subRequests(result[2]);
}
return encoder.encode(
`${esi}`
);
}
async function subRequests(target){
target = esiHost + target
const init = {
method: 'GET',
headers: esiHeaders
}
let response = await fetch(target, init)
if (!response.ok) {
return ''
}
let text = await response.text()
return '<!--esi-->' + text + '<!--/esi-->'
}
I have a base64 string as follow:
I would like this base64 string ( or any base64 String) to be turned into a pdf, then downloaded in React Native. I am using expo for this. I have looked all over the place and could not find an answer.
If you just want to show the pdf in react-native.The react-native-pdf package can do the job.Use a source obj like this.
const source = {uri:"data:application/pdf;base64,JVBERi0xLjcKJc..."};
If you want to download base64 to pdf file, it's also simple.
var RNFetchBlob = require('react-native-fetch-blob').default;
const DocumentDir = RNFetchBlob.fs.dirs.DocumentDir;
let pdfLocation = DocumentDir + '/' + 'test.pdf';
RNFetchBlob.fs.writeFile(pdfLocation, pdf_base64Str, 'base64');
export async function downloadFile(url){
let binary = await getPdfBinary(url);
const base64Str = base64_encode(binary);
const DocumentDir = RNFetchBlob.fs.dirs.DocumentDir;
let pdfLocation = DocumentDir + '/' + 'test.pdf';
RNFetchBlob.fs.writeFile(pdfLocation, base64Str, 'base64');
}
I'm sorry I didn't notice that you are using expo. As I have never written codes about the file system part in the expo. After a lot try, it finally works. But only for android. I think you may hope the pdf file should be saved to an external storage path so the user can see it in the file app. But for ios, It is hard to reach that. A possible way is to use the share dialog. If you just want to save the pdf file to the document folder in the app(not see by the user), FileSystem.writeAsStringAsync is enough!Hope this is helpful!
import * as FileSystem from 'expo-file-system';
import { Platform } from 'react-native';
async function getPdfBinary(url) {
return new Promise((resolve, reject) => {
var xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
xhr.responseType = "arraybuffer"; // get the binary
xhr.setRequestHeader('content-type', 'application/json');
xhr.onload = function (event) {
var arrayBuffer = xhr.response;
var byteArray = new Uint8Array(arrayBuffer);
var len = byteArray.byteLength;
var binary = ""
for (var i = 0; i < len; i++) {
binary += String.fromCharCode(byteArray[i]);
}
resolve(binary);
}
xhr.send();
})
}
function base64_encode(str) {
var c1, c2, c3;
var base64EncodeChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
var i = 0,
len = str.length,
string = '';
while (i < len) {
c1 = str.charCodeAt(i++) & 0xff;
if (i == len) {
string += base64EncodeChars.charAt(c1 >> 2);
string += base64EncodeChars.charAt((c1 & 0x3) << 4);
string += "==";
break;
}
c2 = str.charCodeAt(i++);
if (i == len) {
string += base64EncodeChars.charAt(c1 >> 2);
string += base64EncodeChars.charAt(((c1 & 0x3) << 4) | ((c2 & 0xF0) >> 4));
string += base64EncodeChars.charAt((c2 & 0xF) << 2);
string += "=";
break;
}
c3 = str.charCodeAt(i++);
string += base64EncodeChars.charAt(c1 >> 2);
string += base64EncodeChars.charAt(((c1 & 0x3) << 4) | ((c2 & 0xF0) >> 4));
string += base64EncodeChars.charAt(((c2 & 0xF) << 2) | ((c3 & 0xC0) >> 6));
string += base64EncodeChars.charAt(c3 & 0x3F)
}
return string
}
const downloadForAos = async (pdfBase64Str) => {
const folder = FileSystem.StorageAccessFramework.getUriForDirectoryInRoot("test");
const permissions = await FileSystem.StorageAccessFramework.requestDirectoryPermissionsAsync(folder);
if (!permissions.granted) return;
let filePath = await FileSystem.StorageAccessFramework.createFileAsync(permissions.directoryUri, "test.pdf", "application/pdf");
// let filePath = "content://com.android.externalstorage.documents/tree/primary%3Atest/document/primary%3Atest%2Ftest.txt";
console.log(pdfBase64Str, "====");
try {
await FileSystem.StorageAccessFramework.writeAsStringAsync(filePath, pdfBase64Str, { encoding: FileSystem.EncodingType.Base64 });
alert("download success!")
} catch (err) {
console.log(err);
}
}
const downloadForIos = async () => {
alert("try do it by yourself")
}
export async function downloadPdf(url) {
let binary = await getPdfBinary(url);
console.log(binary, "=====")
const base64Str = base64_encode(binary);
if (Platform.OS === "ios") {
downloadForIos(base64Str);
} else {
downloadForAos(base64Str);
}
}
i have many csv files uploaded in the S3 and i want to send them to dynamodb (json format) how can i use lambda fuction to do it or is there any other solution
Just adding this solution that i have used.
First create you Lambda function, and give it the correct permission to your S3 Bucket and DynamoBD.
Lambda Function index.js
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();
const AWS = require('aws-sdk');
const S3 = new AWS.S3({
maxRetries: 0,
region: 'SET_YOUR_REGION',
});
const insertSuccess = 0;
const insertErrors = 0;
function dynamoResultCallback(err, data) {
if (err) {
insertErrors++;
console.log("Insert Error: \n" + err);
//console.log(err, err.stack); // an error occurred
} else {
insertSuccess++;
}
}
//CONVERT TO JSON FUNCTION
function csvJSON(csv){
var lines=csv.split('\r');
for(var i = 0; i<lines.length; i++){
lines[i] = lines[i].replace(/\s/,'');//delete all blanks
}
var result = [];
var headers=lines[0].split(",");
for(var i=1;i<lines.length;i++){
var obj = {};
var currentline=lines[i].split(",");
for(var j=0;j<headers.length;j++){
//IF OBJECT KEY HAS NO VALUE,
//REMOVE IT OR DYNAMO DB WILL ERROR ON THAT OBJECT
if(currentline[j]!==''){
obj[headers[j].toString()] = currentline[j];
}
}
result.push(obj);
}
return result; //JavaScript object
// JSON.stringify(result); //JSON
}
exports.handler = (event, context, callback) => {
var srcBucket = event.Records[0].s3.bucket.name;
var srcKey = event.Records[0].s3.object.key;
S3.getObject({
Bucket: srcBucket,
Key: srcKey,
}, function(err, data) {
if (err !== null) {
return callback(err, null);
}
var fileData = data.Body.toString('utf-8');
var obj = csvJSON(fileData);
var string = JSON.stringify(obj);
console.log(string);
for (var i = 0; i < obj.length; i++) {
console.log('adding '+ JSON.stringify(obj[i]));
var params = {
Item: obj[i],
TableName: "YOUR_TABLE_NAME"
};
dynamo.putItem(params, dynamoResultCallback);
}
return callback(null, data);
});
};