script to query data from BigQuery, export result to CSV format, and sFTP transfer to destination folder - google-bigquery

Question

I am looking for a reliable script to perform sFTP transfer from Google Cloud Storage bucket to the destination sFTP folder. Currently I'm using pandas/python to query data from Big Query and save the result to Google Storage and it's working flawlessly, and I use Google Cloud function to do the sFTP transfer and it's been working sporadically, sometimes it didn't run, and other times the file transfer was incomplete, and also it takes very long time for it to run (around 15 min via Google could function versus 5 secs if I transfer the same file manually). Attached is the Cloud function currently used. Any help is very much appreciated. Thanks
/**
* Triggered from a change to a Cloud Storage bucket.
*
* #param {!Object} event Event payload.
* #param {!Object} context Metadata for the event.
*/
exports.gcfSendToSftp = (event, context) => {
const Storage = require('#google-cloud/storage');
const os = require("os");
const fs = require("fs");
const Client = require('ssh2-sftp-client');
// Creates a client
const storage = new Storage();
const path = require('path');
const cwd = os.tmpdir()
const MyDate = new Date();
const formatted_year = MyDate.getFullYear();
const formatted_month = ('0' + (MyDate.getMonth()+1)).slice(-2);
const formatted_day = ('0' + MyDate.getDate()).slice(-2);
TRANSACTIONS_LOG_FILENAME='W2G_TransactionsLog_' + formatted_year + formatted_month + formatted_day + '.csv'
destFilename = path.join(cwd, TRANSACTIONS_LOG_FILENAME)
srcFilename=TRANSACTIONS_LOG_FILENAME
async function sendSFTP() {
const stats = fs.statSync(destFilename)
const fileSizeInBytes = stats["size"]
console.log(destFilename + " file size in bytes "+ fileSizeInBytes);
// let data = fs.createReadStream(destFilename);
let data = destFilename;
let remote = '/Home/w2g/W2GInventoryIntegration/'+ srcFilename;
let client = new Client();
client.connect({
host: '111.11.11.11',
port: '22',
username: 'test',
password: 'hokcqqEHH10g',
readyTimeout: 0,
debug: console.log
}).then(() => {
client.fastPut(data, remote);
}).catch(err => {
console.log(err, 'catch error');
});
}
async function downloadFile() {
bucketName='data_export_vw_transactions_log';
const options = {
// The path to which the file should be downloaded, e.g. "./file.txt"
destination: destFilename,
};
// Downloads the file
await storage.bucket(bucketName).file(srcFilename).download(options);
console.log(
`gs://${bucketName}/${srcFilename} downloaded to ${destFilename}.`
);
}
downloadFile().then( () => { sendSFTP() })
};