I am currently using a sheet script to load data from sheets into BigQuery, however when I run the dataUpload() function I get the error below. I have correctly inserted the BQ project id, data id etc. and allowed the required permissions.
Any help will be greatly appreciated. A more refined script will also be great.
8:41:10 AM Error GoogleJsonResponseException: API call to
bigquery.jobs.get failed with error: Not found: Job
big-query-test-xxxxxx:job_0vRPgne07b25coS7TBTuasdB6JSF dataUpload #
Copy of Combined.gs:93 report_submission # Copy of Combined.gs:4
function dataUpload() {
var projectId= 'big-query-test-xxxxxx'; // GCP project name
var datasetId= 'test_data'; // Bigquery dataset name
var tableId= 'Test_Test'; // Bigquery table name
var lastColumn = 15; // Count of column (column A = 1)
var ss = SpreadsheetApp.getActiveSpreadsheet();
var select_sheet = ss.getSheets()[0];
// get name of sheet
var select_sheet = ss.getSheetByName("Sheet1");
// This logs the value in the very last cell of this sheet
var lastRow = select_sheet.getLastRow();
// var lastColumn = select_sheet.getLastColumn();
if (lastRow < 2){
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert("Empty!!");
return ;
}
var range = select_sheet.getRange(1, 1, lastRow, lastColumn);
let rows = range.getValues();
// Normalize the headers (first row) to valid BigQuery column names.
// https://cloud.google.com/bigquery/docs/schemas#column_names
rows[0] = rows[0].map((header) => {
header = header.toLowerCase().replace(/[^\w]+/g, '_');
if (header.match(/^\d/))
header = '_' + header;
return header;
});
// Create the BigQuery load job config. For more information, see:
// https://developers.google.com/apps-script/advanced/bigquery
let loadJob = {
configuration: {
load: {
destinationTable: {
projectId: projectId,
datasetId: datasetId,
tableId: tableId
},
skipLeadingRows: 1,
writeDisposition: 'WRITE_TRUNCATE',
}
}
};
// BigQuery load jobs can only load files, so we need to transform our
// rows (matrix of values) into a blob (file contents as string).
// For convenience, we convert the rows into a CSV data string.
// https://cloud.google.com/bigquery/docs/loading-data-local
let csvRows = rows.map(values =>
// We use JSON.stringify() to add "quotes to strings",
// but leave numbers and booleans without quotes.
// If a string itself contains quotes ("), JSON escapes them with
// a backslash as \" but the CSV format expects them to be
// escaped as "", so we replace all the \" with "".
values.map(value => JSON.stringify(value).replace(/\\"/g, '""'))
);
let csvData = csvRows.map(values => values.join(',')).join('\n');
let blob = Utilities.newBlob(csvData, 'application/octet-stream');
// Run the BigQuery load job.
try {
var result = BigQuery.Jobs.insert(loadJob, projectId, blob);
} catch (e) {
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert(e);
return e;
}
// Get job ID
let json_result = JSON.parse(result);
let job_id = json_result['jobReference']['jobId'];
// Get job info
var job_info = BigQuery.Jobs.get(projectId,job_id);
let json_job_info = JSON.parse(job_info);
// Loop for job Complete
while (json_job_info['status']['state']=='RUNNING'){
job_info = BigQuery.Jobs.get(projectId,job_id);
json_job_info = JSON.parse(job_info);
}
// If job failed, alert error message
if (json_job_info['status']['errors']){
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert(JSON.stringify(json_job_info['status']['errors'][0]['message']));
}
// alert job complete message
else{
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert("Upload Complete");
}
Logger.log(
'Load job started. Click here to check your jobs: ' +
`https://console.cloud.google.com/bigquery?project=${projectId}&page=jobs`
);
}
Related
I used the following code (taken from PDF.CO) to merge multiple pdf files in Google Drive:
/**
* Initial Declaration and References
*/
// Get the active spreadsheet and the active sheet
ss = SpreadsheetApp.getActiveSpreadsheet();
ssid = ss.getId();
// Look in the same folder the sheet exists in. For example, if this template is in
// My Drive, it will return all of the files in My Drive.
var ssparents = DriveApp.getFileById(ssid).getParents();
// Loop through all the files and add the values to the spreadsheet.
var folder = ssparents.next();
/**
* Add PDF.co Menus in Google Spreadsheet
*/
function onOpen() {
var menuItems = [
{name: 'Get All PDF From Current Folder', functionName: 'getPDFFilesFromCurFolder'},
{name: 'Merge PDF URLs Listed In Cell', functionName: 'mergePDFDocuments'}
];
ss.addMenu('PDF.co', menuItems);
}
/**
* Get all PDF files from current folder
*/
function getPDFFilesFromCurFolder() {
var files = folder.getFiles();
var pdfUrlCell = ss.getRange("A4");
var allFileUrls = [];
while (files.hasNext()) {
var file = files.next();
var fileName = file.getName();
if(fileName.endsWith(".pdf")){
// Make File Pulblic accessible with URL so that it can be accessible with external API
var resource = {role: "reader", type: "anyone"};
Drive.Permissions.insert(resource, file.getId());
// Add Url
allFileUrls.push(file.getDownloadUrl());
}
pdfUrlCell.setValue(allFileUrls.join(","));
}
}
function getPDFcoApiKey(){
// Get PDF.co API Key Cell
let pdfCoAPIKeyCell = ss.getRange("B1");
return pdfCoAPIKeyCell.getValue();
}
/**
* Function which merges documents using PDF.co
*/
function mergePDFDocuments() {
// Get Cells for Input/Output
let pdfUrlCell = ss.getRange("A4");
let resultUrlCell = ss.getRange("B4");
let pdfUrl = pdfUrlCell.getValue();
// Prepare Payload
const data = {
"async": true, // As we have large volumn of PDF files, Enabling async mode
"name": "result",
"url": pdfUrl
};
// Prepare Request Options
const options = {
'method' : 'post',
'contentType': 'application/json',
'headers': {
"x-api-key": getPDFcoApiKey()
},
// Convert the JavaScript object to a JSON string.
'payload' : JSON.stringify(data)
};
// Get Response
// https://developers.google.com/apps-script/reference/url-fetch
const resp = UrlFetchApp.fetch('https://api.pdf.co/v1/pdf/merge', options);
// Response Json
const respJson = JSON.parse(resp.getContentText());
if(respJson.error){
console.error(respJson.message);
}
else{
// Job Success Callback
const successCallbackFn = function(){
// Upload file to Google Drive
uploadFile(respJson.url);
// Update Cell with result URL
resultUrlCell.setValue(respJson.url);
}
// Check PDF.co Job Status
checkPDFcoJobStatus(respJson.jobId, successCallbackFn);
}
}
/**
* Checks PDF.co Job Status
*/
function checkPDFcoJobStatus(jobId, successCallbackFn){
// Prepare Payload
const data = {
"jobid": jobId
};
// Prepare Request Options
const options = {
'method' : 'post',
'contentType': 'application/json',
'headers': {
"x-api-key": getPDFcoApiKey()
},
// Convert the JavaScript object to a JSON string.
'payload' : JSON.stringify(data)
};
// Get Response
// https://developers.google.com/apps-script/reference/url-fetch
const resp = UrlFetchApp.fetch('https://api.pdf.co/v1/job/check', options);
// Response Json
const respJson = JSON.parse(resp.getContentText());
if(respJson.status === "working"){
// Pause for 3 seconds
Utilities.sleep(3 * 1000);
// And check Job again
checkPDFcoJobStatus(jobId, successCallbackFn);
}
else if(respJson.status == "success"){
// Invoke Success Callback Function
successCallbackFn();
}
else {
console.error(`Job Failed with status ${respJson.status}`);
}
}
/**
* Save file URL to specific location
*/
function uploadFile(fileUrl) {
var fileContent = UrlFetchApp.fetch(fileUrl).getBlob();
folder.createFile(fileContent);
}
It runs perfectly the first time, but then gives an error:
Exception: Request failed for https://api.pdf.co returned code 402. Truncated server response: {"status":"error","errorCode":402,"error":true,"message":"Not enough credits, subscription expired or metered use is not allowed. Please review cre... (use muteHttpExceptions option to examine full response).
I have the following script where I am getting an error message:
org.mozilla.javascript.EcmaError: TypeError: Cannot set property
"JSZipSync" of undefined to
"org.mozilla.javascript.InterpretedFunction#6e57e155"
(/SuiteScripts/Suitelet to Excel.js#17(eval)#2)
According to Netsuite support, there are no issues with the script and they are not getting this error message
/**
*#NApiVersion 2.x
*#NScriptType Suitelet
*/
define(["N/search", "N/file", "N/https"], function (search, file, https) {
function onRequest(context) {
// Load the xlsx library from the CDN
var response = https.get({
url: "https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.17.0/xlsx.full.min.js",
});
var script = response.body;
eval(script);
// Replace this with the ID of the saved search you want to run
var searchId = "customsearch_my_saved_search";
// Run the search and get the results
var searchResult = search.load({
id: searchId,
});
var searchRows = searchResult.run().getRange({
start: 0,
end: 1000,
});
// Create an array to hold the data for the Excel file
var data = [];
// Add the column names as the first row
var columnNames = [];
searchResult.columns.forEach(function (column) {
columnNames.push(column.label);
});
data.push(columnNames);
// Add the search results to the data array
searchRows.forEach(function (row) {
var rowData = [];
searchResult.columns.forEach(function (column) {
rowData.push(
row.getValue({
name: column.name,
})
);
});
data.push(rowData);
});
// Create the Excel file
var ws = XLSX.utils.aoa_to_sheet(data);
var wb = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(wb, ws, "Sheet1");
var binaryData = XLSX.write(wb, {
type: "binary",
bookType: "xlsx",
});
var fileName = "search_results.xlsx";
var folderId = -4; // -4 represents the "Home" folder in the file cabinet
var file = file.create({
name: fileName,
fileType: file.Type.EXCEL,
contents: binaryData,
folder: folderId,
});
// Send the file as a response to the user
context.response.writeFile({
file: file,
isInline: true,
});
});
}
return {
onRequest: onRequest,
};
});
I can't tell where I am going wrong?
The JSZipSync is a function in the xlsx.full.min.js library which I believe I have loaded correctly.
I have also tried storing this file in the filing cabinet and referencing that instead of a link to the CDN website.
The same error message is generated in both cases
It seems quite new, but just hoping someone here has been able to use nodejs to write directly to BigQuery storage using #google-cloud/bigquery-storage.
There is an explanation of how the overall backend API works and how to write a collection of rows atomically using BigQuery Write API but no such documentation for nodejs yet. A recent release 2.7.0 documents the addition of said feature but there is no documentation, and the code is not easily understood.
There is an open issue requesting an example but thought I'd try my luck to see if anyone has been able to use this API yet.
Suppose you have a BigQuery table called student with three columns id,name and age. Following steps will get you to load data into the table with nodejs storage write api.
Define student.proto file as follows
syntax = "proto2";
message Student {
required int64 id = 1;
optional string name = 2;
optional int64 age = 3;
}
Run the following at the command prompt
protoc --js_out=import_style=commonjs,binary:. student.proto
It should generate student_pb.js file in the current directory.
Write the following js code in the current directory and run it
const {BigQueryWriteClient} = require('#google-cloud/bigquery-storage').v1;
const st = require('./student_pb.js')
const type = require('#google-cloud/bigquery-storage').protos.google.protobuf.FieldDescriptorProto.Type
const mode = require('#google-cloud/bigquery-storage').protos.google.cloud.bigquery.storage.v1.WriteStream.Type
const storageClient = new BigQueryWriteClient();
const parent = `projects/${project}/datasets/${dataset}/tables/student`
var writeStream = {type: mode.PENDING}
var student = new st.Student()
var protoDescriptor = {}
protoDescriptor.name = 'student'
protoDescriptor.field = [{'name':'id','number':1,'type':type.TYPE_INT64},{'name':'name','number':2,'type':type.TYPE_STRING},{'name':'age','number':3,'type':type.TYPE_INT64}]
async function run() {
try {
var request = {
parent,
writeStream
}
var response = await storageClient.createWriteStream(request);
writeStream = response[0].name
var serializedRows = []
//Row 1
student.setId(1)
student.setName('st1')
student.setAge(15)
serializedRows.push(student.serializeBinary())
//Row 2
student.setId(2)
student.setName('st2')
student.setAge(15)
serializedRows.push(student.serializeBinary())
var protoRows = {
serializedRows
}
var proto_data = {
writerSchema: {protoDescriptor},
rows: protoRows
}
// Construct request
request = {
writeStream,
protoRows: proto_data
};
// Insert rows
const stream = await storageClient.appendRows();
stream.on('data', response => {
console.log(response);
});
stream.on('error', err => {
throw err;
});
stream.on('end', async () => {
/* API call completed */
try {
var response = await storageClient.finalizeWriteStream({name: writeStream})
response = await storageClient.batchCommitWriteStreams({parent,writeStreams: [writeStream]})
}
catch(err) {
console.log(err)
}
});
stream.write(request);
stream.end();
}
catch(err) {
console.log(err)
}
}
run();
Make sure your environment variables are set correctly to point to the file containing google cloud credentials.
Change project and dataset values accordingly.
I want to retrieve data via Jenkins API using Google Sheet Script and store it in Google Sheet
1) Pull Jenkins Job Builds using Jenkins API to Google Sheet - DONE
2) Store data to Google Sheet ???
(need only "builds.subBuilds.buildNumber" and "builds.subBuilds.duration" values)
(need to correct mistake in the script)
function getJenkinsBuilds() {
// get the jenkins job
var response = UrlFetchApp.fetch('http://jenkins.[domain].co/job/Build+Deploy/api/json', {
'method': 'get',
'muteHttpExceptions' : true,
'headers' : {'Authorization' : 'Basic [tokan]'},
});
// parse the json reply and return builds
var data = JSON.parse(response);
var builds = data["builds"];
Logger.log(builds);
return builds;
};
// store predefined parameters from builds in the spreadsheet
function setDataToTable() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Jenkins');
var cell = sheet.getRange("A1");
var rows = [['buildNumber','duration'],['','']]; // I GUESS THE MISTAKE IS HERE?
sheet.getRange(cell.getRow(), cell.getColumn(), rows.length, rows[0].length).setValues(rows);
}
Actual result:
Log shows retrieved array with Builds objects, i.e.:
[19-10-10 16:18:16:937 AEDT] [{number=2081, subBuilds=[{jobName=...
'Jenkins' spreadsheet is empty.
Expected result:
Store "builds.subBuilds.buildNumber" and "builds.subBuilds.duration" values
in the Google Sheet ('Jenkins' spreadsheet), i.e.:
buildNumber duration
123 15sec
456 16sec
... ...
I was able to make it working in the next way:
function getJenkinsBuilds()
{
// get jenkins builds
var response = UrlFetchApp.fetch('http://jenkins.[domain].co/job/Build+Deploy/api/json', {
'method': 'get',
'muteHttpExceptions' : true,
'headers' : {'Authorization' : 'Basic [token]'}
});
// parse the json reply
var data = JSON.parse(response);
var builds = data["builds"];
var number = data['builds'][0]['number'];
var url = data['builds'][0]['url'];
Logger.log(number);
Logger.log(url);
// fill in the spreadsheet with data
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Jenkins');
var cell = sheet.getRange('A1');
var rows = [['BUILD']];
for (var i = 0; i < builds.length; i++)
{
var number = data['builds'][i]['number'];
var url = data['builds'][i]['url'];
rows.push(['=HYPERLINK("'+url+'","'+number+'")']);
Logger.log(number);
Logger.log(url);
sheet.getRange(cell.getRow(), cell.getColumn(), rows.length, rows[0].length).setValues(rows);
}
};
In Google BigQuery WebUI, it shows query result screen after executing a query, and it shows the button of "Save as Google Sheets". I like this feature but would like to automate this, is there such function through the REST API that I could do?
It doesn’t seem like there is a straightforward way to do this directly with the BigQuery API. There are few workarounds for this though:
You can use the BigQuery API to query your data and then the GoogleSheets API to upload it to Google Sheets.
You can use Google Apps Script. If you go to this link, you click on “New Script”, you can run the code below. You can adapt this to your needs. You can also add a trigger to run the script every hour/minute …
Here the code snippet from this link:
function runQuery() {
// Replace this value with the project ID listed in the Google
// Cloud Platform project.
var projectId = 'XXXXXXXX';
var request = {
query: 'SELECT TOP(word, 300) AS word, COUNT(*) AS word_count ' +
'FROM publicdata:samples.shakespeare WHERE LENGTH(word) > 10;'
};
var queryResults = BigQuery.Jobs.query(request, projectId);
var jobId = queryResults.jobReference.jobId;
// Check on status of the Query Job.
var sleepTimeMs = 500;
while (!queryResults.jobComplete) {
Utilities.sleep(sleepTimeMs);
sleepTimeMs *= 2;
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId);
}
// Get all the rows of results.
var rows = queryResults.rows;
while (queryResults.pageToken) {
queryResults = BigQuery.Jobs.getQueryResults(projectId, jobId, {
pageToken: queryResults.pageToken
});
rows = rows.concat(queryResults.rows);
}
if (rows) {
var spreadsheet = SpreadsheetApp.create('BiqQuery Results');
var sheet = spreadsheet.getActiveSheet();
// Append the headers.
var headers = queryResults.schema.fields.map(function(field) {
return field.name;
});
sheet.appendRow(headers);
// Append the results.
var data = new Array(rows.length);
for (var i = 0; i < rows.length; i++) {
var cols = rows[i].f;
data[i] = new Array(cols.length);
for (var j = 0; j < cols.length; j++) {
data[i][j] = cols[j].v;
}
}
sheet.getRange(2, 1, rows.length, headers.length).setValues(data);
Logger.log('Results spreadsheet created: %s',
spreadsheet.getUrl());
} else {
Logger.log('No rows returned.');
}
}