BigQuery + javascript (tabledata.insertAll) - google-bigquery

Here is my code:
function runQuery(){
var json = JSON.stringify({"string": "eight", "number": "8", "id": "8"});
var body = {
"kind": "bigquery#tableDataInsertAllRequest",
"rows": [
{
"insertId": "8",
"json": json
}
]
};
var request = gapi.client.bigquery.tabledata.insertAll({
'projectId': project_id,
'datasetId': 'newTest',
'tableId': 'newTable',
'content': body
});
request.execute(function(response){
console.log(response);
});
}
But this code return error: No records present in table data append request. (400)
Can somebody help me?

Problem solved:
var request = gapi.client.bigquery.tabledata.insertAll({
'projectId': projectId,
'datasetId': datasetId,
'tableId': tableId,
"kind": "bigquery#tableDataInsertAllRequest",
"rows":[
{
"insertId": "10",
"json": {"id": "10", "string": "ten", "number": "10"}
}
]
});

Spent a lot of time on the same issue.
dbepcepk's answer resulted in the error:
"Error: Invalid JSON payload received. Unknown name "rows": Cannot bind query parameter. 'rows' is a message type. Parameters can only be bound to primitive types."
Here is how I got around it:
Note the JSON Key "resource" defining the TableDataInsertAllRequest object
const request ={
"projectId": projectId,
"datasetId": datasetId,
"tableId": tableId,
"resource":{
"kind": "bigquery#tableDataInsertAllRequest",
"skipInvalidRows": false,
"ignoreUnknownValues": false,
"templateSuffix":"",
"rows": [{"json": {"test":true}]
}
}
client.tabledata.insertAll(
request,
(err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
});

You can't append data to the sample data set. Create your own to perform your inserts.

Related

PowerBI Create Dataset from a Postgresql datasource using REST API

I created a datasource behind gateway for using rest API. Datasource got created. However, now I want to add a table(create a dataset) from the created datasource to use it in a report. However, I am getting the below error from API.
{
"error": {
"code": "FailedToDeserializeDatasetError",
"pbi.error": {
"code": "FailedToDeserializeDatasetError",
"parameters": {},
"details": [],
"exceptionCulprit": 1
}
}
}
API request body:
{
"datasources": [
{
"gatewayId":"gateway_id",
"datasourceId": "datasource_id",
"datasourceType": "PostgreSql",
"connectionDetails": "{\"server\":\"server_address\",\"database\":\"database_name\"}",
"credentialType": "Basic",
"credentialDetails": {
"privacyLevel": "None",
"useEndUserOAuth2Credentials": false
}
}
],
"defaultMode": "Push",
"name": "API DS 1",
"tables": [
{
"name":"currency_rates",
"description": "DS Table 1 Demo API",
"columns":[
{
"name":"id",
"dataType":"Int64"
},
{
"name":"traded_on",
"dataType":"DateTime"
},
{
"name":"currency_code",
"dataType": "string"
},
{
"name":"close",
"dataType": "Double"
}
]
}
]
}
Not sure what is wrong here.
API Reference: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-post-dataset-in-group

Transform Array into columns in BigQuery

I have a json string stored in a string column in BigQuery. There is an Array in it. I would like to pick some fields from array and write its value to BQ columns.
For example - Consider a below json stored in BQ
{
"pool": "mypool",
"statusCode": "0",
"payloads": [
{
"name": "request",
"fullpath": "com.gcp.commontools.edlpayload.EDLPayloadManagerTest$Request",
"jsonPayload": {
"body": "{\"data\":\"foo\"}"
},
"orientation": "REQUEST",
"httpTransport": {
"httpMethod": "POST",
"headers": {
"headers": {
"a": "1"
}
},
"sourceEndpoint": "/v1/foobar"
}
},
{
"name": "response",
"fullpath": "com.gcp.commontools.edlpayload.EDLPayloadManagerTest$Response",
"jsonPayload": {
"body": "{\"data\":\"bar\"}"
},
"orientation": "RESPONSE",
"httpTransport": {
"headers": {
"headers": {
"b": "2"
}
},
"httpResponseCode": 200
}
},
{
"name": "attributes",
"fullpath": "java.util.HashMap",
"nameValuePairs": {
"data": {
"one": "1"
}
},
"orientation": "TRANSITORY"
}
],
"uuid": "11EC-C714-8ADE2390-9619-1B80E63968CC",
"payloadName": "my-overall-name"
}
Consider a target BQ table schema is
pool, requestFullPath, requestPayload, responseFullPath, responsePayload
From the above json, i would like to pick few json elements and map there value to a column in BQ. Please note, array of payload will be dynamic in nature. There can be only 1 payload in the payloads array or there can be multiple. And the order of them is not fixed. For example, request payload can come at [0]th position, 1st position etc.
Consider below
select * from (
select
json_value(json_col, '$.pool') as pool,
json_value(payload, '$.name') as name,
json_value(payload, '$.fullpath') as FullPath,
json_value(payload, '$.jsonPayload.body') as Payload,
from your_table t
, unnest(json_extract_array(json_col, '$.payloads')) payload
)
pivot (any_value(FullPath) as FullPath, any_value(Payload) as Payload for name in ('request', 'response') )
if applied to sample data in your question - output is

Fetch data in a POST Request in asp.net core

I am using an external web link to get data and fetch it to json The reason why I need to handle it by the controller is to filter the data of it. Sadly, an api link was programmatically incorrect because instead of requesting it as GET method, it was programmed as POST method. I had this code simple code below but the return was a header data not the actual data of the api.
[HttpPost, Route("get/subproject")]
public ActionResult subproject()
{
using (var client = new HttpClient())
{
client.BaseAddress = new Uri(#"https://thisisjustasample.com/");
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
HttpResponseMessage hrm = client.PostAsync("/api/new/get/subproject/details/get_dto", null).Result;
return Ok(hrm);
}
}
The output of the code above is this.
{
"version": "1.1",
"content": {
"headers": [
{
"key": "Content-Length",
"value": [
"29942142"
]
},
{
"key": "Content-Type",
"value": [
"application/json; charset=utf-8"
]
},
{
"key": "Expires",
"value": [
"-1"
]
}
]
}
}
What I need is this data below.
{
"sub_project_id": 267892,
"engineeringMigrationId": 0,
"modality_id": 21,
"id": null,
"reportID": null,
"month": null,
"year": null,
"cycle_id": 204
}
Any help would be appreciated. Thanks in advance.
Don't return hrm directly, If you want to get the response data, you need return.
hrm.Content.ReadAsStringAsync().Result
Demo
1.return Ok(hrm);
2.return Ok(hrm.Content.ReadAsStringAsync().Result);

How to convert from blob URL to binary?

I'm using ImageInput component inside an iterator to upload images in my create form and it generates a structure like this:
"data": {
"items": [
{
"id": 1,
"title": "test",
"subTitle": "test",
"additionalAttributes": {
"price": "3452345"
},
"images": [
{
"src": {
"rawFile": {
"path": "test.jpg"
},
"src": "blob:https://localhost:44323/82c04494-244a-49eb-9d0e-6bca5a3469f7",
"title": "test.jpg"
},
"title": "d"
}
]
}
],
"contact": {
"firstName": "test",
"lastName": "test",
"jobTitle": "test",
"emailAddress": "test#test.com",
"phoneNumber": "23234"
},
"theme_id": 1,
"endDate": "2020-06-19T22:27:00.000Z",
"status": "2"
}
}
What I'm trying to do is sending the image to an API for saving in a folder. Blob URL is an internal object in the browser son it can't be used in the API, so I tried to convert the Blob URL into a binary and send to API.
Following the tutorial I can not get the expected result. Here is my code:
I created a new dataProvider like this:
export const PrivateEventProvider = {
create: (resource: string, params: any) => {
convertFileToBase64(params.data.items[0].images[0].src.src).then(
transformedPicture => {
console.log(`transformedPicture: ${transformedPicture}`);
}
);
const convertFileToBase64 = (file: { rawFile: Blob }) =>
new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result);
reader.onerror = reject;
reader.readAsDataURL(file.rawFile);
});
And I have this error
Unhandled Rejection (TypeError): Failed to execute 'readAsDataURL' on
'FileReader': parameter 1 is not of type 'Blob'.
enter image description here
So my question is, which is the correct way of uploading images to a folder using react-admin?

How do you get all the email body parts? And how do you know how many parts exist?

I'm trying to read emails responded by the Gmail API.
I have trouble accessing all the "parts". And don't have great ways to traverse through the response. I'm also lost as to how many parts can exist so that I can make sure I read the different email responses properly. I've shortened the response below...
{ "payload": { "mimeType": "multipart/mixed", "filename": "",
], "body": { "size": 0 }, "parts": [ {
"body": {
"size": 0
},
"parts": [
{
"partId": "0.0",
"mimeType": "text/plain",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/plain; charset=\"us-ascii\""
},
{
"name": "Content-Transfer-Encoding",
"value": "quoted-printable"
}
],
"body": {
"size": 2317,
"data": "RGVhciBNSVQgQ2x1YiBWb2x1bnRlZXJzIGluIEFzaWEsDQoNCkJ5IG5vdyBlYWNoIG9mIHlvdSBzaG91bGQgaGF2ZSByZWNlaXZlZCBpbnZpdGF0aW9ucyB0byB0aGUgcmVjZXB0aW9ucyBpbiBib3RoIFNpbmdhcG9yZSBhbmQgSG9uZyBLb25nIHdpdGggUHJlc2lkZW50IFJlaWYgb24gTm92ZW1iZXIgNyBhbmQgTm92ZW1iZXIg"
}
},
{
"partId": "0.1",
"mimeType": "text/html",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/html; charset=\"us-ascii\""
},
{
"name": "Content-Transfer-Encoding",
"value": "quoted-printable"
}
],
"body": {
"size": 9116,
"data": "PGh0bWwgeG1sbnM6dj0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTp2bWwiIHhtbG5zOm89InVybjpzY2hlbWFzLW1pY3Jvc29mdC1jb206b2ZmaWNlOm9mZmljZSIgeG1sbnM6dz0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTpvZmZpY2U6d29yZCIgeG1sbnM6bT0iaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS9vZmZpY2UvMjA"
}
}
] }, {
"partId": "1",
"mimeType": "text/plain",
"filename": "",
"body": {
"size": 411,
"data": "X19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX18NClRoYW5rIHlvdSBmb3IgYWxsb3dpbmcgdXMgdG8gcmVhY2ggeW91IGJ5IGVtYWlsLCB0aGUgbW9zdCBpbW1lZGlhdGUgbWVhbnMgZm9yIHNoYXJpbmcgaW5mb3JtYXRpb24gd2l0aCBNSVQgYWx1bW5pLiANCklmIHlvdSB3b3VsZCBsaWtlIHRvIHVuc3Vic2NyaWJlIGZyb20gdGhpcyBtYWlsaW5nIGxpc3Qgc2VuZCBhIGJsYW5rIGVtYWlsIHRvIGxpc3RfdW5zdWJzY3JpYmVAYWx1bS5taXQuZWR1IGFuZCBwdXQgdGhlIGxpc3QgbmFtZSBpbiB0aGUgc3ViamVjdCBsaW5lLg0KRm9yIGV4YW1wbGU6DQpUbzogbGlzdF91bnN1YnNjcmliZUBhbHVtLm1pdC5lZHUNCkNjOg0KU3ViamVjdDogYXNpYW9mZg0K"
} } ] } }
Is there something I'm missing?
A MIME message is not just an array it's a full blown tree structure. So you'll have to traverse it to correctly handle it. Luckily JSON parsers are plentiful and the problem can easily be handled with recursion. In many languages there exist very useful email parsing libraries that can make accessing traditional parts (e.g. the text/plain or text/html displayable part, or attachments) not too laborious.
You'll have to set up walker functions to traverse through the json and pick out the bits you are after. Here is a part of what I wrote. This may help you jumpstart your code. NOTE: this is used inside of wordpress...hence the special jQuery call. Not needed if you do not need to use jquery inside wordpress.
function makeApiCall() {
gapi.client.load('gmail', 'v1', function() {
//console.log('inside call: '+myquery);
var request = gapi.client.gmail.users.messages.list({
'userId': 'me',
'q': myquery
});
request.execute(function(resp) {
jQuery(document).ready(function($) {
//console.log(resp);
//$('.ASAP-emailhouse').height(300);
$.each(resp.messages, function(index, value){
messageId = value.id;
var messagerequest = gapi.client.gmail.users.messages.get({
'userId': 'me',
'id': messageId
});//end var message request
messagerequest.execute(function(messageresp) {
//console.log(messageresp);
$.each(messageresp, responsewalker);
function responsewalker(key, response){
messagedeets={};
$.each(messageresp.payload.headers, headerwalker);
function headerwalker(headerkey, header){
if(header.name =='Date'){
d = new Date(header.value);
var curr_date = d.getDate();
var curr_month = d.getMonth() + 1; //Months are zero based
var curr_year = d.getFullYear();
var formatteddate = curr_month+'/'+curr_date+'/'+curr_year;
messagedeets['date']=formatteddate;
//$('.ASAP-emailhouse').append('<p>'+header.value+'</p>');
}
if(header.name =='Subject'){
//console.log(header.value);
messagedeets.subject=header.value;
}
}
messagedeets.body = {};
$.each(messageresp.payload.parts, walker);
function walker(partskey, value) {
//console.log(value.body);
if (value.body.data !== "undefined") {
//console.log(value.body);
var messagebody = atob(value.body.data);
messagedeets.body.partskey = messagebody;
}
console.log(messagedeets);
$('.ASAP-emailhouse').append('<div class="messagedeets"><p class="message-date">'+messagedeets.date+': <span class="message-subject">'+messagedeets.subject+'</span></p><p>'+messagedeets.body.partskey+'</p></div>');
}//end responsewalker
//$('.ASAP-emailhouse').append('</li>');
}
//$('.ASAP-emailhouse').append('</ul>');
});//end message request
});//end each message id
});//end jquery wrapper for wordpress
});//end request execute list messages
});//end gapi client load gmail
}
The MIME parts you are looking for are in an array. JSON does not tell you up front how many items are in an array. Even MIME itself does not provide a way of knowing how many parts are present without looking at the entire message. You will just have to traverse the entire array to know how many parts are in it, and process each part as you encounter it.
To know how much parts exists, you can just use the Length property.
Example :
json.payload.parts.length
For your example, this property is 2 because there are 2 parts.