Listing public datasets via API - google-bigquery

Is there a way to include public datasets that are visible under a project in BQ cloud console (i.e. publicdata:samples) in datasets.list() / projects.list() ?

Yes - just use project "publicdata" and the datasetId of "samples" in your API calls.
import httplib2
from apiclient import errors
service = build('bigquery', 'v2')
# Auth (however your app does this)
tables = service.tables()
table_list = tables.list(projectId='publicdata',
datasetId='samples').execute()
From the API Console:
GET https://www.googleapis.com/bigquery/v2/projects/publicdata/datasets/samples/tables
{
"kind": "bigquery#tableList",
"etag": "\"asdasdsadsadsad\"",
"tables": [
{
"kind": "bigquery#table",
"id": "publicdata:samples.github_nested",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "github_nested"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.github_timeline",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "github_timeline"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.gsod",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "gsod"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.natality",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "natality"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.shakespeare",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "shakespeare"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.trigrams",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "trigrams"
},
"type": "TABLE"
},
{
"kind": "bigquery#table",
"id": "publicdata:samples.wikipedia",
"tableReference": {
"projectId": "publicdata",
"datasetId": "samples",
"tableId": "wikipedia"
},
"type": "TABLE"
}
],
"totalItems": 7
}

Related

graphql refetchQueries is not working in react-native

I am using react-native with apollo-client. Others queries are working fine with refetch but this refetch results in:
`
WARN Unknown query {
"kind": "Document",
"definitions": [
{
"kind": "OperationDefinition",
"operation": "query",
"variableDefinitions": [
{
"kind": "VariableDefinition",
"variable": {
"kind": "Variable",
"name": {
"kind": "Name",
"value": "where"
}
},
"type": {
"kind": "NonNullType",
"type": {
"kind": "NamedType",
"name": {
"kind": "Name",
"value": "AppointmentWhereInput"
}
}
},
"directives": []
}
],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "appointments"
},
"arguments": [
{
"kind": "Argument",
"name": {
"kind": "Name",
"value": "where"
},
"value": {
"kind": "Variable",
"name": {
"kind": "Name",
"value": "where"
}
}
}
],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "id"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "date"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "slot"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "paymentMode"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "status"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "allergy"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "appointmentType"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "clinic"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "report"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "amount"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "patientStatus"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "pregnancyMonth"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "appointmentNumber"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "registrationNumber"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "RegistrationNumber"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "name"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "mobile"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "age"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "gender"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "appointemntCount"
},
"arguments": [
{
"kind": "Argument",
"name": {
"kind": "Name",
"value": "where"
},
"value": {
"kind": "ObjectValue",
"fields": [
{
"kind": "ObjectField",
"name": {
"kind": "Name",
"value": "status"
},
"value": {
"kind": "ObjectValue",
"fields": [
{
"kind": "ObjectField",
"name": {
"kind": "Name",
"value": "equals"
},
"value": {
"kind": "StringValue",
"value": "confirmed",
"block": false
}
}
]
}
}
]
}
}
],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "appointemnt"
},
"arguments": [
{
"kind": "Argument",
"name": {
"kind": "Name",
"value": "where"
},
"value": {
"kind": "ObjectValue",
"fields": [
{
"kind": "ObjectField",
"name": {
"kind": "Name",
"value": "status"
},
"value": {
"kind": "ObjectValue",
"fields": [
{
"kind": "ObjectField",
"name": {
"kind": "Name",
"value": "equals"
},
"value": {
"kind": "StringValue",
"value": "confirmed",
"block": false
}
}
]
}
}
]
}
}
],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "date"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "id"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "status"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "prescription"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "prescription"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "url"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "bloodReport"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "bloodReport"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "url"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "ultraSoundReport"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "ultraSoundReport"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "url"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "pathology"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "id"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "diagnosticsCenter"
},
"arguments": [],
"directives": [],
"selectionSet": {
"kind": "SelectionSet",
"selections": [
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "id"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "createdAt"
},
"arguments": [],
"directives": []
},
{
"kind": "Field",
"name": {
"kind": "Name",
"value": "__typename"
}
}
]
}
}
]
}
}
],
"loc": {
"start": 0,
"end": 924
}
} requested in refetchQueries options.include array

GKE - Automatic replacement of Instance Group Template

Current Situation: We're having a Kubernetes Cluster in Google (GKE). There is an instance group that refers an instance template defining the VMs.
Problem: Some days ago we had a spontaneous outage. I could find out that our instance group referred a new instance template, that was missing some access rights leading to that outage. Changing the instance template back to the original one solved the issue.
What I'm still wondering: We did neither change the template for the instance group nor did we create the instance template that was used.
I'm the only person managing the cluster, so I'm pretty sure that this happened automatically. How could this happen?
Searching for a reason:
I was researching on that topic but I could not find any related issues. I discovered three logs that might be interesting, but I cannot find a reason for that.
gke-<MY_PROJECT>-default-pool-4f1a9017 is the new instance template created.
All three logs are direct subsequent logs:
Log1 - ERROR - beta.compute.instanceGroupManagers.insert:
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"status": {
"code": 6,
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
},
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "beta.compute.instanceGroupManagers.insert",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.insert",
"targetSize": "0",
"baseInstanceName": "gke-<MY_PROJECT>-default-pool-4f1a9017"
},
"response": {
"#type": "type.googleapis.com/error",
"error": {
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists",
"errors": [
{
"reason": "alreadyExists",
"domain": "global",
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
}
],
"code": 409
}
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"project_id": "<MY_PROJECT>",
"location": "<MY_ZONE>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "ERROR",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"receiveTimestamp": "<TIMESTAMP>"
}
Log2 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.update",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
},
"response": {
"targetId": "8142394286096021440",
"#type": "type.googleapis.com/operation",
"id": "<OPERATION_ID2>",
"insertTime": "<TIMESTAMP>",
"operationType": "compute.instanceGroupManagers.setInstanceTemplate",
"targetLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"zone": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>",
"name": "<OPERATION_ID>",
"progress": "0",
"selfLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID>",
"status": "RUNNING",
"user": "service-<ID>#container-engine-robot.iam.gserviceaccount.com",
"selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID2>",
"startTime": "<TIMESTAMP>"
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"location": "<MY_ZONE>",
"project_id": "<MY_PROJECT>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"first": true
},
"receiveTimestamp": "<TIMESTAMP>"
}
Log3 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1"
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"project_id": "<MY_PROJECT>",
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>",
"location": "<MY_ZONE>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"last": true
},
"receiveTimestamp": "<TIMESTAMP>"
}

How to add data gateway to SQL connector with ARM template

I'm trying to connect to a database through a data gateway (SQL Server Connector) with ARM templates. But I'm not sure if I miss something because I'm getting connection error with the gateway.
This is what I have so far in my api connection:
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"connections_sql_name": {
"defaultValue": "sql",
"type": "String"
},
"connections_sql_displayName": {
"defaultValue": "displaynameDB",
"type": "String"
},
"server": {
"defaultValue": "SERV01",
"type": "String"
},
"database": {
"defaultValue": "DB01",
"type": "String"
},
"authType": {
"defaultValue": "windows",
"type": "String"
},
"username": {
"defaultValue": "USER01",
"type": "String"
},
"password": {
"defaultValue": "PASS123",
"type": "String"
}
},
"variables": {},
"resources": [
{
"type": "Microsoft.Web/connections",
"apiVersion": "2016-06-01",
"name": "[parameters('connections_sql_name')]",
"location": "northeurope",
"properties": {
"displayName": "[parameters('connections_sql_displayName')]",
"customParameterValues": {},
"parameterValues": {
"server": "[parameters('server')]",
"database": "[parameters('database')]",
"authType": "[parameters('authType')]",
"username": "[parameters('username')]",
"password": "[parameters('password')]"
},
"api": {
"id": "[concat('/subscriptions/{sub-id}/providers/Microsoft.Web/locations/northeurope/managedApis/', parameters('connections_sql_name'))]"
}
}
}
]
}
And this is a part of my logic app under inputs:
"gateway": {
"gatewaySettings": {
"connectionDetails": [
"[parameters('gatewayServer')]",
"[parameters('gatewayDatabase')]"
],
"credentialType": "Windows",
"dataSourceType": "sql"
},
"type": "gatewaySetting"
},
Any help is appreciated! :)
Try removing the gateway block from the Logic App definition and changing the connection definition to this:
{
"type": "Microsoft.Web/connections",
"apiVersion": "2016-06-01",
"name": "[parameters('connections_sql_name')]",
"location": "northeurope",
"properties": {
"displayName": "[parameters('connections_sql_displayName')]",
"customParameterValues": {
},
"parameterValues": {
"server": "[parameters('server')]",
"database": "[parameters('database')]",
"authType": "[parameters('authType')]",
"username": "[parameters('username')]",
"password": "[parameters('password')]",
"gateway": {
"id": "/subscriptions/{sub-id}/resourceGroups/{gateway-resource-group-name}/providers/Microsoft.Web/connectionGateways/{gateway-name}"
}
},
"api": {
"id": "[concat('/subscriptions/{sub-id}/providers/Microsoft.Web/locations/northeurope/managedApis/', parameters('connections_sql_name'))]"
}
}
}
The documentation isn't very helpful regarding the gateway property in connection resources.

Deploying a marketplace Connector from Azure Resource Group template

I'm using the Azure Resource Group project template in Visual studio to deploy two API Apps and a Logic App. I want one of those API Apps to be a Blob Connector from the marketplace. What I need is the uri of the .zip package for the connector, as shown here:
{
"apiVersion": "2014-06-01",
"name": "MSDeploy",
"type": "Extensions",
"dependsOn": [
//........
],
"properties": {
"packageUri": "https://auxmktplceprod.blob.core.windows.net/packages/UmbracoCms.WebPI.7.2.5.zip",
"dbType": "SQL",
(source)
I tried this solution, but that cmdlet is now deprecated. Is there any way to get these URIs?
-Thanks!
I found a way of deploying custom api app with Marketplace apps.
Below is a sample script to just guide you
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"blobConnectorName": {
"type": "string",
"minLength": 1,
"defaultValue" : "mytestblobconnector"
},
"blobStorageAccount": {
"type": "string",
"minLength": 1,
"defaultValue" : "mystorage.blob.core.windows.net"
},
"blobStorageKey": {
"type": "securestring",
"minLength": 1,
"defaultValue" : "storgekey"
},
"blobContainerName": {
"type": "string",
"minLength": 1,
"defaultValue" : "mycontainer"
},
"gatewayName": {
"type": "string",
"minLength": 1,
"defaultValue" : "myblobconnectorgateway"
},
"logicAppName": {
"type": "string",
"minLength": 1,
"defaultValue" : "testinglogicapp"
},
"svcPlanName": {
"type": "string",
"minLength": 1,
"defaultValue" : "myresourcegrpserviceplan"
},
"sku": {
"type": "string",
"defaultValue": "Basic",
"allowedValues": [
"Free",
"Basic",
"Standard",
"Premium"
]
},
"svcPlanSize": {
"defaultValue": "0",
"type": "string",
"allowedValues": [
"0",
"1",
"2"
]
},
"gatewayToApiAppSecret": {
"defaultValue": "0000000000000000000000000000000000000000000000000000000000000000",
"type": "securestring"
}
},
"variables": {
"$packageId": "Microsoft.ApiApp",
"$nugetFeed": "http://apiapps-preview.nuget.org/api/v2/"
},
"resources": [
{
"type": "Microsoft.Web/serverfarms",
"apiVersion": "2015-04-01",
"name": "[parameters('svcPlanName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "AppServicePlan"
},
"properties": {
"name": "[parameters('svcPlanName')]",
"sku": "[parameters('sku')]",
"workerSize": "[parameters('svcPlanSize')]",
"numberOfWorkers": 1
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2015-04-01",
"name": "[parameters('gatewayName')]",
"location": "[resourceGroup().location]",
"kind": "gateway",
"tags": {
"displayName": "GatewayHost"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/gateway",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
}
}
],
"dependsOn": [
"[concat(resourceGroup().id, '/providers/Microsoft.Web/serverfarms/', parameters('svcPlanName'))]"
],
"properties": {
"name": "[parameters('gatewayName')]",
"gatewaySiteName": "[parameters('gatewayName')]",
"serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"siteConfig": {
"appSettings": [
{
"name": "ApiAppsGateway_EXTENSION_VERSION",
"value": "latest"
},
{
"name": "EmaStorage",
"value": "D:\\home\\data\\apiapps"
},
{
"name": "WEBSITE_START_SCM_ON_SITE_CREATION",
"value": "1"
}
]
}
}
},
{
"type": "Microsoft.AppService/gateways",
"apiVersion": "2015-03-01-preview",
"name": "[parameters('gatewayName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "Gateway"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/gatewaySite",
"dependsOn": [
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
}
},
{
"type": "tokens",
"apiVersion": "2015-03-01-preview",
"location": "[resourceGroup().location]",
"name": "[parameters('logicAppName')]",
"tags": {
"displayName": "AuthenticationToken"
},
"dependsOn": [
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
]
}
],
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('gatewayName'))]"
],
"properties": {
"host": {
"resourceName": "[parameters('gatewayName')]"
}
}
},
{
"type": "Microsoft.Web/sites",
"apiVersion": "2015-04-01",
"name": "[parameters('blobConnectorName')]",
"location": "[resourceGroup().location]",
"kind": "apiApp",
"tags": {
"displayName": "APIAppHost",
"packageId": "AzureStorageBlobConnector"
},
"dependsOn": [
"[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"[resourceId('Microsoft.AppService/gateways', parameters('gatewayName'))]"
],
"resources": [
{
"type": "siteextensions",
"tags": {
"displayName": "APIAppExtension"
},
"apiVersion": "2015-02-01",
"name": "AzureStorageBlobConnector",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
],
"properties": {
"type": "WebRoot",
"feed_url": "[variables('$nugetFeed')]"
}
},
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/apiApp",
"dependsOn": [
"[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.AppService/apiapps', parameters('blobConnectorName'))]"
}
}
],
"properties": {
"gatewaySiteName": "[parameters('gatewayName')]",
"serverFarmId": "[resourceId('Microsoft.Web/serverfarms', parameters('svcPlanName'))]",
"siteConfig": {
"appSettings": [
{
"name": "EMA_MicroserviceId",
"value": "[parameters('blobConnectorName')]"
},
{
"name": "EMA_Secret",
"value": "[parameters('gatewayToAPIappSecret')]"
},
{
"name": "EMA_RuntimeUrl",
"value": "[concat('https://', reference(resourceId('Microsoft.Web/sites', parameters('gatewayName'))).hostNames[0])]"
},
{
"name": "WEBSITE_START_SCM_ON_SITE_CREATION",
"value": "1"
},
{
"name": "BlobConnector_ContainerUrl",
"value": "[concat('https://', parameters('blobStorageAccount'),'/',parameters('blobContainerName'))]"
},
{
"name": "BlobConnector_AccessKey",
"value": "[parameters('blobStorageKey')]"
}
],
"applicationLogs": {
"filesystem": {
"level": "Verbose"
},
"azureTableStorage": {
"level": "Off",
"sasUrl": null
},
"azureBlobStorage": {
"level": "Off",
"sasUrl": null,
"retentionInDays": null
}
}
}
}
},
{
"type": "Microsoft.AppService/apiapps",
"apiVersion": "2015-03-01-preview",
"name": "[parameters('blobConnectorName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "BlobConnector"
},
"resources": [
{
"type": "providers/links",
"apiVersion": "2015-01-01",
"name": "Microsoft.Resources/apiAppSite",
"dependsOn": [
"[resourceId('Microsoft.AppService/apiapps', parameters('blobConnectorName'))]"
],
"properties": {
"targetId": "[resourceId('Microsoft.Web/sites', parameters('blobConnectorName'))]"
}
}
],
"dependsOn": [
"[resourceId('Microsoft.Web/sites/siteextensions', parameters('blobConnectorName'), 'AzureStorageBlobConnector')]"
],
"properties": {
"package": {
"id": "AzureStorageBlobConnector"
},
"host": {
"resourceName": "[parameters('blobConnectorName')]"
},
"gateway": {
"resourceName": "[parameters('gatewayName')]"
},
"dependencies": [ ]
}
},
{
"type": "Microsoft.Logic/workflows",
"apiVersion": "2015-02-01-preview",
"name": "[parameters('logicAppName')]",
"location": "[resourceGroup().location]",
"tags": {
"displayName": "LogicApp"
},
"dependsOn": [
"[resourceId('Microsoft.AppService/apiApps', parameters('blobConnectorName'))]"
],
"properties": {
"sku": {
"name": "[parameters('sku')]",
"plan": {
"id": "[concat(resourceGroup().id, '/providers/Microsoft.Web/serverfarms/',parameters('svcPlanName'))]"
}
},
"definition": {
"$schema": "http://schema.management.azure.com/providers/Microsoft.Logic/schemas/2014-12-01-preview/workflowdefinition.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"token": {
"defaultValue": "[reference(resourceId('Microsoft.AppService/gateways/tokens', parameters('gatewayName'), parameters('logicAppName'))).token]",
"type": "String",
"metadata": {
"token": {
"name": "token"
}
}
},
"runworkflowmanually": {
"defaultValue": true,
"type": "Bool"
}
},
"triggers": { },
"actions": {
"azurestorageblobconnector": {
"type": "ApiApp",
"inputs": {
"apiVersion": "2015-01-14",
"host": {
"id": "[concat(resourceGroup().id, '/providers/Microsoft.AppService/apiApps/',parameters('blobConnectorName'))]",
"gateway": "[concat('https://', reference(resourceId('Microsoft.Web/sites', parameters('gatewayName'))).hostNames[0])]"
},
"operation": "UploadBlob",
"parameters": {
"BlobPath": "myfolder/test.txt",
"BlobContent": {
"Content": "TestMessage",
"ContentTransferEncoding": "None"
},
"Overwrite": true
},
"authentication": {
"type": "Raw",
"scheme": "Zumo",
"parameter": "#parameters('token')"
}
},
"conditions": [ ]
}
},
"outputs": {
}
},
"parameters": { }
}
}
]
}
Just search for "AzureStorageBlobConnector" in the above json to observe the usage which is the package id of the blob connector from marketplace. I found the package id for the blob connector from azure portal by deploying one manually and then checking its settings. Please feel free to post a comment for package id for other market place apps if you face any difficulty.

bigquery - Input contained no data

I'm testing bigquery platform with real traffic of my site (more than 80M of events by day).
I'm uploading gz files using java api, using insert jobs.
In some cases, i've receive this message: Input contained no data
{
"kind": "bigquery#job",
"etag": "\"******************\"",
"id": "*********",
"selfLink": "********",
"jobReference": {
"projectId": "********",
"jobId": "**************"
},
"configuration": {
"load": {
"schema": {
"fields": [
{
"name": "tms",
"type": "TIMESTAMP"
},
{
"name": "page",
"type": "STRING"
},
{
"name": "user_agent",
"type": "STRING"
},
{
"name": "print_id",
"type": "STRING"
},
{
"name": "referer",
"type": "STRING"
},
{
"name": "gtms",
"type": "TIMESTAMP"
},
{
"name": "cookies",
"type": "STRING"
},
{
"name": "ip",
"type": "STRING"
},
{
"name": "site",
"type": "STRING"
},
{
"name": "call_params",
"type": "STRING"
},
{
"name": "domains",
"type": "RECORD",
"mode": "REPEATED",
"fields": [
{
"name": "name",
"type": "STRING"
},
{
"name": "ads",
"type": "RECORD",
"mode": "REPEATED",
"fields": [
{
"name": "id",
"type": "STRING"
},
{
"name": "type",
"type": "STRING"
},
{
"name": "position",
"type": "STRING"
},
{
"name": "strategy",
"type": "STRING"
},
{
"name": "score",
"type": "STRING"
},
{
"name": "cpc",
"type": "STRING"
},
{
"name": "site",
"type": "STRING"
},
{
"name": "categ",
"type": "STRING"
},
{
"name": "cust",
"type": "STRING"
},
{
"name": "campaign",
"type": "STRING"
}
]
}
]
}
]
},
"destinationTable": {
"projectId": "**********",
"datasetId": "*******",
"tableId": "********"
},
"createDisposition": "CREATE_IF_NEEDED",
"writeDisposition": "WRITE_APPEND",
"sourceFormat": "NEWLINE_DELIMITED_JSON"
}
},
"status": {
"state": "DONE",
"errors": [
{
"reason": "invalid",
"message": "Input contained no data"
}
]
},
"statistics": {
"creationTime": "1416491042309",
"startTime": "1416491061440",
"endTime": "1416491076876",
"load": {
"inputFiles": "1",
"inputFileBytes": "0",
"outputRows": "0",
"outputBytes": "0"
}
}
}
And then of this, all my jobs return the same response.
Can anybody tell me what is the reason of this behaviour?
Thanks!!!!
Your job succeeded: there is no "errorResult" field in the status.
First, I understand this mistake: the return of errors and warnings in the job api is, frankly, as clear as mud.
Here's the quick overview:
status.errorResult is where job error is reported. If no errorResult is reported, the job succeeded.
status.errors is where individual errors and warnings are reported.
Please reference the documentation https://cloud.google.com/bigquery/docs/reference/v2/jobs and search for status.errorResult and status.errors.
Most people don't hit this problem since a job only encountering a warning is pretty rare.
Ok, the problem was very simple: the gz file.
Thanks!