Snowflake Error parsing JSON: unfinished string, pos <number> - sql

I am trying to query a varchar column called "JsonCode" from the "Weather" table in snowflake. The "JsonCode" column looks like this:
{
"Date": "2019-11-07T12:28:18",
"CurrentTemp": "47°F",
"WeatherIconStatus": "clear-day",
"LowTemp": "21°F",
"HighTemp": "50°F",
"WindSpeed": "6 mph",
"TempCategory": "Hot",
"ForecastData": [
{
"Date": "2019-11-08T00:00:00",
"WeatherIconStatus": "clear-day",
"LowTemp": "26°F",
"HighTemp": "51°F",
"WindSpeed": "3 mph"
},
{
"Date": "2019-11-09T00:00:00",
"WeatherIconStatus": "clear-day",
"LowTemp": "28°F",
"HighTemp": "56°F",
"WindSpeed": "7 mph"
}
],
"PrecipitationReportData": {
"ReportDateTimeAsDate": "2019-11-07T04:45:14",
"PrecipitationConditions": "",
"ForecastText": "Clear",
"NewPrecipitationReadings": {
"Overnight": {
"PrecipitationReading": "0in"
},
"TwentyFourHours": {
"PrecipitationReading": "0in"
},
"FortyEightHours": {
"PrecipitationReading": "0in"
},
"SevenDays": {
"PrecipitationReading": "0in"
}
},
"AreaReadings": {
"City": {
"PrecipitationReading": "0in"
},
"Town": {
"PrecipitationReading": "0in"
},
"Highway": {
"PrecipitationReading": null
}
},
"SeasonPrecipitation": {
"PrecipitationReading": "44in"
}
},
"Links": [
{
"Href": "https://weather.com",
"Rel": "self",
"Method": "GET"
}
]
}
I am specifically trying to grab the "WindSpeed" data from 2019-11-07. I created a query that looks like this.
SELECT value:WindSpeed::varchar FROM "DATABASE"."DBO"."WEATHER"
, lateral flatten(input => PARSE_JSON(JsonCode):windspeed);
The response I received is
Error parsing JSON: unfinished string, pos <number>
I have been unable to find any documentation around this error. Any help is appreciated.
Note: I cut out some of the Json since there is multiple days of forecasted data.

This should work without the need of a lateral flatten:
select parse_json(j):WindSpeed
from data
With the posted sample JSON:
with data as (
select $${
"Date": "2019-11-07T12:28:18",
"CurrentTemp": "47°F",
"WeatherIconStatus": "clear-day",
"LowTemp": "21°F",
"HighTemp": "50°F",
"WindSpeed": "6 mph",
"TempCategory": "Hot",
"ForecastData": [
{
"Date": "2019-11-08T00:00:00",
"WeatherIconStatus": "clear-day",
"LowTemp": "26°F",
"HighTemp": "51°F",
"WindSpeed": "3 mph"
},
{
"Date": "2019-11-09T00:00:00",
"WeatherIconStatus": "clear-day",
"LowTemp": "28°F",
"HighTemp": "56°F",
"WindSpeed": "7 mph"
}
],
"PrecipitationReportData": {
"ReportDateTimeAsDate": "2019-11-07T04:45:14",
"PrecipitationConditions": "",
"ForecastText": "Clear",
"NewPrecipitationReadings": {
"Overnight": {
"PrecipitationReading": "0in"
},
"TwentyFourHours": {
"PrecipitationReading": "0in"
},
"FortyEightHours": {
"PrecipitationReading": "0in"
},
"SevenDays": {
"PrecipitationReading": "0in"
}
},
"AreaReadings": {
"City": {
"PrecipitationReading": "0in"
},
"Town": {
"PrecipitationReading": "0in"
},
"Highway": {
"PrecipitationReading": null
}
},
"SeasonPrecipitation": {
"PrecipitationReading": "44in"
}
},
"Links": [
{
"Href": "https://weather.com",
"Rel": "self",
"Method": "GET"
}
]
}$$ j
)
select parse_json(j):WindSpeed
from data

Related

GKE - Automatic replacement of Instance Group Template

Current Situation: We're having a Kubernetes Cluster in Google (GKE). There is an instance group that refers an instance template defining the VMs.
Problem: Some days ago we had a spontaneous outage. I could find out that our instance group referred a new instance template, that was missing some access rights leading to that outage. Changing the instance template back to the original one solved the issue.
What I'm still wondering: We did neither change the template for the instance group nor did we create the instance template that was used.
I'm the only person managing the cluster, so I'm pretty sure that this happened automatically. How could this happen?
Searching for a reason:
I was researching on that topic but I could not find any related issues. I discovered three logs that might be interesting, but I cannot find a reason for that.
gke-<MY_PROJECT>-default-pool-4f1a9017 is the new instance template created.
All three logs are direct subsequent logs:
Log1 - ERROR - beta.compute.instanceGroupManagers.insert:
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"status": {
"code": 6,
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
},
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "beta.compute.instanceGroupManagers.insert",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.insert",
"targetSize": "0",
"baseInstanceName": "gke-<MY_PROJECT>-default-pool-4f1a9017"
},
"response": {
"#type": "type.googleapis.com/error",
"error": {
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists",
"errors": [
{
"reason": "alreadyExists",
"domain": "global",
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
}
],
"code": 409
}
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"project_id": "<MY_PROJECT>",
"location": "<MY_ZONE>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "ERROR",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"receiveTimestamp": "<TIMESTAMP>"
}
Log2 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.update",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
},
"response": {
"targetId": "8142394286096021440",
"#type": "type.googleapis.com/operation",
"id": "<OPERATION_ID2>",
"insertTime": "<TIMESTAMP>",
"operationType": "compute.instanceGroupManagers.setInstanceTemplate",
"targetLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"zone": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>",
"name": "<OPERATION_ID>",
"progress": "0",
"selfLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID>",
"status": "RUNNING",
"user": "service-<ID>#container-engine-robot.iam.gserviceaccount.com",
"selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID2>",
"startTime": "<TIMESTAMP>"
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"location": "<MY_ZONE>",
"project_id": "<MY_PROJECT>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"first": true
},
"receiveTimestamp": "<TIMESTAMP>"
}
Log3 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1"
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"project_id": "<MY_PROJECT>",
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>",
"location": "<MY_ZONE>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"last": true
},
"receiveTimestamp": "<TIMESTAMP>"
}

Create Delivery Plan styling rules using Azure Devops REST Apis

I am trying to create Delivery plan in the Azure Devops project using Azure Devops REST Apis. I have used following method to create the same.
https://learn.microsoft.com/en-us/rest/api/azure/devops/work/plans/create?view=azure-devops-rest-6.0
POST https://dev.azure.com/{organization}/{project}/_apis/work/plans?api-version=6.0
and I am sending following data in the request body properties
{
"properties": {
"teamBacklogMappings": [
{
"teamId": "09d57738-697f-4433-abdd-b80a2bc6337b",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "5df45eec-4108-474a-8d93-bc09c0b9037e",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "e8ed402b-68e7-4140-96f2-07790a08788b",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "14425694-efa5-454e-811c-e9e03d79198f",
"categoryReferenceName": "Microsoft.RequirementCategory"
},
{
"teamId": "b02690e9-1f48-421a-a918-3b23cc9a1b73",
"categoryReferenceName": "Microsoft.RequirementCategory"
}
],
"cardSettings": {
"fields": {
"showId": true,
"showAssignedTo": true,
"assignedToDisplayFormat": "avatarOnly",
"showState": true,
"showTags": true,
"showParent": false,
"showEmptyFields": true,
"showChildRollup": true,
"additionalFields": null,
"coreFields": [
{
"referenceName": "System.AssignedTo",
"displayName": "Assigned To",
"fieldType": "string",
"isIdentity": true
},
{
"referenceName": "System.Id",
"displayName": "ID",
"fieldType": "integer",
"isIdentity": false
},
{
"referenceName": "System.State",
"displayName": "State",
"fieldType": "string",
"isIdentity": false
},
{
"referenceName": "System.Tags",
"displayName": "Tags",
"fieldType": "plainText",
"isIdentity": false
}
]
}
},
"markers": [],
"styleSettings": [
{
"name": "BLOCKER",
"isEnabled": "True",
"filter": "[System.Tags] CONTAINS 'BLOCKER'",
"clauses": [
{
"fieldName": "System.Tags",
"logicalOperator": "AND",
"operator": "CONTAINS",
"value": "BLOCKER"
}
],
"settings": {
"background-color": "#E60017",
"title-color": "#000000"
}
},
{
"name": "New",
"isEnabled": "True",
"filter": "[System.State] = 'New'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "New"
}
],
"settings": {
"background-color": "#AAAAAA",
"title-color": "#000000"
}
},
{
"name": "Dev Completed",
"isEnabled": "True",
"filter": "[System.State] = 'Development Completed'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Development Completed"
}
],
"settings": {
"background-color": "#D7E587",
"title-color": "#000000"
}
},
{
"name": "Deployed to QA",
"isEnabled": "True",
"filter": "[System.State] = 'Deployed to QA (SIT)'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Deployed to QA (SIT)"
}
],
"settings": {
"background-color": "#C3D84C",
"title-color": "#000000"
}
},
{
"name": "Deployed to UAT",
"isEnabled": "True",
"filter": "[System.State] = 'Deployed to UAT'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Deployed to UAT"
}
],
"settings": {
"background-color": "#60AF49",
"title-color": "#000000"
}
},
{
"name": "Deployed to PROD",
"isEnabled": "True",
"filter": "[System.State] = 'Completed'",
"clauses": [
{
"fieldName": "System.State",
"logicalOperator": "AND",
"operator": "=",
"value": "Completed"
}
],
"settings": {
"background-color": "#00643A",
"title-color": "#000000"
}
}
],
"tagStyleSettings": []
}
}
However Styling rules are not getting created in the project.
Got this done by using Update for the Delivery plan immediately after creating the same. Update it with same properties though you will have to add revision property to the request body.
https://learn.microsoft.com/en-us/rest/api/azure/devops/work/plans/update?view=azure-devops-rest-6.0
https://dev.azure.com/{organization}/{project}/_apis/work/plans/{id}?api-version=6.0
Create Delivery Plan doesn't accept style settings, from the UI you can notice this:
Just get the plan id and the response from the Create API and then use them in the Update API, this is the only way.

Elasticsearch to SQL equivalent, get the first and last documents with filter

I have an issue with elasticsearch documents. So the goal is I need to get the latest data close from index stocks and also the last data close from stocks within a specified date. Here's the example of the SQL equivalent:
SELECT (
SELECT `close` FROM `stocks` WHERE date >= 2022-06-01 ORDER by date DESC LIMIT 1
) as last_close,
(
SELECT `close` FROM `stocks` ORDER by date ASC LIMIT 1
) as latest_close FROM stocks
That's the goal I need to achieve, for the rest I think I don't need to share because the bottleneck is on that issue.
Edited: This is the mappings on my index stocks in elasticsearch:
{
"stocks": { -
"mappings": { -
"properties": { -
"avg": { -
"type": "double"
},
"board": { -
"type": "text"
},
"book": { -
"type": "double"
},
"change": { -
"type": "double"
},
"chg": { -
"type": "double"
},
"close": { -
"type": "double"
},
"date": { -
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"der": { -
"type": "double"
},
"eps": { -
"type": "double"
},
"fve": { -
"type": "double"
},
"fvei": { -
"type": "double"
},
"group": { -
"type": "keyword"
},
"high": { -
"type": "double"
},
"low": { -
"type": "double"
},
"open": { -
"type": "double"
},
"paid_up_cap_shares": { -
"type": "text",
"fields": { -
"keyword": { -
"type": "keyword",
"ignore_above": 256
}
}
},
"pbv": { -
"type": "double"
},
"peg_analysis": { -
"type": "text"
},
"peg_ratio": { -
"type": "double"
},
"per": { -
"type": "double"
},
"prev": { -
"type": "double"
},
"roe": { -
"type": "double"
},
"stock": { -
"type": "keyword"
},
"trade_freq": { -
"type": "double"
},
"trade_val": { -
"type": "double"
},
"trade_vol": { -
"type": "double"
}
}
}
}
}
And here's the document example for the index:
{
"_index": "stocks",
"_id": "6odITIEBRQt2Zq4UUGu3",
"_score": 1.0,
"_source": {
"date": "2022-06-10 13:23:36",
"fvei": "112833.74",
"pbv": "0.0",
"prev": "97",
"book": "-1.185716459E8",
"roe": "-7.87",
"der": "-6.87",
"high": "91",
"avg": "91.0",
"fve": "91.0",
"low": "91",
"stock": "WINR",
"per": "0.0",
"close": "91",
"trade_vol": "46283600",
"group": "IDXPROPERT",
"paid_up_cap_shares": "0.1",
"trade_val": "4211807600",
"chg": "-6.59",
"change": "-6.0",
"peg_ratio": "0.0",
"eps": "9327072.3",
"trade_freq": "1433",
"peg_analysis": "negative growth",
"board": "RG",
"open": "91"
}
},
{
"_index": "stocks",
"_id": "7IdITIEBRQt2Zq4UUGu3",
"_score": 1.0,
"_source": {
"date": "2022-06-10 13:23:36",
"fvei": "66215.12",
"pbv": "0.0",
"prev": "685",
"book": "1946574.58",
"roe": "22.08",
"der": "3.62",
"high": "685",
"avg": "677.5",
"fve": "680.0",
"low": "670",
"stock": "TLDN",
"per": "0.0",
"close": "680",
"trade_vol": "577600",
"group": "IDXNONCYC",
"paid_up_cap_shares": "540.65",
"trade_val": "393001500",
"chg": "-0.73",
"change": "-5.0",
"peg_ratio": "0.0",
"eps": "429847.27",
"trade_freq": "117",
"peg_analysis": "negative growth",
"board": "RG",
"open": "685"
}
},
I have found the answer by using top_hits:
{
"query": {
"bool": {
"must": [
{
// range filter
"range": {
"date": {
"gte": "now-89d",
"lte": "now+1d"
}
}
}
]
}
},
"aggs": {
"group": {
"terms": {
"field": "stock"
},
"aggs": {
"last_doc": {
"top_hits": {
"size": 1,
"sort": [
{
"date": {
"order": "desc"
}
}
],
"_source": {
"includes": ["close"]
}
}
},
"first_doc": {
"top_hits": {
"size": 1,
"sort": [
{
"date": {
"order": "asc"
}
}
],
"_source": {
"includes": ["close"]
}
}
}
}
}
}
}

Remove Object Using Conditional Filtering - Mule 3

I need to do a conditional filtering using Mule 3. A sample payload is given below:
{
"MainNode": {
"ID": "123",
"MainNodekey1": "1",
"sysDetail": [
{
"sysDetail11": "localhost1",
"sysDetail21": "443",
"country": [
{
"k1": "country",
"value": [
"IN"
]
}
]
},
{
"sysDetail21": "localhost2",
"sysDetail22": "443",
"country": [
{
"k2": "country",
"value": [
"RSA",
"UK",
"SL"
]
}
]
}
],
"sysDetai2": {},
"sysDetai3": {}
},
"MainCode": "AAA",
"MainTyoe": "BBB",
"MyArray": [
{
"ObjId": "100",
"Obj1": {
"SubObj11": {},
"SubObj12": {},
"ObjCountry": "IN",
"ObjId": "A100"
},
"Obj1Source": {}
},
{
"ObjId": "200",
"Obj2": {
"SubObj21": {},
"SubObj22": {},
"ObjCountry": "IN",
"ObjId": "B100"
},
"Obj2Source": {}
}
]
}
The output that I need is to filter the main payload by applying a filter like below:
payload.MainNode.sysDetail.country..value == payload.Obj1.ObjCountry (which will be constant for all array elements)
Expected output is below:
{
"MainNode": {
"ID": "123",
"MainNodekey1": "1",
"sysDetail": [
{
"sysDetail11": "localhost1",
"sysDetail21": "443",
"country": [
{
"k1": "country",
"value": [
"IN"
]
}
]
}
],
"sysDetai2": {},
"sysDetai3": {}
},
"MainCode": "AAA",
"MainTyoe": "BBB",
"MyArray": [
{
"ObjId": "100",
"Obj1": {
"SubObj11": {},
"SubObj12": {},
"ObjCountry": "IN",
"ObjId": "A100"
},
"Obj1Source": {}
},
{
"ObjId": "200",
"Obj2": {
"SubObj21": {},
"SubObj22": {},
"ObjCountry": "IN",
"ObjId": "B100"
},
"Obj2Source": {}
}
]
}

Azure Data Factory Pipeline errorCode

I have a relatively simple process set up in Azure Data Factory to copy, cleanse and process some log files from a chatbot which has been running fine until I recently started getting the following errorCode:
"errorCode": "InvalidTemplate",
"message": "Unable to process expressions for action 'EvaluatefinaliseTSCRPTS': 'The function 'bool' was invoked with a parameter that is not valid. The value cannot be converted to the target type",
"failureType": "UserError",
"target": "finaliseTSCRPTS",
"details": ""
I can't seem to identify the error in the ADF process despite going through the code for my process below:
"name": "SearchBot dailyTranscripts",
"properties": {
"activities": [
{
"name": "MST Validation",
"type": "Validation",
"dependsOn": [],
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"timeout": "0.00:00:30",
"sleep": 10,
"childItems": true
}
},
{
"name": "Get MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false,
"translator": {
"type": "TabularTranslator",
"mappings": [
{
"source": {
"path": "$['type']"
},
"sink": {
"path": "$['type']"
}
},
{
"source": {
"path": "$['timestamp']"
},
"sink": {
"path": "$['timestamp']"
}
},
{
"source": {
"path": "$['id']"
},
"sink": {
"path": "$['id']"
}
},
{
"source": {
"path": "$['channelId']"
},
"sink": {
"path": "$['channelId']"
}
},
{
"source": {
"path": "$['serviceUrl']"
},
"sink": {
"path": "$['serviceUrl']"
}
},
{
"source": {
"path": "$['from']['id']"
},
"sink": {
"path": "$['from']['id']"
}
},
{
"source": {
"path": "$['from']['aadObjectId']"
},
"sink": {
"path": "$['from']['aadObjectId']"
}
},
{
"source": {
"path": "$['from']['role']"
},
"sink": {
"path": "$['from']['role']"
}
},
{
"source": {
"path": "$['from']['name']"
},
"sink": {
"path": "$['from']['name']"
}
},
{
"source": {
"path": "$['conversation']['conversationType']"
},
"sink": {
"path": "$['conversation']['conversationType']"
}
},
{
"source": {
"path": "$['conversation']['tenantId']"
},
"sink": {
"path": "$['conversation']['tenantId']"
}
},
{
"source": {
"path": "$['conversation']['id']"
},
"sink": {
"path": "$['conversation']['id']"
}
},
{
"source": {
"path": "$['recipient']['id']"
},
"sink": {
"path": "$['recipient']['id']"
}
},
{
"source": {
"path": "$['recipient']['name']"
},
"sink": {
"path": "$['recipient']['name']"
}
},
{
"source": {
"path": "$['recipient']['aadObjectId']"
},
"sink": {
"path": "$['recipient']['aadObjectId']"
}
},
{
"source": {
"path": "$['recipient']['role']"
},
"sink": {
"path": "$['recipient']['role']"
}
},
{
"source": {
"path": "$['channelData']['tenant']['id']"
},
"sink": {
"path": "$['channelData']['tenant']['id']"
}
},
{
"source": {
"path": "$['text']"
},
"sink": {
"path": "$['text']"
}
},
{
"source": {
"path": "$['inputHint']"
},
"sink": {
"path": "$['inputHint']"
}
},
{
"source": {
"path": "$['replyToId']"
},
"sink": {
"path": "$['replyToId']"
}
},
{
"source": {
"path": "$['textFormat']"
},
"sink": {
"path": "$['textFormat']"
}
},
{
"source": {
"path": "$['localTimestamp']"
},
"sink": {
"path": "$['localTimestamp']"
}
},
{
"source": {
"path": "$['locale']"
},
"sink": {
"path": "$['locale']"
}
},
{
"source": {
"path": "$['value']"
},
"sink": {
"path": "$['value']"
}
},
{
"source": {
"path": "$['valueType']"
},
"sink": {
"path": "$['valueType']"
}
},
{
"source": {
"path": "$['name']"
},
"sink": {
"path": "$['name']"
}
},
{
"source": {
"path": "$['label']"
},
"sink": {
"path": "$['label']"
}
}
]
}
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
]
},
{
"name": "finaliseTSCRPTS",
"type": "IfCondition",
"dependsOn": [
{
"activity": "MST Validation",
"dependencyConditions": [
"Completed"
]
},
{
"activity": "Get MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"expression": {
"value": "activity('MST Validation').output.exists",
"type": "Expression"
},
"ifTrueActivities": [
{
"name": "Combine TSCRPTS",
"type": "Copy",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat(formatDateTime(utcnow(), 'yyyy-MM-dd'),'.json')"
}
}
]
},
{
"name": "Delete Staging",
"type": "Delete",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "transcriptsStaging",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "Get monthlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Combine TSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotDailyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#concat('2019-',formatDateTime(utcnow(), 'MM'),'-??.json')"
}
}
],
"outputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Get yearlyTSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Get monthlyTSCRPTS",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "SearchBotMonthlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": {
"value": "#concat(formatDateTime(utcnow(), 'yyyy'),'-??.json')",
"type": "Expression"
}
}
}
],
"outputs": [
{
"referenceName": "SearchBotYearlyTranscripts",
"type": "DatasetReference",
"parameters": {
"sourceFileName": "#dataset().sourceFileName"
}
}
]
},
{
"name": "Copy MST-TSCRPTS",
"type": "Copy",
"dependsOn": [
{
"activity": "Delete Staging",
"dependencyConditions": [
"Succeeded"
]
}
],
"policy": {
"timeout": "0.00:01:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"source": {
"type": "JsonSource",
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true,
"wildcardFileName": "*.json",
"enablePartitionDiscovery": false
}
},
"sink": {
"type": "JsonSink",
"storeSettings": {
"type": "AzureBlobStorageWriteSettings",
"copyBehavior": "MergeFiles"
},
"formatSettings": {
"type": "JsonWriteSettings",
"quoteAllText": true
}
},
"enableStaging": false
},
"inputs": [
{
"referenceName": "teamsLogs",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "transcriptsHistory",
"type": "DatasetReference"
}
]
},
{
"name": "Delete MST-TSCRPTS",
"type": "Delete",
"dependsOn": [
{
"activity": "Copy MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed"
]
}
],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"dataset": {
"referenceName": "teamsLogs",
"type": "DatasetReference"
},
"enableLogging": false,
"storeSettings": {
"type": "AzureBlobStorageReadSettings",
"recursive": true
}
}
},
{
"name": "run learningList",
"type": "ExecutePipeline",
"dependsOn": [
{
"activity": "Delete MST-TSCRPTS",
"dependencyConditions": [
"Succeeded",
"Failed",
"Skipped"
]
}
],
"userProperties": [],
"typeProperties": {
"pipeline": {
"referenceName": "runLearningList",
"type": "PipelineReference"
},
"waitOnCompletion": true
}
}
]
}
}
],
"parameters": {
"sourceFileName": {
"type": "string",
"defaultValue": "#concat(formatDateTime(utcnow(),'yyyy-MM-dd'),'.json')"
}
},
"annotations": []
},
"type": "Microsoft.DataFactory/factories/pipelines"
There is no expression 'EvaluatefinaliseTSCRPTS' and I can't find a 'bool' function either. The only hint I've found was a previous question here
You have an IF activity named "finaliseTSCRPTS", so "EvaluatefinaliseTSCRPTS" is most likely the internal name of the function that performs the IF condition check. The message seems to indicate that it cannot evaluate your expression "activity('MST Validation').output.exists".