MuleSoft DataWeave to roll up joined query results - mule

I have returned some data from a database call which joins several tables together. The current payload looks something like this:
[
{ "ID": 11111
, "Ref": "123ABC"
, "Year": 1994
, "Title": "A title"
, "Author": "Joe Bloggs"
},
{ "ID": 11111
, "Ref": "123ABC"
, "Year": 1994
, "Title": "A title"
, "Author": "John Smith"
},
{ "ID": 22222
, "Ref": "456DEF"
, "Year": 2018
, "Title": "Another title"
, "Author": "Lucy Jones"
}
]
Can anyone help with the DataWeave required to translate it to this:
[
{ "ID": 11111
, "Ref": "123ABC"
, "Year": 1994
, "Title": "A title"
, "Authors": [ {"Name": "Joe Bloggs"}, {"Name": "John Smith"} ]
},
{ "ID": 22222
, "Ref": "456DEF"
, "Year": 2018
, "Title": "Another title"
, "Authors": [ {"Name": "Lucy Jones"} ]
}
]
I've tried playing around with distinctBy and filter, but have only managed to get this so far:
%dw 2.0
output application/json
var myVar = payload distinctBy () ->
{ ID: $.ID
, Ref: $.Ref
, Year: $.Year
, Title: $.Title
}
---
myVar map () -> using (thisID = myVar.ID)
{ ID: $.ID
, Ref: $.Ref
, Year: $.Year
, Title: $.Title
, Authors: payload map {Name: $.Author}
//, Authors: payload filter ($.*ID contains thisID) map {Name: $.Author}
}
Result:
[
{
"ID": 11111,
"Ref": "123ABC",
"Year": 1994,
"Title": "A title",
"Authors": [
{
"Name": "Joe Bloggs"
},
{
"Name": "John Smith"
},
{
"Name": "Lucy Jones"
}
]
},
{
"ID": 22222,
"Ref": "456DEF",
"Year": 2018,
"Title": "Another title",
"Authors": [
{
"Name": "Joe Bloggs"
},
{
"Name": "John Smith"
},
{
"Name": "Lucy Jones"
}
]
}
]

Assuming fields other than the Author will be same for a particular ID, you can use groupBy to make groups for each ID.
Then map the grouped values by updating the Author to be equal to the combined value of all the Author in the group.
%dw 2.0
output application/json
---
payload groupBy $.ID
pluck ((groupedValues) -> {
(groupedValues[0] - "Author"),
Author: groupedValues map {Name: $.Author}
})
Also note that using keyword is deprecated in Mule 4, so avoid using it. You should instead use the do block for it

Related

select node value from json column type

A table I called raw_data with three columns: ID, timestamp, payload, the column paylod is a json type having values such as:
{
"data": {
"author_id": "1461871206425108480",
"created_at": "2022-08-17T23:19:14.000Z",
"geo": {
"coordinates": {
"type": "Point",
"coordinates": [
-0.1094,
51.5141
]
},
"place_id": "3eb2c704fe8a50cb"
},
"id": "1560043605762392066",
"text": " ALWAYS # London, United Kingdom"
},
"matching_rules": [
{
"id": "1560042248007458817",
"tag": "london-paris"
}
]
}
From this I want to select rows where the coordinates is available, such as [-0.1094,51.5141]in this case.
SELECT *
FROM raw_data, json_each(payload)
WHERE json_extract(json_each.value, '$.data.geo.') IS NOT NULL
LIMIT 20;
Nothing was returned.
EDIT
NOT ALL json objects have the coordinates node. For example this value:
{
"data": {
"author_id": "1556031969062010881",
"created_at": "2022-08-18T01:42:21.000Z",
"geo": {
"place_id": "006c6743642cb09c"
},
"id": "1560079621017796609",
"text": "Dear Desperate sister say husband no dey oo."
},
"matching_rules": [
{
"id": "1560077018183630848",
"tag": "kaduna-kano-katsina-dutse-zaria"
}
]
}
The correct path is '$.data.geo.coordinates.coordinates' and there is no need for json_each():
SELECT *
FROM raw_data
WHERE json_extract(payload, '$.data.geo.coordinates.coordinates') IS NOT NULL;
See the demo.

DataWeave unflatten mapping

Need some help in writing DataWeave for this Salesforce payload.
Please help with creating a DataWeave for this input and expected output.
INPUT
[
{
"Order": {
"Account_Name__c": "test danish 002",
"Id": "8018s000004mpqjAAA",
"type": "Order"
},
"UnitPrice": "13.45",
"Product2": {
"Id": null,
"type": "Product2",
"Name": "* 1 LEG FOR MESAHORNO STAND110"
},
"Id": "8028s000000kyUDAAY",
"type": "OrderItem",
"OrderItemNumber": "0000051590"
},
{
"Order": {
"Account_Name__c": "test danish 002",
"Id": "8018s000004mpqjAAA",
"type": "Order"
},
"UnitPrice": "308.45",
"Product2": {
"Id": null,
"type": "Product2",
"Name": "* 2 Uprights incl castors f RTWZ 600 Var"
},
"Id": "8028s000000kyUEAAY",
"type": "OrderItem",
"OrderItemNumber": "0000051591"
},
{
"Order": {
"Account_Name__c": "test danish 002",
"Id": "8018s000004mpqjAAA",
"type": "Order"
},
"UnitPrice": "31.0",
"Product2": {
"Id": null,
"type": "Product2",
"Name": "* 25 Comp Glass Rack - 3 Ext - Red Base"
},
"Id": "8028s000000kyUFAAY",
"type": "OrderItem",
"OrderItemNumber": "0000051592"
},
{
"Order": {
"Account_Name__c": "test danish 002",
"Id": "8018s000004mpqjAAA",
"type": "Order"
},
"UnitPrice": "31.65",
"Product2": {
"Id": null,
"type": "Product2",
"Name": "* 36 Comp Glass Rack 3 Ext - Gray"
},
"Id": "8028s000000kyUGAAY",
"type": "OrderItem",
"OrderItemNumber": "0000051593"
}
]
This is the Salesforce Query on OrderLineItems but the transformed payload needs to be done on Order Object
and the output would look like
Output
{
"Id":"8018s000004mpqjAAA",
"Order Name": "test danish 002",
"Line Items": [
{
"Id":"8028s000000kyUDAAY",
"ItemNumber":"0000051590",
"Name":"* 1 LEG FOR MESAHORNO STAND110"
},
{
"Id":"8028s000000kyUEAAY",
"ItemNumber":"0000051591",
"Name":"* 2 Uprights incl castors f RTWZ 600 Var"
},
{
"Id":"8028s000000kyUFAAY",
"ItemNumber":"0000051592",
"Name":"* 25 Comp Glass Rack - 3 Ext - Red Base"
},
{
"Id":"8028s000000kyUGAAY",
"ItemNumber":"0000051593",
"Name":"* 36 Comp Glass Rack 3 Ext - Gray"
}
]
}
Try like this:
%dw 2.0
output application/json
---
{
"Id" : payload[0].Order.Id,
"Order Name": payload[0].Order.Account_Name__c,
"Line Items" : payload map{
Id: $.Id,
ItemNumber : $.OrderItemNumber,
"Line Items" : $.Product2.Name
}
}
Assuming that all the order IDs are same you can just map the Id directly from payload and use the map on payload to get the Line Items
%dw 2.0
output application/json
---
{
Id: payload[0].Order.Id,
"Order Name": payload[0].Order.Account_Name__c,
"Line Items": payload map {
Id: $.Id,
ItemNumber: $.OrderItemNumber,
Name: $.Product2.Name
}
}
Here is one more option that you can try when ordered Id's are not the same
%dw 2.0
output application/json
---
payload groupBy ((item, index) -> item.Order.Id)
mapObject ((value, key, index) ->
{"Id": value.Order.Id[0],
"Order Name": value.Order.Account_Name__c[0],
"Line Items": value map ({
Id: $.Id,
ItemNumber: $.OrderItemNumber,
Name: $.Product2.Name
})}
)

Querying/flattening multi-level nested JSON in Cosmos DB when array name is random on one level deep down

I would like to get some help to write a query to get a flat JSON to feed Power BI.
The data store is a Cosmos DB container with JSON. The structure is fixed.
The example JSON stucture is this:
"orderid": "999888777",
"name": "dwayne.johnson",
"mainData": {
"giftData": {
"name": "Rocketship",
"giftDetails": {
"**123456789**": {
"boxInfo": [
{
"name": "Leonardo Da Vinci Big Box",
"city": "London",
"country": "United Kingdom",
"currency": "EUR",
"price": 99.43,
}
],
"colorScheme": [
{
"name": "Red Apple Delight",
"code": "123AAA",
}
],
"CourierInfo": [
{
"name": "UPS",
"code": "BBBA333",
}
],
"orderid": "444333434",
"name": "chris.rock",
"mainData": {
"giftData": {
"name": "Junglemaster XXL",
"giftDetails": {
"**32432423424**": {
"boxInfo": [
{
"name": "Gerba Tool Big O",
"city": "München",
"country": "Germany",
"currency": "EUR",
"price": 103.22,
}
],
"colorScheme": [
{
"name": "Green Mix",
"code": "999DDD",
}
],
"CourierInfo": [
{
"name": "GLS",
"code": "CCCGV3",
}
],
I would like to get to following output:
orderid
currency*
price**
999888777
EUR
99.43
444333434
EUR
103.22
*The path would be giftData.giftDetails.123456789 or 32432423424 or any random.boxInfo.currency
**The path would be giftData.giftDetails.123456789 or 32432423424 or any random.boxInfo.price
The question is how can I select the currency and price info 4 levels deep, when the container array name is a random number (in bold)?
Thank you very much for your time and help!

Select data from Json array MS SQL Server

I have to select data from Json like this:
[
{
"id": 10100,
"externalId": "100000035",
"name": "Test1",
"companyId": 10099,
"phone": "0738003811",
"email": "test#Test.com",
"mainAddress": {
"county": "UK",
"province": "test",
"zipCode": "01234",
"city": "test",
"street": "test",
"gln": "44,37489331;26,21941193",
"country": {
"iso2": "UK",
"iso3": "UK"
}
},
"active": false,
"main": true,
"stores": [
"Test"
],
"attributes": [
{
"attributeId": 1059,
"attributeName": "CH6 name",
"attributeExternalId": null,
"attributeValueId": 74292,
"attributeValueType": "MONO_LINGUAL",
"attributeValueEid": null,
"attributePlainValue": "Unknown"
},
{
"attributeId": 1061,
"attributeName": "BD",
"attributeExternalId": null,
"attributeValueId": 81720,
"attributeValueType": "MONO_LINGUAL",
"attributeValueEid": null,
"attributePlainValue": "Not assigned"
}
],
"daysSinceLastOrder": null
},
{
"id": 62606,
"externalId": "VL_LC_000190",
"name": "Test",
"companyId": 17793,
"phone": "44333424",
"email": "test#email.com",
"mainAddress": {
"firmName": "test",
"county": "test",
"province": "test",
"zipCode": "247555",
"city": "test",
"street": "test",
"gln": "44.8773851;23.9223518",
"country": {
"iso2": "RO",
"iso3": "ROU"
},
"phone": "07547063789"
},
"active": true,
"main": false,
"stores": [
"Valcea"
],
"attributes": [
{
"attributeId": 1042,
"attributeName": "Type of location",
"attributeExternalId": "TYPE_OF_DIVISION",
"attributeValueId": 34506,
"attributeValueType": "MONO_LINGUAL",
"attributeValueEid": "Small OTC (<40mp)",
"attributePlainValue": "Small OTC (<40mp)"
},
{
"attributeId": 17,
"attributeName": "Limit for payment",
"attributeExternalId": "LIMIT_FOR_PAYMENT_IN_DAYS",
"attributeValueId": 59120,
"attributeValueType": "NUMBER",
"attributeValueEid": null,
"attributePlainValue": "28"
}
],
"daysSinceLastOrder": 147
}
]
I know how to select data from simple json object using "FROM OPENJSON",
but now I have to select a
AttributeValueId, AttributeId and AttributeName, attributePlainValue and CompanyId for each Attribute. So I dont know how to select data from attributes array and then how to join to this CompanyId which is one level up.
Maybe someone knows how write this query.
As mentioned by #lptr in the comments:
You need to pass the result of one OPENJSON to another, using CROSS APPLY. You can select a whole JSON object or array as a property, by using the syntax AS JSON
select
t1.companyid,
t2.*
from openjson(#j)
with (
companyId int,
attributes nvarchar(max) as json
) as t1
cross apply openjson(t1.attributes)
with
(
attributeId int,
attributeName nvarchar(100),
attributeValueId nvarchar(100),
attributePlainValue nvarchar(100)
) as t2;
db<>fiddle
For example, you can use code like this.
f1.metaData->"$.identity.customerID" = '.$customerID.'

Transform JSON response with lodash

I'm new in lodash (v3.10.1), and having a hard time understanding.
Hope someone can help.
I have an input something like this:
{
{"id":1,"name":"Matthew","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":2,"name":"Mark","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":3,"name":"Luke","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":4,"name":"John","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}},
{"id":5,"name":"Paul","company":{"id":1,"name":"abc","industry":{"id":5,"name":"Medical"}}}
];
I would like to output this or close to this:
{
"industries": [
{
"industry":{
"id":5,
"name":"Medical",
"companies": [
{
"company":{
"id":1,
"name":"abc",
"employees": [
{"id":1,"name":"Matthew"},
{"id":2,"name":"Mark"},
{"id":3,"name":"Luke"},
{"id":4,"name":"John"},
{"id":5,"name":"Paul"}
]
}
}
]
}
}
]
}
Here's something that gets you close to what you want. I structured the output to be an object instead of an array. You don't need the industries or industry properties in your example output. The output structure looks like this:
{
"industry name": {
"id": "id of industry",
"companies": [
{
"company name": "name of company",
"id": "id of company",
"employees": [
{
"id": "id of company",
"name": "name of employee"
}
]
}
]
}
}
I use the _.chain function to wrap the collection with a lodash wrapper object. This enables me to explicitly chain lodash functions.
From there, I use the _.groupBy function to group elements of the collection by their industry name. Since I'm chaining, I don't have to pass in the array again to the function. It's implicitly passed via the lodash wrapper. The second argument of the _.groupBy is the path to the value I want to group elements by. In this case, it's the path to the industry name: company.industry.name. _.groupBy returns an object with each employee grouped by their industry (industries are keys for this object).
I then do use _.transform to transform each industry object. _.transform is essentially _.reduce except that the results returned from the _.transform function is always an object.
The function passed to the _.transform function gets executed against each key/value pair in the object. In the function, I use _.groupBy again to group employees by company. Based off the results of _.groupBy, I map the values to the final structure I want for each employee object.
I then call the _.value function because I want to unwrap the output collection from the lodash wrapper object.
I hope this made sense. If it doesn't, I highly recommend reading Lo-Dash Essentials. After reading the book, I finally got why lodash is so useful.
"use strict";
var _ = require('lodash');
var emps = [
{ "id": 1, "name": "Matthew", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 2, "name": "Mark", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 3, "name": "Luke", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 4, "name": "John", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } },
{ "id": 5, "name": "Paul", "company": { "id": 1, "name": "abc", "industry": { "id": 5, "name": "Medical" } } }
];
var result = _.chain(emps)
.groupBy("company.industry.name")
.transform(function(result, employees, industry) {
result[industry] = {};
result[industry].id = _.get(employees[0], "company.industry.id");
result[ industry ][ 'companies' ] = _.map(_.groupBy(employees, "company.name"), function( employees, company ) {
return {
company: company,
id: _.get(employees[ 0 ], 'company.id'),
employees: _.map(employees, _.partialRight(_.pick, [ 'id', 'name' ]))
};
});
return result;
})
.value();
Results from your example are as follows:
{
"Medical": {
"id": 5,
"companies": [
{
"company": "abc",
"id": 1,
"employees": [
{
"id": 1,
"name": "Matthew"
},
{
"id": 2,
"name": "Mark"
},
{
"id": 3,
"name": "Luke"
},
{
"id": 4,
"name": "John"
},
{
"id": 5,
"name": "Paul"
}
]
}
]
}
}
If you ever wanted the exact same structure as in the questions, I solved it using the jsonata library:
(
/* lets flatten it out for ease of accessing the properties*/
$step1 := $ ~> | $ |
{
"employee_id": id,
"employee_name": name,
"company_id": company.id,
"company_name": company.name,
"industry_id": company.industry.id,
"industry_name": company.industry.name
},
["company", "id", "name"] |;
/* now the magic begins*/
$step2 := {
"industries":
[($step1{
"industry" & $string(industry_id): ${
"id": $distinct(industry_id)#$I,
"name": $distinct(industry_name),
"companies": [({
"company" & $string(company_id): {
"id": $distinct(company_id),
"name": $distinct(company_name),
"employees": [$.{
"id": $distinct(employee_id),
"name": $distinct(employee_name)
}]
}
} ~> $each(function($v){ {"company": $v} }))]
}
} ~> $each(function($v){ {"industry": $v} }))]
};
)
You can see it in action on the live demo site: https://try.jsonata.org/VvW4uTRz_