Grouping some result sql fields into a common object - sql

I want to understand if it is possible using the sqlalchemy core syntax to get the given data in the same format as with pure SQL.
DB relations
events = Table(
"events",
metadata,
Column("id", Integer(), primary_key=True),
......
Column("location_id", ForeignKey(locations.c.id, ondelete="SET NULL"), nullable=False),
Column("activities_id", ForeignKey(activities.c.id, ondelete="SET NULL"), nullable=False),
)
locations = Table(
"locations",
metadata,
Column("id", Integer(), primary_key=True),
....
....
)
activities = Table(
"activities",
metadata,
Column("id", Integer(), primary_key=True),
....
)
I want to group the fields from the tables locations and activities into a separate group. In plain SQL this query runs like this (using Postgresql)
SELECT json_build_object('id', e.id, 'title', e.title, 'creator', json_agg(c), " \
'activitie', json_agg(a), 'users', jsonb_agg(u), 'location', jsonb_agg(l)) " \
AS event FROM events AS e " \
LEFT JOIN event_users AS eu ON e.id = eu.events_id " \
LEFT JOIN users AS u ON eu.users_id = u.id " \
LEFT JOIN users AS c ON e.creator = c.id " \
LEFT JOIN activities AS a ON e.activities_id = a.id " \
LEFT JOIN locations AS l ON e.location_id = l.id " \
WHERE e.id = {_id} GROUP BY e.id
And the result is
{'event':
'{
"id" : 1, "title" : "Test",
"creator" : [{"id":1,"created_at":"2021-03-03T23:39:23.469751+03:00","email":"test#email","phone":"232323","hashed_password":"sdsdsds","is_active":true}],
"activitie" : [{"id":1,"name":"basketball","is_active":true}],
"users" : [null],
"location" : [{"id": 3, "lat": 54.49142965, "city": "Berlin", "long": 26.9173560217231, "street": "Stephans", "building": "12"}]
}'
}
As you can see each Foreignkey field is grouped under a common key.
Now I'm trying to do something similar on the sqlalchemy core (without the user table)
query = (
select(
[
events.c.id,
events.c.title,
locations.c.city,
locations.c.street,
locations.c.building,
activities.c.name,
]
)
.select_from(
events.join(locations).join(activities)
)
.where(
and_(
events.c.id == pk,
locations.c.id == events.c.location_id,
activities.c.id == events.c.activities_id)
)
.order_by(desc(events.c.created_at))
)
print(query)
ev = dict(await database.fetch_one(query))
And get the result
{'id': 1, 'title': 'Test', 'city':Berlin', street': 'Stephans', 'building': '12', 'name': 'basketball'}
How to group result like ?
{
'id': 1,
'title': 'Test',
'location': [
'city': 'Berlin'
'street': 'Stephans',
'building': '12',
],
'activity': [
'name': 'basketball'
]
}
p.s. sql query with #van's code
SELECT json_build_object(:json_build_object_2, events.id, :json_build_object_3, events.title, :json_build_object_4, json_agg(json_build_object(:json_build_object_5, locations.city, :json_build_object_6, locations.street, :json_build_object_7, locations.building)), :json_build_object_8, json_agg(json_build_object(:json_build_object_9, locations.id, :json_build_object_10, locations.lat, :json_build_object_11, locations.long, :json_build_object_12, locations.city, :json_build_object_13, locations.street, :json_build_object_14, locations.building)), :json_build_object_15, json_agg(json_build_object(:json_build_object_16, activities.name))) AS json_build_object_1

Below query should do the job:
from sqlalchemy import func
# ...
query = (
select(
[
func.json_build_object(
"id",
events.c.id,
"title",
events.c.title,
"location",
func.json_agg(
func.json_build_object(
"city",
locations.c.city,
"street",
locations.c.street,
"building",
locations.c.building,
)
),
"location_all_columns_example",
func.json_agg(func.json_build_object(
*itertools.chain(*[(_.name, _) for _ in locations.c])
)),
"activity",
func.json_agg(
func.json_build_object(
"name",
activities.c.name,
)
),
)
]
)
.select_from(events.join(locations).join(activities))
.where(
and_(
events.c.id == pk,
locations.c.id == events.c.location_id,
activities.c.id == events.c.activities_id,
)
)
.order_by(desc(events.c.created_at))
.group_by(events.c.id) # !!! <- IMPORTANT
)
Please note that you need the group_by clause.

Related

how can I make result of ARRAY_AGG() function json parsable

I have a query that selects the rows from joined table as an array using ARRAY_AGG() function.
select
entity_number,
ARRAY_AGG('{"property_id":"'||property_id||'","value":"'||value||'"}') entity_properties from entities
join entity_properties
on entities.id = entity_properties.entity_id
where entities.id in (
select entity_id from entity_properties
where value = '6258006d824a25dabdb39a79.pdf'
)
group by entities.id;
what I get is:
[
{
"entity_number":"P1718238009-1",
"entity_properties":"[
\"{\"property_id\":\"006109cd-a100-437c-a683-f13413b448e6\",\"value\":\"Rozilik berildi\"}\",
\"{\"property_id\":\"010f5e23-d66f-4414-b54b-9647afc6762b\",\"value\":\"6258006d824a25dabdb39a79.pdf\"}\",
\"{\"property_id\":\"0a01904e-1ca0-40ef-bbe1-c90eaddea3fc\",\"value\":\"6260c9e9b06e4c2cc492c470_2634467.pdf\"}\"
]"
}
]
As you can see, it is not json parsable
To parse entity_properties as array of objects I need the data in this format
[
{
"entity_number":"P1718238009-1",
"entity_properties":[
{"property_id":"006109cd-a100-437c-a683-f13413b448e6","value":"Rozilik berildi"},
{"property_id":"010f5e23-d66f-4414-b54b-9647afc6762b","value":"6258006d824a25dabdb39a79.pdf"},
{"property_id":"0a01904e-1ca0-40ef-bbe1-c90eaddea3fc","value":"6260c9e9b06e4c2cc492c470_2634467.pdf"}
]
}
]
Can I achieve what I want with ARRAY_AGG()? How?
If not, what approach should I take?
Try using json_agg and json_build_object function
like this:
select
entity_number,
json_agg(json_build_object('property_id', property_id, 'value', value)) entity_properties from entities
join entity_properties
on entities.id = entity_properties.entity_id
where entities.id in (
select entity_id from entity_properties
where value = '6258006d824a25dabdb39a79.pdf'
)
group by entities.id;
Using a simplified sample data this query provides the first step of the aggregation
with tab as (
select * from (values
(1,'a','x'),
(1,'b','y'),
(2,'c','z')
) tab(entity_number,property_id,value)
)
select
entity_number,
json_agg( json_build_object('property_id', property_id, 'value', value)) entity_properties
from tab
group by 1
;
entity_number|entity_properties |
-------------+----------------------------------------------------------------------------+
1|[{"property_id" : "a", "value" : "x"}, {"property_id" : "b", "value" : "y"}]|
2|[{"property_id" : "c", "value" : "z"}]
Additional aggregation returns the final json array
with tab as (
select * from (values
(1,'a','x'),
(1,'b','y'),
(2,'c','z')
) tab(entity_number,property_id,value)
),
tab2 as (
select
entity_number,
json_agg( json_build_object('property_id', property_id, 'value', value)) entity_properties
from tab
group by 1
)
select
json_agg(
json_build_object(
'entity_number',
entity_number,
'entity_properties',
entity_properties
)
)
from tab2
[
{
"entity_number": 1,
"entity_properties": [
{
"value": "x",
"property_id": "a"
},
{
"value": "y",
"property_id": "b"
}
]
},
{
"entity_number": 2,
"entity_properties": [
{
"value": "z",
"property_id": "c"
}
]
}
]
Note that I used jsonb_pretty to format the output.

How to transform a postgresql select with join to a json object?

I want to transform the result from a select with joins into a json object. I mean this query:
select
cm.*,
e.*,
u.*,
from
chat_messages cm,
events e,
users u
where
cm.event_id = e.id
and cm.user_id = u.id
should output this:
{
"id": 1,
"message": "whatever",
"time": "2021-12-02T00:21:10.571848",
"user": {
"id": 35,
"name": "John Smith"
},
"event": {
"id": 19,
"name": "Test event",
"time": "2021-09-22T00:00:00-03:00",
"local": "Planet Earth"
}
}
(there are more fields than these. I'm just making the example simple)
I found a solution this way:
select
json_build_object(
'id', cm.id,
'message', cm.message,
'time', cm.time,
'user', to_json(u.*),
'event', to_json(e.*)
)
from
chat_messages cm,
events e,
users u
where
cm.event_id = e.id
and cm.user_id = u.id
But I think there should be a much better way to do this. Imagine that chat_messages had a lot more fields. It would be lengthy to describe field by field.
What I want is a way to for the query to transform subqueries in json without me describing field by field.
Anyone knows a better way to do this ?
According to, Postgres document you can use the row_to_json function to transfer row to JSON and then append each table rows with an alias to be detected by row_to_json
with cte as (
select
cm.*,
e as event,
u as user
from
chat_messages cm,
events e,
users u
where
cm.event_id = e.id
and cm.user_id = u.id
)
select row_to_json(c) from cte c;

Sequelize Include, join, limit associated tables

I have a problem with the sequelize query, I think they should give the same rerult, but they don't. Can someone please explain this situation?
The first case:
getLatestStation = (stationId) => {
return models.MonitoringDataInfo.findAll({
where: {stationId :stationId},
attributes: ['id', 'stationId', 'sentAt'],
order : [['sentAt', 'DESC']],
limit: 1,
include : [{model: models.MonitoringData, attributes: ['idData', 'indicator', 'value', 'unit', 'sensorStatus']}]
})
}
The first result:
The first query:
SELECT [MonitoringDataInfo].*, [MonitoringData].[id] AS [MonitoringData.id], [MonitoringData].[idData] AS [MonitoringData.idData], [MonitoringData].[indicator] AS [MonitoringData.indicator], [MonitoringData].[value] AS [MonitoringData.value], [MonitoringData].[unit] AS [MonitoringData.unit], [MonitoringData].[sensorStatus] AS [MonitoringData.sensorStatus] FROM (SELECT [MonitoringDataInfo].[id], [MonitoringDataInfo].[stationId], [MonitoringDataInfo].[sentAt] FROM [monitoring_data_info] AS [MonitoringDataInfo] WHERE [MonitoringDataInfo].[stationId] = N'mkkneh3uBrAkw9hepMlF' ORDER BY [MonitoringDataInfo].[sentAt] DESC OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY) AS [MonitoringDataInfo] LEFT OUTER JOIN [monitoring_data] AS [MonitoringData] ON [MonitoringDataInfo].[id] = [MonitoringData].[idData] ORDER BY [MonitoringDataInfo].[sentAt] DESC;
The second case:
getLatestStation = (stationId) => {
return models.Station.findAll({
attributes: ['id', 'name', 'address', 'envIndex'],
where : {
id : stationId,
publicStatus : 1
},
include : [{
model: models.MonitoringDataInfo,
limit: 1,
// where : {
// stationId: stationId
// },
attributes: ['id', 'stationId', 'sentAt'],
order : [['sentAt', 'DESC']],
limit : 1,
include : [{model: models.MonitoringData, attributes: ['idData', 'indicator', 'value', 'unit', 'sensorStatus']}]
}]
})
}
The second result:
The second query:
SELECT [Station].[id], [Station].[name], [Station].[address], [Station].[envIndex] FROM [stations] AS [Station] WHERE [Station].[id] = N'mkkneh3uBrAkw9hepMlF' AND [Station].[publicStatus] = 1;
SELECT [MonitoringDataInfo].[id], [MonitoringDataInfo].[stationId], [MonitoringDataInfo].[sentAt], [MonitoringData].[id] AS [MonitoringData.id], [MonitoringData].[idData] AS [MonitoringData.idData], [MonitoringData].[indicator] AS [MonitoringData.indicator], [MonitoringData].[value] AS [MonitoringData.value], [MonitoringData].[unit] AS [MonitoringData.unit], [MonitoringData].[sensorStatus] AS [MonitoringData.sensorStatus] FROM [monitoring_data_info] AS [MonitoringDataInfo] LEFT OUTER JOIN [monitoring_data] AS [MonitoringData] ON [MonitoringDataInfo].[id] = [MonitoringData].[idData] WHERE [MonitoringDataInfo].[stationId] IN (N'mkkneh3uBrAkw9hepMlF') ORDER BY [MonitoringDataInfo].[sentAt] DESC, [MonitoringDataInfo].[id] OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY;
The first query does not limit records from MonitoringData in contrast to second one.
Try to indicate separate:true in include:
include : [{model: models.MonitoringData, separate: true, attributes: ['idData', 'indicator', 'value', 'unit', 'sensorStatus']}]

SQL Inner join with For JSON hierarchy

I'm trying to join two SQL tables with inner join and then return them as JSON from my procedure.
My select statement is:
SELECT
#CustomerAddressesJSON =
(SELECT
Address.AddressID, Address.CustomerID,
Address.AddressTypeID, Address.IsPrimary,
CountryID, StateID, CountyID, DistrictID,
StreetID, StreetNumber, PostalCode,
AdditionalInformation, AddressImageID,
CreatedOn, CreatedBy
FROM
[sCustomerManagement].[tCustomerAddresses] Address
INNER JOIN
[sCustomerManagement].[tAddresses] AddressDetails ON Address.AddressID = AddressDetails.AddressID
WHERE
CustomerID = #CustomerID
FOR JSON AUTO)
and the result is like this:
"customerAddressesJSON": "[ {
"AddressID": 1,
"CustomerID": 1,
"AddressTypeID": "T",
"IsPrimary": true,
"AddressDetails": [
{
"CountryID": 1,
"StateID": 1,
"CountyID": 1,
"DistrictID": 1,
"StreetID": 1,
"StreetNumber": "125",
"PostalCode": "1000",
"AdditionalInformation": "Metro Sofia",
"CreatedOn": "2017-10-24T11:46:20.1933333",
"CreatedBy": 24
}
]
}, {
"AddressID": 2,
"CustomerID": 1,
"AddressTypeID": "T",
"IsPrimary": true,
"AddressDetails": [
{
"CountryID": 1,
"StateID": 1,
"CountyID": 1,
"DistrictID": 1,
"StreetID": 1,
"StreetNumber": "125",
"PostalCode": "1000",
"AdditionalInformation": "Metro Sofia",
"CreatedOn": "2017-10-24T11:46:20.1933333",
"CreatedBy": 24
}
]
}
The problem is that I don't want the information in the array AddressDetails to be nested. Is it possible the information there to be outside, so I can receive 2 flat objects, without nested information ?
Thanks
Consider using the PATH mode with dot syntax and map all fields to Address as discussed in docs.
SELECT
#CustomerAddressesJSON =
(SELECT
a.AddressID AS 'Address.AddressID', a.CustomerID AS 'Address.CustomerID',
a.AddressTypeID AS 'Address.AddressTypeID', a,IsPrimary AS 'Address.IsPrimary',
d.CountryID AS 'Address.CountryID', d.StateID AS 'Address.StateID',
d.CountyID AS 'Address.CountyID', d.DistrictID AS 'Address.DistrictID',
d.StreetID As 'Address.StreetID', d.StreetNumber AS 'Address.StreetNumber',
d.PostalCode AS 'Address.PostalCode',
d.AdditionalInformation AS 'Address.AdditionalInformation',
d.AddressImageID AS 'Address.AddressImageID',
d.CreatedOn AS 'Address.CreatedOn', d.CreatedBy AS 'Address.CreatedBy'
FROM
[sCustomerManagement].[tCustomerAddresses] a
INNER JOIN
[sCustomerManagement].[tAddresses] d ON a.AddressID = d.AddressID
WHERE
a.CustomerID = #CustomerID
FOR JSON PATH)
Alternatively, use a derived table:
SELECT
#CustomerAddressesJSON =
(SELECT m.*
FROM
(SELECT a.AddressID, a.CustomerID, a.AddressTypeID, a,IsPrimary,
d.CountryID, d.StateID, d.CountyID, d.DistrictID,
d.StreetID, d.StreetNumber, d.PostalCode,
d.AdditionalInformation, d.AddressImageID,
d.CreatedOn, d.CreatedBy
FROM
[sCustomerManagement].[tCustomerAddresses] a
INNER JOIN
[sCustomerManagement].[tAddresses] d ON a.AddressID = d.AddressID
WHERE
a.CustomerID = #CustomerID
) AS m
FOR JSON AUTO)

Format nested JSON object from Postgres query

I want to get a JSON object formatted similar to this:
{
"username": "USERNAME",
"teamname": "TEAMNAME",
"logs": [
{
"log": {
"log_id": 29,
"person_id": 3,
"activity_id": 3,
"shoe_id": null,
"logdate": "2016-11-29",
"distance": null,
"activitytime": null,
"sleep": null,
"heartrate": null,
"logtitle": null,
"description": null
},
"activity": "Swim",
"comments": {
"comment_id": 1,
"description": "This is a comment",
"person_id": 1,
"log_id": 29
}
}]
}
Currently I have everything formatted correctly except the comments. Here is the SQL query I am using:
SELECT p.username, t.teamname, json_agg(json_build_object('log', l.*, 'comments', c.*, 'activity', a.activity)) as logs
FROM person_tbl p
INNER JOIN log_tbl l ON p.person_id = l.person_id
INNER JOIN activity_tbl a ON l.activity_id = a.activity_id
INNER JOIN comment_tbl c ON c.log_id = l.log_id
INNER JOIN person_team_tbl pt ON p.person_id = pt.person_id
INNER JOIN team_tbl t on t.team_id = pt.team_id
WHERE t.team_id = 5
AND l.logdate > NOW()::date - 7
GROUP BY p.username, t.teamname
ORDER BY p.username
I'm having trouble getting the comments of each log. Right now, it is returning every comment and repeating the logs (they are not associated).
Also, how could I get this query to return the username and teamname when everything else is null (like when there are no logs in the past week)?
Without an SQLfiddle we do not know what your data (and structure) is so it is difficult to answer your question.
For the NULL case - please modify the WHERE clause like this (deliberately not using COALESCE)
WHERE t.team_id = 5 AND (l.logdate IS NULL OR l.logdate > NOW()::date - INTERVAL '7 day')