Splitting time series with Google BigQuery by field value - sql

Question

I have a dataset in Google Bigquery with vehicle positions over time and the direction they are going relative to a base, like
time | x | y | direction | vehicle_id
-----|-----|-----|-----------|-----------
0:00 | ... | ... | returning | 100
0:00 | ... | ... | returning | 200
0:00 | ... | ... | exploring | 300
0:05 | ... | ... | returning | 100
0:05 | ... | ... | exploring | 200
0:05 | ... | ... | exploring | 300
0:10 | ... | ... | exploring | 100
0:10 | ... | ... | exploring | 200
0:10 | ... | ... | exploring | 300
0:15 | ... | ... | exploring | 100
0:15 | ... | ... | exploring | 200
0:15 | ... | ... | returning | 300
I'm able to aggregate by vehicle easily, but I can't come up with a query that can break each vehicle series into 'trips', consisting of sequential occurrences of 'returning' or 'exploring'. I have read about analytic functions but none seem to fit the bill.
SELECT
vehicle_id,
ARRAY_AGG(
STRUCT(direction, time, x, y)
ORDER BY time) as series
FROM t
GROUP BY vehicle_id;
[
{
"vehicle_id": 100,
"series":
[
{"direction": "returning", "time": "0:00", "x": ..., "y": ...},
{"direction": "returning", "time": "0:05", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:10", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:15", "x": ..., "y": ...}
]
},
{
"vehicle_id": 200,
"series":
[
{"direction": "returning", "time": "0:00", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...}
]
},
{
"vehicle_id": 300,
"series":
[
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...},
{"direction": "exploring", "time": "0:00", "x": ..., "y": ...},
{"direction": "returning", "time": "0:00", "x": ..., "y": ...}
]
}
]
What I really want is to have a sequence of trips by vehicle, where each trip has a direction and a series of (t, x, y) positions. Is that possible to do?

Answer 1

Below is for BigQuery Standard SQL and uses pure SQL to achieve the very same result
#standardSQL
SELECT vehicle_id, ARRAY_AGG(STRUCT(direction, trip)) trips
FROM (
SELECT vehicle_id, direction, ARRAY_AGG(STRUCT(time, x, y) ORDER BY time) trip
FROM dataset
GROUP BY vehicle_id, direction
)
GROUP BY vehicle_id
If to apply to sample data from your question as in example below
#standardSQL
WITH dataset AS (
SELECT
TIMESTAMP '2019-09-07 00:00:00' AS time,
0.1 AS x, 0.1 AS y, 'returning' AS direction,
100 AS vehicle_id
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:00', 0.2, 0.2, 'returning', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:00', 0.3, 0.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.1, 1.1, 'returning', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.2, 1.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.3, 1.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.1, 2.1, 'exploring', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.2, 2.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.3, 2.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.1, 3.1, 'exploring', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.2, 3.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.3, 3.3, 'returning', 300
)
SELECT vehicle_id, ARRAY_AGG(STRUCT(direction, trip)) trips
FROM (
SELECT vehicle_id, direction, ARRAY_AGG(STRUCT(time, x, y) ORDER BY time) trip
FROM dataset
GROUP BY vehicle_id, direction
)
GROUP BY vehicle_id
result is

Answer 2

I wasn't able to come up with a pure SQL solution, but Bigquery does provide a way to execute arbitrary processing within itself in the form of user-defined functions (UDF).
By aggregating the entire series of a vehicle into an array, we can feed it into a Javascript function that performs the necessary logic and splits the series in a sequence of trips.
CREATE TEMPORARY FUNCTION split_trips(
series ARRAY<STRUCT<direction STRING,
time TIMESTAMP,
x FLOAT64,
y FLOAT64>>)
RETURNS ARRAY<STRUCT<direction STRING,
trip ARRAY<STRUCT<time TIMESTAMP,
x FLOAT64,
y FLOAT64>>>>
LANGUAGE js AS """
if (series.length == 0) {
return [];
}
let result = [];
let trip = [];
for (let i = 0; i < series.length-1; i++) {
let {direction, time, x, y} = series[i];
trip.push({time: time, x: x, y: y});
if (direction == series[i+1].direction) {
continue;
}
result.push({direction: direction, trip: trip});
trip = [];
}
let lastEntry = series[series.length-1];
trip.push({time: lastEntry.time, x: lastEntry.x, y: lastEntry.y});
result.push({direction: lastEntry.direction, trip: trip});
return result;
""";
WITH dataset AS (
SELECT
TIMESTAMP '2019-09-07 00:00:00' AS time,
0.1 AS x, 0.1 AS y, 'returning' AS direction,
100 AS vehicle_id
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:00', 0.2, 0.2, 'returning', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:00', 0.3, 0.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.1, 1.1, 'returning', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.2, 1.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:05', 1.3, 1.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.1, 2.1, 'exploring', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.2, 2.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:10', 2.3, 2.3, 'exploring', 300
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.1, 3.1, 'exploring', 100
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.2, 3.2, 'exploring', 200
UNION ALL SELECT TIMESTAMP '2019-09-07 00:00:15', 3.3, 3.3, 'returning', 300
),
by_vehicle AS (
SELECT
vehicle_id,
ARRAY_AGG(STRUCT(direction, time, x, y)
ORDER BY TIME) AS series
FROM dataset
GROUP BY vehicle_id
)
SELECT
vehicle_id,
split_trips(series) AS trips
FROM by_vehicle
The Bigquery documentation notes that each function invocation can produce at most 5 MiB of data, so considering the timestamp (64 bits) and two floats (64 bits each) that gives us at most ~200K entries to manipulate at once for each vehicle.