BigQuery SQL : Rolling count distinct bounded between two conditions - sql

Question

I am trying to find the rolling countdistinct of ip_var bounded between two events (in two different columns in Bigquery SQL).
eg i have a table of the form :
id TIME_STAMP event_1 event_2 ip_var
A 1 0 0 1
A 2 1 0 1
A 2 0 0 2
A 3 0 0 2
A 4 0 0 3
A 5 0 1 4
A 6 0 0 1
A 7 0 0 1
B 1 0 0 2
B 2 0 0 2
B 2 1 0 3
B 3 0 0 3
B 4 0 0 3
B 4 0 1 4
B 6 0 0 5
B 7 0 0 6
For each id , i need the countdistinct of ip_var when the event_1 happens till event_2 happens , its always guaranteed that even2 happens after event_1.
I have tried using rolling count for the problem without much success.
Final output looks like
id bounded_count
A 2
B 1

Answer 1

Below is for BigQuery Standard SQL
#standardSQL
SELECT id, COUNT(DISTINCT ip_var) bounded_count
FROM (
SELECT *,
COUNTIF(event_1 = 1) OVER(win ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) grp,
COUNTIF(event_1 = 1) OVER(win ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) != COUNTIF(event_2 = 1) OVER(win) qualify
FROM `project.dataset.table`
WINDOW win AS (PARTITION BY id ORDER BY time_stamp)
)
WHERE qualify
GROUP BY id, grp
if to apply to sample data from your question - result is
Row id bounded_count
1 A 2
2 B 1
Note: above solution also works in case if you have multiple qualified pairs, like in below example (same code, I just added more rows into sample data)
#standardSQL
WITH `project.dataset.table` AS (
SELECT 'A' id, 1 time_stamp, 0 event_1, 0 event_2, 1 ip_var UNION ALL
SELECT 'A', 2, 1, 0, 1 UNION ALL
SELECT 'A', 2, 0, 0, 2 UNION ALL
SELECT 'A', 3, 0, 0, 2 UNION ALL
SELECT 'A', 4, 0, 0, 3 UNION ALL
SELECT 'A', 5, 0, 1, 4 UNION ALL
SELECT 'A', 6, 0, 0, 1 UNION ALL
SELECT 'A', 7, 0, 0, 1 UNION ALL
SELECT 'A', 12, 1, 0, 1 UNION ALL
SELECT 'A', 13, 0, 0, 2 UNION ALL
SELECT 'A', 14, 0, 0, 3 UNION ALL
SELECT 'A', 15, 0, 0, 4 UNION ALL
SELECT 'A', 16, 0, 0, 5 UNION ALL
SELECT 'A', 17, 0, 1, 1 UNION ALL
SELECT 'A', 18, 0, 0, 1 UNION ALL
SELECT 'B', 1, 0, 0, 2 UNION ALL
SELECT 'B', 2, 0, 0, 2 UNION ALL
SELECT 'B', 2, 1, 0, 3 UNION ALL
SELECT 'B', 3, 0, 0, 3 UNION ALL
SELECT 'B', 4, 0, 0, 3 UNION ALL
SELECT 'B', 5, 0, 1, 4 UNION ALL
SELECT 'B', 6, 0, 0, 5 UNION ALL
SELECT 'B', 7, 0, 0, 6
)
SELECT id, COUNT(DISTINCT ip_var) bounded_count, grp
FROM (
SELECT *,
COUNTIF(event_1 = 1) OVER(win ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) grp,
COUNTIF(event_1 = 1) OVER(win ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) != COUNTIF(event_2 = 1) OVER(win) qualify
FROM `project.dataset.table`
WINDOW win AS (PARTITION BY id ORDER BY time_stamp)
)
WHERE qualify
GROUP BY id, grp
with result
Row id bounded_count grp
1 A 2 1
2 A 4 2
3 B 1 1

Answer 2

Hmmm . . . You can use window functions to calculate the timestamps for each event. The rest is just filtering and aggregation:
WITH t as (
SELECT "A" as id, 1 as time_stamp, 0 as event_1, 0 as event_2, 1 as ip_var UNION ALL
SELECT "A", 2, 1, 0, 1 UNION ALL
SELECT "A", 2, 0, 0, 2 UNION ALL
SELECT "A", 3, 0, 0, 2 UNION ALL
SELECT "A", 4, 0, 0, 3 UNION ALL
SELECT "A", 5, 0, 1, 4 UNION ALL
SELECT "A", 6, 0, 0, 1 UNION ALL
SELECT "A", 7, 0, 0, 1 UNION ALL
SELECT "B", 1, 0, 0, 2 UNION ALL
SELECT "B", 2, 0, 0, 2 UNION ALL
SELECT "B", 2, 1, 0, 3 UNION ALL
SELECT "B", 3, 0, 0, 3 UNION ALL
SELECT "B", 4, 0, 0, 3 UNION ALL
SELECT "B", 4, 0, 1, 4 UNION ALL
SELECT "B", 6, 0, 0, 5 UNION ALL
SELECT "B", 7, 0, 0, 6
)
select id, count(distinct ip_var) as bounded_count
from (select t.*,
min(case when event_1 = 1 then time_stamp end) over (partition by id) as timestamp_1,
max(case when event_2 = 1 then time_stamp end) over (partition by id) as timestamp_2
from t
) t
where time_stamp > timestamp_1 and time_stamp < timestamp_2
group by id

Answer 3

One way to do it is:
Find out start_time and end_time for each ID
For each ID, filter out events that are not in counting window
Count distinct ip_var
In order to print out intermediate step, I used temp table to demonstrate the idea. You should make second temp table id_start_end a WITH clause to be more efficient.
CREATE TEMP TABLE t as
SELECT "A" id, 1 time_stamp, 0 event_1, 0 event_2, 1 ip_var UNION ALL
SELECT "A", 2, 1, 0, 1 UNION ALL
SELECT "A", 2, 0, 0, 2 UNION ALL
SELECT "A", 3, 0, 0, 2 UNION ALL
SELECT "A", 4, 0, 0, 3 UNION ALL
SELECT "A", 5, 0, 1, 4 UNION ALL
SELECT "A", 6, 0, 0, 1 UNION ALL
SELECT "A", 7, 0, 0, 1 UNION ALL
SELECT "B", 1, 0, 0, 2 UNION ALL
SELECT "B", 2, 0, 0, 2 UNION ALL
SELECT "B", 2, 1, 0, 3 UNION ALL
SELECT "B", 3, 0, 0, 3 UNION ALL
SELECT "B", 4, 0, 0, 3 UNION ALL
SELECT "B", 4, 0, 1, 4 UNION ALL
SELECT "B", 6, 0, 0, 5 UNION ALL
SELECT "B", 7, 0, 0, 6;
CREATE TEMP TABLE id_start_end AS
SELECT ids.id, t_start.time_stamp as start_time, t_end.time_stamp as end_time FROM
(SELECT DISTINCT id FROM t) ids
JOIN t AS t_start ON ids.id = t_start.id AND t_start.event_1 = 1
JOIN t AS t_end ON ids.id = t_end.id AND t_end.event_2 = 1;
SELECT * FROM id_start_end;
SELECT t.id, COUNT(DISTINCT ip_var)
FROM t JOIN id_start_end
ON t.id = id_start_end.id
AND t.time_stamp < id_start_end.end_time
AND t.time_stamp > id_start_end.start_time
GROUP BY t.id
Output table id_start_end:
+----+------------+----------+
| id | start_time | end_time |
+----+------------+----------+
| A | 2 | 5 |
| B | 2 | 4 |
+----+------------+----------+
Final output:
+----+-----+
| id | f0_ |
+----+-----+
| B | 1 |
| A | 2 |
+----+-----+