WITH yourTable AS (
SELECT 1 AS id, '2013,1625,1297,7634' AS string_col UNION ALL
SELECT 2, '1,2,3,4,5'
)
SELECT id,
(SELECT ARRAY_AGG(CAST(num AS INT64))
FROM UNNEST(SPLIT(string_col)) AS num
) AS num,
ARRAY(SELECT CAST(num AS INT64)
FROM UNNEST(SPLIT(string_col)) AS num
) AS num_2
FROM yourTable
This is how exactly my actual table is designed and Now I would like to multiply num*num_2 and then later sum it up. Is there a way to get this into struct format like ID, nums.num,nums.num_2 so that I can simply multiply which gives me the necessary result.
PS: I am looking for solution in the select statement above but not within "with" statement.
Ok, assuming that you really have reason to have your table the way you have (see my comment on your question) - below should work
#standardSQL
SELECT id,
(
SELECT SUM(num * num_2)
FROM (SELECT pos, num FROM UNNEST(num) num WITH OFFSET pos) a
JOIN (SELECT pos_2, num_2 FROM UNNEST(num_2) num_2 WITH OFFSET pos_2) b
ON a.pos = b.pos_2
) mul
FROM yourTable
you can test it with below
#standardSQL
WITH yourTable AS (
SELECT 1 id, [2013,1625,1297,7634] num, [2013,1625,1297,7634] num_2 UNION ALL
SELECT 2, [1,2,3,4,5], [1,2,3,4,5]
)
SELECT id,
(
SELECT SUM(num * num_2)
FROM (SELECT pos, num FROM UNNEST(num) num WITH OFFSET pos) a
JOIN (SELECT pos_2, num_2 FROM UNNEST(num_2) num_2 WITH OFFSET pos_2) b
ON a.pos = b.pos_2
) mul
FROM yourTable
Related
I need to do the following with 2 array fields in the table below. The arrays are of type Struct<String, String>.
Merge the arrays together
If there is a duplicate key between the labels.key and project.key, then I only want to keep the kvp from the labels field
flatten the combined array into a delimited string an order them (so I can group by)
Example Table Data
SELECT 1 as id, ARRAY
[STRUCT("testlabel2" as key, "thisvalueisbetter" as value), STRUCT("testlabel3", "testvalue3")] as labels,
[STRUCT("testlabel2" as key, "testvalue2" as value)] as project
The below query does everything except #2 and I'm not sure how to accomplish that. Does anyone have a suggestion on how to do this?
SELECT
id,
(SELECT STRING_AGG(DISTINCT CONCAT(l.key, ':', l.value) ORDER BY CONCAT(l.key, ':', l.value))
FROM UNNEST(
ARRAY_CONCAT(labels, project)) AS l) AS label,
FROM `mytestdata` AS t
GROUP BY id, label
Currently this query gives the output:
1 testlabel2:testvalue2,testlabel2:thisvalueisbetter,testlabel3:testvalue3
But I'm looking for:
1 testlabel2:thisvalueisbetter,testlabel3:testvalue3
Below is for BigQuery Standard SQL
#standardSQL
SELECT *,
ARRAY(
SELECT AS STRUCT key, ARRAY_AGG(value ORDER BY source LIMIT 1)[OFFSET(0)] AS value
FROM (
SELECT 0 AS source, * FROM t.labels UNION ALL
SELECT 1, * FROM t.project
)
GROUP BY key
) AS combined_array
FROM `project.dataset.table` t
You can test, play with above using sample data from your question as in below example
#standardSQL
WITH `project.dataset.table` AS (
SELECT ARRAY
[STRUCT("testlabel2" AS key, "thisvalueisbetter" AS value), STRUCT("testlabel3", "testvalue3")] AS labels,
[STRUCT("testlabel2" AS key, "testvalue2" AS value)] AS project
)
SELECT *,
ARRAY(
SELECT AS STRUCT key, ARRAY_AGG(value ORDER BY source LIMIT 1)[OFFSET(0)] AS value
FROM (
SELECT 0 AS source, * FROM t.labels UNION ALL
SELECT 1, * FROM t.project
)
GROUP BY key
) AS combined_array
FROM `project.dataset.table` t
with result
Or ... to fully match your expected output - use below
#standardSQL
SELECT *,
(SELECT STRING_AGG(x) FROM (
SELECT CONCAT(key, ':', ARRAY_AGG(value ORDER BY source LIMIT 1)[OFFSET(0)]) x
FROM (
SELECT 0 AS source, * FROM t.labels UNION ALL
SELECT 1, * FROM t.project
)
GROUP BY key
)) AS combined_result
FROM `project.dataset.table` t
with result
I have a bigquery table, which has a column of the repeated data type with a 512 dimensional vector (float).
I would like to run a query which finds the N most similar vectors.
In my case similarity can be simply defined as the inner product of the target vector and each vector in the database.
I have found and run the below query, which generates this across all the combinations in the table:
#standardSQL
CREATE TABLE ml.url_cosine_similarity AS
WITH pairwise AS (
SELECT t1.url AS id_1, t2.url AS id_2
FROM `project.dataset.table` t1
INNER JOIN `project.dataset.table` t2
ON t1.url < t2.url
)
SELECT id_1, id_2, (
SELECT
SUM(value1 * value2)/
SQRT(SUM(value1 * value1))/
SQRT(SUM(value2 * value2))
FROM UNNEST(a.page_vector) value1 WITH OFFSET pos1
JOIN UNNEST(b.page_vector) value2 WITH OFFSET pos2
ON pos1 = pos2
) cosine_similarity
FROM pairwise t
JOIN `project.dataset.table` a ON a.url = id_1
JOIN `project.dataset.table` b ON b.url = id_2
However since I do not have a good grasp of how arrays work in bigquery, I am unsure on how to change this query to take in a target vector, and return N neighbours.
See simplified example - it returns top 3 nearest pairs of vectors in the table
#standardSQL
WITH `project.dataset.table` AS (
SELECT 1 id, [1,2,3,4,5] page_vector UNION ALL
SELECT 2, [1,3,4,5,16] UNION ALL
SELECT 3, [2,3,4,5,6] UNION ALL
SELECT 4, [2,4,6,8,9] UNION ALL
SELECT 5, [1,3,4,5,16] UNION ALL
SELECT 6, [11,12,13,14,15]
)
SELECT a.id id1, b.id id2, (
SELECT
SUM(value1 * value2)/
SQRT(SUM(value1 * value1))/
SQRT(SUM(value2 * value2))
FROM UNNEST(a.page_vector) value1 WITH OFFSET pos1
JOIN UNNEST(b.page_vector) value2 WITH OFFSET pos2
ON pos1 = pos2
) cosine_similarity
FROM `project.dataset.table` a
JOIN `project.dataset.table` b
ON a.id < b.id
ORDER BY cosine_similarity DESC
LIMIT 3
with output
Row id1 id2 cosine_similarity
1 2 5 1.0
2 1 4 0.9986422261219272
3 3 4 0.9962894120648842
If you want to output the nearest vectors (let's say two) for every vector in table - see below example
#standardSQL
WITH `project.dataset.table` AS (
SELECT 1 id, [1,2,3,4,5] page_vector UNION ALL
SELECT 2, [1,3,4,5,16] UNION ALL
SELECT 3, [2,3,4,5,6] UNION ALL
SELECT 4, [2,4,6,8,9] UNION ALL
SELECT 5, [1,3,4,5,16] UNION ALL
SELECT 6, [11,12,13,14,15]
)
SELECT id, ANY_VALUE(page_vector) page_vector,
ARRAY_AGG(
STRUCT(id2 AS id, page_vector2 AS page_vector, cosine_similarity AS cosine_similarity)
ORDER BY cosine_similarity DESC
LIMIT 2
) similar_vectors
FROM (
SELECT a.id, a.page_vector,
b.id id2, b.page_vector page_vector2, (
SELECT
SUM(value1 * value2)/
SQRT(SUM(value1 * value1))/
SQRT(SUM(value2 * value2))
FROM UNNEST(a.page_vector) value1 WITH OFFSET pos1
JOIN UNNEST(b.page_vector) value2 WITH OFFSET pos2
ON pos1 = pos2
) cosine_similarity
FROM `project.dataset.table` a
JOIN `project.dataset.table` b
ON a.id != b.id
)
GROUP BY id
ORDER BY id
this will produce below output
Postgres has an easy function to achieve this, just using the mode() function we can find the most frequent value. Is there something equivalent within Google's Bigquery?
How could be written a query like this in Bigquery?
select count(*),
avg(vehicles) as mean,
percentile_cont(0.5) within group (order by vehicles) as median,
mode() within group (order by vehicles) as most_frequent_value
FROM "driver"
WHERE vehicles is not null;
Below is for BigQuery Standard SQL
Option 1
#standardSQL
SELECT * FROM (
SELECT COUNT(*) AS cnt,
AVG(vehicles) AS mean,
APPROX_TOP_COUNT(vehicles, 1)[OFFSET(0)].value AS most_frequent_value
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
) CROSS JOIN (
SELECT PERCENTILE_CONT(vehicles, 0.5) OVER() AS median
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
LIMIT 1
)
Option 2
#standardSQL
SELECT * FROM (
SELECT COUNT(*) cnt,
AVG(vehicles) AS mean
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
) CROSS JOIN (
SELECT PERCENTILE_CONT(vehicles, 0.5) OVER() AS median
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
LIMIT 1
) CROSS JOIN (
SELECT vehicles AS most_frequent_value
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
GROUP BY vehicles
ORDER BY COUNT(1) DESC
LIMIT 1
)
Option 3
#standardSQL
CREATE TEMP FUNCTION median(arr ANY TYPE) AS ((
SELECT PERCENTILE_CONT(x, 0.5) OVER()
FROM UNNEST(arr) x LIMIT 1
));
CREATE TEMP FUNCTION most_frequent_value(arr ANY TYPE) AS ((
SELECT x
FROM UNNEST(arr) x
GROUP BY x
ORDER BY COUNT(1) DESC
LIMIT 1
));
SELECT COUNT(*) cnt,
AVG(vehicles) AS mean,
median(ARRAY_AGG(vehicles)) AS median,
most_frequent_value(ARRAY_AGG(vehicles)) AS most_frequent_value
FROM `project.dataset.table`
WHERE vehicles IS NOT NULL
and so on ...
You can use APPROX_TOP_COUNT to get top values, e.g.:
SELECT APPROX_TOP_COUNT(vehicles, 5) AS top_five_vehicles
FROM dataset.driver
If you just want the top value, you can select it from the array:
SELECT APPROX_TOP_COUNT(vehicles, 1)[OFFSET(0)] AS most_frequent_value
FROM dataset.driver
No, there is no equivalent of the mode()-function in BigQuery, but you may define one yourself using any of the logics in the other answers to this thread. You could call it like so:
SELECT mode(`an_array`) AS top_count FROM `somewhere_with_arrays`
but this approach lead to multiple by-row sub-queries wihch is terrible for performance, so if you never grinded BQ to a halt before, you can do it with these functions. I it (the second) only for readability in quick-fixes for very small data-sets.
Check out the two UDF:s below. A third approach would be to implement a JS function in which case this oneliner should be usefull
return arr.sort((a,b) => arr.filter(v => v===a).length - arr.filter(v => v===b).length).pop();
This code establishes two mode()-like functions which eat arrays and return most common string:
CREATE TEMPORARY FUNCTION mode1(mystring ANY TYPE)
RETURNS STRING
AS
(
(
SELECT var FROM
( /* Count occurances of each value of input */
SELECT var, COUNT(*) AS n FROM
( /* Unnest and name*/
SELECT var FROM UNNEST(mystring) var
)
GROUP BY var /* Output is one of existing values */
ORDER BY n DESC /* Output is value with HIGHEST n */
) /* -------------------------------- */
LIMIT 1 /* Only ONE string is the output */
)
);
CREATE TEMPORARY FUNCTION mode2(inp ANY TYPE)
RETURNS STRING
AS
(
(
SELECT result.value FROM UNNEST( (SELECT APPROX_TOP_COUNT(v,1) AS result FROM UNNEST(inp) v)) result
)
);
SELECT
inp,
mode1(inp) AS first_logic_output,
mode2(inp) AS second_logic_output
FROM
(
/* Test data */
SELECT ['Erdős','Turán', 'Erdős','Turán','Euler','Erdős'] AS inp
UNION ALL
SELECT ['Euler','Euler', 'Gauss', 'Euler'] AS inp
)
The method I prefer is to query off of an array since you can easily adjust the criteria of the mode. Below are two example using both an offset and a limit method. With the offset you can grab the Nth most/least frequent value.
WITH t AS (SELECT 18 AS length,
'HIGH' as amps,
99.95 price UNION ALL
SELECT 18, "HIGH", 99.95 UNION ALL
SELECT 18, "HIGH", 5.95 UNION ALL
SELECT 18, "LOW", 33.95 UNION ALL
SELECT 18, "LOW", 33.95 UNION ALL
SELECT 18, "LOW", 4.5 UNION ALL
SELECT 3, "HIGH", 77.95 UNION ALL
SELECT 3, "HIGH", 77.95 UNION ALL
SELECT 3, "HIGH", 9.99 UNION ALL
SELECT 3, "LOW", 44.95 UNION ALL
SELECT 3, "LOW", 44.95 UNION ALL
SELECT 3, "LOW", 5.65
)
SELECT
length,
amps,
-- By Limit
(SELECT x FROM UNNEST(price_array) x
GROUP BY x ORDER BY COUNT(*) DESC LIMIT 1 ) most_freq_price,
(SELECT x FROM UNNEST(price_array) x
GROUP BY x ORDER BY COUNT(*) ASC LIMIT 1 ) least_freq_price,
-- By Offset
ARRAY((SELECT x FROM UNNEST(price_array) x
GROUP BY x ORDER BY COUNT(*) DESC))[OFFSET(0)] most_freq_price_offset,
ARRAY((SELECT x FROM UNNEST(price_array) x
GROUP BY x ORDER BY COUNT(*) ASC))[OFFSET(0)] least_freq_price_offset
FROM (
SELECT
length,
amps,
ARRAY_AGG(price) price_array
FROM t
GROUP BY 1,2
)
I have a select statement that brings the values of 5 different columns.
SELECT TOP 1 digit1, digit2, digit3, digit4, digit5 FROM myTable
The result is "25, 36, 1, 63, 12".
My question is, how do I sort these numbers in Ascending order and reassign them to their respectively Ascending columns?
I.E: digit1 = 1, digit2 = 12, digit3 = 25, digit4 = 36, digit5 = 63.
Thanks in advance for the help.
You can do an UNPIVOT followed by a PIVOT
Example
Select *
From (
Select Item = concat('digit',Row_NUmber() over (Order by Value))
,Value
From (Select top 1 * from YourTable) A
UnPivot ( Value for Digits in (digit1,digit2,digit3,digit4,digit5) ) U
) A
Pivot (max(Value) for Item in (digit1,digit2,digit3,digit4,digit5) ) P
Returns
digit1 digit2 digit3 digit4 digit5
1 12 25 36 63
I would use apply:
select v.*
from (select top 1 digit1, digit2, digit3, digit4, digit5 FROM myTable
) t cross apply
(select max(case when seqnum = 1 then digit end) as digit1,
max(case when seqnum = 2 then digit end) as digit2,
max(case when seqnum = 3 then digit end) as digit3,
max(case when seqnum = 4 then digit end) as digit4,
max(case when seqnum = 5 then digit end) as digit5
from (select v.digit, row_number() over (order by v.digit) as seqnum
from (values (digit1), (digit2), (digit3), (digit4), (digit5)
) v(digit)
) v
) v;
If you want to order the results by multiple values, you can use the following syntax.
SELECT TOP 1 digit1, digit2, digit3, digit4, digit5
FROM myTable
ORDER BY digit1, digit2, digit3, digit4, digit5
If the values need manipulation, you may need to create a UDF/CTE or inline expression to determine the 'value' of the columns to order them. Without a full working example (A SQLFiddle would be helpful), it is difficult to be sure.
Using a CTE should allow this to be simplified to something like this. I am assuming that you need this for multiple rows, but simplifying it to a single row is easier than expanding a solution to multiple rows
WITH MyData as (
SELECT ID, digit1 as Digit FROM myTable
UNION ALL
SELECT ID, digit2 FROM myTable
UNION ALL
SELECT ID, digit3 FROM myTable
UNION ALL
SELECT ID, digit4 FROM myTable
UNION ALL
SELECT ID, digit5 FROM myTable
)
SELECT ID, Digit
FROM MyData
ORDER BY ID, Digit
Your data is now in order. A simple PIVOT will give you the output you are looking for.
My table:
ID NUM VAL
1 1 Hello
1 2 Goodbye
2 2 Hey
2 4 What's up?
3 5 See you
If I want to return the max number for each ID, it's really nice and clean:
SELECT MAX(NUM) FROM table GROUP BY (ID)
But what if I want to grab the value associated with the max of each number for each ID?
Why can't I do:
SELECT MAX(NUM) OVER (ORDER BY NUM) FROM table GROUP BY (ID)
Why is that an error? I'd like to have this select grouped by ID, rather than partitioning separately for each window...
EDIT: The error is "not a GROUP BY expression".
You could probably use the MAX() KEEP(DENSE_RANK LAST...) function:
with sample_data as (
select 1 id, 1 num, 'Hello' val from dual union all
select 1 id, 2 num, 'Goodbye' val from dual union all
select 2 id, 2 num, 'Hey' val from dual union all
select 2 id, 4 num, 'What''s up?' val from dual union all
select 3 id, 5 num, 'See you' val from dual)
select id, max(num), max(val) keep (dense_rank last order by num)
from sample_data
group by id;
When you use windowing function, you don't need to use GROUP BY anymore, this would suffice:
select id,
max(num) over(partition by id)
from x
Actually you can get the result without using windowing function:
select *
from x
where (id,num) in
(
select id, max(num)
from x
group by id
)
Output:
ID NUM VAL
1 2 Goodbye
2 4 What's up
3 5 SEE YOU
http://www.sqlfiddle.com/#!4/a9a07/7
If you want to use windowing function, you might do this:
select id, val,
case when num = max(num) over(partition by id) then
1
else
0
end as to_select
from x
where to_select = 1
Or this:
select id, val
from x
where num = max(num) over(partition by id)
But since it's not allowed to do those, you have to do this:
with list as
(
select id, val,
case when num = max(num) over(partition by id) then
1
else
0
end as to_select
from x
)
select *
from list
where to_select = 1
http://www.sqlfiddle.com/#!4/a9a07/19
If you're looking to get the rows which contain the values from MAX(num) GROUP BY id, this tends to be a common pattern...
WITH
sequenced_data
AS
(
SELECT
ROW_NUMBER() OVER (PARTITION BY id ORDER BY num DESC) AS sequence_id,
*
FROM
yourTable
)
SELECT
*
FROM
sequenced_data
WHERE
sequence_id = 1
EDIT
I don't know if TeraData will allow this, but the logic seems to make sense...
SELECT
*
FROM
yourTable
WHERE
num = MAX(num) OVER (PARTITION BY id)
Or maybe...
SELECT
*
FROM
(
SELECT
*,
MAX(num) OVER (PARTITION BY id) AS max_num_by_id
FROM
yourTable
)
AS sub_query
WHERE
num = max_num_by_id
This is slightly different from my previous answer; if multiple records are tied with the same MAX(num), this will return all of them, the other answer will only ever return one.
EDIT
In your proposed SQL the error relates to the fact that the OVER() clause contains a field not in your GROUP BY. It's like trying to do this...
SELECT id, num FROM yourTable GROUP BY id
num is invalid, because there can be multiple values in that field for each row returned (with the rows returned being defined by GROUP BY id).
In the same way, you can't put num inside the OVER() clause.
SELECT
id,
MAX(num), <-- Valid as it is an aggregate
MAX(num) <-- still valid
OVER(PARTITION BY id), <-- Also valid, as id is in the GROUP BY
MAX(num) <-- still valid
OVER(PARTITION BY num) <-- Not valid, as num is not in the GROUP BY
FROM
yourTable
GROUP BY
id
See this question for when you can't specify something in the OVER() clause, and an answer showing when (I think) you can: over-partition-by-question