bigquery joining a field and json unnested resulted only left hand side table - sql

Question

Need help to get the select statement of normal text record and json unnest answers.
I am getting only left hand normal text record. Am I missing some thing?.
#standardSQL
CREATE TEMP FUNCTION jsonunnest(input STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
return JSON.parse(input).map(j=>JSON.stringify(j));
""";
WITH `Impact_JSON` AS (
SELECT
Impact_Question_id,
Impact_Question_text, json,
ROW_NUMBER() OVER (PARTITION BY bdmp_id, DATE(Impact_Question_aktualisiert_am_ts)
ORDER BY
Impact_Question_aktualisiert_am_ts DESC) AS ROW
FROM
`<project.dataset.table` basetable
),
json_answers AS (
SELECT
regexp_replace(SPLIT(ANY_VALUE(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')),'_')[SAFE_OFFSET(1)], "[^0-9]+"," " ) AS Interview_ID,
regexp_replace(SPLIT(ANY_VALUE(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')),'_')[SAFE_OFFSET(3)], "[^0-9]+"," " ) AS Quest_ID,
STRING_AGG(DISTINCT(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')), ',\n')
AS Impact_antwort_id,
STRING_AGG(DISTINCT(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_daten_typ')),',\n')
AS Impact_reply_data_type,
IFNULL(JSON_EXTRACT_SCALAR(Impact, '$.Impact_topic_text'), 'Empty') AS Impact_topic_text,
IFNULL(JSON_EXTRACT_SCALAR(Impact, '$.Impact_reply_get'), 'Empty') AS Impact_reply_get,
FROM `Impact_JSON`,
UNNEST(jsonunnest(JSON_EXTRACT(json, '$.reply'))) Impact
GROUP by 5,6
),
Impact_Question_id_TBL AS (
select Impact_Question_id from `Impact_JSON` AS C
)
SELECT
Impact_Question_id
FROM
`Impact_JSON` AS T
left join
json_answers as J
ON
(SAFE_CAST(J.Interview_ID as INT64))
=
T.Impact_Question_id
The left hand side records and right hand side records in same table should be captured.

Answer 1

What do you expect as output?
For me the udf did not work. Otherwise I generated some sample data and shorten your query for testing. This had been a good starting point for a question!
Also I changed the join to a full outer join.
For each dataset there is a number in column Impact_Question_id. There is json column containing a nested array data structure. The json data is unnested and group by the column Impact_reply_get. The first part of the column Impact_antwort_id is extracted and named Interview_ID. Because of the grouping, you select a random value. Then you join by this to the master table to the column Impact_Question_id.
The random selecting by any_value of the column Impact_antwort_id (Interview_ID) could be the issue for the mismatch. I would also group by this value and risk double matches.
#standardSQL
CREATE TEMP FUNCTION jsonunnest(input STRING)
RETURNS ARRAY<STRING>
LANGUAGE js AS """
try{
return [].concat(JSON.parse(input) || [] ).map(j=>JSON.stringify(j));
} catch(e) {return ["no",JSON.parse(input)]}
""";
WITH
basetable as (Select row_number() over () Impact_Question_id,
"txt" as Impact_Question_text,
json
#1 as bdmp_id,
#current_date() Impact_Question_aktualisiert_am_ts,
from unnest([ '{"reply":[{"Impact_antwort_id":"anytext_2","Impact_reply_get":"ok test 2"}]}','{"reply":[{"Impact_antwort_id":"anytext_3"}]}']) as json
),
`Impact_JSON` AS (
SELECT
Impact_Question_id,
Impact_Question_text, json,
#ROW_NUMBER() OVER (PARTITION BY bdmp_id, DATE(Impact_Question_aktualisiert_am_ts)
# ORDER BY
# Impact_Question_aktualisiert_am_ts DESC) AS ROW
FROM
basetable
),
json_answers AS (
SELECT
regexp_replace(SPLIT(ANY_VALUE(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')),'_')[SAFE_OFFSET(1)], "[^0-9]+"," " ) AS Interview_ID,
# regexp_replace(SPLIT(ANY_VALUE(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')),'_')[SAFE_OFFSET(3)], "[^0-9]+"," " ) AS Quest_ID,
# STRING_AGG(DISTINCT(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_id')), ',\n')
# AS Impact_antwort_id,
#STRING_AGG(DISTINCT(JSON_EXTRACT_SCALAR(Impact, '$.Impact_antwort_daten_typ')),',\n')
# AS Impact_reply_data_type,
# IFNULL(JSON_EXTRACT_SCALAR(Impact, '$.Impact_topic_text'), 'Empty') AS Impact_topic_text,
IFNULL(JSON_EXTRACT_SCALAR(Impact, '$.Impact_reply_get'), 'Empty') AS Impact_reply_get,
string_agg(Impact) as impact_parsed_json,
FROM `Impact_JSON`,
UNNEST(jsonunnest(JSON_EXTRACT(json, '$.reply'))) Impact
GROUP by 2 #5,6
)
SELECT
*
FROM
`Impact_JSON` AS T
full join
json_answers as J
ON
(SAFE_CAST(J.Interview_ID as INT64))
=
T.Impact_Question_id