Avoid handling invalid codepoint in Bigquery - google-bigquery

Question

I'm using the following SQL code in order to decode UTF-8 chars in Big query:
#standardSQL
CREATE TEMP FUNCTION DecodeUnicode(s STRING) AS (
(SELECT CODE_POINTS_TO_STRING(ARRAY_AGG(CAST(CONCAT('0x', x) AS INT64)))
FROM UNNEST(SPLIT(s, '\\u')) AS x
WHERE x != ''
)
);
WITH normal AS (
select '\\u05DE\\u05EA\\u05DE\\u05D8\\u05D9\\u05E7\\u05D4123' as edited
), uchars AS (
SELECT DISTINCT
c,
DecodeUnicode(c) uchar
FROM normal,
UNNEST(REGEXP_EXTRACT_ALL(edited, r'(\\u[ABCDEF0-9]{4,8})')) c
)
SELECT
edited,
STRING_AGG(IFNULL(uchar, x), '' ORDER BY pos) decoded
FROM(
SELECT
edited,
pos,
SUBSTR(edited,
SUM(CASE char WHEN '' THEN 1 ELSE 6 END)
OVER(PARTITION BY edited ORDER BY pos) - CASE char WHEN '' THEN 0 ELSE 5 END,
CASE char WHEN '' THEN 1 ELSE 6 END) x,
uchar
FROM normal ,
UNNEST(REGEXP_EXTRACT_ALL(edited, r'(\\u[ABCDEF0-9]{4,8})|.')) char WITH
OFFSET AS pos LEFT JOIN uchars u ON u.c = char
)
GROUP BY edited
The problem is that some of the values I'm handling are not valid when using the function above ('DecodeUnicode')
- for example this part 'u05D4123' is not charbase valid.
What can I change in my code that when I have such values the function will not handle it and therefore I will not get the 'Invalid codepoint' Error that I get now?

Answer 1

One of the option is to use SAFE.CODE_POINTS_TO_STRING instead of CODE_POINTS_TO_STRING but then you still will need to eliminate not handled code from result - for example using regexp as it is in below example
#standardSQL
CREATE TEMP FUNCTION DecodeUnicode(s STRING) AS (
(SELECT SAFE.CODE_POINTS_TO_STRING(ARRAY_AGG(CAST(CONCAT('0x', x) AS INT64)))
FROM UNNEST(SPLIT(s, '\\u')) AS x
WHERE x != ''
)
);
WITH normal AS (
SELECT '\\u05DE\\u05EA\\u05DE\\u05D8\\u05D9\\u05E7\\u05D4123' AS edited
), uchars AS (
SELECT DISTINCT
c,
DecodeUnicode(c) uchar
FROM normal,
UNNEST(REGEXP_EXTRACT_ALL(edited, r'(\\u[ABCDEF0-9]{4,8})')) c
)
SELECT
edited,
STRING_AGG(IFNULL(uchar, x), '' ORDER BY pos) decoded,
REGEXP_REPLACE(STRING_AGG(IFNULL(uchar, x), '' ORDER BY pos) ,r'\\u[ABCDEF0-9]{4,8}', '') decoded_and_fixed
FROM(
SELECT
edited,
pos,
SUBSTR(edited,
SUM(CASE char WHEN '' THEN 1 ELSE 6 END)
OVER(PARTITION BY edited ORDER BY pos) - CASE char WHEN '' THEN 0 ELSE 5 END,
CASE char WHEN '' THEN 1 ELSE 6 END) x,
uchar
FROM normal ,
UNNEST(REGEXP_EXTRACT_ALL(edited, r'(\\u[ABCDEF0-9]{4,8})|.')) char WITH
OFFSET AS pos LEFT JOIN uchars u ON u.c = char
)
GROUP BY edited
with result
Row edited decoded decoded_and_fixed
1 \u05DE\u05EA\u05DE\u05D8\u05D9\u05E7\u05D4123 מתמטיק\u05D4 מתמטיק