Safe casting a regexp match from REGEXP_REPLACE - google-bigquery

Question

I have a table with some code points (e.g. &#38) which I want to strip out from a text value in BigQuery.
My strategy is to use a regexp replace on the number replacing the number with the valid character.
If I try:
WITH items as (SELECT "Test & " as item)
SELECT
CODE_POINTS_TO_STRING([SAFE_CAST(REGEXP_EXTRACT(item, r"&#([0-9]{2})") AS INT64)]) as test_replace
FROM items
This will produce the output that I want for the entry
[
{
"test_replace": "&"
}
]
If I try:
WITH items as (SELECT "Test & " as item)
SELECT
REGEXP_REPLACE(
item,
r"&#([0-9]{2});",
CODE_POINTS_TO_STRING([SAFE_CAST("\\1" as INT64)])
) as full_replace
FROM items
This will produce a null output
[
{
"full_replace": null
}
]
However if I hard code the value in:
WITH items as (SELECT "Test & " as item)
SELECT
REGEXP_REPLACE(
item,
r"&#([0-9]{2});",
CODE_POINTS_TO_STRING([SAFE_CAST("38" as INT64)])
) as full_replace
FROM items
This works.
[
{
"full_replace": "Test & "
]
I know that the regexp is evaluating correctly as if I try:
WITH items as (SELECT "Test & " as item)
SELECT
REGEXP_REPLACE(
item,
r"&#([0-9]{2});",
CONCAT("\\1", "test")
) as part_replace
FROM ITEMS
This will return:
[
{
"part_replace": "Test 38test "
}
]
My question is therefore, how do I get the SAFE_CAST() Function to evaluate the regexp match (it seems to be evaluating the string literal).

Answer 1

I have a table with some code points (e.g. &#38) which I want to strip out from a text value in BigQuery.
Try approach as in below example
#standardSQL
CREATE TEMP FUNCTION multiReplace(item STRING, arr ARRAY<STRUCT<x STRING, y STRING>>)
RETURNS STRING
LANGUAGE js AS """
for (i = 0; i < arr.length; i++) {
item = item.replace(arr[i].x, arr[i].y)
};
return item;
""";
WITH items AS (
SELECT "Test & abc ' xyz" AS item UNION ALL
SELECT "abc xyz"
)
SELECT item, multiReplace(item, points) full_replace
FROM (
SELECT
item,
ARRAY(
SELECT AS STRUCT val, CODE_POINTS_TO_STRING([SAFE_CAST(SUBSTR(val, -3, 2) AS INT64)]) point
FROM UNNEST(REGEXP_EXTRACT_ALL(item, r'(&#[0-9]{2};)')) val
) points
FROM items
)
with result
Row item full_replace
1 Test & abc ' xyz Test & abc ' xyz
2 abc xyz abc xyz
Option 2
While the simplest way to approach above is
#standardSQL
CREATE TEMP FUNCTION multiReplace(item STRING)
RETURNS STRING
LANGUAGE js AS """
var decodeHtmlEntity = function(str) {
return str.replace(/&#([0-9]{2});/g, function(match, dec) {
return String.fromCharCode(dec);
});
};
return decodeHtmlEntity(item);
""";
WITH items AS (
SELECT "Test & abc ' xyz" AS item UNION ALL
SELECT "abc xyz"
)
SELECT item, multiReplace(item) full_replace
FROM items
with the same output