Communication channel error when running BigQuery udf - google-bigquery

Question

I recently posted on stack overflow about an OOM error while running many udfs on a single table (bigquery udf out of memory issues). This error seems to have been partially fixed, however, I am running into a new error when running a udf on a 10,000 row table. Here is the error message:
Error: An error occurred while communicating with a subprocess. message: "Communication channel error during 4 command" sandbox_process_error { }
Error Location: User-defined function
Job ID: broad-cga-het:bquijob_32bc01d_1569f11b8a2
The error does not occur when I remove the emit statement in the udf, so the error must be occurring when the udf tries to write back to a different table.
Here is a copy of the udf itself:
bigquery.defineFunction(
'permute',
['obj_nums','num_obj_per_indiv','row_number'], // Names of input columns
[{"name": "num_cooccurrences_list","type": "string","mode":"nullable"}], // Output schema
permute
);
function permute(row, emit) {
var obj_ids = row['obj_nums'].split(",").map(function (x) {
return parseInt(x, 10);
});
var num_obj_per_indiv = row['num_obj_per_indiv'].split(",").map(function (x) {
return parseInt(x, 10);
});
var row_number = row['row_number']
// randomly shuffle objs using Durstenfeld shuffle algorithm
obj_ids = shuffle_objs(obj_ids);
// form dictionary of obj_pairs from obj_ids
var perm_run_obj_set = new Set(obj_ids);
var perm_run_obj_unique = Array.from(perm_run_obj_set);
perm_run_obj_unique.sort();
var perm_run_obj_pairs_dict = {};
output = {}
for (var i = 0; i < perm_run_obj_unique.length - 1; i++) {
for (var j = i + 1; j < perm_run_obj_unique.length; j++) {
var obj_pair = [perm_run_obj_unique[i],perm_run_obj_unique[j]].sort().join("_")
perm_run_obj_pairs_dict[obj_pair] = 0
}
}
// use fixed number of objs per indiv and draw from shuffled objs
var perm_cooccur_dict = {};
//num_obj_per_indiv = num_obj_per_indiv.slice(0,3);
for(var index in num_obj_per_indiv) {
var obj_count = num_obj_per_indiv[index]
var perm_run_objs = [];
for(var j = 0; j < obj_count; j++) {
perm_run_objs.push(obj_ids.pop());
}
perm_run_objs = new Set(perm_run_objs);
perm_run_objs = Array.from(perm_run_objs)
while(perm_run_objs.length > 1) {
current_obj = perm_run_objs.pop()
for(var pair_obj_ind in perm_run_objs) {
var pair_obj = perm_run_objs[pair_obj_ind]
var sorted_pair = [current_obj,pair_obj].sort().join("_")
perm_run_obj_pairs_dict[sorted_pair] += 1
// console.log({"obj_pair":[current_obj,pair_obj].sort().join("_"),"perm_run_id":row_number})
// emit({"obj_pair":[current_obj,pair_obj].sort().join("_"),"perm_run_id":row_number});
}
}
}
// emit({"obj_pair":[current_obj,pair_obj].sort().join("_"),"perm_run_id":row_number});
// form output dictionary
num_cooccur_output = ""
for (var obj_pair in perm_run_obj_pairs_dict) {
//emit({"obj_pair":obj_pair,"num_cooccur":perm_run_obj_pairs_dict[obj_pair]});
num_cooccur_output += String(perm_run_obj_pairs_dict[obj_pair])
num_cooccur_output += ","
}
num_cooccur_output = num_cooccur_output.substring(0, num_cooccur_output.length - 1);
emit({"num_cooccurrences_list":num_cooccur_output});
}
/**
* Randomize array element order in-place.
* Using Durstenfeld shuffle algorithm.
*/
function shuffle_objs(obj_array) {
for (var i = obj_array.length - 1; i > 0; i--) {
var j = Math.floor(Math.random() * (i + 1));
var temp = obj_array[i];
obj_array[i] = obj_array[j];
obj_array[j] = temp;
}
return obj_array;
}
Any help would be greatly appreciated!
Thank you,
Daniel

Answer 1

Not that this directly answers your original question, but I'm not convinced that you need a UDF for this type of transformation. For example, using standard SQL (uncheck "Use Legacy SQL" under "Show Options") you can perform an array transformation such as:
WITH T AS (SELECT [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] AS arr)
SELECT
x,
new_x
FROM T,
UNNEST(
ARRAY(SELECT AS STRUCT
x,
arr[OFFSET(CAST(RAND() * (off + 1) AS INT64))] AS new_x
FROM UNNEST(arr) AS x WITH OFFSET off));
+---+-------+
| x | new_x |
+---+-------+
| 0 | 1 |
| 1 | 2 |
| 2 | 0 |
| 3 | 2 |
| 4 | 4 |
| 5 | 0 |
| 6 | 5 |
| 7 | 4 |
| 8 | 8 |
| 9 | 3 |
+---+-------+
I can explain more if it helps, but the gist of the query is that it randomizes the elements in arr using the formula from your UDF above. The FROM T, UNNEST(... unrolls the elements of the array to make them easier to see, but I could have alternatively done:
WITH T AS (SELECT [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] AS arr)
SELECT
ARRAY(SELECT AS STRUCT
x,
arr[OFFSET(CAST(RAND() * (off + 1) AS INT64))] AS new_x
FROM UNNEST(arr) AS x WITH OFFSET off)
FROM T;
This gives an array of structs as output, where each x is associated with new_x.