split CLOB field into table like structure - sql

Question

I have field in a Oracle database of type CLOB. I would like to split this field in several columns and rows. Here's an example of the content:
4:true5:false24:<p>option sample 1.</p>4:true22:<p>option sample 2.</p>5:false23:<p>option sample 3.</p>5:false22:<p>option sample 4.</p>5:false
The result should look like this:
ID LEVEL ANSWER_OPTION VALUE
1 3 option sample 3 false
1 4 option sample 4 false
2 3 option sample 3 false
4 3 option sample 3 true
3 2 option sample 2 false
1 2 option sample 2 true
2 1 option sample 1 true
2 4 option sample 4 false
4 1 option sample 1 false
2 2 option sample 2 false
4 2 option sample 2 false
1 1 option sample 1 false
3 4 option sample 4 false
4 4 option sample 4 false
3 3 option sample 3 false
3 1 option sample 1 true
We have made the following statement which created the result above.
with guest_string as
( select qsn.id id
, dbms_lob.substr( qsn.guest, 2000, 1 ) answer_options
from mneme_question qsn
where qsn.id < 10
)
select distinct id
, level
, substr(regexp_substr( answer_options'<p>[^<]+', 1, level, 'i'), 4) ANSWER_OPTION
, substr(regexp_substr( answer_options, '(true|false)', regexp_instr( answer_options, '</p>', 1, 1), level, 'i'), 1) VALUE
from guest_string
connect by regexp_substr( answer_options, '<p>[^<]+', 1, level, 'i') is not null
The problem with this code is it takes way to long to split all records we have. We had to cut it off at 10 rows (5 row take 0.25 sec, 10 takes 16 seconds, 15 rows takes about over 2,5 minutes). We currently have 30000 rows and they will grow. At the moment we cannot change the software to change the datamodel, so we will have to do this ad hoc.
Our current approach is to create a procedure that will be called for each record, but it would be better to have a faster parsing. Does anybody have a suggestion how to create a script that can do this in a reasonable time. We can do this at night time, but preferably it shouldn't take longer than 2 minutes.
BTW in the future we could build some sort of mechanism to determine which records have already been parsed, but that also requires some form of detecting already parsed fields that have changed in the mean time. We don't have time to do that yet, so for now we need crude parsing as fast as possible.
Thanks

Answer 1

Here is a different approach.
The code may not be as nice as it can, and you will probably have to fix some small things...
I checked it on 11g (couldn't find 10g) and used your input as the values in my clob column.
For 10 rows (all with the same input as you gave as example),
original query: 9.8 sec
new query: 0.08 sec
I used a pipelined function, here is the code:
create or replace type t_parse is object(idd number, levell number, answer_option varchar2(128), valuee varchar2(4000));
/
create or replace type tab_parse is table of t_parse;
/
create or replace function split_answers return tab_parse
pipelined is
cursor c is
select * from mneme_question;
str_t clob;
phraseP varchar2(128);
phraseV varchar2(8);
i1s number;
i1e number;
i2s number;
levell number;
begin
for r in c loop
str_t := r.guest;
levell := 1;
while str_t is not null loop
i1s := dbms_lob.instr(str_t, '<p>', 1, 1) + 3;
if i1s = 3 then
str_t := '';
else
i1e := dbms_lob.instr(str_t, '</p>', 1, 1);
phraseP := dbms_lob.substr(str_t, i1e - i1s, i1s);
str_t := dbms_lob.substr(str_t, offset => i1e + 4);
i2s := dbms_lob.instr(str_t, 'true', 1, 1) ;
if i2s = 0 then
i2s := dbms_lob.instr(str_t, 'false', 1, 1) ;
if i2s = 0 then
str_t := '';
else
phraseV := dbms_lob.substr(str_t, 5, i2s);
pipe row(t_parse(r.id, levell, phraseP, phraseV));
levell := levell + 1;
end if;
else
phraseV := dbms_lob.substr(str_t, 4, i2s);
pipe row(t_parse(r.id, levell, phraseP, phraseV));
levell := levell + 1;
end if;
end if;
end loop;
end loop;
return;
end split_answers;
/
new query should be like:
select * from table(split_answers);

Answer 2

There is some function whis returning pipelined table:
CREATE OR REPLACE FUNCTION split_clob(p_clob IN CLOB DEFAULT NULL,
p_varchar IN VARCHAR2 DEFAULT NULL,
p_separator IN VARCHAR2)
RETURN varchar_set
PIPELINED IS
l_clob CLOB;
BEGIN
l_clob := nvl(p_clob,
to_clob(p_varchar));
FOR rec IN (
WITH vals AS
(SELECT CAST(TRIM(regexp_substr(l_clob,
'[^'||p_separator||']+',
1,
levels.column_value)) AS
VARCHAR2(320)) AS val
FROM TABLE(CAST(MULTISET
(SELECT LEVEL
FROM dual
CONNECT BY LEVEL <=
length(regexp_replace(l_clob,
'[^'||p_separator||']+')) + 1) AS
sys.odcinumberlist)) levels)
SELECT val FROM vals)
LOOP
PIPE ROW(rec.val);
END LOOP;
RETURN;
END;
U can use it like this:
-with table with clob
SELECT t2.column_value FROM my_table t, TABLE(split_clob(p_clob => t.clob_column,p_separator => ',')) t2
-or with varchar too
SELECT * FROM TABLE(split_clob(p_varchar => '1,2,3,4,5,6, 7, 8, 9', p_separator => ','))