Can I tokenize a string using t-SQL - sql

Question

I was wondering if I have SQL Server 2008 table that was created like this:
CREATE TABLE tbl (id INT PRIMARY KEY,
dvt NVARCHAR(32),
d0 TINYINT,
d1 TINYINT,
d2 TINYINT);
INSERT INTO tbl (id, dvt, d0, d1, d2)
VALUES(1, '1', NULL, NULL, NULL);
INSERT INTO tbl (id, dvt, d0, d1, d2)
VALUES(2, '', NULL, NULL, NULL);
INSERT INTO tbl (id, dvt, d0, d1, d2)
VALUES(3, '2,5', NULL, NULL, NULL);
INSERT INTO tbl (id, dvt, d0, d1, d2)
VALUES(4, '13, 34, 45, 5', NULL, NULL, NULL);
INSERT INTO tbl (id, dvt, d0, d1, d2)
VALUES(5, '1,8, 10', NULL, NULL, NULL);
I need to take the string from the 'dvt' column and split it into 'd0', 'd1' and 'd2' columns. The 'dvt' value can be separated by commas.
I can do this using C# and a tokenization function but I was wondering if it's possible to do the same using SQL?
Columns BEFORE:
1, "1", NULL, NULL, NULL
2, "", NULL, NULL, NULL
3, "2,5", NULL, NULL, NULL
4, "13, 34, 45, 5", NULL, NULL, NULL
5, "1,8, 10", NULL, NULL, NULL
Columns AFTER:
1, "1", 1, NULL, NULL
2, "", NULL, NULL, NULL
3, "2,5", 2, 5, NULL
4, "13, 34, 45, 5", 13, 34, 45 -- 5 is discarded
5, "1,8, 10", 1, 8, 10

Answer 1

The main problem with this type of code is re-use of calculations.
SQL Server is good at caching results (If you type the exact same CHARINDEX() caluculation 5 times, it only calculates once and re-uses that result 4 times).
That's little consolation for the poor coder who has to type or maintain that code though.
SQL Server 2005 onward has CROSS APPLY that does help somewhat. The logic is repeated, but the results can be referenced repeatedly, rather that the calculation typed repeatedly.
SELECT
*,
SUBSTRING(dvt, 1, ISNULL(comma1.pos-1, LEN(dvt)) ) AS item1,
SUBSTRING(dvt, comma1.pos+1, ISNULL(comma2.pos-1, LEN(dvt))-comma1.pos) AS item2,
SUBSTRING(dvt, comma2.pos+1, ISNULL(comma3.pos-1, LEN(dvt))-comma2.pos) AS item3
FROM
(
SELECT 'ab,c,def,hij' AS dvt
UNION ALL
SELECT 'xyz,abc' AS dvt
)
AS data
OUTER APPLY
(SELECT NULLIF(CHARINDEX(',', data.dvt, 1 ), 0) AS pos ) AS comma1
OUTER APPLY
(SELECT NULLIF(CHARINDEX(',', data.dvt, comma1.pos+1), 0) AS pos WHERE comma1.pos > 0) AS comma2
OUTER APPLY
(SELECT NULLIF(CHARINDEX(',', data.dvt, comma2.pos+1), 0) AS pos WHERE comma2.pos > 0) AS comma3
OUTER APPLY
(SELECT NULLIF(CHARINDEX(',', data.dvt, comma3.pos+1), 0) AS pos WHERE comma3.pos > 0) AS comma4
Another option is to simply write a table valued user defined function that does this (even when the result of the function is always one row). Then you simply CROSS APPLY that function.

Answer 2

Try something like this
;WITH Vals AS (
SELECT id,
dvt,
CAST('<r>'+REPLACE(dvt,',','</r><r>')+'</r>' AS XML).query('/r[1]').value('.','varchar(max)') d1,
CAST('<r>'+REPLACE(dvt,',','</r><r>')+'</r>' AS XML).query('/r[2]').value('.','varchar(max)') d2,
CAST('<r>'+REPLACE(dvt,',','</r><r>')+'</r>' AS XML).query('/r[3]').value('.','varchar(max)') d3
FROM tbl
)
SELECT id,
dvt,
CASE WHEN d1 = '' THEN NULL ELSE d1 END d1,
CASE WHEN d2 = '' THEN NULL ELSE d2 END d2,
CASE WHEN d3 = '' THEN NULL ELSE d3 END d3
FROM Vals

Answer 3

It's possible.
You could do it with some repetitive calls to CHARINDEX and checking for nulls but it may be better and clearer to write a FUNCTION to split the string.

Answer 4

I needed String tokenizer for Sybase; separation by 1 or more spaces in name data
Name date clean and has no commas or other special characters
declare #test varchar(60)
select #test=str_replace(lower(rtrim('Jayanta Narayan Choudhuri'))," ",",")
exec sp_splitwords #test
This is based on a neat hint by Kenny Lucas from http://www.sql9.com/?id=102
drop proc sp_splitwords
go
create proc sp_splitwords(#instr varchar(80)) as
begin
declare #pos int,
#word varchar(80),
#list varchar(81)
create table #words(word varchar(80))
select #list = #instr + ','
set #pos = patindex('%,,%',#list)
while #pos > 0
begin
select #list = str_replace(#list,',,',',')
set #pos = patindex('%,,%',#list)
end
set #pos = patindex('%,%',#list)
while #pos > 0
begin
set #word = substring(#list, 1,#pos-1)
set #list = substring(#list, #pos+1,len(#list)-#pos)
if NOT( #word is null OR LEN(#word) = 0 )
insert into #words (word) values (#word)
set #pos = patindex('%,%',#list)
end
select * from #words
order by len(word) desc
drop table #words
end
I could port Metaphone SQL Function to Sybase
http://www.sqlteam.com/forums/topic.asp?TOPIC_ID=125724
Sybase allows recursion in functions from a beautiful workaround
http://www.sypron.nl/quiz2008a.html#jan08
CREATE FUNCTION Metaphone2 (#str VARCHAR(100))
RETURNS VARCHAR(25) AS
BEGIN
RETURN #str
END
DROP FUNCTION Metaphone2
GO
CREATE FUNCTION Metaphone2 (#str VARCHAR(100))
RETURNS VARCHAR(25) AS
BEGIN
RETURN dbo.Metaphone(#str)
END
Changed 1 line of Function pasted from http://www.sqlteam.com/forums/topic.asp?TOPIC_ID=125724
Combination of metaphone and string tokenizer means I can fuzzy search names
first middle and surname and rotations thereof