Replace consecutive duplicate letters with a single letter - sql

Question

How do I create a function in SQL Server 2017 that identifies when a string contains duplicate consecutive letters (a-z) and replaces those duplicate letters with a single instance of that letter?
Here are some examples of what should happen:
CompanyAAABCD -> CompanyABCD
CommpanyABYTTT -> CompanyABYT
Company11111 -> Company11111

Answer 1

alter function fn_RemoveDuplicateChar(#name varchar(200))
RETURNS VARCHAR(200)
as
begin
declare #strPosition int=1;
declare #strlen int=0;
declare #finalstr varchar(200)='';
declare #str varchar(200)='';
declare #fstr varchar(200)='';
select #strlen = (select len(#name))
while #strPosition<=#strlen
begin
select #fstr = SUBSTRING(#name, #strPosition, 1)
select #str = SUBSTRING(#finalstr, len(#finalstr), 1)
If #fstr <> #str or ( ISNUMERIC(#fstr)=1 and ISNUMERIC(#str)=1)
set #finalstr = #finalstr + #fstr
set #strPosition =#strPosition+1
end
return (select #finalstr)
end
go
select dbo.fn_RemoveDuplicateChar('CompanyAAABCD')
select dbo.fn_RemoveDuplicateChar('CommpanyABYTTT')
select dbo.fn_RemoveDuplicateChar('Company11111')

Answer 2

If you just wanted a single round of replacement (i.e. aaabbbb becomes aabb) then you could use this:
CREATE OR ALTER FUNCTION dbo.RemoveDuplicates (#value varchar(200))
RETURNS VARCHAR(200)
WITH SCHEMABINDING
AS
BEGIN
DECLARE #result varchar(200) = #value;
DECLARE #i int = 65;
-- a-z is ASCII 65-90
WHILE #i < 90
BEGIN
SET #result = REPLACE(#result, CHAR(#i) + CHAR(#i), CHAR(#i));
SET #i += 1
END;
RETURN #result;
END;
GO
But it seems you need a recursive replacement, so that every character that has the same before it is removed.
So we can use this version, which is similar to the other answer.
CREATE OR ALTER FUNCTION dbo.RemoveDuplicates (#value varchar(200))
RETURNS varchar(200)
WITH SCHEMABINDING
AS
BEGIN
DECLARE #c char(1);
DECLARE #cLast char(1) = LEFT(#value, 1);
DECLARE #result varchar(200) = #cLast;
DECLARE #strlen int = LEN(#value);
DECLARE #i int = 2;
WHILE (#i < #strlen)
BEGIN
SET #c = SUBSTRING(#value, #i, 1);
IF (#c <> #cLast)
SET #result += #c;
SET #i += 1
END;
RETURN #result;
END;
GO
I rewrote this as an inline Table-Valued Function, and found it significantly faster. Here are two versions of that, depending whether you can use STRING_AGG
CREATE OR ALTER FUNCTION dbo.RemoveDuplicatesXML (#value varchar(200))
RETURNS TABLE
WITH SCHEMABINDING
AS RETURN
(
WITH L1 AS (SELECT n FROM (VALUES(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1)) v(n)),
L2 AS (SELECT 1 n FROM L1 A CROSS JOIN L1 B),
Nums AS (SELECT ROW_NUMBER() OVER (ORDER BY (SELECT 1)) rn FROM L2),
Chars AS (SELECT TOP(LEN(#value)) rn FROM Nums)
SELECT (
SELECT SUBSTRING(#value, rn, 1)
FROM Chars
WHERE rn = 1 OR SUBSTRING(#value, rn - 1, 1) <> SUBSTRING(#value, rn, 1)
ORDER BY rn
FOR XML PATH(''), TYPE
).value('text()[1]','nvarchar(max)') Result
);
GO
CREATE OR ALTER FUNCTION dbo.RemoveDuplicatesAGG (#value varchar(200))
RETURNS TABLE
WITH SCHEMABINDING
AS RETURN
(
WITH L1 AS (SELECT n FROM (VALUES(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1),(1)) v(n)),
L2 AS (SELECT 1 n FROM L1 A CROSS JOIN L1 B),
Nums AS (SELECT ROW_NUMBER() OVER (ORDER BY (SELECT 1)) rn FROM L2),
Chars AS (SELECT TOP(LEN(#value)) rn FROM Nums)
SELECT STRING_AGG(SUBSTRING(#value, rn, 1), '') WITHIN GROUP (ORDER BY rn) Result
FROM Chars
WHERE rn = 1 OR SUBSTRING(#value, rn - 1, 1) <> SUBSTRING(#value, rn, 1)
);
GO
This utilizes Itzik Ben-Gan's famous inline tally-table method to break out the string into single characters. You will need another CROSS JOIN or more (1) if you have more than 256 characters.
You have two methods to use this, the performance should be identical
Either as a scalar subquery
SELECT (SELECT * FROM RemoveDuplicatesAGG(t.MyString) Result
FROM myTable t
Or as an APPLY
SELECT d.Result
FROM myTable t
CROSS APPLY RemoveDuplicatesAGG(t.MyString) d

Answer 3

I know I'm a little late here but if performance is important then you can use the fastest "de-duplicator" in the game (the function, removeDupesExcept8K, is at the end of this post.) It takes an input string and a pattern representing what you want deduplicated; in the example below I'm saying "deduplicate anything that's not between A to Z.
DECLARE #string VARCHAR(8000) = 'AAABBBCCC999';
SELECT rd.NewString FROM samd.removeDupesExcept8K(#string, '[^A-Z]') AS rd;
Returns: ABC999
Let's compare fn_RemoveDuplicateChar from B.Muthamizhselvi above to the one at the end of the post.
Performance test:
--==== Test Data
SELECT TOP(10000)
ID = IDENTITY(INT,1,1),
String = REPLACE(REPLACE(REPLACE(NEWID(),'A',0),'B',0),'-','AAA')
INTO #strings
FROM sys.all_columns, sys.all_columns b;
GO
--==== Performance Test
PRINT CHAR(13)+'dbo.fn_RemoveDuplicateChar'+CHAR(13)+REPLICATE('-',90);
GO
DECLARE #st DATETIME = GETDATE(), #x VARCHAR(100);
SELECT #x = dbo.fn_RemoveDuplicateChar(s.String)
FROM #strings AS s
PRINT DATEDIFF(MS,#st,GETDATE());
GO 3
PRINT CHAR(13)+'samd.removeDupChar8K - Serial'+CHAR(13)+REPLICATE('-',90);
GO
DECLARE #st DATETIME = GETDATE(), #x VARCHAR(100);
SELECT #x = rd.NewString
FROM #strings AS s
CROSS APPLY samd.removeDupesExcept8K(s.String,'[^A-Z]') AS rd
OPTION (MAXDOP 1);
PRINT DATEDIFF(MS,#st,GETDATE());
GO 3
PRINT CHAR(13)+'samd.removeDupChar8K - Parallel'+CHAR(13)+REPLICATE('-',90);
GO
DECLARE #st DATETIME = GETDATE(), #x VARCHAR(100);
SELECT #x = rd.NewString
FROM #strings AS s
CROSS APPLY samd.removeDupesExcept8K(s.String,'[^A-Z]') AS rd
OPTION (QUERYTRACEON 8649);
PRINT DATEDIFF(MS,#st,GETDATE());
GO 3
As you'll see below, removeDupesExcept8K is twice as fast with a serial execution plan (one CPU) and more than 10X faster with a parallel plan. No need to test fn_RemoveDuplicateChar with a parallel plan, scalar UDFs can't go parallel unless inlined.
Test Results:
dbo.fn_RemoveDuplicateChar
------------------------------------------------------------------------------------------
Beginning execution loop
1110
1106
1093
Batch execution completed 3 times.
samd.removeDupChar8K - Serial
------------------------------------------------------------------------------------------
Beginning execution loop
563
560
593
Batch execution completed 3 times.
samd.removeDupChar8K - Parallel
------------------------------------------------------------------------------------------
Beginning execution loop
91
91
93
Batch execution completed 3 times.
The Function
IF OBJECT_ID('samd.removeDupesExcept8K') IS NOT NULL DROP FUNCTION samd.removeDupesExcept8K;
GO
CREATE FUNCTION samd.removeDupesExcept8K(#string varchar(8000), #preserved varchar(50))
/*****************************************************************************************
[Purpose]:
A purely set-based inline table valued function (iTVF) that accepts and input strings
(#string) and a pattern (#preserved) and removes all duplicate characters in #string that
do not match the #preserved pattern.
[Author]:
Alan Burstein
[Compatibility]:
SQL Server 2008+
[Syntax]:
--===== Autonomous use
SELECT rd.newString
FROM samd.removeDupesExcept8K(#string, #preserved) AS rd;
--===== Use against a table
SELECT st.SomeColumn1, rd.newString
FROM SomeTable AS st
CROSS
APPLY samd.removeDupesExcept8K(st.SomeColumn1, #preserved) AS rd;
Parameters:
#string = varchar(8000); Input string to be "cleaned"
#preserved = varchar(50); the pattern to preserve. For example, when #preserved='[0-9]'
only non-numeric characters will be removed
[Return Types]:
Inline Table Valued Function returns:
newString = varchar(8000); the string with duplicate characters removed
[Developer Notes]:
1. Requires NGrams8K. The code for NGrams8K can be found here:
http://www.sqlservercentral.com/articles/Tally+Table/142316/
2. This function is what is referred to as an "inline" scalar UDF." Technically it's an
inline table valued function (iTVF) but performs the same task as a scalar valued user
defined function (UDF); the difference is that it requires the APPLY table operator
to accept column values as a parameter. For more about "inline" scalar UDFs see this
article by SQL MVP Jeff Moden: http://www.sqlservercentral.com/articles/T-SQL/91724/
and for more about how to use APPLY see the this article by SQL MVP Paul White:
http://www.sqlservercentral.com/articles/APPLY/69953/.
Note the above syntax example and usage examples below to better understand how to
use the function. Although the function is slightly more complicated to use than a
scalar UDF it will yield notably better performance for many reasons. For example,
unlike a scalar UDFs or multi-line table valued functions, the inline scalar UDF does
not restrict the query optimizer's ability generate a parallel query execution plan.
3. removeDupesExcept8K is deterministic; for more about deterministic and nondeterministic
functions see https://msdn.microsoft.com/en-us/library/ms178091.aspx
[Examples]:
--===== 1. Examples...
DECLARE #string varchar(8000) = '!!!aa###bb!!!';
BEGIN
--===== 1.1. Remove all duplicate characters
SELECT f.newString
FROM samd.removeDupesExcept8K(#string,'') f; -- Returns: !a#b!
--===== 1.2. Remove all non-alphabetical duplicates
SELECT f.newString
FROM samd.removeDupesExcept8K(#string,'[a-z]') f; -- Returns: !aa#bb!
--===== 1.3. Remove only alphabetical duplicates
SELECT f.newString
FROM samd.removeDupesExcept8K(#string,'[^a-z]') f; -- Returns: !!!a###b!!!
END
---------------------------------------------------------------------------------------
[Revision History]:
Rev 00 - 20160720 - Initial Creation - Alan Burstein
****************************************************************************************/
RETURNS TABLE WITH SCHEMABINDING AS RETURN
SELECT newString =
(
SELECT ng.token+''
FROM samd.NGrams8K(#string,1) AS ng
WHERE ng.token <> SUBSTRING(#string, ng.position+1,1) -- exclude chars = the next char
OR ng.token LIKE #preserved -- preserve characters that match the #preserved pattern
ORDER BY ng.position
FOR XML PATH(''),TYPE
).value('(text())[1]','varchar(8000)'); -- using Wayne Sheffield’s concatenation logic