Searching for non-visible characters - sql

Question

I'm troubleshooting some strangeness in output of my SQL Server when I want to get records that are not null or empty string:
SELECT myString
FROM myTable
WHERE myString IS NOT NULL OR myString != ''
In addition to records that obviously fit (string values returned to SSMS's grid) I'm seeing records where the grid cell is blank. When I select the cell and try to copy it, my clipboard manager (clipmate.com) complains that the data is invalid.
If I output the query to file instead of to grid and then inspect via Hex char mode, sure enough - there are characters I would not expect (much less want) to be in there.
Collation of my database collation value shows as: SQL_Latin1_General_CP1_CI_AS
How would I go about eliminating any/all non-visible characters?

Answer 1

can try this, will remove any characters that are not printable.
CREATE FUNCTION [dbo].[RemoveNonPrintableChars]
(
#p_string varchar(max)
)
RETURNS varchar(max)
AS
BEGIN
declare #l_pos int = 1
declare #l_str varchar(max) = ''
while (#l_pos <= len(#p_string))
begin
if (ascii(substring(#p_string,#l_pos,1)) >=32)
begin
set #l_str=#l_str+substring(#p_string,#l_pos,1)
end
set #l_pos = #l_pos+1
end
return #l_str
END
declare
#l_str varchar(max) = 'andrew'--select char(7)
select dbo.[RemoveNonPrintableChars] (#l_str)
set #l_str = 'andrew'+char(7)-- add NP character
select dbo.[RemoveNonPrintableChars] (#l_str)

Answer 2

If by chance you can't use a UDF.
You may notice that I don't just strip the control characters here, I replace them with a space so not to concatenate strings.
Declare #YourTable table (SomeField varchar(50))
Insert Into #YourTable values
('Michael'+char(13)+char(10)+'LastName')
Select A.*
,B.Value
From #YourTable A
Cross Apply (
Select Value = replace(replace((
Select ''+C
From (
Select N,C=case when ASCII(Substring(A.SomeField,N ,1))>31 then Substring(A.SomeField,N ,1) else '{--space--}' end
From ( Select Top (Len(A.SomeField)) N=Row_Number() Over (Order By Number) From master..spt_values ) N
) C Order by N
For XML Path('') ) ,'{--space--}',' '),' ',' ')
) B
Returns
SomeField Value
Michael Michael LastName
LastName
EDIT
However, If you do want a UDF consider the following non-linear approach
CREATE FUNCTION [dbo].[udf-Str-Strip-Control](#S varchar(max))
Returns varchar(max)
Begin
;with cte1(N) As (Select 1 From (Values(1),(1),(1),(1),(1),(1),(1),(1),(1),(1)) N(N)),
cte2(C) As (Select Top (32) Char(Row_Number() over (Order By (Select NULL))-1) From cte1 a,cte1 b)
Select #S = Replace(#S,C,' ')
From cte2
Return LTrim(RTrim(Replace(Replace(#S,' ',' '),' ',' ')))
End
--Select [dbo].[udf-Str-Strip-Control]('Michael'+char(13)+char(10)+'LastName') --Returns: Michael LastName

Answer 3

I like John's answers better, and would probably modify them if you were looking to control whether or not to include certain control characters. This is the function I used in the past to clean up some strings.
create function dbo.fnCleanVarchar (
#StringParameter varchar(max)
, #CleanStyle tinyint = 1
) returns varchar(max) as
begin;
if #StringParameter is null
return null;
if #CleanStyle > 3 set #CleanStyle = 1;
declare #StringReturn varchar(max);
declare #StringLength int;
declare #CharacterCode int;
declare #CharacterCodePosition int;
set #StringReturn = '';
set #StringLength = len(#StringParameter);
set #CharacterCodePosition = 1;
while #CharacterCodePosition <= #StringLength
begin
set #CharacterCode = ascii(substring(#stringParameter , #CharacterCodePosition , 1))
-- Removes Unprintable Characters 0-8,12,14-31
-- If Style = 1, Remove Unprintable Characters except Tab (9), New Line (10), Carraige Return (13)
-- If Style = 2, Remove Unprintable Characters except character 9 (Tab)
-- If Style = 3, Remove Unprintable Characters and character 9 (Tab)
set #StringReturn = #StringReturn + case
when #CharacterCode >31
then char(#CharacterCode)
when #Style = 3
then ''
when #Style = 2 and #CharacterCode = 9
then char(9)
when #Style = 1 and #CharacterCode in ( 9 , 10 , 13 )
then char(#CharacterCode)
else ''
end;
set #CharacterCodePosition = #CharacterCodePosition + 1
end;
if len(#StringReturn) = 0
return null;
return #StringReturn
end;

Answer 4

Checking for non-visible fields is directly related to find non-visible characters, so consider these two notes:
Note 1: SQL Server will auto-trimming spaces in clauses so N' ' = N'' is true, and any continues strings of empty characters;
Empty characters are a character that is equal to N''.
Note 2: There are 65536 Unicode characters, you can view them with a query like this:
WITH CTE(i, c) AS (
SELECT 0, NCHAR(0) COLLATE SQL_Latin1_General_CP1_CI_AS --I add COLLATE to express your collation but I think it is optional
UNION ALL
SELECT i+1, NCHAR(i+1) COLLATE SQL_Latin1_General_CP1_CI_AS
FROM CTE
WHERE i < 65535
)
SELECT *
FROM CTE
OPTION ( MaxRecursion 0 );
Some of those are not visible and empty like NCHAR(0), NCHAR(12288), ...,
Some of those are not visible and not empty like NCHAR(1), ...,
Some of those are visible and empty like NCHAR(502), ... !!!!.
So if your field is a nvarchar string, you will have a big problem to filter not visible characters, and for varchar strings you have a problem but more little than that.
Side note: You can use COALESCE(myString, '') != '' instead of yours ;).
SUMMARY :
For a little pin don't create a hammer machine!.
When this behaviors are not so important for you and your project don't try to change or handle or create your own equal string function ;).
A sample function to check if a string is visible(returns 1) or not(returns 0) can be like this:
CREATE FUNCTION IsVisible ( #string varchar(max) )
RETURNS bit
AS
BEGIN
DECLARE #pString varchar(max) = #string;
WITH InvisibleChars AS (
SELECT c COLLATE SQL_Latin1_General_CP1_CI_AS AS c
FROM (VALUES (CHAR(0)), (CHAR(1)), (CHAR(9)), (CHAR(10)),
(CHAR(11)), (CHAR(12)), (CHAR(13)), (CHAR(28)),
(CHAR(29)), (CHAR(30)), (CHAR(31)), (CHAR(32)),
(CHAR(160)) -- Above characters are non-visibles
) t(c)
)
SELECT #pString = REPLACE(#pString, c, '')
FROM InvisibleChars;
RETURN CASE WHEN #pString = '' THEN 0 ELSE 1 END;
END
GO