How can I implement Stored Procedure that accept dynamic search criteria? - sql

Suppose I have the below table ( TestTable ) :
ID , SystemID , UserID ( all columns are of type int )
I want to write a stored procedure that should accept a string parameter; its value like ((5 and 6) or 7) to return all users that apply the below queries :
Select * From TestTable Where SystemID = 5
Intersect
Select * From TestTable Where SystemID = 6
and the above result is union with
Select * From TestTable Where SystemID = 7
SP must accept any combination like (((4 or 5) and 6) or 8) , (((5 or 9) or 8) and 10) .. etc
How can I implement that ?
Update : my issue isn't how to split the string .. but how can i make dynamic sql to implement it's logical mean

DECLARE #param NVARCHAR(MAX) = N'4 or 5 and 6 or 8 and 10';
DECLARE
#sql NVARCHAR(MAX) = N'',
#q NVARCHAR(MAX) = N'SELECT UserID FROM dbo.TestTable WHERE SystemID = ';
SELECT #sql = #q + REPLACE(REPLACE(#param, ' or ', '
UNION ALL ' + #q),
' and ', '
INTERSECT ' + #q);
PRINT #sql;
-- EXEC sp_executesql #sql;
Results:
SELECT UserID FROM dbo.TestTable WHERE SystemID = 4
UNION ALL SELECT UserID FROM dbo.TestTable WHERE SystemID = 5
INTERSECT SELECT UserID FROM dbo.TestTable WHERE SystemID = 6
UNION ALL SELECT UserID FROM dbo.TestTable WHERE SystemID = 8
INTERSECT SELECT UserID FROM dbo.TestTable WHERE SystemID = 10
Now, whether this query yields the results you're actually after, I have no idea, but I believe it meets the requirements as stated.

Try this... I have little changed Aaron Bertrand's query.
DECLARE #param NVARCHAR(MAX) = N'(((4 or 5) and 6) or 8)';
DECLARE #sql NVARCHAR(MAX) = N'',
#q NVARCHAR(MAX) = N'SELECT * FROM dbo.TestTable WHERE SystemID = ',
#paranth NVARCHAR(100) = substring(#param,0,PATINDEX('%[0-9]%',#param));
set #param =substring(#param,PATINDEX('%[0-9]%',#param),len(#param)-PATINDEX('%[0-9]%',#param))
SELECT #sql = #q + REPLACE(REPLACE(#param, ' or ', '
UNION ALL ' + #q),
' and ', '
INTERSECT ' + #q);
set #sql=#paranth+#sql
if (isnull(#paranth,'')<>'')
set #sql=#sql+')'
PRINT #sql;

You could use a CSV to Integer table value function inside your SP. You have to create the CsvToInt function first of course. Then you can use it inside your Stored Procedure to turn a parameter into an integer list. As pointed out this only suits the "Or" component of your dynamic search criteria.
You can use this in conjunction with EXEC or sp_executesql. Which will allow you to add sql as a parameter.
SET #myBaseQuery = 'SELECT * FROM TestTable WHERE SystemId = ' + #myParam
EXECUTE(#myBaseQuery)
or
SELECT * FROM TestTable WHERE SystemID IN (SELECT IntValue FROM dbo.CsvToInt('2,3,4,5,6'))
-- use your parameters
CREATE FUNCTION [dbo].[CsvToInt] ( #Array VARCHAR(1000))
RETURNS #IntTable TABLE
(IntValue INT)
AS
BEGIN
DECLARE #separator CHAR(1)
SET #separator = ','
DECLARE #separator_position INT
DECLARE #array_value VARCHAR(1000)
SET #array = #array + ','
While patindex('%,%' , #array) <> 0
BEGIN
SELECT #separator_position = patindex('%,%' , #array)
SELECT #array_value = LEFT(#array, #separator_position - 1)
INSERT #IntTable
VALUES (CAST(#array_value AS INT))
SELECT #array = stuff(#array, 1, #separator_position, '')
END
RETURN
END
CsvToInt function taken from http://www.summit-pro.com/blog/2010/05/18/csv-list-to-int-sql-function/

Related

map a string list in sqlserver like listagg

i try conver some string such as '1,2,3' to 'a,b,c' with the anwser:
select stuff(
(
select ',' + realname from sys_user
where ','+'1,2,3'+',' like '%,'+cast(u_id as varchar(10))+',%' for xml path('')
),1,1,'')
charindex is well done. but i want to create a more common function, so that i can convert in any relation such that.
i try a function :
create function [dbo].[fn_enum2str]
(
#enum as varchar(1000),
#table_name as varchar(100),
#origin_field as varchar(100),
#target_field as varchar(100)
)
as
begin
declare #result varchar(1000)
declare #sqlstr nvarchar(1000)
set #sqlstr = 'set #result = ('
set #sqlstr = #sqlstr + 'select stuff('
set #sqlstr = #sqlstr + '(select '','' + ' +#target_field+ ' from ' + #table_name
set #sqlstr = #sqlstr + ' where '','+#enum+','' like ''%,''+cast('+#origin_field+' as varchar)+'',%'' for xml path(''''))'
set #sqlstr = #sqlstr + ',1,1,''''))'
exec(#sqlstr)
return #result
end
it faild with error, as you know, it is not allow to exec a dynamic sql in function.
i want to
select dbo.fn_enum2str(a.uids,'sys_user','u_id', 'realname') from my_table a
--output 'a,b,c'
so, in my question, how can i create a function or a proc to deal it ?
Suppose you have SQL-SERVER2016 you can use string_split like this:
Test data
CREATE TABLE [dbo].[stringlist]([Numbers] [nvarchar](50) NULL)
Insert into dbo.Stringlist(numbers)
values('1,2,3,4,5,10')
SQL Function
alter function dbo.HinkyBase26( #Value as varchar(250) ) returns VarChar(250) as
begin
--declare #Value as varchar(50) = '13,14,1,2,5,14'
-- Notes: 'A' = 0. Negative numbers are not handled.
declare #Result as VarChar(250) = '';
declare #stringsplit table (numbers nvarchar(50),Letters varchar(1))
insert into #stringsplit(numbers,Letters)
select numbers = #Value ,CHAR(64 + value) as Letters from string_split(#Value,',')
select #Result = Letter from (
select numbers,Letter = STUFF((Select ', ' + Letters
from #stringsplit b
where b.numbers = a.numbers
FOR XML PATH('')),1,2,'')
from #stringsplit a
group by numbers
)z
return #Result
end
Execution of function
SELECT TOP (1000) [Numbers],dbo.HinkyBase26(Numbers)
FROM [LegOgSpass].[dbo].[stringlist]
SQL Stored Proc
Create PROC dbo.usp_convertnumberstostring
#stringvalue nvarchar(250)
AS
BEGIN
Create table #stringsplit (numbers nvarchar(50),Letters varchar(1))
insert into #stringsplit(numbers,Letters)
SELECT Numbers = #stringvalue,CHAR(64 + value) as Letters
from string_split(#stringvalue,',')
select numbers,Letter = STUFF((Select DISTINCT ', ' + Letters
from #stringsplit b
where b.numbers = a.numbers
FOR XML PATH('')),1,2,'')
from #stringsplit a
group by numbers
drop table #stringsplit
END
Execute SP
DECLARE #RC int
DECLARE #stringvalue nvarchar(250) = '1,5,6'
-- TODO: Set parameter values here.
EXECUTE #RC = [dbo].[usp_convertnumberstostring]
#stringvalue
GO
Result
SQL Script
Create table #stringsplit (numbers nvarchar(50),Letters varchar(1))
insert into #stringsplit(numbers,Letters)
SELECT Numbers,CHAR(64 + value) as Letters
FROM [LegOgSpass].[dbo].[stringlist] a
cross apply string_split(numbers,',')
select numbers,Letter = STUFF((Select DISTINCT ', ' + Letters
from #stringsplit b
where b.numbers = a.numbers
FOR XML PATH('')),1,2,'')
from #stringsplit a
group by numbers
Drop table #stringsplit
CREATE function [dbo].[fn_enum2str]
(
#enum as varchar(1000),
#table_name as varchar(100)
)
returns varchar(1000)
as
begin
declare #result varchar(1000)
if #enum is null
return ''
if #table_name = 'sys_user'
set #result = (
select stuff(
(
select ',' + realname from sys_user
where ','+#enum+',' like '%,'+cast(u_id as varchar(10))+',%' for xml path('')
),1,1,''
)
)
if #table_name = 'sys_attachment'
set #result = (
select stuff(
(
select ',/' + filepath from sys_attachment
where ','+#enum+',' like '%,'+cast(aid as varchar(10))+',%' for xml path('')
),1,1,''
)
)
return #result
end
GO
only way to deal it what i can think of, to switch which sql will be exec by a flag. when other relation apearance, add it to the switch list.
select
dbo.fn_enum2str(a.uids, 'sys_user') as names,
dbo.fn_enum2str(a.attachids, 'sys_attachment') as filepaths
from my_table a
so that it can be overlay. yes, it is difficult to remember stuff or for xml path or listagg(oracle), and result to a long sql, and i am lazy.😄
if you have any anwser better, tell me, thanks.

Fill dynamic array with values from an array

I have to execute a similar query on many tables. Instead of writing an n amount of almost similar similar queries, I would like to use a dynamic query. Pseudo:
array = (
'table_a' => 'value_a',
'table_b' => 'value_b',
'table_c' => 'value_c'
);
foreach (array as table => value)
exec(
'select ' + #value + ' into #' + #table + ' from ' + #table
);
end
Is something like this possible in SQL Server 2008 RE? Any help would be greatly appreciated!
You can do this in SQL Server using something like the following script:
CREATE TABLE #temp (id INT IDENTITY(1,1), tablename VARCHAR(50))
INSERT INTO #temp
( tablename )
VALUES ( 'table_a' ),('table_b'),('table_c')
DECLARE #sql NVARCHAR(MAX)
DECLARE #tblcount INT, #i INT = 1
SELECT #tblcount = MAX(id) FROM #temp
WHILE #i <= #tblcount
BEGIN
SELECT #sql = 'SELECT t.somefield, t.otherfield as ' + tablename + ' INTO #some_temptable_' + tablename + ' FROM #mytable t where SomeField like ''1''' FROM #temp WHERE id = #i
EXEC sp_executesql #sql
--SELECT #sql
SELECT #i = #i + 1
END
DROP TABLE #temp

Dynamic queries and Dynamic Where clauses Solution

This week, I found myself in need of some dynamic queries. Now, dynamic queries and dynamic where clauses are nothing new and well documented all over the web. Yet, I needed something more. I needed a fluid way of pulling new where fields to the client and allowing the users to make as many filters as needed. Even have multiple filters on a single field. Even more so, I needed to have access to all the possible operators within SQL server. The following is code is one way to make this happen. I will attempt to point out highlights of the code with the complete code at the bottom.
Hope you enjoy the code.
REQUIREMENTS
The solution will never allow SQL injections. (No exec(command) can be used)
The caller of the stored procedure could be anything.
The data set must come from a Stored Procedure.
Any field can be filtered as many times as needed, with just about any operation.
Any combination of filters should be allowed.
The stored procedure should allow for mandatory parameters
First, let us look over the parameters.
CREATE PROCEDURE [dbo].[MyReport]
-- Add the parameters for the stored procedure here
#p_iDistributorUID INT, -- manditory
#p_xParameters XML = null --optional parameters (hostile)
The first parameter must always be sent, in this demo we have a distributor id that must be sent in. The second parameter is an XML document. These are the “Dynamic Where Clauses” and we consider these potential sql injections, or as I perceive this parameter as hostile.
<root>
<OrFilters>
<AndFilter Name="vcinvoicenumber" Operator="2" Value="inv12"/>
<AndFilter Name="vcID" Operator="1" Value="asdqwe"/>
</OrFilters>
<OrFilters>
<AndFilter Name="iSerialNumber" Operator="1" Value="123456"/>
</OrFilters>
NAME= field name(you could just use the object_id if you want to obfuscate)
OPERATOR = SQL operators such as <,>,=,like,ect.
VALUE is what the users has entered.
Here is what the final code would look like.
Select *
FROM someTable
Where (
vcinvoicenumber like ‘inv12%’
and vcID = ‘asdqwe’
)
Or
(
iSerialNumber = ‘123456’
)
First thing is to find out how many “OrFilters” tags there are.
SELECT #l_OrFilters = COUNT(1)
FROM #p_xParameters.nodes('/root/OrFilters') Tab(Col)
Next we need a temp table to hold the values in the XML doc.
CREATE TABLE #temp
(keyid int IDENTITY(1,1) NOT NULL,value varchar(max))
We now create a cursor for the first “OrFilters”tag.
DECLARE OrFilter_cursor CURSOR LOCAL
FOR
SELECT Tab.Col.value('#Name','varchar(max)') AS Name
,Tab.Col.value('#Operator','Smallint') AS Operator
,Tab.Col.value('#Value','varchar(max)') AS Value
FROM #p_xParameters.nodes('/root/OrFilters[sql:variable("#l_OrFilters")]/AndFilter') Tab(Col)
To make sure we have a valid field, we check against the system tables.
SELECT #l_ParameterInName = [all_columns].Name
,#l_ParameterDataType= [systypes].Name
,#l_ParameterIsVariable= Variable
,#l_ParameterMax_length=max_length
,#l_ParameterpPrecision=precision
,#l_ParameterScale =[all_columns].scale
FROM [AprDesktop].[sys].[all_views]
INNER JOIN [AprDesktop].[sys].[all_columns]
ON [all_views].object_id = [all_columns].object_id
INNER JOIN [AprDesktop].[sys].[systypes]
ON [all_columns].system_type_id = [systypes].xtype
WHERE [all_views].name = 'vw_CreditMemo_Lists'
and [all_columns].Name= #l_Name
Now we save the parameter to the temp table
IF ##ROWCOUNT = 1
BEGIN
INSERT INTO #temp (value) SELECT #l_Value
SET #l_FilterKey = ##IDENTITY
.
.
.
We make a call to a function that will actually build the where clauses.
SET #l_TemporaryWhere +=
dbo.sfunc_FilterWhereBuilder2(#l_Operator
,#l_ParameterInName
,#l_TemporaryWhere
,CAST(#l_FilterKey AS VARCHAR(10))
,#l_ParameterDataType
,#l_ParameterVariable)
Looking at this Function, you can see we used a case statement to genereate the where clause string.
set #l_CastToType = ' CAST( VALUE as ' + #p_DataType + #p_PrecisionScale + ') '
set #l_CastToString = ' CAST( '+#p_Field+' as VARCHAR(MAX)) '
-- Add the T-SQL statements to compute the return value here
SELECT #l_Return =
CASE
--EQUAL
--ex: vcUID = (select value FROM #temp where keyid = 1)
WHEN #p_Command = 1
THEN #p_Field + ' = (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--BEGIN WITH
--ex:vcInvoiceNumber LIKE (select value+'%' FROM #temp where keyid = 2)
WHEN #p_Command = 2
THEN #l_CastToString +' LIKE (select value+'+ QUOTENAME('%','''') +' FROM #temp where keyid = ' + #p_KeyValue + ')'
.
.
.
And finally call the sp_execute.
EXECUTE sp_executesql #l_SqlCommand ,#l_Parameters, #p_iDistributorUID
CALLING CODE
DECLARE #return_value int
DECLARE #myDoc xml
SET #myDoc =
'<root>
<OrFilters>
<AndFilter Name="vcinvoicenumber" Operator="1" Value="123"/>
</OrFilters>
</root>'
EXEC #return_value = [dbo].[spp_CreditMemo_Request_List_v2]
#p_siShowView = 1,
#p_iDistributorUID = 3667,
#p_xParameters = #myDoc
SELECT 'Return Value' = #return_value
MAIN STORED PROCEDURE
ALTER PROCEDURE [dbo].[MyReport]
-- Add the parameters for the stored procedure here
#p_iDistributorUID INT , --manditory
#p_xParameters XML = null --optional parameters(hostile)
AS
BEGIN
-- SET NOCOUNT ON added to prevent extra result sets from
-- interfering with SELECT statements.
SET NOCOUNT ON;
DECLARE #l_TemporaryWhere NVARCHAR(MAX)
-- declare variables
DECLARE #l_SqlCommand NVARCHAR(MAX)
DECLARE #l_Parameters NVARCHAR(MAX)
DECLARE #l_WhereClause NVARCHAR(MAX)
DECLARE #l_OrFilters INT
--cursor variables
DECLARE #l_Name VARCHAR(MAX)
DECLARE #l_Operator SMALLINT
DECLARE #l_Value VARCHAR(MAX)
--variables from the database views
DECLARE #l_ParameterInName NVARCHAR(128)
DECLARE #l_ParameterDataType NVARCHAR(128)
DECLARE #l_ParameterIsVariable BIT
DECLARE #l_ParameterMax_length SMALLINT
DECLARE #l_ParameterpPrecision TINYINT
DECLARE #l_ParameterScale TINYINT
--the variable that holds the latest ##identity
DECLARE #l_FilterKey INT
--init local variables
SET #l_SqlCommand =''
SET #l_Parameters =''
SET #l_WhereClause =''
BEGIN TRY
--verify manditory variables
if #p_iDistributorUID is null
raiserror('Null values not allowed for #p_iDistributorUID', 16, 1)
--Build the base query
-- only the fields needed in the tile should be selected
SET #l_SqlCommand =
' SELECT * ' +
' FROM vw_Lists '
--how many "OR" filters are there
SELECT #l_OrFilters = COUNT(1)
FROM #p_xParameters.nodes('/root/OrFilters') Tab(Col)
--create a temp table to
--hold the parameters to send into the sp
CREATE TABLE #temp
(
keyid int IDENTITY(1,1) NOT NULL,value varchar(max)
)
--Cycle through all the "OR" Filters
WHILE #l_OrFilters > 0
BEGIN
SET #l_TemporaryWhere = '';
--Create a cursor of the Next "OR" filter
DECLARE OrFilter_cursor CURSOR LOCAL
FOR
SELECT Tab.Col.value('#Name','varchar(max)') AS Name
,Tab.Col.value('#Operator','Smallint') AS Operator
,Tab.Col.value('#Value','varchar(max)') AS Value
FROM #p_xParameters.nodes('/root/OrFilters[sql:variable("#l_OrFilters")]/AndFilter') Tab(Col)
OPEN OrFilter_cursor
FETCH NEXT FROM OrFilter_cursor
INTO #l_Name, #l_Operator,#l_Value
WHILE ##FETCH_STATUS = 0
BEGIN
--verify the parameter actual exists
-- and get parameter details
SELECT #l_ParameterInName = [all_columns].Name
,#l_ParameterDataType= [systypes].Name
,#l_ParameterIsVariable= Variable
,#l_ParameterMax_length=max_length
,#l_ParameterpPrecision=precision
,#l_ParameterScale =[all_columns].scale
FROM [AprDesktop].[sys].[all_views]
INNER JOIN [sys].[all_columns]
ON [all_views].object_id = [all_columns].object_id
INNER JOIN [sys].[systypes]
ON [all_columns].system_type_id = [systypes].xtype
WHERE [all_views].name = 'vw_CreditMemo_Lists'
and [all_columns].Name= #l_Name
--if the paremeter exists, create a where clause
-- if the parameters does not exists, possible injection
IF ##ROWCOUNT = 1
BEGIN
--insert into the temp table the parameter value
--NOTE: we have turned in the ##identity as the key
INSERT INTO #temp (value) SELECT #l_Value
SET #l_FilterKey = ##IDENTITY
-- if the parameter is variable in length, add the length
DECLARE #l_ParameterVariable VARCHAR(1000)
IF #l_ParameterIsVariable = 1
BEGIN
SET #l_ParameterVariable ='(' + CAST(#l_ParameterMax_length as VARCHAR(MAX)) + ') '
END
ELSE
BEGIN
SET #l_ParameterVariable = ''
END
-- create the where clause for this filter
SET #l_TemporaryWhere +=
dbo.sfunc_FilterWhereBuilder2(#l_Operator
,#l_ParameterInName
,#l_TemporaryWhere
,CAST(#l_FilterKey AS VARCHAR(10))
,#l_ParameterDataType
,#l_ParameterVariable)
END
FETCH NEXT FROM OrFilter_cursor
INTO #l_Name, #l_Operator,#l_Value
END
-- clean up the cursor
CLOSE OrFilter_cursor
DEALLOCATE OrFilter_cursor
--add the and filers
IF #l_TemporaryWhere != ''
BEGIN
--if the where clause is not empty, we need to add an OR
IF #l_WhereClause != ''
BEGIN
SET #l_WhereClause += ' or ';
END
--add temp to where clause including the
SET #l_WhereClause += '(' + #l_TemporaryWhere + ')';
END
--get the next AND set
SET #l_OrFilters = #l_OrFilters - 1
END
--generate the where clause
IF #l_WhereClause != ''
BEGIN
SET #l_WhereClause ='('+ #l_WhereClause + ') AND '
END
--add in the first mandatory parameter
SET #l_WhereClause += ' vw_CreditMemo_Lists.iDistributorUID = #l_iDistributorUID '
SET #l_Parameters += '#l_iDistributorUID int'
--do we need to attach the where clause
if #l_WhereClause IS NOT NULL AND RTRIM(LTRIM(#l_WhereClause)) != ''
BEGIN
SET #l_SqlCommand += ' WHERE '+ #l_WhereClause;
END
print #l_SqlCommand
--query for the data
EXECUTE sp_executesql #l_SqlCommand ,#l_Parameters, #p_iDistributorUID
END TRY
BEGIN CATCH
DECLARE #ErrorUID int;
DECLARE #ErrorMessage NVARCHAR(4000);
DECLARE #ErrorSeverity INT;
DECLARE #ErrorState INT;
SELECT
#ErrorMessage = ERROR_MESSAGE(),
#ErrorSeverity = ERROR_SEVERITY(),
#ErrorState = ERROR_STATE();
--write the to stored procedure log
EXEC #ErrorUID = spp_Errors_CreateEntry #l_SqlCommand
-- Use RAISERROR inside the CATCH block to return error
-- information about the original error that caused
-- execution to jump to the CATCH block.
RAISERROR (#ErrorUID, -- Message text.
#ErrorSeverity, -- Severity.
#ErrorState -- State.
);
IF(CURSOR_STATUS('LOCAL','OrFilter_cursor') >= 0)
BEGIN
CLOSE OrFilter_cursor
END
IF(CURSOR_STATUS('LOCAL','OrFilter_cursor') = -1)
BEGIN
DEALLOCATE OrFilter_cursor
END
END CATCH
return
END
FUNCTION
ALTER FUNCTION [dbo].[sfunc_FilterWhereBuilder2]
(
#p_Command SMALLINT ,
#p_Field VARCHAR(1000) ,
#p_WhereClause VARCHAR(MAX) ,
#p_KeyValue VARCHAR(10) ,
#p_DataType VARCHAR(100) = NULL ,
#p_PrecisionScale VARCHAR(100) = NULL
)
RETURNS VARCHAR(MAX)
AS
BEGIN
-- Declare the return variable here
DECLARE #l_Return VARCHAR(MAX)
DECLARE #l_CastToType VARCHAR(4000)
DECLARE #l_CastToString VARCHAR(MAX)
set #l_CastToType = ' CAST( VALUE as ' + #p_DataType + #p_PrecisionScale + ') '
set #l_CastToString = ' CAST( '+#p_Field+' as VARCHAR(MAX)) '
-- Add the T-SQL statements to compute the return value here
SELECT #l_Return =
CASE
--EQUAL
--ex: vcBurnUID = (select value FROM #temp where keyid = 1)
WHEN #p_Command = 1
THEN #p_Field + ' = (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--BEGIN WITH
--ex:vcInvoiceNumber LIKE (select value+'%' FROM #temp where keyid = 2)
WHEN #p_Command = 2
THEN #l_CastToString +' LIKE (select value+'+ QUOTENAME('%','''') +' FROM #temp where keyid = ' + #p_KeyValue + ')'
--END WITH
--ex:vcInvoiceNumber LIKE (select '%'+value FROM #temp where keyid = 2)
WHEN #p_Command = 4
THEN #l_CastToString +' LIKE (select '+ QUOTENAME('%','''') +'+value FROM #temp where keyid = ' + #p_KeyValue + ')'
--END WITH
--ex:vcInvoiceNumber LIKE (select '%'+value+'%' FROM #temp where keyid = 2)
WHEN #p_Command = 8
THEN #l_CastToString +' LIKE (select '+ QUOTENAME('%','''') +'+value+'+ QUOTENAME('%','''') +' FROM #temp where keyid = ' + #p_KeyValue + ')'
--greater than
--ex: iSerialNumber > (select CAST(value as INT) FROM #temp where keyid = 1)
WHEN #p_Command = 16
THEN #p_Field +' > (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--greater than equal
--ex: iSerialNumber >= (select CAST(value as INT) FROM #temp where keyid = 1)
WHEN #p_Command = 32
THEN #p_Field +' >= (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--Less than
--ex: iSerialNumber < (select CAST(value as INT) FROM #temp where keyid = 1)
WHEN #p_Command = 64
THEN #p_Field +' < (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--less than equal
--ex: iSerialNumber <= (select CAST(value as INT) FROM #temp where keyid = 1)
WHEN #p_Command = 128
THEN #p_Field +' <= (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--less than equal
--ex: iSerialNumber != (select CAST(value as INT) FROM #temp where keyid = 1)
WHEN #p_Command = 256
THEN #p_Field +' != (select '+#l_CastToType+' FROM #temp where keyid = ' + #p_KeyValue + ')'
--default to an empty string
ELSE ''
END
if #l_Return != '' and LEN(#p_WhereClause) > 1
begin
set #l_Return = ' AND ' + #l_Return
end
-- Return the result of the function
RETURN #l_Return
END

Improve SQL Server query to convert arbitrary table to JSON

After a lot of searching and piecing together the very excellent techniques for converting result sets using the FOR XML and .nodes() commands that are around the web, I was able to create this single query (not a stored procedure) which does a reasonably good job of converting any arbitrary SQL query to a JSON array.
The query will encode each data row as a single JSON object with a leading comma.
The data rows are wrapped by brackets and the whole result set is then expected to be exported to a file.
I'd like to see if anyone out there can see ways to improve its performance?
Here's the query with a sample table:
declare #xd table (col1 varchar(max), col2 int, col3 real, colNull int)
insert into #xd
select '', null, null, null
UNION ALL select 'ItemA', 123, 123.123, null
UNION ALL select 'ItemB', 456, 456.456, null
UNION ALL select '7890', 789, 789.789, null
select '[{}'
UNION ALL
select ',{' + STUFF((
(select ','
+ '"' + r.value('local-name(.)', 'varchar(max)') + '":'
+ case when r.value('./#xsi:nil', 'varchar(max)') = 'true' then 'null'
when isnumeric(r.value('.', 'varchar(max)')) = 1
then r.value('.', 'varchar(max)')
else '"' + r.value('.', 'varchar(max)') + '"'
end
from rows.nodes('/row/*') as x(r) for xml path(''))
), 1, 1, '') + '}'
from (
-- Arbitrary query goes here, (fields go where t.* is, table where #xd t is)
select (select t.* for xml raw,type,elements XSINIL) rows
from #xd t
) xd
UNION ALL
select ']'
My biggest critique of it, is that it's insanely slow.
It currently takes about 3:30 for ~42,000 rows.
My other big critique is that it currently assumes that everything that looks like a number is a number. It doesn't try to discover column type in the least (and I'm not even sure if it can).
A final minor critique is that the first data row will have a comma up front and technically it shouldn't. To compensate for that it requires that empty JSON object in the first row that starts the JSON array.
Other critiques (preferably with solutions) invited, the only real limitation I have is that the solution be decently repeatable on many arbitrary SQL queries without having to explicitly identify the column names.
I'm using SQL Server 2012.
Thanks and to anyone else like me who was looking for a generalized SQL Results -> JSON Array converter, ENJOY!
I say if you really want to kick up performance, use metaprogramming. The example below tries this with 40,000 rows and returns results in less than a second (not counting inserting the initial 40k rows, which in this example only takes about 2 seconds). It also takes into account your data types to not enclose numbers in quotes.
declare #xd table (col1 varchar(max), col2 int, col3 real, colDate datetime, colNull int);
declare #i int = 0;
while #i < 10000 begin
set #i += 1;
insert into #xd
select '', null, null, null, null
union all select 'ItemA', 123, 123.123, getDate(), null
union all select 'ItemB', 456, 456.456, getDate(), null
union all select '7890', 789, 789.789, getDate(), null;
end;
select *
into #json_base
from (
-- Insert SQL Statement here
select * from #xd
) t;
declare #columns table (
id int identity primary key,
name sysname,
datatype sysname,
is_number bit,
is_date bit);
insert into #columns(name, datatype, is_number, is_date)
select columns.name, types.name,
case when number_types.name is not NULL
then 1 else 0
end as is_number,
case when date_types.name is not NULL
then 1 else 0
end as is_date
from tempdb.sys.columns
join tempdb.sys.types
on (columns.system_type_id = types.system_type_id)
left join (values ('int'), ('real'), ('numeric'),
('decimal'), ('bigint'), ('tinyint')) as number_types(name)
on (types.name = number_types.name)
left join (values ('date'), ('datetime'), ('datetime2'),
('smalldatetime'), ('time'), ('datetimeoffset')) as date_types(name)
on (types.name = date_types.name)
where object_id = OBJECT_ID('tempdb..#json_base');
declare #field_list varchar(max) = STUFF((
select '+'',''+' + QUOTENAME(QUOTENAME(name, '"') + ':', '''')
+ '+' + case when is_number = 1
then 'COALESCE(LTRIM('
+ QUOTENAME(name) + '),''null'')'
when is_date = 1
then 'COALESCE(QUOTENAME(LTRIM(convert(varchar(max), '
+ QUOTENAME(name) + ', 126)),''"''),''null'')'
else 'COALESCE(QUOTENAME('
+ QUOTENAME(name) + ',''"''),''null'')'
end
from #columns
for xml path('')),
1, 5, '');
create table #json_result (
id int identity primary key,
line varchar(max));
declare #sql varchar(max) = REPLACE(
'insert into #json_result '
+ 'select '',{''+{f}+''}'' '
+ 'from #json_base', '{f}', #field_list);
exec(#sql);
update #json_result
set line = STUFF(line, 1, 1, '')
where id = 1;
select '['
UNION ALL
select line
from #json_result
UNION ALL
select ']';
drop table #json_base;
drop table #json_result;
From Firoz Ansari:
CREATE PROCEDURE [dbo].[GetJSON] (
#ParameterSQL AS VARCHAR(MAX)
)
AS
BEGIN
DECLARE #SQL NVARCHAR(MAX)
DECLARE #XMLString VARCHAR(MAX)
DECLARE #XML XML
DECLARE #Paramlist NVARCHAR(1000)
SET #Paramlist = N'#XML XML OUTPUT'
SET #SQL = 'WITH PrepareTable (XMLString) '
SET #SQL = #SQL + 'AS ( '
SET #SQL = #SQL + #ParameterSQL+ ' FOR XML RAW, TYPE, ELEMENTS '
SET #SQL = #SQL + ') '
SET #SQL = #SQL + 'SELECT #XML = XMLString FROM PrepareTable '
EXEC sp_executesql #SQL, #Paramlist, #XML=#XML OUTPUT
SET #XMLString = CAST(#XML AS VARCHAR(MAX))
DECLARE #JSON VARCHAR(MAX)
DECLARE #Row VARCHAR(MAX)
DECLARE #RowStart INT
DECLARE #RowEnd INT
DECLARE #FieldStart INT
DECLARE #FieldEnd INT
DECLARE #Key VARCHAR(MAX)
DECLARE #Value VARCHAR(MAX)
DECLARE #StartRoot VARCHAR(100); SET #StartRoot = ''
DECLARE #EndRoot VARCHAR(100); SET #EndRoot = ''
DECLARE #StartField VARCHAR(100); SET #StartField = ''
SET #RowStart = CharIndex(#StartRoot, #XMLString, 0)
SET #JSON = ''
WHILE #RowStart > 0
BEGIN
SET #RowStart = #RowStart+Len(#StartRoot)
SET #RowEnd = CharIndex(#EndRoot, #XMLString, #RowStart)
SET #Row = SubString(#XMLString, #RowStart, #RowEnd-#RowStart)
SET #JSON = #JSON+'{'
-- for each row
SET #FieldStart = CharIndex(#StartField, #Row, 0)
WHILE #FieldStart > 0
BEGIN
-- parse node key
SET #FieldStart = #FieldStart+Len(#StartField)
SET #FieldEnd = CharIndex(#EndField, #Row, #FieldStart)
SET #Key = SubString(#Row, #FieldStart, #FieldEnd-#FieldStart)
SET #JSON = #JSON+'"'+#Key+'":'
-- parse node value
SET #FieldStart = #FieldEnd+1
SET #FieldEnd = CharIndex('0 SET #JSON = SubString(#JSON, 0, LEN(#JSON))
SET #JSON = #JSON+'},'
--/ for each row
SET #RowStart = CharIndex(#StartRoot, #XMLString, #RowEnd)
END
IF LEN(#JSON) > 0 SET #JSON = SubString(#JSON, 0, LEN(#JSON))
SET #JSON = '[' + #JSON + ']'
SELECT #JSON
END

String concatenation in SQL server

Consider a situation we have two variables in SQL Server 2005's SP as below,
#string1 = 'a,b,c,d'
#string2 = 'c,d,e,f,g'
Is there a solution to get a new string out of that like (#string1 U #string2) without using any loops. i.e the final string should be like,
#string3 = 'a,b,c,d,e,f,g'
In case you need to do this as a set and not one row at a time. Given the following split function:
USE tempdb;
GO
CREATE FUNCTION dbo.SplitStrings(#List nvarchar(max))
RETURNS TABLE
AS
RETURN ( SELECT Item FROM
( SELECT Item = x.i.value(N'./text()[1]', N'nvarchar(max)')
FROM ( SELECT [XML] = CONVERT(xml, '<i>'
+ REPLACE(#List,',', '</i><i>') + '</i>').query('.')
) AS a CROSS APPLY [XML].nodes('i') AS x(i) ) AS y
WHERE Item IS NOT NULL
);
GO
Then with the following table and sample data, and string variable, you can get all of the results this way:
DECLARE #foo TABLE(ID INT IDENTITY(1,1), col NVARCHAR(MAX));
INSERT #foo(col) SELECT N'c,d,e,f,g';
INSERT #foo(col) SELECT N'c,e,b';
INSERT #foo(col) SELECT N'd,e,f,x,a,e';
DECLARE #string NVARCHAR(MAX) = N'a,b,c,d';
;WITH x AS
(
SELECT f.ID, c.Item FROM #foo AS f
CROSS APPLY dbo.SplitStrings(f.col) AS c
), y AS
(
SELECT ID, Item FROM x
UNION
SELECT x.ID, s.Item
FROM dbo.SplitStrings(#string) AS s
CROSS JOIN x
)
SELECT ID, Items = STUFF((SELECT ',' + Item
FROM y AS y2 WHERE y2.ID = y.ID
FOR XML PATH(''), TYPE).value(N'./text()[1]', N'nvarchar(max)'), 1, 1, N'')
FROM y
GROUP BY ID;
Results:
ID Items
-- ----------
1 a,b,c,d,e,f,g
2 a,b,c,d,e
3 a,b,c,d,e,f,x
On newer versions (SQL Server 2017+), the query is much simpler, and you don't need to create your own custom string-splitting function:
;WITH x AS
(
SELECT f.ID, c.value FROM #foo AS f
CROSS APPLY STRING_SPLIT
(
CONCAT(f.col, N',', #string), N','
) AS c GROUP BY f.ID, c.value
)
SELECT ID, STRING_AGG(value, N',')
WITHIN GROUP (ORDER BY value)
FROM x GROUP BY ID;
Example db<>fiddle
Now that all said, what you really should do is follow the previous advice and store these things in a related table in the first place. You can use the same type of splitting methodology to store the strings separately whenever an insert or update happens, instead of just dumping the CSV into a single column, and your applications shouldn't really have to change the way they're passing data into your procedures. But it sure will be easier to get the data out!
EDIT
Adding a potential solution for SQL Server 2008 that is a bit more convoluted but gets things done with one less loop (using a massive table scan and replace instead). I don't think this is any better than the solution above, and it is certainly less maintainable, but it is an option to test out should you find you are able to upgrade to 2008 or better (and also for any 2008+ users who come across this question).
SET NOCOUNT ON;
-- let's pretend this is our static table:
CREATE TABLE #x
(
ID int IDENTITY(1,1),
col nvarchar(max)
);
INSERT #x(col) VALUES(N'c,d,e,f,g'), (N'c,e,b'), (N'd,e,f,x,a,e');
-- and here is our parameter:
DECLARE #string nvarchar(max) = N'a,b,c,d';
The code:
DECLARE #sql nvarchar(max) = N'DECLARE #src TABLE(ID INT, col NVARCHAR(32));
DECLARE #dest TABLE(ID int, col nvarchar(32));';
SELECT #sql += '
INSERT #src VALUES(' + RTRIM(ID) + ','''
+ REPLACE(col, ',', '''),(' + RTRIM(ID) + ',''') + ''');'
FROM #x;
SELECT #sql += '
INSERT #dest VALUES(' + RTRIM(ID) + ','''
+ REPLACE(#string, ',', '''),(' + RTRIM(ID) + ',''') + ''');'
FROM #x;
SELECT #sql += '
WITH x AS (SELECT ID, col FROM #src UNION SELECT ID, col FROM #dest)
SELECT DISTINCT ID, Items = STUFF((SELECT '','' + col
FROM x AS x2 WHERE x2.ID = x.ID FOR XML PATH('''')), 1, 1, N'''')
FROM x;'
EXEC sys.sp_executesql #sql;
GO
DROP TABLE #x;
This is much trickier to do in 2005 (though not impossible) because you need to change the VALUES() clauses to UNION ALL...
Two ways you can do that:
Build a CLR function to do the job for you. Move the logic back to .NET code which is much easier platform for string manipulation.
If you have to use SQL Server, then you will need to:
"explode" the two strings into two tables, this function might help: http://blog.logiclabz.com/sql-server/split-function-in-sql-server-to-break-comma-separated-strings-into-table.aspx
Get a unique list of strings from the two tables. (simple query)
"implode" the two string tables into a variable (http://stackoverflow.com/questions/194852/concatenate-many-rows-into-a-single-text-string)
Found this function dbo.Split in a related answer, which you can use like this:
declare #string1 nvarchar(50) = 'a,b,c,d'
declare #string2 nvarchar(50) = 'c,d,e,f,g'
select * from dbo.split(#string1, ',')
select * from dbo.split(#string2, ',')
declare #data nvarchar(100) = ''
select #data = #data + ',' + Data from (
select Data from dbo.split(#string1, ',')
union
select Data from dbo.split(#string2, ',')
) as d
select substring(#data, 2, LEN(#data))
The last SELECT returns
a,b,c,d,e,f,g
How about
set #string3 = #string1+','+#string2
Sorry, wasn't clear you wanted only unique occurrences. What version of SQL server are you using? String manipulation functions vary per version.
If you don't mind a UDF to split the string, try this:
CREATE FUNCTION dbo.Split
(
#RowData nvarchar(2000),
#SplitOn nvarchar(5)
)
RETURNS #RtnValue table
(
Id int identity(1,1),
Data nvarchar(100)
)
AS
BEGIN
Declare #Cnt int
declare #data varchar(100)
Set #Cnt = 1
While (Charindex(#SplitOn,#RowData)>0)
Begin
Insert Into #RtnValue (data)
Select ltrim(rtrim(Substring(#RowData,1,Charindex(#SplitOn,#RowData)-1)))
Set #RowData = Substring(#RowData,Charindex(#SplitOn,#RowData)+1,len(#RowData))
Set #Cnt = #Cnt + 1
End
Insert Into #RtnValue (data)
Select Data = ltrim(rtrim(#RowData))
Return
END
and the code to use the UDF
go
#string1 = 'a,b,c,d'
#string2 = 'c,d,e,f,g'
declare #string3 varchar(200)
set #string3 = ''
select #string3 = #string3+data+','
from ( select data,min(id) as Id from dbo.split(#string1+','+#string2,',')
group by data ) xx
order by xx.id
print left(#string3,len(#string3)-1)
The following SQL function will convert a comma separated list to a table variable...
CREATE FUNCTION [dbo].[udfCsvToTable]( #CsvString VARCHAR( 8000))
-- Converts a comman separated value into a table variable
RETURNS #tbl TABLE( [Value] VARCHAR( 100) COLLATE DATABASE_DEFAULT NOT NULL)
AS BEGIN
DECLARE #Text VARCHAR( 100)
SET #CsvString = RTRIM( LTRIM( #CsvString))
SET #CsvString = REPLACE( #CsvString, CHAR( 9), '')
SET #CsvString = REPLACE( #CsvString, CHAR( 10), '')
SET #CsvString = REPLACE( #CsvString, CHAR( 13), '')
IF LEN( #CsvString) < 1 RETURN
WHILE LEN( #CsvString) > 0 BEGIN
IF CHARINDEX( ',', #CsvString) > 0 BEGIN
SET #Text = LEFT( #CsvString, CHARINDEX( ',', #CsvString) - 1)
SET #CsvString = LTRIM( RTRIM( RIGHT( #CsvString, LEN( #CsvString) - CHARINDEX( ',', #CsvString))))
END
ELSE BEGIN
SET #Text = #CsvString
SET #CsvString = ''
END
INSERT #tbl VALUES( LTRIM( RTRIM( #Text)))
END
RETURN
END
You can then union the two tables together, like so...
SELECT * FROM udfCsvToTable('a,b,c,d')
UNION
SELECT * FROM udfCsvToTable('c,d,e,f,g')
Which will give you a result set of:
a
b
c
d
e
f
g