Pivot TSQL table with dynamic columns - sql

I am passing up JSON as a parameter to a SQL stored proc. I use a function that takes a JSON dictionary and creates a table with key value pairs (two columns) that I then use with a COALESCE to create dynamic sql for an INSERT statement. This works fine for a single dictionary, but I need to also be able to send up JSON string that contains an array of dictionaries. Right now, my parse function gives me this table variable:
What I need is a table variable like this:
I can get the column names from the first table with this:
SELECT DISTINCT element name from #JSONTable
I should mention that these elementname's can and will change. I will not know the number of distinct elementname values.
UPDATE - Using Umair's answer, I am getting close:
DECLARE #JSONString AS VARCHAR(MAX)
SET #JSONString = '[{"ravid":3,"ravversion":2,"taskid":3},{"ravid":4,"ravversion":7,"taskid":99}]'
IF OBJECT_ID('tempdb..#JSONTable') IS NOT NULL
DROP TABLE #JSONTable
CREATE TABLE #JSONTable
(
elementname VARCHAR(500) ,
elementvalue VARCHAR(500)
)
INSERT INTO #JSONTable
( elementname ,
elementvalue
)
SELECT NAME ,
StringValue
FROM dbo.parseJSON(#JSONString)
WHERE LEN(StringValue) > 0 AND NAME IS NOT NULL
--declare a csv variable with all the distinct elements
DECLARE #csv NVARCHAR(max) = STUFF(
(
SELECT ',' + elementname
FROM (
SELECT DISTINCT elementname
FROM #JSONTable
) AS e
FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'),
1,
1,
''
);
DECLARE #sql NVARCHAR(MAX) = '
SELECT *
FROM (
SELECT *, Row = ROW_NUMBER() OVER (PARTITION BY elementname ORDER BY elementname)
FROM #JSONTable
) AS t
PIVOT (
MAX(elementvalue)
FOR elementname IN (' + #csv + ')
) AS p
'
EXEC sp_executesql #sql
But the dictionary values don't correspond to the key. Here is the results of Umair's current answer:

How about
--declare a csv variable with all the distinct elements
DECLARE #csv NVARCHAR(max) = STUFF(
(
SELECT ',' + elementname
FROM (
SELECT DISTINCT elementname
FROM #JSONTable
) AS e
FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'),
1,
1,
''
);
DECLARE #sql NVARCHAR(MAX) = '
SELECT *
FROM (
SELECT *, Row = ROW_NUMBER() OVER (PARTITION BY elementname ORDER BY elementvalue)
FROM #JSONTable
) AS t
PIVOT (
MAX(elementvalue)
FOR elementname IN (' + #csv + ')
) AS p
'
EXEC sp_executesql #sql
i.e dynamic pivoting :)
Edit:
Since you do not want to aggregate by anything in the pivot, I added a row number function to assign each distinct elementname a sequentially increasing id (based on element value). This will essentially group the pivot by this row column, producing all the required rows.

Related

Extract all records from a JSON column, using JSON type

I have a couple tables (see reproducible code at the bottom):
tbl1_have
id json_col
1 {"a_i":"a","a_j":1}
1 {"a_i":"b","a_j":2}
2 {"a_i":"c","a_j":3}
2 {"a_i":"d","a_j":4}
tbl2_have
id json_col
1 [{"a_i":"a","a_j":1},{"a_i":"b","a_j":2}]
2 [{"a_i":"c","a_j":3},{"a_i":"d","a_j":4}]
I wish to extract all json columns without providing explicit data type conversion for each columns since in my use case the names and amounts of nested attributes vary.
The expected output is the same for both cases:
tbl_want
id a_i a_j
1 a 1
1 b 2
2 c 3
2 d 4
with a_i and a_j correctly stored as a character and numeric column, which mean I'd like to map json types to SQL types (say INT and VARCHAR() here) automatically.
The following gets me half way for both tables:
SELECT id, a_i, a_j FROM tbl2_have CROSS APPLY OPENJSON(json_col)
WITH(a_i VARCHAR(100), a_j INT)
id a_i a_j
1 1 a 1
2 1 b 2
3 2 c 3
4 2 d 4
How can I work around mentioning the types explicitly in with() ?
reproducible code :
CREATE TABLE tbl1_have (id INT, json_col VARCHAR(100))
INSERT INTO tbl1_have VALUES
(1, '{"a_i":"a","a_j":1}'),
(1, '{"a_i":"b","a_j":2}'),
(2, '{"a_i":"c","a_j":3}'),
(2, '{"a_i":"d","a_j":4}')
CREATE TABLE tbl2_have (id INT, json_col VARCHAR(100))
INSERT INTO tbl2_have VALUES
(1, '[{"a_i":"a","a_j":1},{"a_i":"b","a_j":2}]'),
(2, '[{"a_i":"c","a_j":3},{"a_i":"d","a_j":4}]')
SELECT id, a_i, a_j FROM tbl1_have CROSS APPLY OPENJSON(json_col)
WITH(a_i VARCHAR(100), a_j INT)
SELECT id, a_i, a_j FROM tbl2_have CROSS APPLY OPENJSON(json_col)
WITH(a_i VARCHAR(100), a_j INT)
I am assuming that you don't know the name and type of keys in advance. You need to use dynamic SQL.
You first need to use OPENJSON without the WITH clause on the {objects} like so:
select string_agg(quotename(k) + case t
when 0 then ' nchar(1)' -- javascript null
when 1 then ' nvarchar(max)' -- javascript string
when 2 then ' float' -- javascript number
when 3 then ' bit' -- javascript boolean
else ' nvarchar(max) as json' -- javascript array or object
end, ', ') within group (order by k)
from (
select j2.[key], max(j2.[type])
from test
cross apply openjson(case when json_col like '{%}' then '[' + json_col + ']' else json_col end) as j1
cross apply openjson(j1.value) as j2
group by j2.[key]
) as kt(k, t)
The inner query gives you the name and type of all the keys across all json values in the table. The outer query builds the WITH clause for dynamic SQL.
The rest is relatively straight forward, use the generated clause in your dynamic SQL. Here is the complete example:
declare #table_name nvarchar(100) = 'test';
declare #with_clause nvarchar(100);
declare #query1 nvarchar(999) = N'select #with_clause_temp = string_agg(quotename(k) + case t
when 0 then '' nchar(1)''
when 1 then '' nvarchar(max)''
when 2 then '' float''
when 3 then '' bit''
else '' nvarchar(max) as json''
end, '', '') within group (order by k)
from (
select j2.[key], max(j2.[type])
from ' + quotename(#table_name) + '
cross apply openjson(case when json_col like ''{%}'' then ''['' + json_col + '']'' else json_col end) as j1
cross apply openjson(j1.value) as j2
group by j2.[key]
) as kt(k, t)';
exec sp_executesql #query1, N'#with_clause_temp nvarchar(100) out', #with_clause out;
declare #query2 nvarchar(999) = N'select id, j.*
from ' + quotename(#table_name) + '
cross apply openjson(json_col)
with (' + #with_clause + ') as j';
exec sp_executesql #query2;
Demo on db<>fiddle
I have found a solution that maybe works for your use case. I am no SQL-expert by any means, and i did not manage to automatically detect the datatypes of the dynamic columns. But i found a solution for your two examples.
First i tried to get all column names dynamically from the json_col. I found an answer on stackoverflow and got this piece of code:
STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname) FROM #tmpTbl FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
This will output all column names as a string separated by commas, in your example: ' [a_i], [a_j]'. This can then be used to dynamically SELECT columns.
As already mentioned above, i was not able to write a datatype detection algorithm. I just hardcoded the columns to have nvarchar(100) as datatype.
To dynamically get the column-names with the corresponding datatype (hardcoded as nvarchar(100)) i used a slightly modified version of above query:
STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname)+' nvarchar(100)' FROM #tmpTbl FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
Then i just used them in the WITH-CLAUSE.
Full version for the table tbl1_have
DECLARE #cols NVARCHAR(MAX), #colsWithType NVARCHAR(MAX), #query NVARCHAR(MAX);
DROP TABLE IF EXISTS #tmpTbl
SELECT outerTable.[id] AS columnid, innerTable.[key] AS columnname, innerTable.[value] AS columnvalue
INTO #tmpTbl
FROM tbl1_have outerTable CROSS APPLY OPENJSON(json_col) AS innerTable
SELECT * FROM #tmpTbl
SET #cols = STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname) FROM #tmpTbl FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
SET #colsWithType = STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname)+' nvarchar(100)' FROM #tmpTbl FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
SET #query = N'SELECT id, '+#cols+' FROM tbl1_have CROSS APPLY OPENJSON(json_col)
WITH('+#colsWithType+')';
exec sp_executesql #query
Full Version for the table tbl2_have:
DECLARE #cols NVARCHAR(MAX), #colsWithType NVARCHAR(MAX), #query NVARCHAR(MAX);
DROP TABLE IF EXISTS #tmpTbl
DROP TABLE IF EXISTS #tmpTbl2
SELECT *
INTO #tmpTbl
FROM tbl2_have CROSS APPLY OPENJSON(json_col)
SELECT outerTable.[id] AS columnid, innerTable.[key] AS columnname, innerTable.[value] AS columnvalue
INTO #tmpTbl2
FROM #tmpTbl outerTable CROSS APPLY OPENJSON([value]) AS innerTable
SELECT * FROM #tmpTbl
SELECT * FROM #tmpTbl2
SET #cols = STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname) FROM #tmpTbl2 FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
SET #colsWithType = STUFF(
(
SELECT DISTINCT ', '+QUOTENAME(columnname)+' nvarchar(100)' FROM #tmpTbl2 FOR XML PATH(''), TYPE
).value('.', 'nvarchar(max)'), 1, 1, '');
SET #query = N'SELECT id, '+#cols+' FROM tbl2_have CROSS APPLY OPENJSON(json_col)
WITH('+#colsWithType+')';
exec sp_executesql #query
Would using the Value returned from OPENJSON work? It probably maps to a string data type, however, you do not have to know the type upfront. The official doc of the OPENJSON rowset function indicates that it returns a Key:Value pair as well as a Type for each parse. The Type value may be useful, however, it determines the datatype while parsing. I bet that Value is always a string type, as it would have to be.
;WITH X AS
(
SELECT id, a_i=J.[Key], a_j=J.[Value] FROM #tbl2_have CROSS APPLY OPENJSON(json_col) J
)
SELECT
id,
a_i=MAX(CASE WHEN J.[Key]='a_i' THEN J.[Value] ELSE NULL END),
a_j=MAX(CASE WHEN J.[Key]='a_j' THEN J.[Value] ELSE NULL END)
FROM X CROSS APPLY OPENJSON(X.a_j) J
GROUP BY
id,a_i,a_j

How to create temp table with dynamic SQL query result

I have this stored procedure:
Declare #MarketID AS NVARCHAR(MAX) = '1.136529848';
Declare #UserID AS NVARCHAR(MAX) = '6a309d84-d1c6-434d-b9df-4f96a74da912';
DECLARE #colsSelect AS NVARCHAR(MAX);
DECLARE #colsTemp AS NVARCHAR(MAX);
DECLARE #query AS NVARCHAR(MAX);
SELECT
#colsSelect = STUFF((SELECT distinct ',' +
'''''' + ' as ' + QUOTENAME(name)
FROM RunnersInfoes AS t
WHERE marketID = #MarketID
FOR XML PATH(''), TYPE).value('.', 'NVARCHAR(MAX)') , 1, 1, '');
PRINT #colsSelect
SET #query= ';WITH cte AS
(
SELECT
id, ParentId, 0 AS Level, Share, AccountTypeName, FirstName
FROM
dbo.View_UserProfile
WHERE
View_UserProfile.id = ' + '''' + #UserID + '''' +'
UNION ALL
SELECT
t.id, t.ParentId, Level + 1 AS Level, t.Share, t.AccountTypeName, t.FirstName
FROM
View_UserProfile t
INNER JOIN
cte ON t.ParentId = cte.id
)
SELECT
ID, AccountTypeName AS Type, FirstName AS Name, ' + #colsSelect + '
FROM cte AS t'
EXECUTE (#query)
and it's generating this result:
I want to create temp table or variable type table for following result , remember the column of this result are dynamically rendered. Sometimes result returns more columns and sometimes with less but first 3 columns remain the same for every result. So kindly help for creating dynamic table inside the stored procedure.
You can do:
SELECT ID
, AccountTypeName AS Type
, FirstName AS Name
, ' + #colsSelect + '
INTO ##TEMPTABLE
FROM cte AS t
Since you execute this dynamically, you cannot use #TEMPTABLE because a local temp table will only exist in the scope of the query that defines it. Using ## creates a global temp table which will be accessible outside the scope of the dynamic query.
Please use the SELECT - INTO clause for your use case as given below
SELECT * INTO #temptable FROM cte
To create a temp table that is filled by a dynamic query, use global temp tables like this example.
For the select ... into ... statement to work, you need to make sure every column from the select has a name.
declare #query varchar(1000) = 'select 1 as ID, ''test'' as Column_1 into ##mytable'
exec (#Query)
select * from ##mytable
drop table ##mytable
Do not forget to drop the temp table when your done.

Moving Column names to rows

I have such a table with date as column name
but I would like to have these dates in one column in more than one row something like this:
Date Index
20170806 9206
20170813 8041
20170820 8861
20170827 8356
How can I do it in SQL Server
If you would like to go for more dynamic solution rather than hard coding all columns, the below scripts should work:
IF OBJECT_ID('TestTable') IS NOT NULL
DROP TABLE TestTable;
CREATE TABLE TestTable
(
[20170806] INT NOT NULL,
[20170813] INT NOT NULL,
[20170820] INT NOT NULL,
[20170827] INT NOT NULL
)
INSERT INTO TestTable VALUES (9206, 8041, 8861, 8356)
DECLARE #cols NVARCHAR(MAX),
#sql NVARCHAR(MAX)
SELECT #cols = COALESCE(#cols + ',', '') + QUOTENAME(c.COLUMN_NAME)
FROM INFORMATION_SCHEMA.[COLUMNS] AS c
WHERE c.TABLE_NAME = 'TestTable'
SET #sql = '
SELECT [Date],
[Index]
FROM TestTable
UNPIVOT([Index] FOR [Date] IN ('+ #cols +')) AS up'
exec sp_executesql #sql;
You can use UNPIVOT for this.
SELECT * FROM MyTable
UNPIVOT([Date] For [Index] IN( [20170806], [20170813], [20170820], [20170827])) UNPVT
In addition, if you want to make it dynamically, you can use this query too.
DECLARE #ColNames NVARCHAR(MAX)
= STUFF(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(
(SELECT T.* FROM (VALUES(1)) AS DUMY(ID) LEFT JOIN MyTable T ON 1=2 FOR XML AUTO, ELEMENTS XSINIL )
,'<T xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">','')
,'</T>','')
,'_x0032_','2')
,' xsi:nil="true"/>','] ')
,'<',',[') ,1,1,'')
DECLARE #SqlQ NVARCHAR(MAX)
= 'SELECT * FROM MyTable UNPIVOT([Date] For [Index] IN( ' + #ColNames + ')) UNPVT'
EXEC sp_executesql #SqlQ
You could use pivot such as:
However, i dont know your exact table names
select field_names
from table_name
pivot
( index
for index in ( Date, Index)
) pivot
but a useful article to follow is
"https://technet.microsoft.com/en-us/library/ms177410(v=sql.105).aspx"
Designed the table (variable) structure as below.
Hopefully, it matches your table structure.
DECLARE #data TABLE
(
[20170806] INT,
[20170813] INT,
[20170820] INT,
[20170827] INT
)
INSERT INTO #data VALUES (9206, 8041, 8861, 8356)
SELECT * FROM #data
You can use UNPIVOT statement for this. If you have dynamic columns, check this.
SELECT [Date],[Index]
FROM
#data
UNPIVOT
(
[Index] FOR [Date] IN ([20170806], [20170813], [20170820], [20170827])
) AS unpivotTable;
and the output is

Returning the results of a select statement that uses a FOR XML PATH('') to condense to one row in a Stored Procedure

I need to pull several values out of some XML that I have and return it as one string from a stored procedure. I have created the select statement that will pull all of the rows into one.
This is the code that I currently have.
CREATE FUNCTION openrpt.getSearchTermNames (#searchID NVARCHAR(MAX))
RETURNS VARCHAR(MAX) AS
BEGIN
DECLARE #xmlData XML
DECLARE #Names XML
SET #xmlData =
(
SELECT QueryXML
FROM ConflictsSearchTerms
WHERE ConflictsSearchTerms.[ID] = #searchID);
WITH xmlnamespaces ( default 'http://schemas.datacontract.org/2004/07/IntApp.Wilco.Model.Conflicts.Searches', 'http://schemas.microsoft.com/2003/10/Serialization/Arrays' AS d2p1, 'http://www.w3.org/2001/XMLSchema-instance' AS i )
RETURN
(
SELECT '; ' + temp.value('.', 'NVARCHAR(MAX)')
FROM #xmlData.nodes('/ConflictsSearchTermQuery/TermItems/d2p1:string') AS XMLDATA(temp)
UNION
SELECT '; ' + temp.value('d2p1:Value[1]', 'NVARCHAR(MAX)') AS [Test]
FROM #xmlData.nodes('/ConflictsSearchTermQuery/CorporateTreeCompaniesById/d2p1:KeyValueOfstringstring') AS XMLDATA(temp) FOR XML PATH('')
)
The select statemt will correctly return a list of the items from those two fields in the XML seperated by semi-colons but I am getting a syntax error at the return . I also get an error that says
The FOR XML clause is invalid in views, inline functions, derived tables, and subqueries when they contain a set operator. To work around, wrap the SELECT containing a set operator using derived table syntax and apply FOR XML on top of it.
Try this:
CREATE FUNCTION openrpt.getSearchTermNames
(
#searchID NVARCHAR(MAX)
)
RETURNS NVARCHAR(MAX)
AS
BEGIN
DECLARE #xmlData XML
DECLARE #Names XML
DECLARE #x XML
SET #xmlData = ( SELECT QueryXML
FROM ConflictsSearchTerms
WHERE ConflictsSearchTerms.[ID] = #searchID
);
WITH XMLNAMESPACES ( DEFAULT 'http://schemas.datacontract.org/2004/07/IntApp.Wilco.Model.Conflicts.Searches', 'http://schemas.microsoft.com/2003/10/Serialization/Arrays' AS d2p1, 'http://www.w3.org/2001/XMLSchema-instance' AS i )
SELECT #x =
( SELECT * FROM (
SELECT '; ' + temp.value('.', 'NVARCHAR(MAX)') AS [Test]
FROM #xmlData.nodes('/ConflictsSearchTermQuery/TermItems/d2p1:string') AS XMLDATA(temp)
UNION
SELECT '; ' + temp.value('d2p1:Value[1]', 'NVARCHAR(MAX)')
FROM #xmlData.nodes('/ConflictsSearchTermQuery/CorporateTreeCompaniesById/d2p1:KeyValueOfstringstring') AS XMLDATA(temp))t
FOR XML PATH('')
)
RETURN CAST(#x AS NVARCHAR(MAX))
END

How to to select two columns (Name, Value) and return as single result?

So I have a table that has a column for Name, and a column for Value and an ID. There can be multiple rows for the same ID. I would like to create a select that will return a single row for each ID and the values in the Name column would be the column name, and the Value would be the value. Example:
CREATE TABLE dbo.Attribute
(
AttributeID int NOT NULL,
Name varchar(20) NOT NULL,
Value varchar(20) NOT NULL
) ;
Data:
{1,"Color", "Blue"},{1,"Material", "leather"}
Would like Select to return:
[AttributeID:1, Color:Blue, Material: leather]
I have been playing with PIVOT and UNPIVOT but not getting what I need.
Thanks to #Mihai's link. I was able to do what I
DECLARE #cols AS NVARCHAR(MAX),
#query AS NVARCHAR(MAX);
SET #cols = STUFF((SELECT distinct ',' + QUOTENAME(c.Name)
FROM VariantAttribute c
FOR XML PATH(''), TYPE
).value('.', 'NVARCHAR(MAX)')
,1,1,'')
set #query = 'SELECT variantid, ' + #cols + ' from
(
select variantid
, Name
, value
from VariantAttribute
) x
pivot
(
max([Value])
for Name in (' + #cols + ')
) p '
execute(#query)
Before I get more slack about how I am storing this data. There is not a way out of it at the moment, but this query is to push this data to Azure Search (uses Elastic search) to be able to easily search on this data.