Operand type clash: varchar is incompatible with User-Defined Table Type - sql

I created a User-Defined Table Type:
CREATE TYPE dbo.ListTableType AS TABLE(
ITEM varchar(500) NULL
)
I leverage it in a function:
CREATE FUNCTION dbo.fn_list_to_string
(
#LIST dbo.ListTableType READONLY
)
RETURNS varchar(max)
AS
BEGIN
DECLARE #RESULT varchar(max)
SET #RESULT = ''
DECLARE #NL AS CHAR(2) = CHAR(13) + CHAR(10)
SELECT #RESULT = #RESULT + ITEM + #NL FROM #LIST
SET #RESULT = SUBSTRING(#RESULT, 1, LEN(#RESULT) - 1)
RETURN #RESULT
END
Finally, I try to use this function in a simple select:
SELECT
P.PROGRAM_ID,
PROGRAM_NAME,
PROGRAM_DESC,
P.STATUS_ID,
STATUS_DESC,
P.CONTACT_SID,
I.FIRST_NAME + ' ' + I.LAST_NAME as CONTACT_NAME,
P.CLARITY_ID,
dbo.fn_list_to_string(
( SELECT CONVERT(varchar,CLARITY_ID) as ITEM
FROM dbo.MUSEUM_PROGRAM_PROJECTS as A
JOIN dbo.MUSEUM_PROJECTS as B on B.PROJECT_ID = A.PROJECT_ID
WHERE PROGRAM_ID = P.PROGRAM_ID )
) as PROJECT_CLARITY_IDS
FROM dbo.MUSEUM_PROGRAMS as P
LEFT JOIN dbo.MUSEUM_PROGRAM_STATUS_TYPES as S on S.STATUS_ID = P.STATUS_ID
LEFT JOIN dbo.v_IDVAULT_ENRICHED_CURRENT_EMPLOYEES as I on I.[SID] = P.CONTACT_SID
But I get this error:
Operand type clash: varchar is incompatible with ListTableType
Any idea why? Also if there's another [more elegant] way to achieve what I'm trying to do I'm open to suggestions as well! Thanks in advance!

Here is a simple demonstration of the FOR XML PATH technique which does all of this with a very simple subquery and no table types or extremely inefficient multi-statement table-valued functions etc.
USE tempdb;
GO
CREATE TABLE dbo.P(Program_ID INT);
CREATE TABLE dbo.M(Clarity_ID INT, Program_ID INT);
INSERT dbo.P VALUES(1),(2),(3),(4);
INSERT dbo.M VALUES(1,1),(1,2),(2,3),(3,2),(1,4),(4,1);
SELECT
P.PROGRAM_ID,
PROJECT_CLARITY_IDS = STUFF((
SELECT CHAR(13)+CHAR(10)+CONVERT(VARCHAR(12),Clarity_ID)
FROM dbo.M WHERE Program_ID = p.Program_ID
FOR XML PATH(''), TYPE).value('.[1]','nvarchar(max)'),1,2,'')
FROM dbo.P AS p;
SQLfiddle demo
The output doesn't look right in SQLfiddle or in results to grid in Management Studio, because they strip out carriage returns/line feeds for display purposes, but you can replace CHAR(13)+CHAR(10) with two commas or semi-colons or something to verify that there are two characters there.

Using STUFF..FOR XML PATH construct for concatanation in combination with CTE will get the results you'd like. Something like this:
WITH CTE_PROJECT_CLARITIES AS
(
SELECT DISTINCT PROGRAM_ID
, STUFF((
SELECT CHAR(13) + CHAR(10) + CONVERT(varchar(11),CLARITY_ID)
FROM dbo.MUSEUM_PROGRAM_PROJECTS as A
JOIN dbo.MUSEUM_PROJECTS as B on B.PROJECT_ID = A.PROJECT_ID
WHERE A.PROGRAM_ID = X.PROGRAM_ID
FOR XML PATH ('')),1,2,'') AS PROJECT_CLARITY_IDS
FROM MUSEUM_PROGRAM_PROJECTS X
)
SELECT
P.PROGRAM_ID,
PROGRAM_NAME,
PROGRAM_DESC,
P.STATUS_ID,
STATUS_DESC,
P.CONTACT_SID,
I.FIRST_NAME + ' ' + I.LAST_NAME as CONTACT_NAME,
P.CLARITY_ID,
X.PROJECT_CLARITY_IDS
FROM dbo.MUSEUM_PROGRAMS as P
LEFT JOIN dbo.MUSEUM_PROGRAM_STATUS_TYPES as S on S.STATUS_ID = P.STATUS_ID
LEFT JOIN dbo.v_IDVAULT_ENRICHED_CURRENT_EMPLOYEES as I on I.[SID] = P.CONTACT_SID
LEFT JOIN CTE_PROJECT_CLARITIES X ON X.PROGRAM_ID = p.PROGRAM_ID
SQLFiddle DEMO (not sure if I got the columns right, but you'll get the idea)

Related

Mask values in a SQL Query string for SQL Server

I am a SQL Server DBA. I would like to write a procedure which I can provide to rest of my team where they can view the text for currently running queries on the server (Similar to how we view in sp_who2) but with all the values masked.
Examples:
Query text
Query text after Masking
Select * from sometable where rating = '4'
Select * from sometable where rating = '****'
Select name, id from sometable where id = '3233'
Select name, id from sometable where id = '****'
UPDATE Customers SET ContactName = 'Alfred Schmidt' WHERE CustomerID = 1;
UPDATE Customers SET ContactName = '****' WHERE CustomerID = ****;
INSERT INTO Customers (CustomerName, ContactName) VALUES ('Cardinal', 'Tom B. Erichsen');
INSERT INTO Customers (CustomerName, ContactName) VALUES ('*****', '****');
If I understand correctly your issue.
You can use this query:
select
r.session_id,
r.status,
r.command,
r.cpu_time,
r.total_elapsed_time,
t.text
from sys.dm_exec_requests as r
cross apply sys.dm_exec_sql_text(r.sql_handle) as t
e.g.
I run it on my SQL server right now:
(#P1 nvarchar(5),#P2 bigint,#P3 int,#P4 numeric(28, 12),#P5 nvarchar(5),#P6 datetime,#P7 datetime)
SELECT SUM(A.SETTLEAMOUNTCUR) FROM CUSTSETTLEMENT A,CUSTTRANS B WHERE ((A.DATAAREAID=#P1) AND (((A.TRANSRECID=#P2) AND (A.CANBEREVERSED=#P3)) AND (A.SETTLEAMOUNTCUR<>#P4))) AND ((B.DATAAREAID=#P5) AND (((B.RECID=A.OFFSETRECID) AND (B.TRANSDATE>=#P6)) AND (B.TRANSDATE<=#P7)))
All variables are hidden.
You could try some XML-trickery to handle the strings.
First replace all single quotes with an empty tag <X/> to get a XML that looks like this.
INSERT INTO Customers (CustomerName, ContactName)
VALUES (<X />Cardinal<X />, <X />Tom B. Erichsen<X />);
Then you shred the xml to get the text nodes and the node numbers where mod 2 is 0 is the ones you want to mask.
After that you can rebuild your query string using the mask values.
I have not found a way to deal with numbers other then removing all numbers from the query using Translate or nested replace and that will of course also remove numbers from table names and column names as well.
You could try something like this.
declare #S nvarchar(max);
declare #X xml;
set #S = N'UPDATE Customers SET ContactName = ''Alfred Schmidt'' WHERE CustomerID = 1;';
set #X = replace(#S, '''', '<X/>');
with C as
(
select T.X.value('.', 'nvarchar(max)') as V,
row_number() over(order by T.X) as RN
from #X.nodes('text()') as T(X)
)
select #S = (
select case when C.RN % 2 = 0 then '''*****''' else C.V end
from C
order by C.RN
for xml path(''), type
).value('text()[1]', 'nvarchar(max)');
set #S = translate(#S, '0123456789', '**********')
print #S;
Result:
UPDATE Customers SET ContactName = '*****' WHERE CustomerID = *;
Note: Just realized that this solution does not handle the cases where the string values contains single quotes but I think this is something that possibly can inspire more robust solution so I will leave it here.
sys.sp_get_query_template
fiddle
declare #t nvarchar(max), #p nvarchar(max);
declare #q nvarchar(max) = 'UPDATE Customers SET ContactName = N''Alfred Schmidt'' WHERE CustomerID = 1 AND Rate = 0.75 AND Rver = 0x0102 AND DateCreated = dateadd(day, -10, ''202z0818'')';
exec sys.sp_get_query_template #querytext = #q, #templatetext = #t OUTPUT, #parameters = #p OUTPUT;
select p,
case when tp like '%int' then cast('****' as nvarchar(40))
when tp like 'decimal(%' or tp like 'numeric(%' then '**.**'
when tp like '%binary(%' then '0x****'
when tp like 'n%char%' then 'N''****'''
else '''****'''
end as rv
into #t
from
(
select *, '#'+left(s.value, charindex(' ', s.value+' ')-1) as p, stuff(s.value, 1, charindex(' ', s.value), '') as tp
from string_split(replace(#p, ',#', '#'), '#') as s
where s.value <> ''
) as ss;
update #t
set #t = replace(#t, p, rv);
select #q union all select #t;

Unexpected output when executed

I have a problem in SQL Server, when I try to select two columns.
Here is the code I have written:
declare #d varchar(max)
set #d = ''
begin
select #d = #d + convert(nvarchar(MAX), R.Data, 121) + ('<table></td><td>' + convert(nvarchar(MAX), C.No) + '</td><td>' + C.Nome + '</td><td>' + convert(nvarchar(MAX), V.IdVendedor) + '</td><td>' + convert(nvarchar(MAX), V.Vendnm) + '</td></tr></table>')
from dbo.Reclamacoes R
Inner Join dbo.PHC_CLIENTES_SAMSYS C On R.IdCliente = C.Id
Inner Join dbo.PHC_VENDEDORES_SAMSYS V on R.IdVendedor = V.IdVendedor
select #d As Data, #d As HTML_COLUMN
end
It shows like this:
2019-02-07 00:00:00.0000000 " <table></td><td>2762.00</td><td>REGINA & MIGUEL, LDA. ( REMI )</td><td>78.00</td><td>AndreiaVeloso</td></tr></table> "
As you see, data is mixed with what is supposed to appear in the other column (I put in "" what it's supposed to appear in the other column).
And when I execute the column know as "Data" is mixed with " HTML_COLUMN.
And I'm trying to get two columns just with one variable.
Can someone tell me if is possible in my case, to have a query showing two columns when having just one variable? Thanks in advance.
You can make #d a Table Variable and put two columns in it.
Example from MSDN:
DECLARE #MyTableVar table(
EmpID int NOT NULL,
OldVacationHours int,
NewVacationHours int,
ModifiedDate datetime);
Try this..
DECLARE #HTML NVARCHAR(MAX) = '';
SET #HTML
= N'<table border="1" >'
+ N'<tr><th>Data</th><th>Nome</th><th>IdVendedor</th><th>Vendnm</th>'
+ CAST((
SELECT
td = R.Data
, ''
, td = C.Nome
, ''
, td = V.IdVendedor
, ''
, td = V.Vendnm
FROM dbo.Reclamacoes R
INNER JOIN dbo.PHC_CLIENTES_SAMSYS C ON R.IdCliente = C.Id
INNER JOIN dbo.PHC_VENDEDORES_SAMSYS V ON R.IdVendedor = V.IdVendedor
FOR XML PATH('tr'), TYPE
) AS NVARCHAR(MAX)) + N'</table>';
SELECT #HTML;

How to get column-level dependencies in a view

I've made some research on the matter but don't have solution yet. What I want to get is column-level dependencies in a view. So, let's say we have a table like this
create table TEST(
first_name varchar(10),
last_name varchar(10),
street varchar(10),
number int
)
and a view like this:
create view vTEST
as
select
first_name + ' ' + last_name as [name],
street + ' ' + cast(number as varchar(max)) as [address]
from dbo.TEST
What I'd like is to get result like this:
column_name depends_on_column_name depends_on_table_name
----------- --------------------- --------------------
name first_name dbo.TEST
name last_name dbo.TEST
address street dbo.TEST
address number dbo.TEST
I've tried sys.dm_sql_referenced_entities function, but referencing_minor_id is always 0 there for views.
select
referencing_minor_id,
referenced_schema_name + '.' + referenced_entity_name as depends_on_table_name,
referenced_minor_name as depends_on_column_name
from sys.dm_sql_referenced_entities('dbo.vTEST', 'OBJECT')
referencing_minor_id depends_on_table_name depends_on_column_name
-------------------- --------------------- ----------------------
0 dbo.TEST NULL
0 dbo.TEST first_name
0 dbo.TEST last_name
0 dbo.TEST street
0 dbo.TEST number
The same is true for sys.sql_expression_dependencies and for obsolete sys.sql_dependencies.
So do I miss something or is it impossible to do?
There're some related questions (Find the real column name of an alias used in a view?), but as I said - I haven't found a working solution yet.
EDIT 1: I've tried to use DAC to query if this information is stored somewhere in System Base Tables but haven't find it
This solution could answer your question only partially. It won't work for columns that are expressions.
You could use sys.dm_exec_describe_first_result_set to get column information:
#include_browse_information
If set to 1, each query is analyzed as if it has a FOR BROWSE option on the query. Additional key columns and source table information are returned.
CREATE TABLE txu(id INT, first_name VARCHAR(10), last_name VARCHAR(10));
CREATE TABLE txd(id INT, id_fk INT, address VARCHAR(100));
CREATE VIEW v_txu
AS
SELECT t.id AS PK_id,
t.first_name AS name,
d.address,
t.first_name + t.last_name AS name_full
FROM txu t
JOIN txd d
ON t.id = d.id_fk
Main query:
SELECT name, source_database, source_schema,
source_table, source_column
FROM sys.dm_exec_describe_first_result_set(N'SELECT * FROM v_txu', null, 1) ;
Output:
+-----------+--------------------+---------------+--------------+---------------+
| name | source_database | source_schema | source_table | source_column |
+-----------+--------------------+---------------+--------------+---------------+
| PK_id | fiddle_0f9d47226c4 | dbo | txu | id |
| name | fiddle_0f9d47226c4 | dbo | txu | first_name |
| address | fiddle_0f9d47226c4 | dbo | txd | address |
| name_full | null | null | null | null |
+-----------+--------------------+---------------+--------------+---------------+
DBFiddleDemo
It is a solution based on query plan. It has some adventages
almost any select queries can be processed
no SchemaBinding
and disadventages
has not been tested properly
can become broken suddenly if Microsoft change XML query plan.
The core idea is that every column expression inside XML query plan is defined in "DefinedValue" node. First subnode of "DefinedValue" is a reference to output column and second one is a expression. The expression computes from input columns and constant values.
As mentioned above It's based only on empirical observation and needs to be tested properly.
It's a invocation example:
exec dbo.GetColumnDependencies 'select * from dbo.vTEST'
target_column_name | source_column_name | const_value
---------------------------------------------------
address | Expr1007 | NULL
name | Expr1006 | NULL
Expr1006 | NULL | ' '
Expr1006 | [testdb].[dbo].first_name | NULL
Expr1006 | [testdb].[dbo].last_name | NULL
Expr1007 | NULL | ' '
Expr1007 | [testdb].[dbo].number | NULL
Expr1007 | [testdb].[dbo].street | NULL
It's code.
First of all get XML query plan.
declare #select_query as varchar(4000) = 'select * from dbo.vTEST' -- IT'S YOUR QUERY HERE.
declare #select_into_query as varchar(4000) = 'select top (1) * into #foo from (' + #select_query + ') as src'
, #xml_plan as xml = null
, #xml_generation_tries as tinyint = 10
;
while (#xml_plan is null and #xml_generation_tries > 0) -- There is no guaranty that plan will be cached.
begin
execute (#select_into_query);
select #xml_plan = pln.query_plan
from sys.dm_exec_query_stats as qry
cross apply sys.dm_exec_sql_text(qry.sql_handle) as txt
cross apply sys.dm_exec_query_plan(qry.plan_handle) as pln
where txt.text = #select_into_query
;
end
if (#xml_plan is null
) begin
raiserror(N'Can''t extract XML query plan from cache.' ,15 ,0);
return;
end
;
Next is a main query. It's biggest part is recursive common table expression for column extraction.
with xmlnamespaces(default 'http://schemas.microsoft.com/sqlserver/2004/07/showplan'
,'http://schemas.microsoft.com/sqlserver/2004/07/showplan' as shp -- Used in .query() for predictive namespace using.
)
, cte_column_dependencies as
(
The seed of recursion is a query that extracts columns for #foo table that store 1 row of interested select query.
select
(select foo_col.info.query('./ColumnReference') for xml raw('shp:root') ,type) -- Becouse .value() can't extract attribute from root node.
as target_column_info
, (select foo_col.info.query('./ScalarOperator/Identifier/ColumnReference') for xml raw('shp:root') ,type)
as source_column_info
, cast(null as xml) as const_info
, 1 as iteration_no
from #xml_plan.nodes('//Update/SetPredicate/ScalarOperator/ScalarExpressionList/ScalarOperator/MultipleAssign/Assign')
as foo_col(info)
where foo_col.info.exist('./ColumnReference[#Table="[#foo]"]') = 1
The recursive part searches for "DefinedValue" node with depended column and extract all "ColumnReference" and "Const" subnodes that used in column expression. It's over complicated by XML to SQL conversions.
union all
select
(select internal_col.info.query('.') for xml raw('shp:root') ,type)
, source_info.column_info
, source_info.const_info
, prev_dependencies.iteration_no + 1
from #xml_plan.nodes('//DefinedValue/ColumnReference') as internal_col(info)
inner join cte_column_dependencies as prev_dependencies -- Filters by depended columns.
on prev_dependencies.source_column_info.value('(//ColumnReference/#Column)[1]' ,'nvarchar(4000)') = internal_col.info.value('(./#Column)[1]' ,'nvarchar(4000)')
and exists (select prev_dependencies.source_column_info.value('(.//#Schema)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Schema)[1]' ,'nvarchar(4000)'))
and exists (select prev_dependencies.source_column_info.value('(.//#Database)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Database)[1]' ,'nvarchar(4000)'))
and exists (select prev_dependencies.source_column_info.value('(.//#Server)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Server)[1]' ,'nvarchar(4000)'))
cross apply ( -- Becouse only column or only constant can be places in result row.
select (select source_col.info.query('.') for xml raw('shp:root') ,type) as column_info
, null as const_info
from internal_col.info.nodes('..//ColumnReference') as source_col(info)
union all
select null as column_info
, (select const.info.query('.') for xml raw('shp:root') ,type) as const_info
from internal_col.info.nodes('..//Const') as const(info)
) as source_info
where source_info.column_info is null
or (
-- Except same node selected by '..//ColumnReference' from its sources. Sorry, I'm not so well to check it with XQuery simple.
source_info.column_info.value('(//#Column)[1]' ,'nvarchar(4000)') <> internal_col.info.value('(./#Column)[1]' ,'nvarchar(4000)')
and (select source_info.column_info.value('(//#Schema)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Schema)[1]' ,'nvarchar(4000)')) is null
and (select source_info.column_info.value('(//#Database)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Database)[1]' ,'nvarchar(4000)')) is null
and (select source_info.column_info.value('(//#Server)[1]' ,'nvarchar(4000)') intersect select internal_col.info.value('(./#Server)[1]' ,'nvarchar(4000)')) is null
)
)
Finally, It's select statement that convert XML to appropriate human text.
select
-- col_dep.target_column_info
--, col_dep.source_column_info
--, col_dep.const_info
coalesce(col_dep.target_column_info.value('(.//shp:ColumnReference/#Server)[1]' ,'nvarchar(4000)') + '.' ,'')
+ coalesce(col_dep.target_column_info.value('(.//shp:ColumnReference/#Database)[1]' ,'nvarchar(4000)') + '.' ,'')
+ coalesce(col_dep.target_column_info.value('(.//shp:ColumnReference/#Schema)[1]' ,'nvarchar(4000)') + '.' ,'')
+ col_dep.target_column_info.value('(.//shp:ColumnReference/#Column)[1]' ,'nvarchar(4000)')
as target_column_name
, coalesce(col_dep.source_column_info.value('(.//shp:ColumnReference/#Server)[1]' ,'nvarchar(4000)') + '.' ,'')
+ coalesce(col_dep.source_column_info.value('(.//shp:ColumnReference/#Database)[1]' ,'nvarchar(4000)') + '.' ,'')
+ coalesce(col_dep.source_column_info.value('(.//shp:ColumnReference/#Schema)[1]' ,'nvarchar(4000)') + '.' ,'')
+ col_dep.source_column_info.value('(.//shp:ColumnReference/#Column)[1]' ,'nvarchar(4000)')
as source_column_name
, col_dep.const_info.value('(/shp:root/shp:Const/#ConstValue)[1]' ,'nvarchar(4000)')
as const_value
from cte_column_dependencies as col_dep
order by col_dep.iteration_no ,target_column_name ,source_column_name
option (maxrecursion 512) -- It's an assurance from infinite loop.
All what you need is mentioned into definition of view.
so we can extract this information via following the next steps:-
Assign the view definition into a string variable.
Split it with (,) comma.
Split the alias with (+) plus operator via using CROSS APPLY with XML.
use the system tables for getting the accurate information like original table.
Demo:-
Create PROC psp_GetLevelDependsView (#sViewName varchar(200))
AS
BEGIN
Declare #stringToSplit nvarchar(1000),
#name NVARCHAR(255),
#dependsTableName NVARCHAR(50),
#pos INT
Declare #returnList TABLE ([Name] [nvarchar] (500))
SELECT TOP 1 #dependsTableName= table_schema + '.'+ TABLE_NAME
FROM INFORMATION_SCHEMA.VIEW_COLUMN_USAGE
select #stringToSplit = definition
from sys.objects o
join sys.sql_modules m on m.object_id = o.object_id
where o.object_id = object_id( #sViewName)
and o.type = 'V'
WHILE CHARINDEX(',', #stringToSplit) > 0
BEGIN
SELECT #pos = CHARINDEX(',', #stringToSplit)
SELECT #name = SUBSTRING(#stringToSplit, 1, #pos-1)
INSERT INTO #returnList
SELECT #name
SELECT #stringToSplit = SUBSTRING(#stringToSplit, #pos+1, LEN(#stringToSplit)-#pos)
END
INSERT INTO #returnList
SELECT #stringToSplit
select COLUMN_NAME , b.Name as Expression
Into #Temp
FROM INFORMATION_SCHEMA.COLUMNS a , #returnList b
WHERE TABLE_NAME= #sViewName
And (b.Name) like '%' + ( COLUMN_NAME) + '%'
SELECT A.COLUMN_NAME as column_name,
Split.a.value('.', 'VARCHAR(100)') AS depends_on_column_name , #dependsTableName as depends_on_table_name
Into #temp2
FROM
(
SELECT COLUMN_NAME,
CAST ('<M>' + REPLACE(Expression, '+', '</M><M>') + '</M>' AS XML) AS Data
FROM #Temp
) AS A CROSS APPLY Data.nodes ('/M') AS Split(a);
SELECT b.column_name , a.COLUMN_NAME as depends_on_column_name , b.depends_on_table_name
FROM INFORMATION_SCHEMA.VIEW_COLUMN_USAGE a , #temp2 b
WHERE VIEW_NAME= #sViewName
and b.depends_on_column_name like '%' + a.COLUMN_NAME + '%'
drop table #Temp
drop table #Temp2
END
Test:-
exec psp_GetLevelDependsView 'vTest'
Result:-
column_name depends_on_column_name depends_on_table_name
----------- --------------------- --------------------
name first_name dbo.TEST
name last_name dbo.TEST
address street dbo.TEST
address number dbo.TEST
I was playing around with this but didn't have time to go any further. Maybe this will help:
-- Returns all table columns called in the view and the objects they pull from
SELECT
v.[name] AS ViewName
,d.[referencing_id] AS ViewObjectID
,c.[name] AS ColumnNames
,OBJECT_NAME(d.referenced_id) AS ReferencedTableName
,d.referenced_id AS TableObjectIDsReferenced
FROM
sys.views v
INNER JOIN sys.sql_expression_dependencies d ON d.referencing_id = v.[object_id]
INNER JOIN sys.objects o ON d.referencing_id = o.[object_id]
INNER JOIN sys.columns c ON d.referenced_id = c.[object_id]
WHERE v.[name] = 'vTEST'
-- Returns all output columns in the view
SELECT
OBJECT_NAME([object_id]) AS ViewName
,[object_id] AS ViewObjectID
,[name] AS OutputColumnName
FROM sys.columns
WHERE OBJECT_ID('vTEST') = [object_id]
-- Get the view definition
SELECT
VIEW_DEFINITION
FROM INFORMATION_SCHEMA.VIEWS
WHERE TABLE_NAME = 'vTEST'
Unfortunately, SQL Server does not explicitly store mapping between source table columns and view columns. I suspect the main reason is simply due to the potential complexity of views (expression columns, functions called on those columns, nested queries etc.).
The only way that I can think of to determine the mapping between view columns and source columns would be to either parse the query associated to the view or parse the execution plan of the view.
The approach I have outlined here focuses on the second option and relies on the fact that SQL Server will avoid generating output lists for columns not required by a query.
The first step is to get the list of dependent tables and their associated columns required for the view. This can be achieved via the standard system tables in SQL Server.
Next, we enumerate all of the view’s columns via a cursor.
For each view column, we create a temporary wrapper stored procedure that only selects the single column in question from view. Because only a single column is requested SQL Server will only retrieve the information needed to output that single view column.
The newly created procedure will run the query in format only mode and will therefore not cause any actual I/O operations on the database, but it will generate an estimated execution plan when executed. After the query plan is generate, we query the output lists from the execution plan. Since we know which view column was selected we can now associate the output list to view column in question. We can further refine the association by only associating columns that form part of our original dependency list, this will eliminate expression outputs from the result set.
Note that with this method if the view needs to join different tables together to generate the output then all columns required to generate the output will be returned even if it is not directly used in the column expression since it is still in directly required.
The following stored procedure demonstrates the above implementation method:
CREATE PROCEDURE ViewGetColumnDependencies
(
#viewName NVARCHAR(50)
)
AS
BEGIN
CREATE TABLE #_suppress_output
(
result NVARCHAR(500) NULL
);
DECLARE #viewTableColumnMapping TABLE
(
[ViewName] NVARCHAR(50),
[SourceObject] NVARCHAR(50),
[SourceObjectColumnName] NVARCHAR(50),
[ViewAliasColumn] NVARCHAR(50)
)
-- Get list of dependent tables and their associated columns required for the view.
INSERT INTO #viewTableColumnMapping
(
[ViewName]
,[SourceObject]
,[SourceObjectColumnName]
)
SELECT v.[name] AS [ViewName]
,'[' + OBJECT_NAME(d.referenced_major_id) + ']' AS [SourceObject]
,c.[name] AS [SourceObjectColumnName]
FROM sys.views v
LEFT OUTER JOIN sys.sql_dependencies d ON d.object_id = v.object_id
LEFT OUTER JOIN sys.columns c ON c.object_id = d.referenced_major_id AND c.column_id = d.referenced_minor_id
WHERE v.[name] = #viewName;
DECLARE #aliasColumn NVARCHAR(50);
-- Next, we enumerate all of the views columns via a cursor.
DECLARE ViewColumnNameCursor CURSOR FOR
SELECT aliases.name AS [AliasName]
FROM sys.views v
LEFT OUTER JOIN sys.columns AS aliases on v.object_id = aliases.object_id -- c.column_id=aliases.column_id AND aliases.object_id = object_id('vTEST')
WHERE v.name = #viewName;
OPEN ViewColumnNameCursor
FETCH NEXT FROM ViewColumnNameCursor
INTO #aliasColumn
DECLARE #tql_create_proc NVARCHAR(MAX);
DECLARE #queryPlan XML;
WHILE ##FETCH_STATUS = 0
BEGIN
/*
For each view column, we create a temporary wrapper stored procedure that
only selects the single column in question from view. The stored procedure
will run the query in format only mode and will therefore not cause any
actual I/O operations on the database, but it will generate an estimated
execution plan when executed.
*/
SET #tql_create_proc = 'CREATE PROCEDURE ___WrapView
AS
SET FMTONLY ON;
SELECT CONVERT(NVARCHAR(MAX), [' + #aliasColumn + ']) FROM [' + #viewName + '];
SET FMTONLY OFF;';
EXEC (#tql_create_proc);
-- Execute the procedure to generate a query plan. The insert into the temp table is only done to
-- suppress the empty result set from being displayed as part of the output.
INSERT INTO #_suppress_output
EXEC ___WrapView;
-- Get the query plan for the wrapper procedure that was just executed.
SELECT #queryPlan = [qp].[query_plan]
FROM [sys].[dm_exec_procedure_stats] AS [ps]
JOIN [sys].[dm_exec_query_stats] AS [qs] ON [ps].[plan_handle] = [qs].[plan_handle]
CROSS APPLY [sys].[dm_exec_query_plan]([qs].[plan_handle]) AS [qp]
WHERE [ps].[database_id] = DB_ID() AND OBJECT_NAME([ps].[object_id], [ps].[database_id]) = '___WrapView'
-- Drop the wrapper view
DROP PROCEDURE ___WrapView
/*
After the query plan is generate, we query the output lists from the execution plan.
Since we know which view column was selected we can now associate the output list to
view column in question. We can further refine the association by only associating
columns that form part of our original dependency list, this will eliminate expression
outputs from the result set.
*/
;WITH QueryPlanOutputList AS
(
SELECT T.X.value('local-name(.)', 'NVARCHAR(max)') as Structure,
T.X.value('./#Table[1]', 'NVARCHAR(50)') as [SourceTable],
T.X.value('./#Column[1]', 'NVARCHAR(50)') as [SourceColumnName],
T.X.query('*') as SubNodes
FROM #queryPlan.nodes('*') as T(X)
UNION ALL
SELECT QueryPlanOutputList.structure + N'/' + T.X.value('local-name(.)', 'nvarchar(max)'),
T.X.value('./#Table[1]', 'NVARCHAR(50)') as [SourceTable],
T.X.value('./#Column[1]', 'NVARCHAR(50)') as [SourceColumnName],
T.X.query('*')
FROM QueryPlanOutputList
CROSS APPLY QueryPlanOutputList.SubNodes.nodes('*') as T(X)
)
UPDATE #viewTableColumnMapping
SET ViewAliasColumn = #aliasColumn
FROM #viewTableColumnMapping CM
INNER JOIN
(
SELECT DISTINCT QueryPlanOutputList.Structure
,QueryPlanOutputList.[SourceTable]
,QueryPlanOutputList.[SourceColumnName]
FROM QueryPlanOutputList
WHERE QueryPlanOutputList.Structure like '%/OutputList/ColumnReference'
) SourceColumns ON CM.[SourceObject] = SourceColumns.[SourceTable] AND CM.SourceObjectColumnName = SourceColumns.SourceColumnName
FETCH NEXT FROM ViewColumnNameCursor
INTO #aliasColumn
END
CLOSE ViewColumnNameCursor;
DEALLOCATE ViewColumnNameCursor;
DROP TABLE #_suppress_output
SELECT *
FROM #viewTableColumnMapping
ORDER BY [ViewAliasColumn]
END
The stored procedure can now be executed as follow:
EXEC dbo.ViewGetColumnDependencies #viewName = 'vTEST'

Call SQL Functions after PIVOT

I have a stored procedure that's taking a very long time because I have 2 function calls that are being called before a PIVOT, which means it's calling the functions 5 times for each record rather than once for each record. How can I get rewrite my query so that the 2 function calls right at the end of the query are run after the Pivot rather than before?
Here's the query
CREATE TABLE #Temp
(
ServiceRecordID INT,
LocationStd VARCHAR(1000),
AreaServedStd VARCHAR(1000),
RegionalLimited BIT,
Region VARCHAR(255),
Visible BIT
)
DECLARE #RegionCount INT
SELECT #RegionCount = COUNT(RegionID) FROM Regions WHERE SiteID = #SiteID AND RegionID % 100 != 0
INSERT INTO #Temp
SELECT TOP (#RegionCount * 100) SR.ServiceRecordID, SR.LocationStd, SR.AreaServedStd, SR.RegionalLimited, R.Region,
CASE WHEN (ISNULL(R_SR.RegionID,0) = 0 AND ISNULL(R_SR_Serv.RegionID,0) = 0) THEN 0 ELSE 1 END AS Visible
FROM ServiceRecord SR
INNER JOIN Sites S ON SR.SiteID = S.SiteID
INNER JOIN Regions R ON R.SiteID = S.SiteID
LEFT OUTER JOIN lkup_Region_ServiceRecord R_SR ON R_SR.RegionID = R.RegionID AND R_SR.ServiceRecordID = SR.ServiceRecordID
LEFT OUTER JOIN lkup_Region_ServiceRecord_Serv R_SR_Serv ON R_SR_Serv.RegionID = R.RegionID AND R_SR_Serv.ServiceRecordID = SR.ServiceRecordID AND SR.RegionalLimited = 0
WHERE SR.SiteID = #SiteID
AND R.RegionID % 100 != 0
ORDER BY SR.ServiceRecordID
DECLARE #RegionList varchar(2000),#SQL varchar(max)
SELECT #RegionList = STUFF((SELECT DISTINCT ',[' + Region + ']' FROM #Temp ORDER BY ',[' + Region + ']' FOR XML PATH('')),1,1,'')
SET #SQL='SELECT * FROM
(SELECT ServiceRecordID,
dbo.fn_ServiceRecordGetServiceName(ServiceRecordID,'''') AS ServiceName,
LocationStd,
AreaServedStd,
RegionalLimited,
Region As Region,
dbo.fn_GetOtherRegionalSitesForServiceRecord(ServiceRecordID) AS OtherSites,
CAST(Visible AS INT) AS Visible FROM #Temp) B PIVOT(MAX(Visible) FOR Region IN (' + #RegionList + ')) A'
EXEC(#SQL)
Move the function calls after the PIVOT:
SET #SQL='
SELECT
A.*,
N.ServiceName,
S.OtherSites
FROM
(
SELECT
ServiceRecordID,
LocationStd,
AreaServedStd,
RegionalLimited,
Region,
CAST(Visible AS INT) AS Visible
FROM #Temp
) B
PIVOT(MAX(Visible) FOR Region IN (' + #RegionList + ')) A
OUTER APPLY (
SELECT dbo.fn_ServiceRecordGetServiceName(A.ServiceRecordID,'''')
) N (ServiceName)
OUTER APPLY (
SELECT dbo.fn_GetOtherRegionalSitesForServiceRecord(A.ServiceRecordID)
) S (OtherSites);
';
Or just put them in the outer SELECT:
SET #SQL='
SELECT
A.*,
ServiceName = dbo.fn_ServiceRecordGetServiceName(A.ServiceRecordID,''''),
OtherSites = dbo.fn_GetOtherRegionalSitesForServiceRecord(A.ServiceRecordID)
FROM
(
SELECT
ServiceRecordID,
LocationStd,
AreaServedStd,
RegionalLimited,
Region,
CAST(Visible AS INT) AS Visible
FROM #Temp
) B
PIVOT(MAX(Visible) FOR Region IN (' + #RegionList + ')) A
';
If you can possibly convert those functions to be table-valued rowset-returning consisting of a single SELECT statement, you may get a huge performance improvement as well.
CREATE FUNCTION dbo.fn_ServiceRecordGetServiceName2(
#ServiceRecordID itn
)
RETURNS TABLE
AS
RETURN ( -- single select statement
SELECT ServiceName = Blah
FROM dbo.Gorp
WHERE Gunk = 'Ralph'
);
Then
OUTER APPLY dbo.fn_ServiceRecordGetServiceName(ServiceRecordID,'''') N
And N.ServiceName will return the value(s).
Also, it is not correct to tack on square brackets to convert data values to valid sysnames. You should use the QuoteName function. This will ensure your system doesn't break no matter WHAT crazy value is entered 13 years from now (think 'Taiwan [North]'):
STUFF((SELECT DISTINCT ',' + QuoteName(Region) FROM #Temp ...
Note:
Since you said that this is for display in a web page, you don't even need to do the pivoting on the server. Instead, return 2 rowsets to the client, one with the Site data and one with the column data for the Regions. You would need an additional pass through every row in the Region rowset to find out all the regions needed, but this can be done very quickly. Finally, adjust your program code to step through the Region rows as needed for each matching Site, and created your output.
One reason this is worth the investment is that if your application grows in size, you can always throw another web server at the problem, but it's a lot harder to throw another database at it. A new web server will cost less than continually beefing up your SQL Server.
P.S. Even dynamic SQL is easier to deal with when you format it well. :)

lucene, or sql fulltext?

I want to create a search website to search docs (all kinds of formats including pdf), images, videos, and audio. I also want to be able to filter my search results based on some criteria like author name, date, etc.
I'm doing this in .NET, so what's the easiest way to get up and running? SQL fulltext searching seems tempting because I'm familiar with sql, and plus since I want to filter my search results, it will be easy to store the filter fields for each item.
If your primary concern is getting it up and running quickly and easily, then SQL fulltext search is definitely the way to go.
Lucene.NET has its advantages, but it is by no means a walk in the park to set up correctly. The documentation is a bit lacking and there are a very limited number of examples on the web.
Stored procedure for snippets:
CREATE PROCEDURE SimpleCommentar
#SearchTerm nvarchar(100),
#Style nvarchar(200)
AS
BEGIN
CREATE TABLE #match_docs
(
doc_id bigint NOT NULL PRIMA
);
INSERT INTO #match_docs
(
doc_id
)
SELECT DISTINCT
Commentary_ID
FROM Commentary
WHERE FREETEXT
(
Commentary,
#SearchTerm,
LANGUAGE N'English'
);
DECLARE #db_id int = DB_ID(),
#table_id int = OBJECT_ID(N'
#column_id int =
(
SELECT
column_id
FROM sys.columns
WHERE object_id = OBJECT_I
AND name = N'Commentary'
);
SELECT
s.Commentary_ID,
t.Title,
MIN
(
N'...' + SUBSTRING
(
REPLACE
(
c.Commentary,
s.Display_Term,
N'<span style="' + #Style + '">' + s.Display_Term + '</span>'
),
s.Pos - 512,
s.Length + 1024
) + N'...'
) AS Snippet
FROM
(
SELECT DISTINCT
c.Commentary_ID,
w.Display_Term,
PATINDEX
(
N'%[^a-z]' + w.Display_Term + N'[^a-z]%',
c.Commentary
) AS Pos,
LEN(w.Display_Term) AS Length
FROM sys.dm_fts_index_keywords_by_document
(
#db_id,
#table_id
) w
INNER JOIN dbo.Commentary c
ON w.document_id = c.Commentary_ID
WHERE w.column_id = #column_id
AND EXISTS
(
SELECT 1
FROM #match_docs m
WHERE m.doc_id = w.document_id
)
AND EXISTS
(
SELECT 1
FROM sys.dm_fts_parser
(
N'FORMSOF(FREETEXT, "' + #SearchTerm + N'")',
1033,
0,
1
) p
WHERE p.Display_Term = w.Display_Term
)
) s
INNER JOIN dbo.Commentary c
ON s.Commentary_ID = c.Commentary_ID
INNER JOIN dbo.Book_Commentary bc
ON c.Commentary_ID = bc.Commentary_ID
INNER JOIN dbo.Book_Title bt
ON bc.Book_ID = bt.Book_ID
INNER JOIN dbo.Title t
ON bt.Title_ID = t.Title_ID
WHERE t.Is_Primary_Title = 1
GROUP BY
s.Commentary_ID,
t.Title;
DROP TABLE #match_docs;
END;