SQL Server 2008 - For XML Path is very slow - sql

I need help to increase the performance of my query, the example is below. I have a SELECT query list multiple fields of CUSTOMER table, this is really fast, about 15 ms. However, when I include the statement below using FOR XML PATH to grab the Customer POs (multiple) and combine them into one column, it is very slow, but it works.
Any suggestion how to increase the performance, but still getting the same results (combine the Customer POs into one column)? A sample code would be appreciated.
Select
Col1, Col2,
(SELECT
STUFF((SELECT ', ' + CustomerPO
FROM dbo.Tbl_CustomerPO
WHERE CustomerID = cus.CustomerID
FOR XML PATH('')), 1, 1, '')
) AS CustomerPOs
FROM Tbl_Customer cus
Thank you,

Try something like this ....
With a TINY recordset the "string concatenating" function method appears better from an execution plan cost, IO, and Time bases.
IF OBJECT_ID(N'fnConcatenateCustPOs', N'fN') IS NOT NULL
BEGIN
DROP FUNCTION dbo.fnConcatenateCustPOs
END
GO
CREATE FUNCTION dbo.fnConcatenateCustPOs
(
#CustomerID INT
)
RETURNS nvarchar(max)
--WITH ENCRYPTION
AS
BEGIN
DECLARE #StrFP nvarchar(3750)
--DECLARE #Custpo TABLE(CustomerPOId INT, CustomerID INT)
SET #StrFP = ''
SET #StrFP = ''
SELECT #StrFP = + #StrFP + ',' + CAST(CustomerPOId AS nvarchar(50))
FROM Custpo co
WHERE co.CustomerID = #CustomerID
RETURN SUBSTRING(#StrFP, 2, LEN(#StrFP))
END
GO
IF OBJECT_ID(N'Cust', N'U') IS NOT NULL
BEGIN
DROP TABLE Cust
END
IF OBJECT_ID(N'Custpo', N'U') IS NOT NULL
BEGIN
DROP TABLE CustPO
END
CREATE TABLE Cust (CustomerID INT)
CREATE TABLE CustPO (CustomerPOId INT, CustomerId INT)
INSERT Cust
SELECT 1
UNION
SELECT 2
INSERT CustPO
SELECT 10, 1
UNION
SELECT 20, 1
UNION
SELECT 30, 2
UNION
SELECT 31, 2
SET STATISTICS IO ON
SET STATISTICS TIME ON
SELECT CustomerId, dbo.fnConcatenateCustPOs(CustomerID)
FROM Cust cus
Select
CustomerID,
(SELECT
STUFF((SELECT ', ' + CAST(CustomerPOId AS nvarchar(50))
FROM dbo.CustPO
WHERE CustomerID = cus.CustomerID
FOR XML PATH('')), 1, 1, '')
) AS CustomerPOs
FROM Cust cus
SET STATISTICS IO OFF
SET STATISTICS TIME OFFTry:
Select
Col1, Col2,
(
STUFF((', ' + CustomerPO
FOR XML PATH('')), 1, 1, '')
) AS CustomerPOs
FROM Tbl_Customer cus
INNER JOIN dbo.Tbl_CustomerPO cpo ON cus.CustomerID = cpo.CustomerID

You're introducing a JOIN to your query, which will inherently affect performance.
If you index the joining field CustomerID you can speed up this query. Not much else to do here.
Note: Since you're prefixing your CustomerPo list with a comma and a space, you should use:
FOR XML PATH('')), 1, 2, '')
If you don't want your resulting strings to all start with a space.

Related

Mask values in a SQL Query string for SQL Server

I am a SQL Server DBA. I would like to write a procedure which I can provide to rest of my team where they can view the text for currently running queries on the server (Similar to how we view in sp_who2) but with all the values masked.
Examples:
Query text
Query text after Masking
Select * from sometable where rating = '4'
Select * from sometable where rating = '****'
Select name, id from sometable where id = '3233'
Select name, id from sometable where id = '****'
UPDATE Customers SET ContactName = 'Alfred Schmidt' WHERE CustomerID = 1;
UPDATE Customers SET ContactName = '****' WHERE CustomerID = ****;
INSERT INTO Customers (CustomerName, ContactName) VALUES ('Cardinal', 'Tom B. Erichsen');
INSERT INTO Customers (CustomerName, ContactName) VALUES ('*****', '****');
If I understand correctly your issue.
You can use this query:
select
r.session_id,
r.status,
r.command,
r.cpu_time,
r.total_elapsed_time,
t.text
from sys.dm_exec_requests as r
cross apply sys.dm_exec_sql_text(r.sql_handle) as t
e.g.
I run it on my SQL server right now:
(#P1 nvarchar(5),#P2 bigint,#P3 int,#P4 numeric(28, 12),#P5 nvarchar(5),#P6 datetime,#P7 datetime)
SELECT SUM(A.SETTLEAMOUNTCUR) FROM CUSTSETTLEMENT A,CUSTTRANS B WHERE ((A.DATAAREAID=#P1) AND (((A.TRANSRECID=#P2) AND (A.CANBEREVERSED=#P3)) AND (A.SETTLEAMOUNTCUR<>#P4))) AND ((B.DATAAREAID=#P5) AND (((B.RECID=A.OFFSETRECID) AND (B.TRANSDATE>=#P6)) AND (B.TRANSDATE<=#P7)))
All variables are hidden.
You could try some XML-trickery to handle the strings.
First replace all single quotes with an empty tag <X/> to get a XML that looks like this.
INSERT INTO Customers (CustomerName, ContactName)
VALUES (<X />Cardinal<X />, <X />Tom B. Erichsen<X />);
Then you shred the xml to get the text nodes and the node numbers where mod 2 is 0 is the ones you want to mask.
After that you can rebuild your query string using the mask values.
I have not found a way to deal with numbers other then removing all numbers from the query using Translate or nested replace and that will of course also remove numbers from table names and column names as well.
You could try something like this.
declare #S nvarchar(max);
declare #X xml;
set #S = N'UPDATE Customers SET ContactName = ''Alfred Schmidt'' WHERE CustomerID = 1;';
set #X = replace(#S, '''', '<X/>');
with C as
(
select T.X.value('.', 'nvarchar(max)') as V,
row_number() over(order by T.X) as RN
from #X.nodes('text()') as T(X)
)
select #S = (
select case when C.RN % 2 = 0 then '''*****''' else C.V end
from C
order by C.RN
for xml path(''), type
).value('text()[1]', 'nvarchar(max)');
set #S = translate(#S, '0123456789', '**********')
print #S;
Result:
UPDATE Customers SET ContactName = '*****' WHERE CustomerID = *;
Note: Just realized that this solution does not handle the cases where the string values contains single quotes but I think this is something that possibly can inspire more robust solution so I will leave it here.
sys.sp_get_query_template
fiddle
declare #t nvarchar(max), #p nvarchar(max);
declare #q nvarchar(max) = 'UPDATE Customers SET ContactName = N''Alfred Schmidt'' WHERE CustomerID = 1 AND Rate = 0.75 AND Rver = 0x0102 AND DateCreated = dateadd(day, -10, ''202z0818'')';
exec sys.sp_get_query_template #querytext = #q, #templatetext = #t OUTPUT, #parameters = #p OUTPUT;
select p,
case when tp like '%int' then cast('****' as nvarchar(40))
when tp like 'decimal(%' or tp like 'numeric(%' then '**.**'
when tp like '%binary(%' then '0x****'
when tp like 'n%char%' then 'N''****'''
else '''****'''
end as rv
into #t
from
(
select *, '#'+left(s.value, charindex(' ', s.value+' ')-1) as p, stuff(s.value, 1, charindex(' ', s.value), '') as tp
from string_split(replace(#p, ',#', '#'), '#') as s
where s.value <> ''
) as ss;
update #t
set #t = replace(#t, p, rv);
select #q union all select #t;

sql Id concatenation in sequence in a separate column like running total

I need running Id concatenation just like running balance or total..
Concatenate the previous Ids to current Id row wise just like shown in picture
query is
with relation (Id, [orderSequence])
as
(
select Id,cast(Id as varchar(20))
from [ACChartofAccount]
union all
select p.Id, cast(Cast(r.Id as varchar) + ',' + cast(p.Id as varchar) as varchar(20))
from [ACChartofAccount] p
inner join relation r on p.ParentId = r.Id
)
select Id,orderSequence
from relation
order by orderSequence
You can use below query to get above result.
DECLARE #Table TABLE(ID VARCHAR(10));
INSERT INTO #table(ID) VALUES ('320'),(332),(333),(334),(335);
SELECT mt.ID,
STUFF((
SELECT ', ' + ID
FROM #table t
WHERE t.ID <= mt.ID
FOR XML PATH('')), 1, 2, '') AS oldersequence
FROM #table mt
ORDER BY ID

Dynamic SELECT statement, generate columns based on present and future values

Currently building a SELECT statement in SQL Server 2008 but would like to make this SELECT statement dynamic, so the columns can be defined based on values in a table. I heard about pivot table and cursors, but seems kind of hard to understand at my current level, here is the code;
DECLARE #date DATE = null
IF #date is null
set # date = GETDATE() as DATE
SELECT
Name,
value1,
value2,
value3,
value4
FROM ref_Table a
FULL OUTER JOIN (
SELECT
PK_ID ID,
sum(case when FK_ContainerType_ID = 1 then 1 else null) Box,
sum(case when FK_ContainerType_ID = 2 then 1 else null) Pallet,
sum(case when FK_ContainerType_ID = 3 then 1 else null) Bag,
sum(case when FK_ContainerType_ID = 4 then 1 else null) Drum
from
Packages
WHERE
#date between PackageStart AND PackageEnd
group by PK_ID ) b on a.Name = b.ID
where
Group = 0
The following works great for me , but PK_Type_ID and the name of the column(PackageNameX,..) are hard coded, I need to be dynamic and it can build itself based on present or futures values in the Package table.
Any help or guidance on the right direction would be greatly appreciated...,
As requested
ref_Table (PK_ID, Name)
1, John
2, Mary
3, Albert
4, Jane
Packages (PK_ID, FK_ref_Table_ID, FK_ContainerType_ID, PackageStartDate, PackageEndDate)
1 , 1, 4, 1JAN2014, 30JAN2014
2 , 2, 3, 1JAN2014, 30JAN2014
3 , 3, 2, 1JAN2014, 30JAN2014
4 , 4, 1, 1JAN2014, 30JAN2014
ContainerType (PK_ID, Type)
1, Box
2, Pallet
3, Bag
4, Drum
and the result should look like this;
Name Box Pallet Bag Drum
---------------------------------------
John 1
Mary 1
Albert 1
Jane 1
The following code like I said works great, the issue is the Container table is going to grow and I need to replicated the same report without hard coding the columns.
What you need to build is called a dynamic pivot. There are plenty of good references on Stack if you search out that term.
Here is a solution to your scenario:
IF OBJECT_ID('tempdb..##ref_Table') IS NOT NULL
DROP TABLE ##ref_Table
IF OBJECT_ID('tempdb..##Packages') IS NOT NULL
DROP TABLE ##Packages
IF OBJECT_ID('tempdb..##ContainerType') IS NOT NULL
DROP TABLE ##ContainerType
SET NOCOUNT ON
CREATE TABLE ##ref_Table (PK_ID INT, NAME NVARCHAR(50))
CREATE TABLE ##Packages (PK_ID INT, FK_ref_Table_ID INT, FK_ContainerType_ID INT, PackageStartDate DATE, PackageEndDate DATE)
CREATE TABLE ##ContainerType (PK_ID INT, [Type] NVARCHAR(50))
INSERT INTO ##ref_Table (PK_ID,NAME)
SELECT 1,'John' UNION
SELECT 2,'Mary' UNION
SELECT 3,'Albert' UNION
SELECT 4,'Jane'
INSERT INTO ##Packages (PK_ID, FK_ref_Table_ID, FK_ContainerType_ID, PackageStartDate, PackageEndDate)
SELECT 1,1,4,'2014-01-01','2014-01-30' UNION
SELECT 2,2,3,'2014-01-01','2014-01-30' UNION
SELECT 3,3,2,'2014-01-01','2014-01-30' UNION
SELECT 4,4,1,'2014-01-01','2014-01-30'
INSERT INTO ##ContainerType (PK_ID, [Type])
SELECT 1,'Box' UNION
SELECT 2,'Pallet' UNION
SELECT 3,'Bag' UNION
SELECT 4,'Drum'
DECLARE #DATE DATE, #PARAMDEF NVARCHAR(MAX), #COLS NVARCHAR(MAX), #SQL NVARCHAR(MAX)
SET #DATE = '2014-01-15'
SET #COLS = STUFF((SELECT DISTINCT ',' + QUOTENAME(T.[Type])
FROM ##ContainerType T
FOR XML PATH, TYPE).value('.', 'NVARCHAR(MAX)'),1,1,'')
SET #SQL = 'SELECT [Name], ' + #COLS + '
FROM (SELECT [Name], [Type], 1 AS Value
FROM ##ref_Table R
JOIN ##Packages P ON R.PK_ID = P.FK_ref_Table_ID
JOIN ##ContainerType T ON P.FK_ContainerType_ID = T.PK_ID
WHERE #DATE BETWEEN P.PackageStartDate AND P.PackageEndDate) X
PIVOT (COUNT(Value) FOR [Type] IN (' + #COLS + ')) P
'
PRINT #COLS
PRINT #SQL
SET #PARAMDEF = '#DATE DATE'
EXEC SP_EXECUTESQL #SQL, #PARAMDEF, #DATE=#DATE
Output:
Name Bag Box Drum Pallet
Albert 0 0 0 1
Jane 0 1 0 0
John 0 0 1 0
Mary 1 0 0 0
Static Query:
SELECT [Name],[Box],[Pallet],[Bag],[Drum] FROM
(
SELECT *
FROM
(
SELECT rf.Name,cnt.[Type], pk.PK_ID AS PKID, rf.PK_ID AS RFID
FROM ref_Table rf INNER JOIN Packages pk ON rf.PK_ID = pk.FK_ref_Table_ID
INNER JOIN ContanerType cnt ON cnt.PK_ID = pk.FK_ContainerType_ID
) AS SourceTable
PIVOT
(
COUNT(PKID )
FOR [Type]
IN ( [Box],[Pallet],[Bag],[Drum])
) AS PivotTable
) AS Main
ORDER BY RFID
Dynamic Query:
DECLARE #columnList nvarchar (MAX)
DECLARE #pivotsql nvarchar (MAX)
SELECT #columnList = STUFF(
(
SELECT ',' + '[' + [Type] + ']'
FROM ContanerType
FOR XML PATH( '')
)
,1, 1,'' )
SET #pivotsql =
N'SELECT [Name],' + #columnList + ' FROM
(
SELECT *
FROM
(
SELECT rf.Name,cnt.[Type], pk.PK_ID AS PKID, rf.PK_ID AS RFID
FROM ref_Table rf INNER JOIN Packages pk ON rf.PK_ID = pk.FK_ref_Table_ID
INNER JOIN ContanerType cnt ON cnt.PK_ID = pk.FK_ContainerType_ID
) AS SourceTable
PIVOT
(
COUNT(PKID )
FOR [Type]
IN ( ' + #columnList + ')
) AS PivotTable
) AS Main
ORDER BY RFID;'
EXEC sp_executesql #pivotsql
Following my tutorial below will help you to understand the PIVOT functionality:
We write sql queries in order to get different result sets like full, partial, calculated, grouped, sorted etc from the database tables. However sometimes we have requirements that we have to rotate our tables. Sounds confusing?
Let's keep it simple and consider the following two screen grabs.
SQL Table:
Expected Results:
Wow, that's look like a lot of work! That is a combination of tricky sql, temporary tables, loops, aggregation......, blah blah blah
Don't worry let's keep it simple, stupid(KISS).
MS SQL Server 2005 and above has a function called PIVOT. It s very simple to use and powerful. With the help of this function we will be able to rotate sql tables and result sets.
Simple steps to make it happen:
Identify all the columns those will be part of the desired result set.
Find the column on which we will apply aggregation(sum,ave,max,min etc)
Identify the column which values will be the column header.
Specify the column values mentioned in step3 with comma separated and surrounded by square brackets.
So, if we now follow above four steps and extract information from the above sales table, it will be as below:
Year, Month, SalesAmount
SalesAmount
Month
[Jan],[Feb] ,[Mar] .... etc
We are nearly there if all the above steps made sense to you so far.
Now we have all the information we need. All we have to do now is to fill the below template with required information.
Template:
Our SQL query should look like below:
SELECT *
FROM
(
SELECT SalesYear, SalesMonth,Amount
FROM Sales
) AS SourceTable
PIVOT
(
SUM(Amount )
FOR SalesMonth
IN ( [Jan],[Feb] ,[Mar],
[Apr],[May],[Jun] ,[Jul],
[Aug],[Sep] ,[Oct],[Nov] ,[Dec])
) AS PivotTable;
In the above query we have hard coded the column names. Well it's not fun when you have to specify a number of columns.
However, there is a work arround as follows:
DECLARE #columnList nvarchar (MAX)
DECLARE #pivotsql nvarchar (MAX)
SELECT #columnList = STUFF(
(
SELECT ',' + '[' + SalesMonth + ']'
FROM Sales
GROUP BY SalesMonth
FOR XML PATH( '')
)
,1, 1,'' )
SET #pivotsql =
N'SELECT *
FROM
(
SELECT SalesYear, SalesMonth,Amount
FROM Sales
) AS SourceTable
PIVOT
(
SUM(Amount )
FOR SalesMonth
IN ( ' + #columnList +' )
) AS PivotTable;'
EXEC sp_executesql #pivotsql
Hopefully this tutorial will be a help to someone somewhere.
Enjoy coding.

Update data based on parameter with operator from another table

I have 2 tables with sample data below:
tblSale
PartCode PartGroup SaleQty
a FM 600
b MM 202
c SM 10
d NM 0
tblCondition
PartGroup Condition
FM >500
MM >=200
SM >=1
NM 0
in SQL Server stored procedure i want to update PartGroup in tblSale by PartGroup in tblCondition based on sum(SaleQty) and compare with Condition.
Any help please.
UPDATE:
Example:
PartCode 'A' is has PartGroup='FM' and SaleQty=500.
if SaleQty=400 then update PartGroup='MM' based on Condition in tblCondition.
UPDATE tblSale
SET tblSale.PartGroup=tblCondition.PartGroup
WHERE SUM(tblSale.Sale) ??? tblCondition.Condition
I don't think you will be able to do this without dynamic code.
For my solution you will need make some changes/notes:
change PartGroup NM condition 0 to =0
make sure that tblCondition table conditions are inserted from biggest (500) to lowest (0)
First what I do is create CASE for every row from tblCondition table.
Then I SUM data to temp table by PartCode (I split PartCode 'c' to 2 rows for testing)
And for the last, create dynamic code, which will update data
/*
CREATE TABLE #tblSale ( PartCode VARCHAR(10), PartGroup VARCHAR(10), SaleQty INT)
INSERT INTO #tblSale SELECT 'a', 'FM', 600
INSERT INTO #tblSale SELECT 'b', 'MM', 202
INSERT INTO #tblSale SELECT 'c', 'SM', 5
INSERT INTO #tblSale SELECT 'd', 'NM', 0
INSERT INTO #tblSale SELECT 'c', 'SM', 5
CREATE TABLE #tblCondition ( PartGroup VARCHAR(10), Condition VARCHAR(10))
INSERT INTO #tblCondition SELECT 'FM', '>500'
INSERT INTO #tblCondition SELECT 'MM', '>=200'
INSERT INTO #tblCondition SELECT 'SM', '>=1'
INSERT INTO #tblCondition SELECT 'NM', '=0'
*/
--CREATE CASES
DECLARE #CaseStr NVARCHAR(1000) = 'CASE '
SELECT #CaseStr = #CaseStr + '
WHEN SaleSUM ' + Condition + ' THEN '''+ PartGroup + ''' '
FROM #tblCondition
SET #CaseStr = #CaseStr + ' END'
-- SUM data by PartCode
SELECT PartCode, SUM(SaleQty) AS SaleSUM
INTO #tblSaleSUM
FROM #tblSale
GROUP BY PartCode
-- Create dynamic code for update
DECLARE #query NVARCHAR(MAX)
SET #query = N'
UPDATE S
SET S.PartGroup = SS.PartGroup
FROM #tblSale AS S
INNER JOIN
(
SELECT PartCode, ' + #CaseStr + ' AS PartGroup
FROM #tblSaleSUM
) AS SS
ON SS.PartCode = S.PartCode
'
EXEC sp_executesql #query
Use this
update ts set partGroup = something
from tblSale ts
inner join tblCondition tc
on tc.PartGroup=ts.PartGroup
inner join (Select PartGroup, sum(SaleQty) as SumSaleQty
from tblSale
group by PartGroup) as sums
on sums.PartGroup = tc.PartGroup
and sums.SumSaleQty >= tc.Condition

replace value in varchar(max) field with join

I have a table that contains text field with placeholders. Something like this:
Row Notes
1. This is some notes ##placeholder130## this ##myPlaceholder##, #oneMore#. End.
2. Second row...just a ##test#.
(This table contains about 1-5k rows on average. Average number of placeholders in one row is 5-15).
Now, I have a lookup table that looks like this:
Name Value
placeholder130 Dog
myPlaceholder Cat
oneMore Cow
test Horse
(Lookup table will contain anywhere from 10k to 100k records)
I need to find the fastest way to join those placeholders from strings to a lookup table and replace with value. So, my result should look like this (1st row):
This is some notes Dog this Cat, Cow. End.
What I came up with was to split each row into multiple for each placeholder and then join it to lookup table and then concat records back to original row with new values, but it takes around 10-30 seconds on average.
You could try to split the string using a numbers table and rebuild it with for xml path.
select (
select coalesce(L.Value, T.Value)
from Numbers as N
cross apply (select substring(Notes.notes, N.Number, charindex('##', Notes.notes + '##', N.Number) - N.Number)) as T(Value)
left outer join Lookup as L
on L.Name = T.Value
where N.Number <= len(notes) and
substring('##' + notes, Number, 2) = '##'
order by N.Number
for xml path(''), type
).value('text()[1]', 'varchar(max)')
from Notes
SQL Fiddle
I borrowed the string splitting from this blog post by Aaron Bertrand
SQL Server is not very fast with string manipulation, so this is probably best done client-side. Have the client load the entire lookup table, and replace the notes as they arrived.
Having said that, it can of course be done in SQL. Here's a solution with a recursive CTE. It performs one lookup per recursion step:
; with Repl as
(
select row_number() over (order by l.name) rn
, Name
, Value
from Lookup l
)
, Recurse as
(
select Notes
, 0 as rn
from Notes
union all
select replace(Notes, '##' + l.name + '##', l.value)
, r.rn + 1
from Recurse r
join Repl l
on l.rn = r.rn + 1
)
select *
from Recurse
where rn =
(
select count(*)
from Lookup
)
option (maxrecursion 0)
Example at SQL Fiddle.
Another option is a while loop to keep replacing lookups until no more are found:
declare #notes table (notes varchar(max))
insert #notes
select Notes
from Notes
while 1=1
begin
update n
set Notes = replace(n.Notes, '##' + l.name + '##', l.value)
from #notes n
outer apply
(
select top 1 Name
, Value
from Lookup l
where n.Notes like '%##' + l.name + '##%'
) l
where l.name is not null
if ##rowcount = 0
break
end
select *
from #notes
Example at SQL Fiddle.
I second the comment that tsql is just not suited for this operation, but if you must do it in the db here is an example using a function to manage the multiple replace statements.
Since you have a relatively small number of tokens in each note (5-15) and a very large number of tokens (10k-100k) my function first extracts tokens from the input as potential tokens and uses that set to join to your lookup (dbo.Token below). It was far too much work to look for an occurrence of any of your tokens in each note.
I did a bit of perf testing using 50k tokens and 5k notes and this function runs really well, completing in <2 seconds (on my laptop). Please report back how this strategy performs for you.
note: In your example data the token format was not consistent (##_#, ##_##, #_#), I am guessing this was simply a typo and assume all tokens take the form of ##TokenName##.
--setup
if object_id('dbo.[Lookup]') is not null
drop table dbo.[Lookup];
go
if object_id('dbo.fn_ReplaceLookups') is not null
drop function dbo.fn_ReplaceLookups;
go
create table dbo.[Lookup] (LookupName varchar(100) primary key, LookupValue varchar(100));
insert into dbo.[Lookup]
select '##placeholder130##','Dog' union all
select '##myPlaceholder##','Cat' union all
select '##oneMore##','Cow' union all
select '##test##','Horse';
go
create function [dbo].[fn_ReplaceLookups](#input varchar(max))
returns varchar(max)
as
begin
declare #xml xml;
select #xml = cast(('<r><i>'+replace(#input,'##' ,'</i><i>')+'</i></r>') as xml);
--extract the potential tokens
declare #LookupsInString table (LookupName varchar(100) primary key);
insert into #LookupsInString
select distinct '##'+v+'##'
from ( select [v] = r.n.value('(./text())[1]', 'varchar(100)'),
[r] = row_number() over (order by n)
from #xml.nodes('r/i') r(n)
)d(v,r)
where r%2=0;
--tokenize the input
select #input = replace(#input, l.LookupName, l.LookupValue)
from dbo.[Lookup] l
join #LookupsInString lis on
l.LookupName = lis.LookupName;
return #input;
end
go
return
--usage
declare #Notes table ([Id] int primary key, notes varchar(100));
insert into #Notes
select 1, 'This is some notes ##placeholder130## this ##myPlaceholder##, ##oneMore##. End.' union all
select 2, 'Second row...just a ##test##.';
select *,
dbo.fn_ReplaceLookups(notes)
from #Notes;
Returns:
Tokenized
--------------------------------------------------------
This is some notes Dog this Cat, Cow. End.
Second row...just a Horse.
Try this
;WITH CTE (org, calc, [Notes], [level]) AS
(
SELECT [Notes], [Notes], CONVERT(varchar(MAX),[Notes]), 0 FROM PlaceholderTable
UNION ALL
SELECT CTE.org, CTE.[Notes],
CONVERT(varchar(MAX), REPLACE(CTE.[Notes],'##' + T.[Name] + '##', T.[Value])), CTE.[level] + 1
FROM CTE
INNER JOIN LookupTable T ON CTE.[Notes] LIKE '%##' + T.[Name] + '##%'
)
SELECT DISTINCT org, [Notes], level FROM CTE
WHERE [level] = (SELECT MAX(level) FROM CTE c WHERE CTE.org = c.org)
SQL FIDDLE DEMO
Check the below devioblog post for reference
devioblog post
To get speed, you can preprocess the note templates into a more efficient form. This will be a sequence of fragments, with each ending in a substitution. The substitution might be NULL for the last fragment.
Notes
Id FragSeq Text SubsId
1 1 'This is some notes ' 1
1 2 ' this ' 2
1 3 ', ' 3
1 4 '. End.' null
2 1 'Second row...just a ' 4
2 2 '.' null
Subs
Id Name Value
1 'placeholder130' 'Dog'
2 'myPlaceholder' 'Cat'
3 'oneMore' 'Cow'
4 'test' 'Horse'
Now we can do the substitutions with a simple join.
SELECT Notes.Text + COALESCE(Subs.Value, '')
FROM Notes LEFT JOIN Subs
ON SubsId = Subs.Id WHERE Notes.Id = ?
ORDER BY FragSeq
This produces a list of fragments with substitutions complete. I am not an MSQL user, but in most dialects of SQL you can concatenate these fragments in a variable quite easily:
DECLARE #Note VARCHAR(8000)
SELECT #Note = COALESCE(#Note, '') + Notes.Text + COALSCE(Subs.Value, '')
FROM Notes LEFT JOIN Subs
ON SubsId = Subs.Id WHERE Notes.Id = ?
ORDER BY FragSeq
Pre-processing a note template into fragments will be straightforward using the string splitting techniques of other posts.
Unfortunately I'm not at a location where I can test this, but it ought to work fine.
I really don't know how it will perform with 10k+ of lookups.
how does the old dynamic SQL performs?
DECLARE #sqlCommand NVARCHAR(MAX)
SELECT #sqlCommand = N'PlaceholderTable.[Notes]'
SELECT #sqlCommand = 'REPLACE( ' + #sqlCommand +
', ''##' + LookupTable.[Name] + '##'', ''' +
LookupTable.[Value] + ''')'
FROM LookupTable
SELECT #sqlCommand = 'SELECT *, ' + #sqlCommand + ' FROM PlaceholderTable'
EXECUTE sp_executesql #sqlCommand
Fiddle demo
And now for some recursive CTE.
If your indexes are correctly set up, this one should be very fast or very slow. SQL Server always surprises me with performance extremes when it comes to the r-CTE...
;WITH T AS (
SELECT
Row,
StartIdx = 1, -- 1 as first starting index
EndIdx = CAST(patindex('%##%', Notes) as int), -- first ending index
Result = substring(Notes, 1, patindex('%##%', Notes) - 1)
-- (first) temp result bounded by indexes
FROM PlaceholderTable -- **this is your source table**
UNION ALL
SELECT
pt.Row,
StartIdx = newstartidx, -- starting index (calculated in calc1)
EndIdx = EndIdx + CAST(newendidx as int) + 1, -- ending index (calculated in calc4 + total offset)
Result = Result + CAST(ISNULL(newtokensub, newtoken) as nvarchar(max))
-- temp result taken from subquery or original
FROM
T
JOIN PlaceholderTable pt -- **this is your source table**
ON pt.Row = T.Row
CROSS APPLY(
SELECT newstartidx = EndIdx + 2 -- new starting index moved by 2 from last end ('##')
) calc1
CROSS APPLY(
SELECT newtxt = substring(pt.Notes, newstartidx, len(pt.Notes))
-- current piece of txt we work on
) calc2
CROSS APPLY(
SELECT patidx = patindex('%##%', newtxt) -- current index of '##'
) calc3
CROSS APPLY(
SELECT newendidx = CASE
WHEN patidx = 0 THEN len(newtxt) + 1
ELSE patidx END -- if last piece of txt, end with its length
) calc4
CROSS APPLY(
SELECT newtoken = substring(pt.Notes, newstartidx, newendidx - 1)
-- get the new token
) calc5
OUTER APPLY(
SELECT newtokensub = Value
FROM LookupTable
WHERE Name = newtoken -- substitute the token if you can find it in **your lookup table**
) calc6
WHERE newstartidx + len(newtxt) - 1 <= len(pt.Notes)
-- do this while {new starting index} + {length of txt we work on} exceeds total length
)
,lastProcessed AS (
SELECT
Row,
Result,
rn = row_number() over(partition by Row order by StartIdx desc)
FROM T
) -- enumerate all (including intermediate) results
SELECT *
FROM lastProcessed
WHERE rn = 1 -- filter out intermediate results (display only last ones)