I have a stored procedure which uses a while loop to iterate to each record
and do some calculations. The stored procedure has the following logic:
Inserts some records from old_table to new_table for a specific range
(7.8 million records) . insert took 4 minutes.
Then it copies records which have amount > 100 into a temp table
(37 000 records).
create clustered index on the id column of temp table.
start the while loop to iterate to each id column in temp table (total 37 000 records).
for each iteration a stored procedure is called inside the while loop.
The Sp works fine (faster) Up to 5 000 records (it completes in 5 mins)
but after 5 000 there is a delay in processing records. The delay occurs after processing each 100 records. Is there any reason for this delay?
sample structure of the stored procedure:
use my_db
go
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
IF OBJECT_ID('[dbo].[usp_datacopy]') IS NULL -- Check if SP Exists
EXEC('CREATE PROCEDURE [dbo].[usp_datacopy] AS SET NOCOUNT ON;') -- Create dummy/empty SP
GO
ALTER PROCEDURE [dbo].[usp_datacopy]
#Startdate datetime,
#Enddate datetime,
#Percent int
AS
BEGIN
SET NOCOUNT ON;
BEGIN TRY
BEGIN TRAN
DECLARE #PerVal NUMERIC(10,2) = #percent /100.00,--Get the percent value for input.
#wkstartdate DateTime,
#wkenddate Datetime,
#OfficeID int,
#id int = 1,
#spcount int ,
#rdate datetime,
IF OBJECT_ID ('TEMPDB..#temptable') IS NOT NULL
DROP TABLE #temptable
INSERT INTO [dbo].[new_table]
( TimeID, OfficeID, dateval, rHour, Complexity,RowCountval)
SELECT TimeID + 364, OfficeID,dateadd(dd,364,dateval),rHour, Complexity , RowCountval,
FROM [dbo].[old_table] WITH (NOLOCK)
WHERE dateval BETWEEN #Startdate AND #Enddate
--select Day records with value > 100
SELECT ROW_NUMBER() over(order by timeid) id,TimeID, OfficeID, dateval, rHour, Complexity,RowCountval
INTO #temptable
FROM [dbo].[new_table] WITH (NOLOCK)
WHERE RDATE BETWEEN #wkstartdate AND #wkenddate
AND RowCountval > 100
CREATE CLUSTERED INDEX id_index on #temptable (id)
SELECT #spcount = COUNT (1) from #temptable
WHILE #id <= #spcount --37000 records
BEGIN
SELECT #OfficeID = officeid FROM #temptable WHERE id = #id
SELECT #rdate = dateval FROM #temptable WHERE id = #id
EXEC CalcPercentforEachRecord #rdate, #OfficeID, #PerVal
PRINT #ID
SELECT #id = #id + 1
SELECT #OfficeID = NULL
SELECT #rdate = NULL
END
IF ##ERROR = 0
BEGIN
COMMIT TRAN;
END
END TRY
BEGIN CATCH
SELECT ##ERROR AS ERROR,ERROR_MESSAGE() AS ErrorMessage,ERROR_LINE() AS ErrorLineNo
ROLLBACK TRAN;
END CATCH
SET NOCOUNT OFF;
END
Related
I have written this stored procedure for data purge from one of my tables. I have made it configurable to either archive the data or do a hard delete. Since the volume of data is quiet large, i am using loops to do the same.
The delete part of it is working fine However the archive part of it is giving me hard times and i am kind of stuck there with multiple tries.
Here is my SP.
ALTER PROCEDURE [dbo].[spPurgeRecords_new] (
#Age AS INT,
#NumberOfLoops AS BIGINT,
#DeleteSize BIGINT,
#IsArchive BIT
)
AS
SET NOCOUNT ON
SET XACT_ABORT ON
BEGIN
DECLARE #CurrentLoop AS BIGINT;
SET #CurrentLoop = 0
declare #cutoffDate datetime;
declare #maxXDate datetime;
declare #loop varchar(50);
select #cutoffDate = dateadd(year,-#Age,getdate())
select #maxXDate = max(dateCreated) from cbr_audit where dateCreated < #cutoffDate
declare #date varchar(100), #cmd varchar(1000),#archivedate varchar(100)
set #date = (select FORMAT(getdate(), 'yyyyMMdd'));
set #archivedate = (select FORMAT(#maxXDate, 'yyyyMMdd'));
declare #backupTable varchar(100)
set #backupTable = 'cbr_audit_Backup_' + #date;
BEGIN TRY
BEGIN TRANSACTION
WHILE #CurrentLoop < #NumberOfLoops
BEGIN
IF #IsArchive = 1
BEGIN
--Archive the records into a backup table
IF OBJECT_ID (#backupTable, N'U') IS NULL
begin
set #cmd = 'SELECT * INTO [cbr_audit_Backup_'+ #date +'] FROM [cbr_audit] WITH (NOLOCK) where convert(datetime,dateCreated,101) <= CONVERT(DATETIME, ''' + #archivedate + ''', 101)'
exec(#cmd)
end
--Delete the rows from cbr_audit table
DELETE
FROM dbo.cbr_audit
WHERE id IN
(SELECT TOP(#DeleteSize) id
FROM dbo.cbr_audit WITH (NOLOCK)
WHERE dateCreated <= #maxXDate);
END
ELSE
BEGIN
-- Delete the records
DELETE
FROM dbo.cbr_audit
WHERE id IN
(SELECT TOP(#DeleteSize) id
FROM dbo.cbr_audit WITH (NOLOCK)
WHERE dateCreated <= #maxXDate);
END
-- WAITFOR DELAY '00:00:00:500';
SET #CurrentLoop = #CurrentLoop + 1;
set #loop = cast(#currentloop as varchar(50))
RAISERROR (#loop, 0, 1) WITH NOWAIT
END
COMMIT TRANSACTION
END TRY
BEGIN CATCH
--Rollback
RETURN
END CATCH
END
In This SP, in the archive part of code the dynamic sql is not giving any results. DateCreated is of Datetime type. Can someone please help me on this.
Thanks in Advance.
Seems that you are using SQL 2016 or above, try CONCAT, something like:
select concat('SELECT * INTO [cbr_audit_Backup_', #date,'] FROM [cbr_audit] WITH (NOLOCK) where convert(datetime,dateCreated,121) <= ''', CONVERT(varchar(30), #archivedate , 121), '''')
few more things:
I strongly recommend you always to use style 121 (canonical)
or 126 (ISO8601) so you'll SQL will not be confused by mm/dd/yyyy (101) or dd/mm/yyyy (103).
you are deleting by chunks, but the transaction is for all the block. consider doing only transaction for each delete (implicit)
instead insert into and then delete, have a look to OUTPUT clause to DELETE here
I need to update a table in batches, but it does not work. I tried 2 options below.
Both of the options update the first 10 rows but the update is still running. But only 10 rows remain updated.
Seems like update never finishes and count shows more than number of records in the tables to be updated.
Please advise.
-- OPTION #1
SET NOCOUNT OFF
IF OBJECT_ID('tempdb..#Table') IS NOT NULL
BEGIN
DROP TABLE #Table
END
-- select count(*) from #Table where ID = 0
-- select * from #Table
CREATE TABLE #Table ( ID INT )
WHILE (1 = 1)
AND ( Select count(*) from #Table ) < 10000
BEGIN
BEGIN TRANSACTION
INSERT INTO #Table (ID)
VALUES (1)
IF ##ROWCOUNT = 10000 -- terminating condition;
BEGIN
COMMIT TRANSACTION
BREAK
END
END
-- UPDATE
WHILE (1 = 1)
BEGIN
BEGIN TRANSACTION
UPDATE TOP (10) upd
SET ID = 0
FROM #Table upd
IF ##ROWCOUNT = 0 -- terminating condition;
BEGIN
COMMIT TRANSACTION
BREAK
END
END
-- OPTION #2
SET NOCOUNT OFF
IF OBJECT_ID('tempdb..#Table2') IS NOT NULL
BEGIN
DROP TABLE #Table2
END
-- select count(*) from #Table2 where ID = 0
-- select * from #Table2
CREATE TABLE #Table2 ( ID INT )
--DECLARE #rows INT
--DECLARE #count INT
WHILE (1 = 1)
AND ( Select count(*) from #Table2 ) < 10000
BEGIN
BEGIN TRANSACTION
INSERT INTO #Table2 (ID)
VALUES (1)
IF ##ROWCOUNT = 10000 -- terminating condition;
BEGIN
COMMIT TRANSACTION
BREAK
END
END
DECLARE #rows INT
DECLARE #count INT
-- UPDATE
SET #rows = 1
SET #count = 0
WHILE #rows > 0
BEGIN
BEGIN TRANSACTION
UPDATE TOP (10) #Table2 -- upd
SET ID = 0
-- FROM #Table upd
SET #rows = ##ROWCOUNT
SET #count = #count + #rows
RAISERROR('COUNT %d', 0, 1, #count) WITH NOWAIT
COMMIT TRANSACTION
END
OK there were a couple of issues with your code.
You can't use TOP in an update - however its fairly straight forward to restrict the rows with a sub-query as shown.
You were setting all the ID's to 1 therefore there was no way to uniquely identify a row, you could only update all of them. I have assumed that in your real life problem you would have unique ID's and I have modified the code to suit.
I'm unsure about the intention of the various nested transactions, they don't appear to accomplish much and they don't match the logic.
IF OBJECT_ID('tempdb..#Table2') IS NOT NULL
BEGIN
DROP TABLE #Table2;
END
CREATE TABLE #Table2 (ID INT);
DECLARE #Count int = 0;
WHILE (select count(*) from #Table2) < 10000 BEGIN
INSERT INTO #Table2 (ID)
VALUES (#Count)
-- Make sure we have a unique id for the test, else we can't identify 10 records
set #Count = #Count + 1;
END
-- While exists an 'un-updated' record continue
WHILE exists (select 1 from #Table2 where ID > 0) BEGIN
-- Update any top 10 'un-updated' records
UPDATE #Table2 SET
ID = 0
where id in (select top 10 id from #Table2 where ID > 0)
END
DROP TABLE #Table2
How can I get rowcount of all outputs in stored procedure?
e.g.
Create Proc Test
AS Begin
select top 10 * from sysobjects
select top 20* from sysobjects
End
##rowcount will return the outputs row count. But in my case there are 2 outputs. ##rowcount will return only last output only.
exec Test
select ##rowcount
This will return only 20. But I need 10, 20
Any way to achieve?
CREATE PROCEDURE TestRowCount
#rowcount1 INT OUTPUT,
#rowcount2 INT OUTPUT
AS
BEGIN
select top 10 * from sysobjects
SET #rowcount1 = ##ROWCOUNT
select top 20* from sysobjects
SET #rowcount2 = ##ROWCOUNT
END
Calling Procedure as follows :
DECLARE #rowcount1 INT,
#rowcount2 INT;
EXEC dbo.TestRowCount #rowcount1 = #rowcount1 OUTPUT, -- int
#rowcount2 = #rowcount2 OUTPUT -- int
Try this way:
Create Proc Test
AS Begin
select top 10 * from sysobjects
select ##rowcount
select top 20* from sysobjects
select ##rowcount
End
Then execute the query:
exec Test
Function ##ROWCOUNT returns the number of rows affected by the last statement, so one possible approach (to get the total row count from all SELECT statements) is to define an output parameter in your stored procedure:
Procedure:
CREATE PROCEDURE TestRowCount
#RowCount int OUTPUT
AS
BEGIN
SET #RowCount = 0
select top 10 * from sysobjects
SET #RowCount = #RowCount + ##ROWCOUNT
select top 20 * from sysobjects
SET #RowCount = #RowCount + ##ROWCOUNT
END
Result:
DECLARE #RC int
DECLARE #RowCount int
EXECUTE #RC = TestRowCount #RowCount OUTPUT
SELECT #RowCount
If you want to get the count of rows from each SELECT statement, use something like this:
CREATE PROCEDURE TestRowCount
AS
BEGIN
CREATE TABLE #Tmp (RC int)
select top 10 * from sysobjects
INSERT INTO #Tmp (RC) VALUES (##ROWCOUNT)
select top 20 * from sysobjects
INSERT INTO #Tmp (RC) VALUES (##ROWCOUNT)
SELECT * FROM #Tmp
DROP TABLE #Tmp
END
There is need to fetch 100 IDs from Employee table, using cursor, then to execute stored procedure and to put number of rows and execution time for each ID into one temp table. Some idea, how to count number of rows which stored procedure will catch and execution time for that ?
declare #temptable table
( ID nvarchar
, numberOfRows int
, executionTime int)
declare #id nvarchar(15)
declare db_cursor CURSOR FOR
select top 100 NationalIdNumber
from HumanResources.Employee
open db_cursor
fetch next from db_cursor into #id
while ##fetch_status = 0
begin
insert into #temptable
exec [dbo].[uspEmployeeData] #id
fetch next from db_cursor into #id
end
close db_cursor
deallocate db_cursor
I am using SQL Server 2014 Standard edition
You can do something like:
declare #tableresults TABLE (#id INT, row_count INT, durationms int)
DECLARE #starttime DATETIME
open db_cursor
fetch next from db_cursor into #id
while ##fetch_status = 0
begin
SET #starttime = GETDATE()
insert into #temptable
exec [dbo].[uspEmployeeData] #id
INSERT INTO #tableresults
VALUES (#id, ##rowcount, DATEDIFF(millisecond, #starttime, GETDATE() )
fetch next from db_cursor into #id
end
Find my first Cursor query.
Note:
Here I am using Microsoft SQL Server 2012
I take Ids from temp table. Whereas you use actual table.
And you ask temp table as a result. But I create actual table.
You must start from CountAgainstId procedure.
CREATE TABLE #Employee
(
EmpID int
, EmpName varchar (50) NOT NULL
, Salary int NOT NULL
, Address varchar (200) NOT NULL
)
GO
INSERT INTO #Employee(EmpID,EmpName,Salary,Address) VALUES(1,'Mohan',12000,'Noida')
, (1,'Mohan',12000,'Noida')
, (2,'Pavan',25000,'Delhi')
, (3,'Amit',22000,'Dehradun')
, (4,'Sonu',22000,'Noida'), (4,'Sonu',22000,'Noida')
, (5,'Deepak',28000,'Gurgaon'), (5,'Deepak',28000,'Gurgaon'), (5,'Deepak',28000,'Gurgaon')
GO
-- I inserted same rows.
SELECT * FROM #Employee
#Employee:
EmpID EmpName Salary Address
1 Mohan 12000 Noida
1 Mohan 12000 Noida
2 Pavan 25000 Delhi
3 Amit 22000 Dehradun
4 Sonu 22000 Noida
4 Sonu 22000 Noida
5 Deepak 28000 Gurgaon
5 Deepak 28000 Gurgaon
5 Deepak 28000 Gurgaon
Cursor Query:
create procedure RowCounts (
#EmpId int
, #Nos int output
, #RunTime int output
)
as
begin
declare #StartTime datetime
, #EndTime datetime
set #StartTime = (select getdate ())
set #nos = (select COUNT (*) from #employee where EmpID = #EmpId)
set #EndTime = (select GETDATE ())
/*
Do further, what do you want.
*/
set #RunTime = DATEDIFF (MILLISECOND, #StartTime, #EndTime)
end
*****************************************
create procedure CountAgainstId as
begin
declare #RowCount int
, #RunTime int
, #EmpId int
, #i int = 0
declare CountCursors cursor
static for
select empid from #employee
open CountCursors
fetch next from CountCursors into #empid
if OBJECT_ID ('dbo.Summaries') is null -- object_id() is not recognised the temp tables
begin
create table Summaries (Empid int, NoOfRow int, ExecutionTime int)
end
else
begin
truncate table Summaries
end
while ##FETCH_STATUS = 0
begin
if not exists (
select * from Summaries where empid = #EmpId -- for removing the duplicate empid's
)
begin
exec RowCounts #EmpId, #RowCount output, #RunTime output
insert into Summaries
values (#EmpId, #RowCount, #RunTime)
end
fetch next from CountCursors into #empid
end
CLOSE CountCursors
DEALLOCATE CountCursors
select * from Summaries
end
Summaries (Output Table):
Empid NoOfRow ExecutionTime
1 2 0
2 1 0
3 1 0
4 2 0
5 3 0
Let me know, what did you got.
I have a stored procedure that selects some data in a different format from my original table:
USE [VolTracker]
GO
DECLARE #return_value int,
#offset int
SET #offset = 5
WHILE #offset >= 1 BEGIN
EXEC #return_value = [dbo].[sp_getStats]
#Doffset = #offset,
#StartTime = --Some Datetime,
#EndTime = --Some later Datetime,
#Contract = NULL
SET #offset = #offset - 1
END
GO
This specific example selects all 5 of the tables that I would like it to properly. However, I would like all of these tables joined into one table. How might I go about doing this?
Create a table variable that matches the schema of the resultset returned by sp_getStats. Then, insert into this variable within your loop:
...
declare #Stage table (YourColumn1 varchar(10), YourColumn2 int, ...);
WHILE #offset >= 1 BEGIN
INSERT INTO #Stage
EXEC #return_value = [dbo].[sp_getStats]
#Doffset = #offset,
#StartTime = --Some Datetime,
#EndTime = --Some later Datetime,
#Contract = NULL
SET #offset = #offset - 1
END
select * from #Stage;
The above will work to return the union of all the resultsets returned, however if its possible for you to modify the procedure (or create a new one) that can return the complete set without a loop then I would suggest doing do.
Create a temporary table or table variable and insert into table each time you execute the stored procedure. Check this out.
//declare your #temptable here
WHILE #offset >= 1 BEGIN
INSERT INTO #tempTable
EXEC #return_value = [dbo].[sp_getStats]
#Doffset = #offset,
#StartTime = --Some Datetime,
#EndTime = --Some later Datetime,
#Contract = NULL
SET #offset = #offset - 1
END