How to effectively split grouped records into batches - sql

Question

For each group in table I need to split that group into specific amount of records (batches) and mark each record in batch with according batch id.
Right now, my implementation based on cursors is IMHO clumsy. It takes 1 minute to split set of 10 000 rows which is, needless to say, very slow. Any clues how to make that work faster?
Here is test script.
-- Needed to generate big data
DECLARE #Naturals TABLE (ID INT)
INSERT INTO #Naturals (ID)
VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)
DECLARE #TestData TABLE
(
LINK INT,
F_House INT,
F_Batch UNIQUEIDENTIFIER
)
INSERT INTO #TestData (LINK, F_House)
SELECT ROW_NUMBER() OVER (order by T1.ID), ROW_NUMBER() OVER (order by T1.ID) % 5
FROM
#Naturals T1
CROSS JOIN #Naturals T2
CROSS JOIN #Naturals T3
CROSS JOIN #Naturals T4
--CROSS JOIN #Naturals T5 -- that would give us 100 000
-- Finished preparing Data (10 000 rows)
SELECT 'Processing:', COUNT(*) FROM #TestData
DECLARE #batchSize INT -- That would be amount of rows in each batch
SET #batchSize = 50
IF OBJECT_ID('tempdb..#G') IS NOT NULL -- Split set of data into groups. We need to create batches in each group.
DROP TABLE #G
SELECT
buf.F_House, COUNT(*) AS GroupCount
INTO #G
FROM #TestData buf
GROUP BY buf.F_House -- That logic could be tricky one. Right now simplifying
DECLARE #F_House INT -- That would be group key
DECLARE db_cursor CURSOR FOR
SELECT F_House
FROM #G
ORDER BY F_House
OPEN db_cursor FETCH NEXT FROM db_cursor INTO #F_House
WHILE ##FETCH_STATUS = 0
BEGIN
PRINT 'Processing house group: ' + CAST(#F_House AS VARCHAR(10))
-- For each group let's create batches
WHILE EXISTS (SELECT 1 FROM #TestData AS itmds
WHERE itmds.F_House = #F_House
AND itmds.F_Batch IS NULL
)
BEGIN
DECLARE #batchLink UNIQUEIDENTIFIER
SET #batchLink = NEWID()
UPDATE itmds
SET itmds.F_Batch = #batchLink
FROM #TestData AS itmds
WHERE itmds.F_House = #F_House
AND itmds.F_Batch IS NULL
AND itmds.LINK IN
(
SELECT TOP (#batchSize)
sub.LINK
FROM #TestData sub
WHERE sub.F_House = #F_House
AND sub.F_Batch IS NULL
)
END
FETCH NEXT FROM db_cursor INTO #F_House
END
CLOSE db_cursor
DEALLOCATE db_cursor
SELECT
buf.F_House, COUNT(distinct F_Batch) AS BatchCountInHouse
FROM #TestData buf
GROUP BY buf.F_House
ORDER BY buf.F_House
Expected output (considering batchsize = 50)
10 000 rows / 5 houses = 2000 rows/house
2000 rows/house / 50(batchSize) = 40 batches/house

Answer 1

This is set based avoiding a cursor. The assigned F_Batch is a BIGINT:
;with baseRowNum as
(
SELECT LINK, F_House,
-- row number per F_House
Row_Number() Over (PARTITION BY F_House ORDER BY LINK) AS rn
FROM #TestData
)
SELECT *,
-- combine F_House & group number into a unique result
F_House * 10000 +
-- start a new sub group for every F_House or after #batchSize rows
Sum(CASE WHEN rn % #batchSize = 1 THEN 1 ELSE 0 END)
Over (ORDER BY F_House, rn
ROWS Unbounded Preceding) AS F_Batch
FROM baseRowNum
If you really need a UNIQUEINDENTIFIER you can join back:
;with baseRowNums as
(
SELECT LINK, F_House,
-- row number per F_House
Row_Number() Over (PARTITION BY F_House ORDER BY LINK) AS rn
FROM #TestData
)
,batchNums as
(
SELECT *,
-- combine F_House & group number into a unique result
F_House * 10000 +
-- start a new sub group for every F_House or after #batchSize rows
Sum(CASE WHEN rn % #batchSize = 1 THEN 1 ELSE 0 END)
Over (ORDER BY F_House, rn
ROWS Unbounded Preceding) AS F_Batch
FROM baseRowNums
)
,GUIDs as
(
select F_Batch, MAX(newid()) as GUID
from batchNums
group by F_Batch
)
-- select * from
--from batchNums join GUIDs
-- on batchNums.F_Batch = GUIDs.F_Batch
select F_House, GUID, count(*)
from batchNums join GUIDs
on batchNums.F_Batch = GUIDs.F_Batch
group by F_House, GUID
order by F_House, count(*) desc
See Fiddle.

Answer 2

I would use an inner looping inside of a looping referencing a grouping level.
Then you can iterate through from the grouping down into the BatchGrouping. However as you pointed out speed is an issue with table variables and CTE's for that reason I in this case tested with a tempdb # table. This way I could index after the insert and optimize performance. I can run a million rows of aggregation logic in about 16 seconds. I consider that acceptable performance. But my Dev Box is an I7 6700, with 16 gigs of DDR4, and an SSD. Performance times may vary based on hardware obviously.
--Make up some fake data for example
DECLARE
#Start INT = 1
, #End INT = 100000
;
SET NOCOUNT ON;
IF OBJECT_ID('tempdb..#Temp') IS NOT NULL
DROP TABLE tempdb..#Temp
CREATE Table #Temp (Id INT, Grp int, Val VARCHAR(8), BatchGroup int)
WHILE #Start <= #End
BEGIN
INSERT INTO #Temp (Id, Grp, Val)
VALUES (#Start, CAST(RAND() * 8 AS INT) + 1, LEFT(NEWID(), 8))
SELECT #Start += 1;
END
CREATE CLUSTERED INDEX IX_Temp_Grp ON #Temp(Grp, BatchGroup)
--Determine Batch Size You want for groupings
DECLARE #BatchSize INT = 1000;
--Let's randomly mess with groupings
DECLARE #X INT = 1
WHILE #X <= 4
BEGIN
; WITH x AS
(
SELECT TOP (#BatchSize * 4)
Id
, Grp
, Val
FROM #Temp
WHERE Grp = CAST(RAND() * 8 AS INT) + 1
)
UPDATE x
SET Grp = CAST(RAND() * 8 AS INT) + 1
SELECT #X += 1
END
DECLARE
#CurrentGroup INT = 1
, #CurrentBatch INT = 1
WHILE #CurrentGroup <= (SELECT MAX(Grp) FROM #Temp) -- Exists (SELECT 1 FROM #Temp WHERE BatchGroup IS NULL)
BEGIN
WHILE EXISTS (SELECT 1 FROM #Temp WHERE Grp = #CurrentGroup AND BatchGroup IS NULL)
BEGIN
; WITH x AS
(
SELECT TOP (#BatchSize) *
FROM #Temp
WHERE Grp = #CurrentGroup
AND BatchGroup IS NULL
)
update x
SET BatchGroup = #CurrentBatch
SELECT #CurrentBatch += 1;
END
SELECT #CurrentBatch = 1
SELECT #CurrentGroup += 1;
END
--Proof
Select
Grp
, COUNT(DISTINCT Id)
, COUNT(DISTINCT BatchGroup)
From #Temp
GROUP BY Grp

Answer 3

Actually, I've tried NTILE() with cursors and it's quite fast(I mean its faster then 1 minute for 10 000 rows).
10 000 rows for 0 seconds.
100 000 rows for 3 seconds.
1 000 000 rows for 34 seconds.
10 000 000 rows for 6 minutes
Linear grow in complexity which is nice.
-- Needed to generate big data
DECLARE #Naturals TABLE (ID INT)
INSERT INTO #Naturals (ID)
VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10)
DECLARE #TestData TABLE
(
LINK INT,
F_House INT,
F_Batch UNIQUEIDENTIFIER
)
INSERT INTO #TestData (LINK, F_House)
SELECT ROW_NUMBER() OVER (order by T1.ID), ROW_NUMBER() OVER (order by T1.ID) % 5
FROM
#Naturals T1
CROSS JOIN #Naturals T2
CROSS JOIN #Naturals T3
CROSS JOIN #Naturals T4
--CROSS JOIN #Naturals T5 -- that would give us 100 000
-- Finished preparing Data (10 000 rows)
SELECT 'Processing:', COUNT(*) FROM #TestData
DECLARE #batchSize INT -- That would be amount of rows in each batch
SET #batchSize = 50
IF OBJECT_ID('tempdb..#G') IS NOT NULL -- Split set of data into groups. We need to create batches in each group.
DROP TABLE #G
SELECT
buf.F_House, COUNT(*) AS GroupCount
INTO #G
FROM #TestData buf
GROUP BY buf.F_House -- That logic could be tricky one. Right now simplifying
DECLARE #F_House INT -- That would be group key
DECLARE db_cursor CURSOR FOR
SELECT F_House
FROM #G
ORDER BY F_House
OPEN db_cursor FETCH NEXT FROM db_cursor INTO #F_House
WHILE ##FETCH_STATUS = 0
BEGIN
PRINT 'Processing house group: ' + CAST(#F_House AS VARCHAR(10))
DECLARE #rowsInGroup INT
SELECT #rowsInGroup = COUNT(*) FROM #TestData
WHERE F_House = #F_House
IF OBJECT_ID('tempdb..#TileBatch') IS NOT NULL
DROP TABLE #TileBatch
SELECT
T.[NTile], NEWID() AS F_Batch
INTO #TileBatch
FROM
(
SELECT distinct
NTILE(#rowsInGroup / #batchSize) OVER (ORDER BY LINK) AS [NTile]
from
#TestData
WHERE F_House = #F_House
) T
UPDATE D
SET D.F_Batch = B.F_Batch
FROM
#TestData D
INNER JOIN
(
SELECT
*, NTILE(#rowsInGroup / #batchSize) OVER (ORDER BY LINK) AS [NTile]
from
#TestData
WHERE F_House = #F_House
) DT ON D.LINK = DT.LINK
INNER JOIN
#TileBatch B ON DT.[NTile] = B.[NTile]
WHERE D.F_House = #F_House
FETCH NEXT FROM db_cursor INTO #F_House
END
CLOSE db_cursor
DEALLOCATE db_cursor
SELECT
buf.F_House, COUNT(distinct F_Batch) AS BatchCountInHouse
FROM #TestData buf
GROUP BY buf.F_House
ORDER BY buf.F_House