Delete Duplicate Records with Same Values - sql

Question

I have a TSQL statement that is taking several hours to run. I'm sure I need to look into the import process to avoid duplicates being inserted but for the time being I'd just like to remove all records except one with duplicate values. ParameterValueId is the primary key on the table but I have many duplicate entries that need to be deleted. I only need one record for each ParameterId, SiteId, MeasurementDateTime, and ParameterValue. Below is my current method for deleting duplicate records. It finds all values that have a count > 1. It then finds the first Id with those values and deletes all of the records with those values that don't match the first ID found by those values. Besides the print statements is there a more efficient way of doing this. Can I do a way with the cursor at all to improve performance?
BEGIN TRANSACTION
SET NOCOUNT ON
DECLARE #BeginningRecordCount INT
SET #BeginningRecordCount =
(
SELECT COUNT(*)
FROM ParameterValues
)
DECLARE #ParameterId UNIQUEIDENTIFIER
DECLARE #SiteId UNIQUEIDENTIFIER
DECLARE #MeasurementDateTime DATETIME
DECLARE #ParameterValue FLOAT
DECLARE CDuplicateValues CURSOR FOR
SELECT
[ParameterId]
,[SiteId]
,[MeasurementDateTime]
,[ParameterValue]
FROM [ParameterValues]
GROUP BY
[ParameterId]
,[SiteId]
,[MeasurementDateTime]
,[ParameterValue]
HAVING COUNT(*) > 1
OPEN CDuplicateValues
FETCH NEXT FROM CDuplicateValues INTO
#ParameterId
,#SiteId
,#MeasurementDateTime
,#ParameterValue
DECLARE #FirstParameterValueId UNIQUEIDENTIFIER
DECLARE #DuplicateRecordsDeleting INT
WHILE ##FETCH_STATUS <> -1
BEGIN
SET #FirstParameterValueId =
(
SELECT TOP 1 ParameterValueId
FROM ParameterValues
WHERE
ParameterId = #ParameterId
AND SiteId = #SiteId
AND MeasurementDateTime = #MeasurementDateTime
AND ParameterValue = #ParameterValue
)
SET #DuplicateRecordsDeleting =
(
SELECT COUNT(*)
FROM ParameterValues
WHERE
ParameterId = #ParameterId
AND SiteId = #SiteId
AND MeasurementDateTime = #MeasurementDateTime
AND ParameterValue = #ParameterValue
AND ParameterValueId <> #FirstParameterValueId
)
PRINT 'DELETING ' + CAST(#DuplicateRecordsDeleting AS NVARCHAR(50))
+ ' records with values ParameterId : ' + CAST(#ParameterId AS NVARCHAR(50))
+ ', SiteId : ' + CAST (#SiteId AS NVARCHAR(50))
+ ', MeasurementDateTime : ' + CAST(#MeasurementDateTime AS NVARCHAR(50))
+ ', ParameterValue : ' + CAST(#ParameterValue AS NVARCHAR(50))
DELETE FROM ParameterValues
WHERE
ParameterId = #ParameterId
AND SiteId = #SiteId
AND MeasurementDateTime = #MeasurementDateTime
AND ParameterValue = #ParameterValue
AND ParameterValueId <> #FirstParameterValueId
FETCH NEXT FROM CDuplicateValues INTO
#ParameterId
,#SiteId
,#MeasurementDateTime
,#ParameterValue
END
CLOSE CDuplicateValues
DEALLOCATE CDuplicateValues
DECLARE #EndingRecordCount INT
SET #EndingRecordCount =
(
SELECT COUNT(*)
FROM ParameterValues
)
PRINT 'Beginning Record Count : ' + CAST(#BeginningRecordCount AS NVARCHAR(50))
PRINT 'Ending Record Count : ' + CAST(#EndingRecordCount AS NVARCHAR(50))
PRINT 'Total Records Deleted : ' + CAST((#BeginningRecordCount - #EndingRecordCount) AS NVARCHAR(50))
SET NOCOUNT OFF
PRINT 'RUN THE COMMIT OR ROLLBACK STATEMENT AFTER VERIFYING DATA.'
--COMMIT
--ROLLBACK

Answer 1

Use option with CTE and OVER clause. OUTPUT.. INTO clause saves the information from rows affected by an DELETE statement into #delParameterValues table. Further, in the body of procedure, you can use this table to print the affected rows.
DECLARE #delParameterValues TABLE
(
ParameterId UNIQUEIDENTIFIER,
SiteId UNIQUEIDENTIFIER,
MeasurementDateTime DATETIME,
ParameterValue FLOAT,
DeletedRecordCount int
)
;WITH cte AS
(
SELECT *, ROW_NUMBER() OVER (PARTITION BY [ParameterId],[SiteId],[MeasurementDateTime],[ParameterValue] ORDER BY 1/0) AS rn,
COUNT(*) OVER (PARTITION BY [ParameterId],[SiteId],[MeasurementDateTime],[ParameterValue]) AS cnt
FROM [ParameterValues]
)
DELETE cte
OUTPUT DELETED.[ParameterId],
DELETED.[SiteId],
DELETED.[MeasurementDateTime],
DELETED.[ParameterValue],
DELETED.cnt INTO #delParameterValues
WHERE rn != 1
SELECT DISTINCT *
FROM #delParameterValues
Demo on SQLFiddle

Answer 2

you can do it in a single sql:
DELETE p FROM ParameterValues p
LEFT JOIN
(SELECT ParameterId, SiteId, MeasurementDateTime, ParameterValue, MAX(ParameterValueId) AS MAX_PARAM
FROM ParameterValues
GROUP BY ParameterId, SiteId, MeasurementDateTime, ParameterValue
) m
ON m.ParameterId = p.ParameterId
AND m.SiteId = p.SiteId
AND m.MeasurementDateTime = p.MeasurementDateTime
AND m.ParameterValue = p.ParameterValue
AND m.MAX_PARAM = p.ParameterValueId
WHERE m.ParameterId IS NULL
Of course it will not print the output, but you can still print the rows before and after