How to use SQL MERGE to remove duplicates, and update data? - sql

Question

I am doing some house-keeping on duplicate data. I have different tables like Recipes, Ingredients, and RecipeIngredients.
In the Ingredients table, users have previously added multiple ingredients with the same name/title, e.g., "chicken" will appear many times instead of just one. I want to remove the duplicates but still keep a reference to the recipe.
I am trying to use SQL MERGE but it is deleting the wrong data, and I have starred myself blind on it. What am I doing wrong / it's probably just some quick fix??
When I run the code below, I get this table relation:
Chicken Recipe
Chicken
Burger Recipe
Salt, Pepper, Patty
But what I really want is:
Chicken Recipe
Chicken, Salt
Burger Recipe
Salt, Pepper, Patty
The MERGE statement deletes the "Salt" from RecipeIngredient instead of removing the duplicate. What am I doing wrong?
-- create table structure
CREATE TABLE #Recipes (
Id int,
Title nvarchar(50)
)
CREATE TABLE #Ingredients (
Id int,
Title nvarchar(50)
)
CREATE TABLE #RecipeIngredients (
Id int,
Recipe_id int,
Ingredient_id int
)
-- load data
INSERT INTO #Recipes (Id,Title) VALUES (1,'Chicken Recipe');
INSERT INTO #Recipes (Id,Title) VALUES (2,'Burger Recipe');
INSERT INTO #Ingredients (Id,Title) VALUES (1,'Chicken');
INSERT INTO #Ingredients (Id,Title) VALUES (2,'Chicken'); -- duplicate ingredient
INSERT INTO #Ingredients (Id,Title) VALUES (3,'Salt');
INSERT INTO #Ingredients (Id,Title) VALUES (4,'Pepper');
INSERT INTO #Ingredients (Id,Title) VALUES (5,'Patty');
INSERT INTO #RecipeIngredients (Id,Recipe_id,Ingredient_id) VALUES (1,1,2); -- chicken has chicken
INSERT INTO #RecipeIngredients (Id,Recipe_id,Ingredient_id) VALUES (2,1,3); -- chicken has salt
INSERT INTO #RecipeIngredients (Id,Recipe_id,Ingredient_id) VALUES (3,2,3); -- burger has salt
INSERT INTO #RecipeIngredients (Id,Recipe_id,Ingredient_id) VALUES (4,2,4); -- burger has pepper
INSERT INTO #RecipeIngredients (Id,Recipe_id,Ingredient_id) VALUES (5,2,5); -- burger has patty
-- try to clean up
MERGE #RecipeIngredients
USING
(
SELECT MAX(id) as MyId
FROM #Ingredients
GROUP BY Title
) NewIngredients ON #RecipeIngredients.Id = NewIngredients.MyId
WHEN MATCHED THEN
UPDATE SET #RecipeIngredients.Ingredient_id = NewIngredients.MyId
WHEN NOT MATCHED BY SOURCE THEN DELETE;
GO
-- delete duplicate ingredients, i.e., that no longer has a value in table #RecipeIngredients
DELETE FROM #Ingredients WHERE Id NOT IN (SELECT Ingredient_Id FROM #RecipeIngredients)
-- clean up
DROP TABLE #Recipes
DROP TABLE #Ingredients
DROP TABLE #RecipeIngredients

Answer 1

You should first update all the duplicate IDs to the single ID and then do the cleanup.
I've changed the winning ID determination from MAX to MIN since it will leave and have the same value in case there was an insert inbetween (hopefully you increment IDs as identity). Or you may use SNAPSHOT isolation level to enforce the max (or SERIALIZABLE to stop producing more new duplicates during this transaction). Also the cleanup of #Ingredients table should not use in filter, because as of design it is ok to have unused ingredients and users do not want to lose their data. So I've deleted the duplicates in the same way with MIN(id).
This is updated MERGE statement which sets Ingredient_ID to the single value
MERGE #RecipeIngredients as t
USING
(
SELECT id, min(id) over(partition by title) as MaxId
FROM #Ingredients
) as NewIngredients
ON t.Ingredient_Id = NewIngredients.Id
WHEN MATCHED THEN
UPDATE SET t.Ingredient_id = NewIngredients.MaxId;
Then I remove the duplicates from the #RecipeIngredients:
/*Cleanup duplicates from RecipeIngredients*/
delete t from (
select
row_number() over(
partition by
Recipe_id,
Ingredient_id
order by
id asc
) as rn
from #RecipeIngredients
) as t
where rn > 1
And finally cleanup the deduplicated records in #Ingredients table:
delete t from (
select
id,
min(id) over(partition by title) as minid
from #Ingredients
) as t
where id <> minid
And all the code in the db<>fiddle is here
UPD
I've added the more robust way to do a cleanup:
remove duplicates from #Ingredients table first
capture the deleted records
then update Ingredient_Id for that deleted IDs in the #RecipeIngredients table, purging the duplicates in it (that can be created after unification, I do not know if it is the case) with MERGE statement.
Here's the new code and db<>fiddle for it. Also I've added anothed duplicated ingredient in #Ingredients table and another ingredient with different Ingredient_Id in #RecipeIngredients table (to show the MERGE's deletion part).
/*Declare the table for unified ingredients*/
declare #deletedIngredients table (
id int,
unifiedId int
);
/*Cleanup of duplicate ingredients and
catch the deleted records with the corresponding unified Id
*/
with i_del as (
/*Leave only the first (by ID) record with the same name*/
select id, min(id) over(partition by title) as unifiedId
from #Ingredients
)
delete from i
/*Catch the deleted records and corresponding unified Ids*/
output
deleted.id,
i_del.unifiedId
into #deletedIngredients
from #Ingredients as i
join i_del
on i.id = i_del.id
/*Remove only duplicates where Id is not equal to the master record Id*/
where i.id <> i_del.unifiedId
;
/*Then do an update of IDs on the RecipeIngredients
and delete the duplicates from it (that can be created during the unification of Ingredients_Id)
*/
merge into #RecipeIngredients as t
using (
select
ri.id,
i.unifiedid,
/*Number the rows per Receipe_Id and new Ingredient_Id*/
row_number() over(
partition by
ri.Recipe_Id,
i.unifiedId
order by ri.id asc
) as rn
from #RecipeIngredients as ri
join #deletedIngredients as i
on ri.Ingredient_Id = i.id
) as s
on t.id = s.id
/*The first record should have the new unified id*/
when matched and s.rn = 1 then
update set ingredient_id = s.unifiedId
/*And unintentionally created duplicate should be removed*/
when matched and s.rn > 1 then delete
;
commit;