Avoid inserting duplicates when working with composite primary key - sql

Question

I'm writing a query in SQL that transfers data from an Acces DB to a new SQL server management DB.
This transfer is a one time only, so I do not have to worry about the code being super generic.
I'm trying to insert data from a couple of tables (including tables from the Acces DB), but get an error about duplicates on the composite primary key.
My problem is that I can't see why my uses of distinct does not prevent those duplicates.
The table I want to insert into is constructed as so:
The code I'm trying to execute is written as followed:
-- Insert into ComponentSupplier for [Supplier 1]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 1 order no], a.[PDF Data 1 sheet link], a.[Supplier 1 LF], '1'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 1]
-- Insert into ComponentSupplier for [Supplier 2]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 2 order no], a.[PDF Data 2 sheet link], a.[Supplier 2 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 2]
-- Insert into ComponentSupplier for [Supplier 3]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 3 order no], a.[PDF Data 3 sheet link], a.[Supplier 3 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 3]
-- Insert into ComponentSupplier for [Supplier 4]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 4 order no], a.[PDF Data 4 sheet link], a.[Supplier 4 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 4]
-- Insert into ComponentSupplier for [Supplier 5]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 5 order no], a.[PDF Data 5 sheet link], a.[Supplier 5 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 5]
-- Insert into ComponentSupplier for [Supplier 6]
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 6 order no], a.[PDF Data 6 sheet link], a.[Supplier 6 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 6]

Answer 1

The distinct de-duplicates based on all the columns in the select.
The issue could be because you are inserting record using multiple insert..selects, you could try something like this to ensure records are de-duplicated correctly.
The distinct will de-dup withing each select, and the union will de-dup across the select statements:
insert into CDB2020.dbo.ComponentSupplier
(
Supplier_ID,
BK_ID,
Part_ID,
Datasheet,
LF,
Preferred
)
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 1 order no], a.[PDF Data 1 sheet link], a.[Supplier 1 LF], '1'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 1]
union
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 2 order no], a.[PDF Data 2 sheet link], a.[Supplier 2 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 2]
union
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 3 order no], a.[PDF Data 3 sheet link], a.[Supplier 3 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 3]
union
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 4 order no], a.[PDF Data 4 sheet link], a.[Supplier 4 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 4]
union
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 5 order no], a.[PDF Data 5 sheet link], a.[Supplier 5 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 5]
union
select distinct s.Supplier_ID, c.BK_ID, a.[Supplier 6 order no], a.[PDF Data 6 sheet link], a.[Supplier 6 LF], '0'
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 6]
You can use a query like this to find out duplicate records (group by all fields in your composite key):
select s.Supplier_ID, c.BK_ID, count(*)
from AccessDataMigration_1.dbo.[B-K Data] as a
inner join CDB2020.dbo.Components as c on c.BK_ID = a.[B-K no]
inner join CDB2020.dbo.Suppliers as s on s.Name = a.[Supplier 6]
group by s.Supplier_ID, c.BK_ID
having count(*) > 1