I create a SQL procedure to import data from txt file. However there are lots of files(about 80 files) and i cannot list its name. Its name formatted like 'DATA_XXXXXXX.TXT'
DECLARE #sql1 nvarchar(max) = N'BULK INSERT dbo.Student FROM '''
+ 'D:\NEW_FOLDER\DATA_20190222'
+ '.TXT'
+ ''' WITH
(
FIELDTERMINATOR = ''|'',
MAXERRORS = 10000
);';
EXEC sys.sp_executesql #sql1;
I want all data on theses file should be loaded into table. How can I do that?
You could do it with a while loop, insert the values into a temptable and increment the file name variable with dynamic SQL:
CREATE TABLE #TEMP_FILENAMES
(
FILENAME VARCHAR(50)
)
INSERT INTO #TEMP_FILENAMES
VALUES('20190222')
INSERT INTO #TEMP_FILENAMES
VALUES('20190223')
DECLARE INT #YEARMMDD
WHILE EXISTS(SELECT * FROM #TEMP_FILENAMES)
BEGIN
SET #YEARMMDD = (SELECT TOP 1 FILENAME FROM #TEMP_FILENAMES)
DECLARE #sql1 nvarchar(max) = N'BULK INSERT dbo.Student FROM '''
+ 'D:\NEW_FOLDER\DATA_' + #YEARMMDD
+ '.TXT'
+ ''' WITH
(
FIELDTERMINATOR = ''|'',
MAXERRORS = 10000
);';
EXEC sys.sp_executesql #sql1;
DELETE FROM #TEMP_FILENAMES WHERE FILENAME = #YEARMMDD
END
Related
the table that im trying to insert into has an additional first column which is id that doesnt exist in the file. Below is my attempt trying to insert the record from the file(with header row) while inserting to the additional column but im getting repeated syntax error on the command
SELECT #fileName=[FileName],
#email=[Email],
#CustomerId=ImportBy,
#fullPath = 'D:test.txt'
FROM [dbo].[Log_Import]
WHERE INTID = #intId
SET #command = 'BULK INSERT [dbo].[ProcessTesting] SELECT ' + CAST(#intID AS VARCHAR(50)) +', * FROM ''' + #fullPath + ''' WITH ( FIELDTERMINATOR =''\t'', FIRSTROW = 1 )'
EXEC(#command)
You could use OPENROWSET(BULK to insert with a custom INSERT...SELECT statement.
I strongly suggest you use a format file.
SELECT #fileName=[FileName],
#email=[Email],
#CustomerId=ImportBy,
#fullPath = 'D:test.txt'
FROM [dbo].[Log_Import]
WHERE INTID = #intId
SET #command = '
INSERT [dbo].[ProcessTesting] (ID, OtherColumnsHere)
SELECT
' + CAST(#intID AS VARCHAR(50)) +',
OtherColumnsHere
FROM OPENROWSET(BULK ' + QUOTENAME(#fullPath, '''') + ', FORMATFILE = ''YourFormatFile'', FIRSTROW = 1 ) r;
';
EXEC sp_executesql #command;
Note use of QUOTENAME to escape the file name properly
I have been trying my darndest to get the code described below to work. I am very inexpert at MSSMS and SQL. That said, I love the efficiency of SQL databases and would really love to make this code work.
I have tested my CSV files with this code:
BULK INSERT BCPData
FROM 'D:\cheese\bcp_test.csv'
WITH (FIRSTROW = 2,
FIELDTERMINATOR = ','
,ROWTERMINATOR = '0x0a'
);
GO
They import easily and the data appears.
However, if I try to use the code shown below (I need an code that automatically imports multiple CSV files into my table) I only get "NULL" results in the columns.
My query is as follows:
exec ImportFiles 'd:\cheese\' , 'd:\cheese\Archive' , 'bcp*.csv' , 'MergeBCPData'
I run this query after using the following code to create the necessary stored procedures:
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[ImportFiles]') and `OBJECTPROPERTY(id, N'IsProcedure') = 1)`
drop procedure [dbo].[ImportFiles]
GO
create procedure ImportFiles
#FilePath varchar(1000) = 'd:\cheese\' ,
#ArchivePath varchar(1000) = 'd:\cheese\Archive\' ,
#FileNameMask varchar(1000) = 'bcp*.csv' ,
#MergeProc varchar(128) = 'MergeBCPData'
AS
set nocount on
declare #ImportDate datetime
select #ImportDate = getdate()
declare #FileName varchar(1000) ,
#File varchar(1000)
declare #cmd varchar(2000)
create table ##Import (s varchar(8000))
create table #Dir (s varchar(8000))
/*****************************************************************/
-- Import file
/*****************************************************************/
select #cmd = 'dir /B ' + #FilePath + #FileNameMask
delete #Dir
insert #Dir exec master..xp_cmdshell #cmd
delete #Dir where s is null or s like '%not found%'
while exists (select * from #Dir)
begin
select #FileName = min(s) from #Dir
select #File = #FilePath + #FileName
select #cmd = 'bulk insert'
select #cmd = #cmd + ' ##Import'
select #cmd = #cmd + ' from'
select #cmd = #cmd + ' ''' + replace(#File,'"','') + ''''
select #cmd = #cmd + ' with (FIELDTERMINATOR = '','''
select #cmd = #cmd + ',ROWTERMINATOR = ''0x0a''
)'
truncate table ##Import
-- import the data
exec (#cmd)
-- remove filename just imported
delete #Dir where s = #FileName
exec #MergeProc
-- Archive the file
select #cmd = 'move ' + #FilePath + #FileName + ' ' + #ArchivePath + #FileName
exec master..xp_cmdshell #cmd
end
drop table ##Import
drop table #Dir
go
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[MergeBCPData]') and OBJECTPROPERTY(id, N'IsProcedure') = 1)
drop procedure [dbo].[MergeBCPData]
GO
create procedure MergeBCPData
AS
set nocount on
-- insert data to production table
insert BCPData
(
City ,
Visit_Duration_Seconds ,
Timezone ,
Most_Likely_Company
)
select
SUBSTRING('City', 1, 5),
SUBSTRING('Visit_Duration_Seconds', 1, 12),
SUBSTRING('Timezone', 1, 3),
SUBSTRING('Most_Likely_Company',1, 30)
from ##Import
go
Any help would be very appreciated. I'm hopeful it is just an error that my inexperienced eyes are too novel to catch. THANK YOU!
You stated 'I need an code that automatically imports multiple CSV files into my table'. Is there a pattern in the file names that you can exploit? Do the file names have dates in them, per chance? If there is some repeating pattern that you can exploit, like a series of dates, you can loop through all the files in your folder, and append all files to one table, in one go. Check out the code below, and post back if you have questions.
DECLARE #intFlag INT
SET #intFlag = 1
WHILE (#intFlag <=50) – we are running 50 loops...change this as needed
BEGIN
PRINT #intFlag
declare #fullpath1 varchar(1000)
select #fullpath1 = '''\\your_path_here\FTP\' + convert(varchar, getdate()- #intFlag , 112) + '_Daily.csv'''
declare #cmd1 nvarchar(1000)
select #cmd1 = 'bulk insert [dbo].[Daily] from ' + #fullpath1 + ' with (FIELDTERMINATOR = ''\t'', FIRSTROW = 5, ROWTERMINATOR=''0x0a'')'
exec (#cmd1)
SET #intFlag = #intFlag + 1
END
Here are some common date formats.
http://www.sql-server-helper.com/tips/date-formats.aspx
Again, I'm assuming you have dates in your file names.
I cobbled together the code below. I can write file paths into a table but I can't use the paths in the table to bulk load CSV files in a folder. Can some expert here take a look and let me know what's wrong? TIA.
IF OBJECT_ID('tempdb..#DirectoryTree') IS NOT NULL
DROP TABLE #DirectoryTree;
CREATE TABLE #DirectoryTree (
id int IDENTITY(1,1)
,subdirectory nvarchar(512)
,depth int
,isfile bit);
INSERT #DirectoryTree (subdirectory,depth,isfile)
EXEC master.sys.xp_dirtree 'C:\my_path\CSV Files\',1,1;
SELECT * FROM #DirectoryTree
WHERE isfile = 1 AND RIGHT(subdirectory,4) = '.csv'
ORDER BY id;
GO
DROP TABLE ALLFILENAMES
--CREATE TABLE ALLFILENAMES(id VARCHAR(999),subdirectory VARCHAR(255),depth VARCHAR(1),isfile VARCHAR(1))
Select * INTO ALLFILENAMES
From #DirectoryTree
--code above is fine; problems start here
--cursor loop
--bulk insert won't take a variable name, so make a sql and execute it instead:
Declare #sql varchar(8000)
set #sql = 'BULK INSERT BULKACT FROM ''' + 'ALLFILENAMES.subdirectory' + ''' '
+ ' WITH (
DATAFILETYPE = ''char'',
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
The problem is with the Bulk Insert. Here is the error message that I get: Msg 4860, Level 16, State 1, Line 28
Cannot bulk load. The file "ALLFILENAMES.subdirectory" does not exist.
So, 'ALLFILENAMES' is the name of the table and 'subdirectory' is the name of the field that contains all paths to all CSV files.
you need to do a select from that ALLFILENAMES table. You can't just specified the tablename + column name like that and expect it to work
also you need to specify the full path in the FROM file name
and you may use temp table for ALLFILENAMES instead of permanent tble
Declare #sql varchar(max)
select #sql = isnull(#sql , '')
+ 'BULK INSERT BULKACT FROM ''C:\my_path\CSV Files\' + ALLFILENAMES.subdirectory + ''' '
+ ' WITH (
DATAFILETYPE = ''char'',
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
); ' + char(13)
from ALLFILENAMES
print #sql
and there is a WITH (FORMAT = 'CSV'); option for bulk insert from CSV file
Thanks for the help, Sqiurrel. I got the code below to work, and add some comments. It's really ugly though. I guess SQL Server really isn't designed for these kinds of things...
----------------------------------------------------------
-- Create 5 tables and all fields in tables
DECLARE #intFlag INT
SET #intFlag = 1
WHILE (#intFlag <=5)
BEGIN
declare #cmd nvarchar(1000),
#MyTableName nvarchar(100)
print str(#intFlag)
set #MyTableName = 'CSV' + replace(str(#intFlag),' ','')
print #MyTableName
set #cmd = 'CREATE TABLE dbo.' + quotename(#MyTableName, '[') + '(Name varchar(255), Address varchar(255), Age varchar(255), Work varchar(255));';
print #cmd
exec(#cmd)
SET #intFlag = #intFlag + 1
END
GO
----------------------------------------------------------
IF OBJECT_ID('tempdb..#DirectoryTree') IS NOT NULL
DROP TABLE #DirectoryTree;
CREATE TABLE #DirectoryTree (
id int IDENTITY(1,1)
,subdirectory nvarchar(512)
,depth int
,isfile bit);
INSERT #DirectoryTree (subdirectory,depth,isfile)
EXEC master.sys.xp_dirtree 'C:\your_path_here\',1,1;
SELECT * FROM #DirectoryTree
WHERE isfile = 1 AND RIGHT(subdirectory,4) = '.csv'
ORDER BY id;
GO
DROP TABLE ALLFILENAMES
Select * INTO ALLFILENAMES
From #DirectoryTree
----------------------------------------------------------
--cursor loop
--bulk insert won't take a variable name, so make a sql and execute it instead:
Declare #sql varchar(max)
select #sql = isnull(#sql , '')
+ 'BULK INSERT ' + ALLFILENAMES.subdirectory + ' FROM ''C:\your_path_here\' + ALLFILENAMES.subdirectory + ''' '
+ ' WITH (
DATAFILETYPE = ''char'',
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
); ' + char(13)
from ALLFILENAMES
print #sql
exec (#sql)
----------------------------------------
i am trying to bulk insert into Db using sql server 2005
Below is the code.
declare #path varchar(500)
set #path = 'E:\Support\test.csv';
Create table #mytable( name varchar(max), class varchar(max), roll varchar(max) )
BULK INSERT #mytable FROM #path <-- Error line
WITH ( FIELDTERMINATOR = ',', ROWTERMINATOR = '\n' );
Go
select * from #mytable
drop table #mytable
Problem: issue is that my file path is dynamic and comes from a variable instead of hard coding which is not working
If i change the error line to below it works
BULK INSERT #mytable FROM 'E:\Support\test.csv';
Please advise how to fix this
Try to use Dynamic SQL:
declare #sql varchar(max)
set #sql = 'BULK INSERT #mytable FROM ''' + #path + ''' WITH ...
exec (#sql)
DECLARE #path varchar(50) = 'D:\ARQUIVOS_CARGAS\CABOS\FILE.prn'
DECLARE #SQL_BULK VARCHAR(MAX)
SET #SQL_BULK = 'BULK INSERT #TAB FROM ''' + #path + ''' WITH
(
CODEPAGE = ''ACP'',
FIRSTROW = 1,
FIELDTERMINATOR = ''tab'',
ROWTERMINATOR = ''0x0a'',
KEEPNULLS
)'
EXEC (#SQL_BULK)
I am writing a stored procedure which is supposed to take data from a csv file and insert into a table. My problem is that the number of columns in the csv file are not fixed(ie number of columns is variable). So I need some way to create a temporary table with exactly the same number of columns as in the csv file. So that I can use bulk insert.
Well I tried solving the issue by writing a sp which will take the csv file path as parameter and create a table names as temptable with the same format as that of the number of columns in the csv. CSV file looks like
eid,ename,esalary,etemp
1,Mark,1000,
2,Peter,1000,
Stored Proc script
create proc createtable
#path nvarchar(50)
as
begin
declare #execSQL nvarchar(1000)
declare #tempstr varchar(1000)
declare #col varchar(1000)
declare #table nvarchar(1000)
-- Create a temp table to with one column to hold the first row of the csv file
CREATE TABLE #tbl (line VARCHAR(1000))
SET #execSQL =
'BULK INSERT #tbl
FROM ''' + #path + '''
WITH (
FIELDTERMINATOR =''\n'',
FIRSTROW = 1,
ROWTERMINATOR = ''\n'',
LASTROW = 1
)
'
EXEC sp_executesql #stmt=#execSQL
SET #col = ''
SET #tempstr = (SELECT TOP 1 RTRIM(REPLACE(Line, CHAR(9), ',')) FROM #tbl)
DROP TABLE #tbl
WHILE CHARINDEX(',',#tempstr) > 0
BEGIN
SET #col=#col + LTRIM(RTRIM(SUBSTRING(#tempstr, 1, CHARINDEX(',',#tempstr)-1))) + ' varchar(100),'
SET #tempstr = SUBSTRING(#tempstr, CHARINDEX(',',#tempstr)+1, len(#tempstr))
END
SET #col = #col + #tempstr + ' varchar(100)'
if object_id('temptable') is not null
drop table temptable
SET #table = 'create table temptable (' + #col + ')'
EXEC sp_executesql #stmt=#table
-- Load data from csv
SET #execSQL =
'BULK INSERT temptable
FROM ''' + #path + '''
WITH (
FIELDTERMINATOR ='','',
FIRSTROW = 2,
ROWTERMINATOR = ''\n''
)
'
EXEC sp_executesql #stmt=#execSQL
end
improved nadeems script... A little bit more robust.
This code is excelent for loading multiple CSV files without using the default wizzards.
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE proc [dbo].[importeer_csv_as_table]
#path nvarchar(255),
#new_table_name varchar(255),
#field_terminator varchar(255),
#row_terminator varchar(255)
as
begin
declare #execsql nvarchar(max)
declare #tempstr varchar(max)
declare #col varchar(max)
declare #table nvarchar(max)
declare #drop_table varchar(max)
-- Create a temp table to with one column to hold the first row of the csv file
create table #tbl (line varchar(1000))
set #execsql =
'bulk insert #tbl
from ''' + #path + '''
with (
fieldterminator =''' + #row_terminator + ''',
firstrow = 1,
rowterminator = ''' + #row_terminator + ''',
lastrow = 1
)
'
exec sp_executesql #stmt=#execsql
--replace field terminator with comma
update #tbl set line = replace(line, #field_terminator, ',')
set #col = ''
set #tempstr = (select top 1 rtrim(replace(line, char(9), ',')) from #tbl)
drop table #tbl
while charindex(',',#tempstr) > 0
begin
set #col=#col + '[' + ltrim(rtrim(substring(#tempstr, 1, charindex(',',#tempstr)-1))) + '] varchar(max),'
set #tempstr = substring(#tempstr, charindex(',',#tempstr)+1, len(#tempstr))
end
set #col = #col + '[' + #tempstr + '] varchar(max)'
if object_id(#new_table_name) is not null
begin
set #drop_table = 'drop table [' + #new_table_name + ']'
exec sp_executesql #stmt= #drop_table
end
set #table = 'create table [' + #new_table_name + '] (' + #col + ')'
--select #table
exec sp_executesql #stmt=#table
--Load data from csvle
set #execsql =
'bulk insert [' + #new_table_name + ']
from ''' + #path + '''
with (
fieldterminator =''' + #field_terminator + ''',
firstrow = 2,
rowterminator = ''' + #row_terminator + '''
)
'
exec sp_executesql #stmt=#execsql
end
GO
You could use Powershell to process the CSV file, there is an example here which you could probably adapt to take account of the variable number of fields. You can build the SQL to create a table and then issue a bulk load.