I can read data from txt-based file in batch, but I need to import it into temp separated by columns.
DECLARE #SQL VARCHAR(8000)
IF OBJECT_ID('TempDB..#BulkInsert') IS NOT NULL
BEGIN
DROP TABLE #BulkInsert
END
CREATE TABLE #BulkInsert
(
Line VARCHAR(MAX)
)
SET #SQL = 'BULK INSERT #BulkInsert FROM ''C:\fff.txt'' WITH (ROWTERMINATOR = ''\n'' , FIELDTERMINATOR = ''\t'')'
EXEC (#SQL)
SELECT * FROM #BulkInsert
Related
I have been trying my darndest to get the code described below to work. I am very inexpert at MSSMS and SQL. That said, I love the efficiency of SQL databases and would really love to make this code work.
I have tested my CSV files with this code:
BULK INSERT BCPData
FROM 'D:\cheese\bcp_test.csv'
WITH (FIRSTROW = 2,
FIELDTERMINATOR = ','
,ROWTERMINATOR = '0x0a'
);
GO
They import easily and the data appears.
However, if I try to use the code shown below (I need an code that automatically imports multiple CSV files into my table) I only get "NULL" results in the columns.
My query is as follows:
exec ImportFiles 'd:\cheese\' , 'd:\cheese\Archive' , 'bcp*.csv' , 'MergeBCPData'
I run this query after using the following code to create the necessary stored procedures:
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[ImportFiles]') and `OBJECTPROPERTY(id, N'IsProcedure') = 1)`
drop procedure [dbo].[ImportFiles]
GO
create procedure ImportFiles
#FilePath varchar(1000) = 'd:\cheese\' ,
#ArchivePath varchar(1000) = 'd:\cheese\Archive\' ,
#FileNameMask varchar(1000) = 'bcp*.csv' ,
#MergeProc varchar(128) = 'MergeBCPData'
AS
set nocount on
declare #ImportDate datetime
select #ImportDate = getdate()
declare #FileName varchar(1000) ,
#File varchar(1000)
declare #cmd varchar(2000)
create table ##Import (s varchar(8000))
create table #Dir (s varchar(8000))
/*****************************************************************/
-- Import file
/*****************************************************************/
select #cmd = 'dir /B ' + #FilePath + #FileNameMask
delete #Dir
insert #Dir exec master..xp_cmdshell #cmd
delete #Dir where s is null or s like '%not found%'
while exists (select * from #Dir)
begin
select #FileName = min(s) from #Dir
select #File = #FilePath + #FileName
select #cmd = 'bulk insert'
select #cmd = #cmd + ' ##Import'
select #cmd = #cmd + ' from'
select #cmd = #cmd + ' ''' + replace(#File,'"','') + ''''
select #cmd = #cmd + ' with (FIELDTERMINATOR = '','''
select #cmd = #cmd + ',ROWTERMINATOR = ''0x0a''
)'
truncate table ##Import
-- import the data
exec (#cmd)
-- remove filename just imported
delete #Dir where s = #FileName
exec #MergeProc
-- Archive the file
select #cmd = 'move ' + #FilePath + #FileName + ' ' + #ArchivePath + #FileName
exec master..xp_cmdshell #cmd
end
drop table ##Import
drop table #Dir
go
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[MergeBCPData]') and OBJECTPROPERTY(id, N'IsProcedure') = 1)
drop procedure [dbo].[MergeBCPData]
GO
create procedure MergeBCPData
AS
set nocount on
-- insert data to production table
insert BCPData
(
City ,
Visit_Duration_Seconds ,
Timezone ,
Most_Likely_Company
)
select
SUBSTRING('City', 1, 5),
SUBSTRING('Visit_Duration_Seconds', 1, 12),
SUBSTRING('Timezone', 1, 3),
SUBSTRING('Most_Likely_Company',1, 30)
from ##Import
go
Any help would be very appreciated. I'm hopeful it is just an error that my inexperienced eyes are too novel to catch. THANK YOU!
You stated 'I need an code that automatically imports multiple CSV files into my table'. Is there a pattern in the file names that you can exploit? Do the file names have dates in them, per chance? If there is some repeating pattern that you can exploit, like a series of dates, you can loop through all the files in your folder, and append all files to one table, in one go. Check out the code below, and post back if you have questions.
DECLARE #intFlag INT
SET #intFlag = 1
WHILE (#intFlag <=50) – we are running 50 loops...change this as needed
BEGIN
PRINT #intFlag
declare #fullpath1 varchar(1000)
select #fullpath1 = '''\\your_path_here\FTP\' + convert(varchar, getdate()- #intFlag , 112) + '_Daily.csv'''
declare #cmd1 nvarchar(1000)
select #cmd1 = 'bulk insert [dbo].[Daily] from ' + #fullpath1 + ' with (FIELDTERMINATOR = ''\t'', FIRSTROW = 5, ROWTERMINATOR=''0x0a'')'
exec (#cmd1)
SET #intFlag = #intFlag + 1
END
Here are some common date formats.
http://www.sql-server-helper.com/tips/date-formats.aspx
Again, I'm assuming you have dates in your file names.
I'm trying run BULK INSERT in a loop. Loop through each file in some directory ends with no of particular file. Below is my solution
DECLARE #startFlag INT
DECLARE #endFlag INT
DECLARE #fileName varchar(50)
SET #startFlag = 1
SET #endFlag = 10
WHILE (#startFlag <= #endFlag)
BEGIN
SET #fileName = 'c:\path to file\filename_' + cast(#startFlag as varchar) + '.csv'
BULK
INSERT dbo.Intraday
FROM #fileName
WITH
(
FIELDTERMINATOR = '|',
ROWTERMINATOR = '\n'
)
SET #startFlag = #startFlag + 1
END
GO
but seems don't work. Is there anything I've overlooked or another missing stuff I can fix this issue?
You can't use variables or expressions all the places you might like in TSQL. You'll have to use dynamic SQL:
declare #fileName nvarchar(2000) = 'foo.csv'
SET #fileName = 'foo'
declare #sql nvarchar(max) = N'
BULK
INSERT dbo.Intraday
FROM '''+#fileName+'''
WITH
(
FIELDTERMINATOR = ''|'',
ROWTERMINATOR = ''\n''
)';
exec (#sql);
you can not use veritable name after From. you have to provide the name of file after from clause not variable. so you need to make complete bulk insert statement dynamically. please refer below sample code -
declare #sql nvarchar(max)
DECLARE #fileName varchar(50)
set #fileName ='C:\Input.txt'
set #sql = 'BULK
INSERT dbo.Intraday
FROM ''' + #fileName + '''
WITH
(
FIELDTERMINATOR = ''|'',
ROWTERMINATOR = ''\n''
)'
exec(#sql)
I'm using dynamic SQL for bulk insert with a parameter (Bulk insert using stored procedure).
DECLARE #sql NVARCHAR(4000) = 'BULK INSERT TblValues FROM ''' + #FileName + ''' WITH ( FIELDTERMINATOR ='','', ROWTERMINATOR =''\n'' )';
EXEC(#sql);
But... How to avoid SQL injection?
You could use QUOTENAME to surround the file name in single quotes:
DECLARE #sql NVARCHAR(4000) = 'BULK INSERT TblValues FROM ' + QUOTENAME(#FileName,'''') + ' WITH ( FIELDTERMINATOR ='','', ROWTERMINATOR =''\n'' )';
EXEC (#sql);
One way would be to retrieve the file name versus pass it in... something like
DECLARE #fileLocation VARCHAR(128) = '\\some\folder\location'
IF OBJECT_ID('tempdb..#FileNames') IS NOT NULL DROP TABLE #FileNames
CREATE TABLE #FileNames(
id int IDENTITY(1,1)
,subdirectory nvarchar(512)
,depth int
,isfile bit)
INSERT #FileNames(subdirectory,depth,isfile)
EXEC xp_dirtree #fileLocation, 1, 1
Then, in #FileNames will be all the files in that directory (where isfile = 1 of course). Then you can simply query the file name(s) from the temp table.
I want to insert multiple csv file to sqlserver table. I have a procedure for doing that. It works but my all files have same column name. So instead of importing all records it just imports column name which is the first row. If I manually delete first row then it imports the other records, I have thousands of files so I cant work manually. I am posting store procedure here. Please tell me if I can change something to make this work.
ALTER procedure [dbo].[usp_ImportMultipleFiles] #filepath varchar(500),
#pattern varchar(100), #TableName varchar(128)
as
set quoted_identifier off
declare #query varchar(1000)
declare #max1 int
declare #count1 int
Declare #filename varchar(100)
set #count1 =0
create table #x (name varchar(200))
set #query ='master.dbo.xp_cmdshell "dir '+#filepath+#pattern +' /b"'
insert #x exec (#query)
delete from #x where name is NULL
select identity(int,1,1) as ID, name into #y from #x
drop table #x
set #max1 = (select max(ID) from #y)
--print #max1
--print #count1
While #count1 <= #max1
begin
set #count1=#count1+1
set #filename = (select name from #y where [id] = #count1)
set #Query ='BULK INSERT '+ #Tablename + ' FROM '''+ #Filepath+#Filename+'''
WITH ( FIELDTERMINATOR = '','',ROWTERMINATOR = ''\n'')'
--print #query
exec (#query)
insert into logtable (query) select #query
end
drop table #y
You can use the First Row option in your bulk insert statement something like...
BULK INSERT Table_Name
FROM 'C:\FilePath'
WITH
(
FIELDTERMINATOR = ','
,ROWTERMINATOR = '\n'
,FIRSTROW = 2 --<-- This option here
)
Edit to your proc
DECLARE #Query NVARCHAR(MAX);
SET #Query = N'BULK INSERT '+ #Tablename +
N' FROM '''+ #Filepath+#Filename +
N''' WITH ( FIELDTERMINATOR = '',''
,ROWTERMINATOR = ''\n''
,FIRSTROW = 2
)'
i am trying to bulk insert into Db using sql server 2005
Below is the code.
declare #path varchar(500)
set #path = 'E:\Support\test.csv';
Create table #mytable( name varchar(max), class varchar(max), roll varchar(max) )
BULK INSERT #mytable FROM #path <-- Error line
WITH ( FIELDTERMINATOR = ',', ROWTERMINATOR = '\n' );
Go
select * from #mytable
drop table #mytable
Problem: issue is that my file path is dynamic and comes from a variable instead of hard coding which is not working
If i change the error line to below it works
BULK INSERT #mytable FROM 'E:\Support\test.csv';
Please advise how to fix this
Try to use Dynamic SQL:
declare #sql varchar(max)
set #sql = 'BULK INSERT #mytable FROM ''' + #path + ''' WITH ...
exec (#sql)
DECLARE #path varchar(50) = 'D:\ARQUIVOS_CARGAS\CABOS\FILE.prn'
DECLARE #SQL_BULK VARCHAR(MAX)
SET #SQL_BULK = 'BULK INSERT #TAB FROM ''' + #path + ''' WITH
(
CODEPAGE = ''ACP'',
FIRSTROW = 1,
FIELDTERMINATOR = ''tab'',
ROWTERMINATOR = ''0x0a'',
KEEPNULLS
)'
EXEC (#SQL_BULK)