I have a sql query that exports each XML row of a table to individual XML files. The process works perfectly, except it stops and does not process all rows at once. I have to change the row number to start where it left off. It will process all the rows with no errors using this procedure, however I would prefer that it process all rows at once.
This is my query:
DECLARE
#FILENAME VARCHAR(500),
#bcpcmd VARCHAR(2000),
#RN VARCHAR(10),
#i int;
DECLARE #Table table (RN int, IsDone char(1))
INSERT #Table SELECT RN, 0 FROM [x_rpt].[dbo].[x_abc] WHERE RN>=1 --this is where I update RN to process records where it leaves off--
SET #i=0
WHILE #i<= (SELECT COUNT(*) FROM #Table WHERE IsDone = '0')
BEGIN
SELECT TOP 1 #RN=RN FROM #Table WHERE IsDone = 0
SET #FILENAME = '"C:\temp\data\abc\Jan_2016_'+#RN+'.xml"'
SET #bcpcmd = 'BCP "SELECT [XML] from [x_rpt].[dbo].[x_abc] WHERE RN='+#RN+'" queryout "'
SET #bcpcmd = #bcpcmd + #FILENAME + '" -w -T -S "SERVER"'
EXEC master..xp_cmdshell #bcpcmd
UPDATE #Table set IsDone='1' where RN=convert(int,#RN)
SET #i=#i+1
END
Try to use a CURSOR based approach:
CREATE TABLE SomeDB.dbo.MockUp(RN INT,[XML] XML);
INSERT INTO SomeDB.dbo.MockUp VALUES
(1,N'<test RN="1"/>')
,(17,N'<test RN="17"/>')
,(-300,N'<test RN="-300"/>');
DECLARE
#FILENAME VARCHAR(500),
#bcpcmd VARCHAR(2000),
#rn INT;
DECLARE c CURSOR FOR SELECT RN FROM SomeDB.dbo.MockUp
OPEN c;
FETCH NEXT FROM c INTO #rn;
WHILE ##FETCH_STATUS = 0
BEGIN
SET #FILENAME = '"C:\SomePath\Jan_2016_'+CAST(#rn AS VARCHAR(MAX))+'.xml"'
SET #bcpcmd = 'BCP "SELECT [XML] from SomeDB.[dbo].MockUp WHERE RN='+CAST(#rn AS VARCHAR(MAX))+'" queryout "'
SET #bcpcmd = #bcpcmd + #FILENAME + '" -w -T -S ' + ##SERVERNAME
EXEC master..xp_cmdshell #bcpcmd
FETCH NEXT FROM c INTO #rn;
END
CLOSE c;
DEALLOCATE c;
GO
DROP TABLE SomeDB.dbo.MockUp
Adjust path and database / schema / table / column names...
Related
I have been trying my darndest to get the code described below to work. I am very inexpert at MSSMS and SQL. That said, I love the efficiency of SQL databases and would really love to make this code work.
I have tested my CSV files with this code:
BULK INSERT BCPData
FROM 'D:\cheese\bcp_test.csv'
WITH (FIRSTROW = 2,
FIELDTERMINATOR = ','
,ROWTERMINATOR = '0x0a'
);
GO
They import easily and the data appears.
However, if I try to use the code shown below (I need an code that automatically imports multiple CSV files into my table) I only get "NULL" results in the columns.
My query is as follows:
exec ImportFiles 'd:\cheese\' , 'd:\cheese\Archive' , 'bcp*.csv' , 'MergeBCPData'
I run this query after using the following code to create the necessary stored procedures:
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[ImportFiles]') and `OBJECTPROPERTY(id, N'IsProcedure') = 1)`
drop procedure [dbo].[ImportFiles]
GO
create procedure ImportFiles
#FilePath varchar(1000) = 'd:\cheese\' ,
#ArchivePath varchar(1000) = 'd:\cheese\Archive\' ,
#FileNameMask varchar(1000) = 'bcp*.csv' ,
#MergeProc varchar(128) = 'MergeBCPData'
AS
set nocount on
declare #ImportDate datetime
select #ImportDate = getdate()
declare #FileName varchar(1000) ,
#File varchar(1000)
declare #cmd varchar(2000)
create table ##Import (s varchar(8000))
create table #Dir (s varchar(8000))
/*****************************************************************/
-- Import file
/*****************************************************************/
select #cmd = 'dir /B ' + #FilePath + #FileNameMask
delete #Dir
insert #Dir exec master..xp_cmdshell #cmd
delete #Dir where s is null or s like '%not found%'
while exists (select * from #Dir)
begin
select #FileName = min(s) from #Dir
select #File = #FilePath + #FileName
select #cmd = 'bulk insert'
select #cmd = #cmd + ' ##Import'
select #cmd = #cmd + ' from'
select #cmd = #cmd + ' ''' + replace(#File,'"','') + ''''
select #cmd = #cmd + ' with (FIELDTERMINATOR = '','''
select #cmd = #cmd + ',ROWTERMINATOR = ''0x0a''
)'
truncate table ##Import
-- import the data
exec (#cmd)
-- remove filename just imported
delete #Dir where s = #FileName
exec #MergeProc
-- Archive the file
select #cmd = 'move ' + #FilePath + #FileName + ' ' + #ArchivePath + #FileName
exec master..xp_cmdshell #cmd
end
drop table ##Import
drop table #Dir
go
if exists (select * from dbo.sysobjects where id = object_id(N'[dbo].[MergeBCPData]') and OBJECTPROPERTY(id, N'IsProcedure') = 1)
drop procedure [dbo].[MergeBCPData]
GO
create procedure MergeBCPData
AS
set nocount on
-- insert data to production table
insert BCPData
(
City ,
Visit_Duration_Seconds ,
Timezone ,
Most_Likely_Company
)
select
SUBSTRING('City', 1, 5),
SUBSTRING('Visit_Duration_Seconds', 1, 12),
SUBSTRING('Timezone', 1, 3),
SUBSTRING('Most_Likely_Company',1, 30)
from ##Import
go
Any help would be very appreciated. I'm hopeful it is just an error that my inexperienced eyes are too novel to catch. THANK YOU!
You stated 'I need an code that automatically imports multiple CSV files into my table'. Is there a pattern in the file names that you can exploit? Do the file names have dates in them, per chance? If there is some repeating pattern that you can exploit, like a series of dates, you can loop through all the files in your folder, and append all files to one table, in one go. Check out the code below, and post back if you have questions.
DECLARE #intFlag INT
SET #intFlag = 1
WHILE (#intFlag <=50) – we are running 50 loops...change this as needed
BEGIN
PRINT #intFlag
declare #fullpath1 varchar(1000)
select #fullpath1 = '''\\your_path_here\FTP\' + convert(varchar, getdate()- #intFlag , 112) + '_Daily.csv'''
declare #cmd1 nvarchar(1000)
select #cmd1 = 'bulk insert [dbo].[Daily] from ' + #fullpath1 + ' with (FIELDTERMINATOR = ''\t'', FIRSTROW = 5, ROWTERMINATOR=''0x0a'')'
exec (#cmd1)
SET #intFlag = #intFlag + 1
END
Here are some common date formats.
http://www.sql-server-helper.com/tips/date-formats.aspx
Again, I'm assuming you have dates in your file names.
SQL Output file contains nulls but I'm using isnull in my select statement
Here is my code. I am still getting NULLs in my output file for columns Modifer1 and Modifier2.
Select
A.SvcProvSvcId as 'CDMCode',
A.BillName,
A.[CPT/HCPCSCdVal],
isnull(A.[Modifier 1],'') as Modifier1,
isnull(A.[Modifer 2],'') as Modifier2,
A.RevCdVal,
A.Price
into ##temp_CDM
From ##temp_CDMAllPrices A
inner join
(select max(isnull(LastRateChange,getdate())) Last_Price_Chng,ExtrnId,SvcProvSvcId from ##temp_CDMAllPrices group by ExtrnId,SvcProvSvcId) B
on A.ExtrnId = B.ExtrnId
AND A.SvcProvSvcId = B.SvcProvSvcId
AND isnull(A.LastRateChange,getdate()) = B.Last_Price_Chng
DECLARE #FileLocation varchar(255)
DECLARE #OutputFile varchar(255)
DECLARE #FileName varchar(255)
DECLARE #strCommand varchar(8000)
SET #FileLocation = '\\server\LoadFiles\'
SET #OutputFile = 'ExperianCDM'
SET #FileName = #FileLocation + LTRIM(RTRIM(#OutputFile)) + '.txt'
SET #strCommand = 'bcp "SELECT * from [server].[Database].[##temp_CDM]" queryout ' + #filename +' -t "|" -c -T'
RAISERROR(#FileName, 0, 1) WITH NOWAIT
PRINT #strCommand
EXECUTE master.dbo.xp_cmdShell #strCommand
Drop Table ##temp_CDMAllPrices, ##temp_CDM
I have a folder called "Dump." This folder consists of various .CSV Files.
The folder Location is 'C:\Dump'
I want to Import the contents of these files into SQL Server.
I want the rough code along with proper comments so that I understand it.
I have tried a few codes that I found on the Net. But they haven't quite worked out for me for some strange reason.
The steps I would like to have are
Step 1: Copy all the File Names in the folder to a Table
Step 2: Iterate through the table and copy the data from the files using Bulk Insert.
Someone do please help me out on this one. Thanks a lot in advance :)
--BULK INSERT MULTIPLE FILES From a Folder
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Dump\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
--Extras
--delete from ALLFILENAMES where WHICHFILE is NULL
--select * from ALLFILENAMES
--drop table ALLFILENAMES
This will give you separate tables for each file.
--BULK INSERT MULTIPLE FILES From a Folder
drop table allfilenames
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'D:\Benihana\backup_csv_benihana_20191128032207_part_1\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
delete from ALLFILENAMES where WHICHFILE is null
--SELECT replace(whichfile,'.csv',''),* FROM dbo.ALLFILENAMES
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%' order by WHICHFILE desc
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql =
'select * into '+ Replace(#filename, '.csv','')+'
from openrowset(''MSDASQL''
,''Driver={Microsoft Access Text Driver (*.txt, *.csv)}''
,''select * from '+#Path+#filename+''')'
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
For Step 1 Maybe you can look at:
http://www.sql-server-performance.com/forum/threads/copying-filenames-to-sql-table.11546/
or
How to list files inside a folder with SQL Server
and then Step 2
How to cast variables in T-SQL for bulk insert?
HTH
You might need to enable the xp_cmdshell first:
sp_configure 'show advanced options', '1'
RECONFIGURE
go
sp_configure 'xp_cmdshell', '1'
RECONFIGURE
go
And, to enable ad_hoc,
sp_configure 'show advanced options', 1;
RECONFIGURE;
GO
sp_configure 'Ad Hoc Distributed Queries', 1;
RECONFIGURE;
GO
To solve step 1, xp_dirtree can also be used to list all files and folders.
Keep in mind that it is an undocumented function. Security precautions must be considered. Intentionally crafted filenames could be an intrusion vector.
In python you can use d6tstack which makes this simple
import d6tstack
import glob
c = d6tstack.combine_csv.CombinerCSV(glob.glob('*.csv'))
c.to_mssql_combine('mssql+pymssql://usr:pwd#localhost/db', 'tablename')
See SQL examples. It also deals with data schema changes, creates table and allows you to preprocess data. It leverages BULK INSERT so should be just as fast.
to expand upon the answer by SarangArd you can replace temp with the following if your file name matches your table name.
' + Left(#filename, Len(#filename)-4) + '
This code will create a new table per CSV file that is imported.
Best to populate empty database from CSV files.
CREATE TABLE ALLFILENAMES
(
WHICHPATH VARCHAR(255)
,WHICHFILE VARCHAR(255)
)
DECLARE #filename VARCHAR(255),
#path VARCHAR(255),
#sql VARCHAR(8000),
#cmd VARCHAR(1000)
SET #path = 'L:\DATA\SOURCE\CSV\' --PATH TO YOUR CSV FILES (CHANGE TO YOUR PATH)
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES
SET WHICHPATH = #path
WHERE WHICHPATH IS NULL
DECLARE c1 CURSOR
FOR SELECT WHICHPATH
,WHICHFILE
FROM ALLFILENAMES
WHERE WHICHFILE LIKE '%.csv%'
OPEN c1
FETCH NEXT FROM c1 INTO #path,
#filename
WHILE ##fetch_status <> -1
BEGIN
CREATE TABLE #Header
(
HeadString NVARCHAR(MAX)
)
DECLARE #Columns NVARCHAR(MAX) = ''
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #QUERY2 NVARCHAR(MAX) = ''
DECLARE #HeaderQuery NVARCHAR(MAX) = ''
SELECT #HeaderQuery = #HeaderQuery + 'bulk insert #Header from ''' + #path + #filename + '''
with(firstrow=1,lastrow=1)'
EXEC (#HeaderQuery)
SELECT #Columns = (SELECT QUOTENAME(value) + ' nvarchar(max)' + ','
FROM #Header
CROSS APPLY STRING_SPLIT(HeadString,',') FOR xml PATH(''))
IF ISNULL(#Columns,'') <> ''
BEGIN
SET #Columns = LEFT(#Columns,LEN(#Columns) - 1)
SELECT #Query = #Query + 'CREATE TABLE ' + Replace(#filename,'.csv','') + ' (' + replace(#Columns,'"','') + ')'
PRINT #Query
EXEC (#QUERY)
END
SELECT #QUERY2 = #QUERY2 + 'bulk insert ' + replace(Replace(#filename,'.csv',''),'.TPS','') + ' from ''' + #path + #filename + '''
with(firstrow=2,FORMAT=''csv'',FIELDTERMINATOR='','',ROWTERMINATOR=''\n'')'
EXEC (#QUERY2)
DROP TABLE #Header
FETCH NEXT FROM c1 INTO #path,
#filename
END
CLOSE c1
DEALLOCATE c1
I have a business requirement to take a table with columns PrimaryKey, A, B, C, D, E and dump each to a file such as:
filename: Primarykey.txt
(row 1) A
(row 2) B
(row 3) C
etc.
Is there a good way to do this with SQL Server 2008 or should I write a C# program using a datatable? The table I am using has about 200k rows in it.
Thanks
The links below contain some previous posts and another link to a possible solution using SSIS.
You might have to play around until you get what you want.
Good luck.
Some clues here
or
SQL Server Forums - Create multiple files from SSIS
Using xp_cmdshell
If xp_cmdshell is permitted, you might be able to use something like this:
declare #path varchar(200)
declare #schema varchar(100)
declare #pk varchar(100)
declare #sql varchar(2000)
declare #cmd varchar(2500)
set #path = 'C:\your\file\path\'
declare rowz cursor for (select PrimaryKey from TableA)
open rowz
fetch next from rowz into #pk
while ##FETCH_STATUS = 0
begin
set #sql = 'select A, B, C, D, E from TableA where PrimaryKey = ''' + #pk + ''''
set #cmd = 'bcp "' + #sql + '" queryout "' + #path + #pk + '.txt" -T -c -t\n'
exec xp_cmdshell #cmd
fetch next from rowz into #item
end
close rowz
deallocate rowz
Using sqlcmd
Alternately, you can create a series of statements to run through sqlcmd to create the separate files.
First, create a temporary table and add some boilerplate entries:
create table #temp (tkey int, things varchar(max))
insert into #temp (tkey, things) values (0, 'SET NOCOUNT ON;
GO'),(2, ':out stdout')
Then, fill the temporary table with the queries that we need:
declare #dir varchar(250)
declare #sql varchar(5000)
declare #cmd varchar(5500)
declare #schema varchar(100)
declare #pk varchar(100)
set #schema = 'YourSchema'
declare rowz cursor for (select PrimaryKey from "YourSchema"..TableA)
open rowz
fetch next from rowz into #pk
while ##FETCH_STATUS = 0
begin
set #dir = '"C:\your\file\path\'+#pk+'.txt"'
set #sql = '
SELECT A +''
''+ B +''
''+ C +''
''+ D +''
''+ E
FROM "'+#schema+'"..TableA where PrimaryKey = '+#pk
set #cmd = ':out '+#dir+#sql+'
GO'
insert into #temp (tkey,things) values (1, #cmd)
fetch next from rowz into #pk
end
close rowz
deallocate rowz
Next, query the temporary table to get the generated queries:
select things from #temp order by tkey
Finally, copy the results to a file and send this file as input to sqlcmd.exe at the command prompt with the parameter -h -1:
C:\your\file\path>sqlcmd -h -1 -S YourServer -i "script.sql"
I am trying to export a fairly large number of image files, stored internally in an SQL database as binary data.
Being fairly new to writing stored procedures in SQL, I have come across a couple of very useful guides on how this can be archived, but I seem to be missing something.
I am running SQL Server 2008 R2 locally, and I am trying to write the files to a folder on my C:\ drive.
Here is the buisness part of what I have so far:
BEGIN
DECLARE #cmd VARCHAR(8000)
DECLARE #result int
DECLARE curExportBinaryDocs CURSOR FAST_FORWARD FOR
SELECT 'BCP "SELECT Photograph_Data FROM [ALBSCH Trial].[dbo].[Photograph] WHERE Photograph_ID = '
+ CAST(Photograph_ID AS VARCHAR(500)) + '" queryout "' + #OutputFilePath
+ CAST(Photograph_ID AS VARCHAR(500)) + '.jpg"' + ' -n -T'
FROM dbo.Photograph
OPEN curExportBinaryDocs
FETCH NEXT FROM curExportBinaryDocs INTO #cmd
WHILE ##FETCH_STATUS = 0
BEGIN
--PRINT #cmd
EXEC #result = xp_cmdshell #cmd
FETCH NEXT FROM curExportBinaryDocs INTO #cmd
END
CLOSE curExportBinaryDocs
DEALLOCATE curExportBinaryDocs
END
'#result' is always being set to '1' (failed) after the xp_cmdshell call. All the table names/fields are correct, so I suspect there is something wrong with my BCP call, but I am not sure what to try next.
Any help or advice would be very welcome.
Well, first of all.. (and sorry about that ;) ) DON"T USE CURSORS..
and sorry for the caps...
One of the most baddest things about cursors are that they can lock your table. What i always do for these purposes (and which is quite faster), i use a for loop.. like this
declare #totrow int
, #currow int
, #result int
, #nsql nvarchar(max)
declare #sqlStatements table (
Id int identity(1, 1)
, SqlStatement varchar(max)
)
insert
into #sqlStatements
select 'QUERY PART'
from table
set #totrow = ##rowcount
set #currow = 1
while #totrow > 0 and #currow <= #totrow
begin
select #nsql = SqlStatement
from #SqlStatements
where Id = #currow
exec #result = xp_cmdshell #nsql
set #currow = #currow + 1
end
For the next part, does the SQL Server process has enough permission to write to the c: drive? Also, look into your message pane when you execute your code, maybe you can find something there?
What you also can do, try to execute it manually. Just get one BCP statement and execute it with the xp_cmdshell. Does it gives any errors?
Here is my final working procedure and format file. I was not able to find the finer details of
BCP commands, permision settings and format file layouts in one place, so maybe this will be of use to someone.
CREATE PROCEDURE [dbo].[ImgExport]
#OutputFilePath VARCHAR(500) = 'C:\SQLTest\ '
AS
BEGIN
DECLARE #totrow int
DECLARE #currow int
DECLARE #result int
DECLARE #nsql nvarchar(4000)
DECLARE #sqlStatements table (ID int IDENTITY(1, 1), SqlStatement varchar(max))
INSERT
INTO #sqlStatements
SELECT 'BCP "SELECT Photograph_Data FROM [ALBSCH_Trial].[dbo].[Photograph] WHERE Photograph_ID = '''
+ CAST(Photograph_ID AS VARCHAR(500)) + '''" queryout ' + #OutputFilePath
+ CAST(Photograph_ID AS VARCHAR(500)) + '.jpg -S localhost\SQLEXPRESS2008 -T -f C:\SQLTest\Images.fmt'
FROM dbo.Photograph
SET #totrow = ##ROWCOUNT
SET #currow = 1
WHILE #totrow > 0 and #currow <= #totrow
BEGIN
SELECT #nsql = SqlStatement
FROM #sqlStatements
WHERE ID = #currow
EXEC #result = xp_cmdshell #nsql
SET #currow = #currow + 1
END
END
Format file:
9.0
1
1 SQLBINARY 0 0 "\t" 1 Photograph_Data ""
I hope that helps somebody.