I have this code and executes with no issues but i dont see the tables or files that are supposed to be in the 'Names' Database here is the queries
And this is the data im trying to insert in bulk
also here is the code:
CREATE PROCEDURE import_txt_files12
AS
BEGIN
DECLARE #file varchar(255)
DECLARE #path varchar(255)
DECLARE #sql varchar(8000)
DECLARE #tablename varchar(255)
SET #path = 'C:\Users\marti\OneDrive\Desktop\Data Analist course\Datasets\names\'
DECLARE file_cursor CURSOR FOR
SELECT physical_name FROM sys.master_files
WHERE [name] LIKE 'yob%' AND physical_name LIKE #path + '%'
OPEN file_cursor
FETCH NEXT FROM file_cursor INTO #file
WHILE ##FETCH_STATUS = 0
BEGIN
-- Extract table name from file name
SET #tablename = SUBSTRING(#file, LEN(#path), LEN(#file) - LEN(#path) - 4)
SET #sql = 'BULK INSERT [Names].[dbo].[' + #tablename + '] FROM ''' + #file + ''' WITH (FORMAT = ''txt'', FIRSTROW = 1, FIELDTERMINATOR = '','', ROWTERMINATOR = ''\n'')'
EXEC (#sql)
FETCH NEXT FROM file_cursor INTO #file
END
CLOSE file_cursor
DEALLOCATE file_cursor
END
Related
I am trying to insert data from txt files to existing tables dynamically. There is no error shown, it just doesn't stop running. I am not sure what the mistake I am doing in the while loop is.
IF OBJECT_ID('TEMPDB..#FILES') IS NOT NULL DROP TABLE #FILES
CREATE TABLE #FILES
(
FileName VARCHAR(MAX),
DEPTH VARCHAR(MAX),
[FILE] VARCHAR(MAX)
)
INSERT INTO #FILES
EXEC master.dbo.xp_DirTree '\\Server1.newdevlab.local\SQLBackups_2012\Internal\Nimisha\',1,1
Select * from #FILES
DECLARE #FILENAME VARCHAR(MAX),#SQL VARCHAR(MAX)
DECLARE #TABLENAME Varchar(12)
SET NOCOUNT ON;
WHILE EXISTS (SELECT 1 FROM #FILES)
BEGIN
BEGIN TRY
SET #FILENAME = (SELECT TOP 1 FileName FROM #FILES)
SET #TABLENAME = 'Buddy.' + right(#TABLENAME, len(#TABLENAME)-6)
SET #SQL = N'BULK INSERT ' + #TABLENAME + '' +
N' FROM ''\\Server1.newdevlab.local\SQLBackups_2012\Internal\Nimisha\' + #FILENAME +'''
WITH (FIRSTROW = 2, FIELDTERMINATOR = ''\t'', ROWTERMINATOR = ''\n'');'
PRINT #SQL
EXEC(#SQL)
DELETE FROM #FILES WHERE FileName = #FILENAME
END TRY
BEGIN CATCH
PRINT 'Failed processing : ' + #FILENAME
END CATCH
END
USE xxxx
IF OBJECT_ID('TEMPDB..#FILES') IS NOT NULL DROP TABLE #FILES
CREATE TABLE #FILES
(
FileName VARCHAR(100),
DEPTH VARCHAR(100),
[FILE] VARCHAR(MAX)
)
INSERT INTO #FILES
EXEC master.dbo.xp_DirTree '\\server.newdevlab.local\SQLBackups_2012\Internal\xxx\',1,1
Select * from #FILES
DECLARE #FILENAME VARCHAR(MAX) ,#SQLString VARCHAR(MAX)
DECLARE #TABLENAME Varchar(50)
DECLARE file_cursor CURSOR
FOR
SELECT FileName FROM #FILES
OPEN file_cursor
FETCH NEXT
FROM file_cursor
INTO #FileName
WHILE ##FETCH_STATUS = 0
BEGIN
BEGIN TRY
SET #TABLENAME = SUBSTRING(('Buddy.' + substring(#FILENAME,7, len(#FILENAME)-3)), 1 ,LEN(('Buddy.' + substring(#FILENAME,7, len(#FILENAME)-3)))-4)
SET #SQLString = N'BULK INSERT ' + #TABLENAME + '' +
N' FROM ''\\server.newdevlab.local\SQLBackups_2012\Internal\xxx\' + #FILENAME +'''
WITH (FIRSTROW = 2, FIELDTERMINATOR = ''\t'', ROWTERMINATOR = ''\n'');'
SELECT #SQLString
EXEC(#SQLString)
END TRY
BEGIN CATCH
SELECT ERROR_NUMBER(),ERROR_SEVERITY(),ERROR_STATE(),ERROR_PROCEDURE(),ERROR_LINE(),ERROR_MESSAGE(),USER,GETDATE()
END CATCH
FETCH NEXT
FROM file_cursor
INTO #FileName
END
CLOSE file_cursor
DEALLOCATE file_cursor
If your intention is to loop until you process all the records in the table, then I think you go with
get the count of records before the loop and loop until your counter exceeds the total number of records. I know I am not able to resolve your issue but giving a workaround. Hope this is helpful to you.
I'm trying to update all table based on my value. This is a script Task in my SSIS package.
But I get the error:
Multiple-step OLE DB operation generated errors. Check each OLE DB status value, if available. No work was done.
Code looks like this:
DECLARE #Column_name varchar(MAX)
DECLARE #Column_Datatype varchar(max)
SET #COLUMN_NAME='site_id'
SET #COLUMN_DATATYPE='varchar(10)'
------------------------------------------------Code---------------------------------------------------
GO
--Declare Variables
DECLARE #TableName VARCHAR(100)
DECLARE #TableSchema VARCHAR(100)
DECLARE #COLUMN_NAME VARCHAR(50)
DECLARE #COLUMN_NAME_UPDATE VARCHAR(50)
SET #COLUMN_NAME_UPDATE = ?
SET #COLUMN_NAME='site_id'
DECLARE #COLUMN_DATATYPE VARCHAR(50)
SET #COLUMN_DATATYPE='varchar(max)'
--Declare Cursor
DECLARE CUR CURSOR FOR
SELECT TABLE_SCHEMA,
TABLE_NAME
FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_TYPE = 'BASE TABLE'
--OPEN CURSOR
OPEN CUR
--Fetch First Row
FETCH NEXT FROM CUR INTO #TableSchema,#TableName
--Loop
WHILE ##FETCH_STATUS = 0
BEGIN
DECLARE #SQL NVARCHAR(MAX)
SET #SQL= ( SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME=#TableName AND COLUMN_NAME=#COLUMN_NAME
and Table_Schema=#TableSchema)
BEGIN
SET #SQL='UPDATE '+ #TableSchema+'.'+ '[' + #TableName + ']' + ' SET '+#COLUMN_NAME + '='
+ '''' + #COLUMN_NAME_UPDATE + '''' + 'WHERE site_id IS NULL '
PRINT #SQL
EXEC ( #SQL)
END
FETCH NEXT FROM CUR INTO #TableSchema,#TableName
END
--Close and Deallocate Cursor
CLOSE CUR
DEALLOCATE CUR
I use a variable like input.
What's wrong?
I'm trying to loop through a fields defined in a query to an update statement.
I have the following SQL:
Declare #SQL varchar(max)
#SQL= 'Select [a],[b],[c],[d],[e]....[z]
From Table1;'
I want to be able to loop through all the fields [a]-[z] and update via the following statement:
Update Table 1
Set [a] = Case when [a] = 'Not at all' Then 0
when [a] = 'Very Much' Then 10 End
Field names are not actually [a]..[z]; I can't run the the update statement on the whole table, only a specific set of field names.
Struggling to write it programatically in SQL Server.
Declare #SQL varchar(max)
Declare #name varchar(100)
DECLARE #getid CURSOR
Set #getid = cursor for
SELECT name
FROM
sys.dm_exec_describe_first_result_set('Select [a],[b],[c],[d],[e]....[z]
From Table1', NULL, 0)
Open #getid
FETCH NEXT
FROM #getid INTO #name
WHILE ##FETCH_STATUS = 0
BEGIN
SET #SQL = 'Update Table1
Set ' + #name + ' = Case when ' + #name +'= ''Very Much'' Then ''10''
when ' + #name + ' = ''Not at all'' Then ''0''
Else ' + #name + ' End'
Exec(#SQL)
FETCH NEXT
FROM #getid INTO #name
END
CLOSE #getid
DEALLOCATE #getid
Basically dm_exec_describe_first_result_set is grabbing the fieldnames and outputting it as a recordset. Then we are just passing the the each of the records to #name and use it form our update statement and then executing it for each record passed.
Hope this helps someone else! Curious to see if there is a better way.
I think if you want to make it a little more generic I would do something like the following code. This will allow you to not have to write the specific query for every table you want to do this to and you could potentially filter out columns you do not want in the future.
To be clear, I borrowed the SQL to do the actual UPDATE from #Dale-K post and just made it pretty.
DECLARE #strSQL NVARCHAR(1000)
DECLARE #strTable NVARCHAR(100)
DECLARE #strColName VARCHAR(100)
SET #strTable = N'Table1'
CREATE TABLE #COLUMNS(ColName varchar(100))
SET #strSQL = ' select COLUMN_NAME
from INFORMATION_SCHEMA.COLUMNS
where TABLE_NAME = #TableName and DATA_TYPE in (''nvarchar'', ''varchar'')'
INSERT INTO #COLUMNS
EXEC sp_executeSQL #strSQL, N'#TableName nvarchar(100)', #TableName = #strTable
DECLARE csrColumns CURSOR LOCAL FORWARD_ONLY FOR
SELECT ColName FROM #COLUMNS
OPEN csrColumns
FETCH csrColumns INTO #strColName
WHILE ##FETCH_STATUS = 0
BEGIN
SET #strSQL = N'UPDATE ' + #strTable + '
SET ' + #strColName + ' = CASE WHEN ' + #strColName +'= ''Very Much'' THEN ''10''
WHEN ' + #strColName + ' = ''Not at all'' THEN ''0''
ELSE ' + #strColName + ' END'
exec sp_ExecuteSQL #strSQL
FETCH csrColumns INTO #strColName
END
CLOSE csrColumns
DEALLOCATE csrColumns
I have n tables that is unknown before runtime with tablename always being tablename1, tablename2... tablenameN. The first column of each table is always Name. The challenge is to change that column name in each table to Name1, Name2.. NameN. I know I should be using sp_rename and a loop. Having trouble building up the query, I'm pretty new to SQL. Help would be appreciated. THanks
This should do the rename:
DECLARE #counter INT;
DECLARE #tableName NVARCHAR(100);
DECLARE #columnName NVARCHAR(100);
DECLARE #newColumnName NVARCHAR(100);
SET #counter = 1;
WHILE #counter < 65536
BEGIN
SET #tableName = 'tableName' + CAST(#counter AS NVARCHAR)
IF EXISTS(SELECT * FROM sys.tables WHERE name = #tableName)
BEGIN
SET #columnName = #tableName + N'.name';
SET #newColumnName = N'name' + CAST(#counter AS NVARCHAR);
EXEC sp_rename #objname=#columnName, #newName=#newColumnName;
END
ELSE
BEGIN
SET #counter = 65536
END
SET #counter = #counter + 1
END
It's a bit crude though.. and renames only 65535 tables and full amount only when there's none missing in between.
Uncomment sql_exec when you're sure it does what you're expecting :)
DECLARE #TableName sysname, #ColName sysname
DECLARE #num sysname
DECLARE #sql nvarchar(4000)
DECLARE cTables CURSOR FOR SELECT name from dbo.sysobjects where Category = 0 AND type NOT IN (N'F', N'FN', N'IF', N'TF', N'P', N'TR', N'V', N'K') AND name like 'tablename%'
OPEN cTables
FETCH NEXT FROM cTables INTO #TableName
WHILE ##FETCH_STATUS = 0
BEGIN
SET #num = SUBSTRING(#Tablename, 10, 5)
SET #sql = N'sp_RENAME ''' + #TableName + '.[Name]'' , ''[Name' + #num + ']'', ''COLUMN'''
PRINT #sql
-- EXEC sp_sqlexec #sql
FETCH NEXT FROM cTables INTO #TableName
END
CLOSE cTables;
DEALLOCATE cTables;
Here's a SP - give it a try ;-)
CREATE PROCEDURE dbo.Rename
(
#n INT
)
AS
BEGIN
SET NOCOUNT ON
DECLARE #Stmt NVARCHAR(MAX)
DECLARE #i INT
DECLARE #tabname NVARCHAR(MAX)
DECLARE #colname NVARCHAR(MAX)
SET #i = 1
WHILE #i <= #n
BEGIN
SET #tabname = N'tablename' + CAST(#i AS NVARCHAR(MAX))
SET #colname = N'name' + CAST(#i AS NVARCHAR(MAX))
IF EXISTS(SELECT TOP 1 1 FROM sys.tables t WHERE t.name = #tabname)
BEGIN
SET #Stmt = N'EXEC sp_rename ''' + #tabname + '.[name]'', ''' + #colname +''',''COLUMN'''
--PRINT #Stmt
EXEC sp_executesql #Stmt
END
SET #i = #i + 1
END
END
I have a folder called "Dump." This folder consists of various .CSV Files.
The folder Location is 'C:\Dump'
I want to Import the contents of these files into SQL Server.
I want the rough code along with proper comments so that I understand it.
I have tried a few codes that I found on the Net. But they haven't quite worked out for me for some strange reason.
The steps I would like to have are
Step 1: Copy all the File Names in the folder to a Table
Step 2: Iterate through the table and copy the data from the files using Bulk Insert.
Someone do please help me out on this one. Thanks a lot in advance :)
--BULK INSERT MULTIPLE FILES From a Folder
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'C:\Dump\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%'
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql = 'BULK INSERT Temp FROM ''' + #path + #filename + ''' '
+ ' WITH (
FIELDTERMINATOR = '','',
ROWTERMINATOR = ''\n'',
FIRSTROW = 2
) '
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
--Extras
--delete from ALLFILENAMES where WHICHFILE is NULL
--select * from ALLFILENAMES
--drop table ALLFILENAMES
This will give you separate tables for each file.
--BULK INSERT MULTIPLE FILES From a Folder
drop table allfilenames
--a table to loop thru filenames drop table ALLFILENAMES
CREATE TABLE ALLFILENAMES(WHICHPATH VARCHAR(255),WHICHFILE varchar(255))
--some variables
declare #filename varchar(255),
#path varchar(255),
#sql varchar(8000),
#cmd varchar(1000)
--get the list of files to process:
SET #path = 'D:\Benihana\backup_csv_benihana_20191128032207_part_1\'
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES SET WHICHPATH = #path where WHICHPATH is null
delete from ALLFILENAMES where WHICHFILE is null
--SELECT replace(whichfile,'.csv',''),* FROM dbo.ALLFILENAMES
--cursor loop
declare c1 cursor for SELECT WHICHPATH,WHICHFILE FROM ALLFILENAMES where WHICHFILE like '%.csv%' order by WHICHFILE desc
open c1
fetch next from c1 into #path,#filename
While ##fetch_status <> -1
begin
--bulk insert won't take a variable name, so make a sql and execute it instead:
set #sql =
'select * into '+ Replace(#filename, '.csv','')+'
from openrowset(''MSDASQL''
,''Driver={Microsoft Access Text Driver (*.txt, *.csv)}''
,''select * from '+#Path+#filename+''')'
print #sql
exec (#sql)
fetch next from c1 into #path,#filename
end
close c1
deallocate c1
For Step 1 Maybe you can look at:
http://www.sql-server-performance.com/forum/threads/copying-filenames-to-sql-table.11546/
or
How to list files inside a folder with SQL Server
and then Step 2
How to cast variables in T-SQL for bulk insert?
HTH
You might need to enable the xp_cmdshell first:
sp_configure 'show advanced options', '1'
RECONFIGURE
go
sp_configure 'xp_cmdshell', '1'
RECONFIGURE
go
And, to enable ad_hoc,
sp_configure 'show advanced options', 1;
RECONFIGURE;
GO
sp_configure 'Ad Hoc Distributed Queries', 1;
RECONFIGURE;
GO
To solve step 1, xp_dirtree can also be used to list all files and folders.
Keep in mind that it is an undocumented function. Security precautions must be considered. Intentionally crafted filenames could be an intrusion vector.
In python you can use d6tstack which makes this simple
import d6tstack
import glob
c = d6tstack.combine_csv.CombinerCSV(glob.glob('*.csv'))
c.to_mssql_combine('mssql+pymssql://usr:pwd#localhost/db', 'tablename')
See SQL examples. It also deals with data schema changes, creates table and allows you to preprocess data. It leverages BULK INSERT so should be just as fast.
to expand upon the answer by SarangArd you can replace temp with the following if your file name matches your table name.
' + Left(#filename, Len(#filename)-4) + '
This code will create a new table per CSV file that is imported.
Best to populate empty database from CSV files.
CREATE TABLE ALLFILENAMES
(
WHICHPATH VARCHAR(255)
,WHICHFILE VARCHAR(255)
)
DECLARE #filename VARCHAR(255),
#path VARCHAR(255),
#sql VARCHAR(8000),
#cmd VARCHAR(1000)
SET #path = 'L:\DATA\SOURCE\CSV\' --PATH TO YOUR CSV FILES (CHANGE TO YOUR PATH)
SET #cmd = 'dir ' + #path + '*.csv /b'
INSERT INTO ALLFILENAMES(WHICHFILE)
EXEC Master..xp_cmdShell #cmd
UPDATE ALLFILENAMES
SET WHICHPATH = #path
WHERE WHICHPATH IS NULL
DECLARE c1 CURSOR
FOR SELECT WHICHPATH
,WHICHFILE
FROM ALLFILENAMES
WHERE WHICHFILE LIKE '%.csv%'
OPEN c1
FETCH NEXT FROM c1 INTO #path,
#filename
WHILE ##fetch_status <> -1
BEGIN
CREATE TABLE #Header
(
HeadString NVARCHAR(MAX)
)
DECLARE #Columns NVARCHAR(MAX) = ''
DECLARE #Query NVARCHAR(MAX) = ''
DECLARE #QUERY2 NVARCHAR(MAX) = ''
DECLARE #HeaderQuery NVARCHAR(MAX) = ''
SELECT #HeaderQuery = #HeaderQuery + 'bulk insert #Header from ''' + #path + #filename + '''
with(firstrow=1,lastrow=1)'
EXEC (#HeaderQuery)
SELECT #Columns = (SELECT QUOTENAME(value) + ' nvarchar(max)' + ','
FROM #Header
CROSS APPLY STRING_SPLIT(HeadString,',') FOR xml PATH(''))
IF ISNULL(#Columns,'') <> ''
BEGIN
SET #Columns = LEFT(#Columns,LEN(#Columns) - 1)
SELECT #Query = #Query + 'CREATE TABLE ' + Replace(#filename,'.csv','') + ' (' + replace(#Columns,'"','') + ')'
PRINT #Query
EXEC (#QUERY)
END
SELECT #QUERY2 = #QUERY2 + 'bulk insert ' + replace(Replace(#filename,'.csv',''),'.TPS','') + ' from ''' + #path + #filename + '''
with(firstrow=2,FORMAT=''csv'',FIELDTERMINATOR='','',ROWTERMINATOR=''\n'')'
EXEC (#QUERY2)
DROP TABLE #Header
FETCH NEXT FROM c1 INTO #path,
#filename
END
CLOSE c1
DEALLOCATE c1