excel vba - query on a spreadsheet - sql
if i have these 2 tables:
is there some sort of excel vba code (using ADO) that could acheive these desired results which could utilise any query i put in the SQL sheet?
Here's some VBA code that allows you to read an Excel range using the text SQL driver. It's quite a complex example, but I'm guessing that you came here because you're a fairly advanced user with a more complex problem than the examples we see on other sites.
Before I post the code in full, here's the original 'sample usage' comment in the core function, FetchXLRecordSet:
' Sample usage:
'
' Set rst = FetchXLRecordSet(SQL, "TableAccountLookup", "TableCashMap")
'
' Where the query uses two named ranges, "TableAccountLookup" and "TableCashMap"
' as shown in this SQL statement:
'
' SELECT
' B.Legal_Entity_Name, B.Status,
' SUM(A.USD_Settled) As Settled_Cash
' FROM
' [TableAccountLookup] AS A,
' [TableCashMap] AS B
' WHERE
' A.Account IS NOT NULL
' AND B.Cash_Account IS NOT NULL
' AND A.Account = B.Cash_Account
' GROUP BY
' B.Legal_Entity_Name,
' B.Status
It's clunky, forcing you to name the tables (or list the range addresses in full) when you run the query, but it simplifies the code.
Option Explicit
Option Private Module
' ADODB data retrieval functions to support Excel
' Online reference for connection strings:
' http://www.connectionstrings.com/oracle#p15
' Online reference for ADO objects & properties:
' http://msdn.microsoft.com/en-us/library/ms678086(v=VS.85).aspx
' External dependencies:
' Scripting - C:\Program files\scrrun.dll
' ADO - C:\Program files\Common\system\ado\msado27.tlb
Private m_strTempFolder As String
Private m_strConXL As String
Private m_objConnXL As ADODB.Connection
Public Property Get XLConnection() As ADODB.Connection
On Error GoTo ErrSub
' The Excel database drivers have memory problems so we use the text driver
' to read csv files in a temporary folder. We populate these files from
' ranges specified for use as tables by the FetchXLRecordSet() function.
Dim objFSO As Scripting.FileSystemObject
Set objFSO = New Scripting.FileSystemObject
Set m_objConnXL = New ADODB.Connection
' Specify and clear a temporary folder:
m_strTempFolder = objFSO.GetSpecialFolder(2).ShortPath
If Right(m_strTempFolder, 1) <> "\" Then
m_strTempFolder = m_strTempFolder & "\"
End If
m_strTempFolder = m_strTempFolder & "XLSQL"
Application.DisplayAlerts = False
If objFSO.FolderExists(m_strTempFolder) Then
objFSO.DeleteFolder m_strTempFolder
End If
If Not objFSO.FolderExists(m_strTempFolder) Then
objFSO.CreateFolder m_strTempFolder
End If
If Right(m_strTempFolder, 1) <> "\" Then
m_strTempFolder = m_strTempFolder & "\"
End If
' JET OLEDB text driver connection string:
' Provider=Microsoft.Jet.OLEDB.4.0;Data Source=c:\txtFilesFolder\;Extended Properties="text;HDR=Yes;FMT=Delimited";
' ODBC text driver connection string:
' Driver={Microsoft Text Driver (*.txt; *.csv)};Dbq=c:\txtFilesFolder\;Extensions=asc,csv,tab,txt;
m_strConXL = "Provider=Microsoft.Jet.OLEDB.4.0;Data Source=" & m_strTempFolder & ";"
m_strConXL = m_strConXL & "Extended Properties=" & Chr(34) & "text;HDR=Yes;IMEX=1" & Chr(34) & ";"
With m_objConnXL
.CursorLocation = adUseClient
.CommandTimeout = 90
.ConnectionString = m_strConXL
.Mode = adModeRead
End With
If m_objConnXL.State = adStateClosed Then
Application.StatusBar = "Connecting to the local Excel tables"
m_objConnXL.Open
End If
Set XLConnection = m_objConnXL
ExitSub:
Application.StatusBar = False
Exit Property
ErrSub:
MsgPopup "Error connecting to the Excel local data. Please contact Application Support.", vbCritical + vbApplicationModal, "Database connection failure!", 10
Resume ErrEnd
' Resume ExitSub
ErrEnd:
End ' Terminal error. Halt.
End Property
Public Sub CloseConnections()
On Error Resume Next
Set m_objConnXL = Nothing
End Sub
Public Function FetchXLRecordSet(ByVal SQL As String, ParamArray TableNames()) As ADODB.Recordset
' This allows you to retrieve data from Excel ranges using SQL. You
' need to pass additional parameters specifying each range you're using as a table
' so that the these ranges can be saved as csv files in the 'XLSQL' temporary folder
' Note that your query must use the 'table' naming conventions required by the Excel
' database drivers: http://www.connectionstrings.com/excel#20
On Error Resume Next
Dim i As Integer
Dim iFrom As Integer
Dim strRange As String
Dim j As Integer
Dim k As Integer
If IsEmpty(TableNames) Then
TableNames = Array("")
End If
If InStr(TypeName(TableNames), "(") < 1 Then
TableNames = Array(TableNames)
End If
Set FetchXLRecordSet = New ADODB.Recordset
With FetchXLRecordSet
.CacheSize = 8
Set .ActiveConnection = XLConnection
iFrom = InStr(8, SQL, "From", vbTextCompare) + 4
For i = LBound(TableNames) To UBound(TableNames)
strRange = ""
strRange = TableNames(i)
If strRange = "0" Or strRange = "" Then
j = InStr(SQL, "FROM") + 4
j = InStr(j, SQL, "[")
k = InStr(j, SQL, "]")
strRange = Mid(SQL, j + 1, k - j - 1)
End If
RangeToFile strRange
SQL = Left(SQL, iFrom) & Replace(SQL, strRange, strRange & ".csv", iFrom + 1, 1)
SQL = Replace(SQL, "$.csv", ".csv")
SQL = Replace(SQL, ".csv$", ".csv")
SQL = Replace(SQL, ".csv.csv", ".csv")
Next i
.Open SQL, , adOpenStatic, , adCmdText + adAsyncFetch
i = 0
Do While .State > 1
i = (i + 1) Mod 3
Application.StatusBar = "Connecting to the database" & String(i, ".")
Sleep 250
Loop
End With
Application.StatusBar = False
End Function
Public Function ReadRangeSQL(SQL_Range As Excel.Range) As String
' Read a range into a string.
' Each row is delimited with a carriage-return and a line break.
' Empty cells are concatenated into the string as 'Tabs' of four spaces.
'NH Feb 2018: you cannot return more than 32767 chars into a range.
Dim i As Integer
Dim j As Integer
Dim arrCells As Variant
Dim arrRows() As String
Dim arrRowX() As String
Dim strRow As String
Dim boolIndent As Boolean
Const SPACE As String * 1 = " "
Const SPACE4 As String * 4 = " "
Const MAX_LEN As Long = 32767
arrCells = SQL_Range.Value2
If InStr(TypeName(arrCells), "(") Then
ReDim arrRows(LBound(arrCells, 1) To UBound(arrCells, 1))
ReDim arrRowX(LBound(arrCells, 2) To UBound(arrCells, 2))
For i = LBound(arrCells, 1) To UBound(arrCells, 1) - 1
boolIndent = True
For j = LBound(arrCells, 2) To UBound(arrCells, 2)
If isError(arrCells(i, j)) Then
SQL_Range(i, j).Calculate
End If
If Not isError(arrCells(i, j)) Then
arrRowX(j) = arrCells(i, j)
Else
arrRowX(j) = vbNullString
End If
If boolIndent And arrRowX(j) = "" Then
arrRowX(j) = SPACE4
Else
boolIndent = False
End If
Next j
arrRows(i) = Join(arrRowX, SPACE)
If Len(Trim$(arrRows(i))) = 0 Then
arrRows(i) = vbNullString
Else
arrRows(i) = RTrim$(Join(arrRowX, SPACE))
End If
Next i
Erase arrCells
Erase arrRowX
ReadRangeSQL = Join(arrRows, vbCrLf)
Erase arrRows
ReadRangeSQL = Replace(ReadRangeSQL, vbCrLf & vbCrLf, vbCrLf)
Else
ReadRangeSQL = CStr(arrCells)
End If
If Len(ReadRangeSQL) > MAX_LEN Then
' Trip terminating spaces from each row:
Do While InStr(1, ReadRangeSQL, SPACE & vbCrLf, vbBinaryCompare) > 0
ReadRangeSQL = Replace(ReadRangeSQL, SPACE & vbCrLf, vbCrLf)
Loop
End If
If Len(ReadRangeSQL) > MAX_LEN Then
' Reduce the 'tab' size to 2 selectively, after each row's indentation
arrRows = Split(ReadRangeSQL, vbCrLf)
For i = LBound(arrRows) To UBound(arrRows)
If Len(arrRows(i)) > 16 Then
If InStr(12, arrRows(i), SPACE4) > 0 Then
arrRows(i) = Left$(arrRows(i), 12) & Replace(Right$(arrRows(i), Len(arrRows(i)) - 12), SPACE4, SPACE & SPACE)
End If
End If
Next i
ReadRangeSQL = Join(arrRows, vbCrLf)
Erase arrRows
End If
If Len(ReadRangeSQL) > MAX_LEN Then
' Reduce the 'tab' size to 2 indiscriminately. This will make your SQL illegible:
Do While InStr(1, ReadRangeSQL, SPACE4, vbBinaryCompare) > 0
ReadRangeSQL = Replace(ReadRangeSQL, SPACE4, SPACE & SPACE)
Loop
End If
End Function
Public Sub RangeToFile(ByRef strRange As String)
' Output a range to a csv file in a temporary folder created by the XLConnection function
' strRange specifies a range in the current workbook using the 'table' naming conventions
' specified for Excel OLEDB database drivers: http://www.connectionstrings.com/excel#20
' The first row of the range is assumed to be a set of column names.
On Error Resume Next
Dim objFSO As Scripting.FileSystemObject
Dim rng As Excel.Range
Dim strFile As String
Dim arrData As Variant
Dim iRow As Long
Dim jCol As Long
Dim strData As String
Dim strLine As String
strRange = Replace(strRange, "[", "")
strRange = Replace(strRange, "]", "")
If Right(strRange, 1) = "$" Then
strRange = Replace(strRange, "$", "")
Set rng = ThisWorkbook.Worksheets(strRange).UsedRange
Else
strRange = Replace(strRange, "$", "")
Set rng = Range(strRange)
If rng Is Nothing Then
Set rng = ThisWorkbook.Worksheets(strRange).UsedRange
End If
End If
If rng Is Nothing Then
Exit Sub
End If
Set objFSO = New Scripting.FileSystemObject
strFile = m_strTempFolder & strRange & ".csv"
If objFSO.FileExists(strFile) Then
objFSO.DeleteFile strFile, True
End If
If objFSO.FileExists(strFile) Then
Exit Sub
End If
arrData = rng.Value2
With objFSO.OpenTextFile(strFile, ForWriting, True)
' Header row:
strLine = ""
strData = ""
iRow = LBound(arrData, 1)
For jCol = LBound(arrData, 2) To UBound(arrData, 2)
strData = arrData(iRow, jCol)
strData = Replace(strData, Chr(34), Chr(39))
strData = Replace(strData, Chr(10), " ")
strData = Replace(strData, Chr(13), " ")
strData = strData & ","
strLine = strLine & strData
Next jCol
strLine = Left(strLine, Len(strLine) - 1) ' Trim trailing comma
If Len(Replace(Replace(strLine, Chr(34), ""), ",", "")) > 0 Then
.WriteLine strLine
End If
' Rest of the data
For iRow = LBound(arrData, 1) + 1 To UBound(arrData, 1)
strLine = ""
strData = ""
For jCol = LBound(arrData, 2) To UBound(arrData, 2)
If IsError(arrData(iRow, jCol)) Then
strData = "#ERROR"
Else
strData = arrData(iRow, jCol)
strData = Replace(strData, Chr(34), Chr(39))
strData = Replace(strData, Chr(10), " ")
strData = Replace(strData, Chr(13), " ")
strData = Replace(strData, Chr(9), " ")
strData = Trim(strData)
End If
strData = Chr(34) & strData & Chr(34) & "," ' Quotes to coerce all values to text
strLine = strLine & strData
Next jCol
strLine = Left(strLine, Len(strLine) - 1) ' Trim trailing comma
If Len(Replace(Replace(strLine, Chr(34), ""), ",", "")) > 0 Then
.WriteLine strLine
End If
Next iRow
.Close
End With ' textstream object from objFSO.OpenTextFile
Set objFSO = Nothing
Erase arrData
Set rng = Nothing
End Sub
And finally, Writing a Recordset to a Range - the code would be trivial if it wasn't for all the errors you have to handle:
Public Sub RecordsetToRange(rngTarget As Excel.Range, objRecordset As ADODB.Recordset, Optional FieldList As Variant, Optional ShowFieldNames As Boolean = False, Optional Orientation As Excel.XlRowCol = xlRows)
' Write an ADO Recordset to an Excel range in a single 'hit' to the sheet
' Calling function is responsible for setting the record pointer (must not be EOF!)
' The target range is resized automatically to the dimensions of the array, with the top left cell used as the start point.
On Error Resume Next
Dim OutputArray As Variant
Dim i As Integer
Dim iCol As Integer
Dim iRow As Integer
Dim varField As Variant
If objRecordset Is Nothing Then
Exit Sub
End If
If objRecordset.State <> 1 Then
Exit Sub
End If
If objRecordset.BOF And objRecordset.EOF Then
Exit Sub
End If
If Orientation = xlColumns Then
If IsEmpty(FieldList) Or IsMissing(FieldList) Then
OutputArray = objRecordset.GetRows
Else
OutputArray = objRecordset.GetRows(Fields:=FieldList)
End If
Else
If IsEmpty(FieldList) Or IsMissing(FieldList) Then
OutputArray = ArrayTranspose(objRecordset.GetRows)
Else
OutputArray = ArrayTranspose(objRecordset.GetRows(Fields:=FieldList))
End If
End If
ArrayToRange rngTarget, OutputArray
If ShowFieldNames Then
If Orientation = xlColumns Then
ReDim OutputArray(LBound(OutputArray, 1) To UBound(OutputArray, 1), 1 To 1)
iRow = LBound(OutputArray, 1)
If IsEmpty(FieldList) Or IsMissing(FieldList) Then
For i = 0 To objRecordset.Fields.Count - 1
If i > UBound(OutputArray, 1) Then
Exit For
End If
OutputArray(iRow + i, 1) = objRecordset.Fields(i).Name
Next i
Else
If InStr(TypeName(FieldList), "(") < 1 Then
FieldList = Array(FieldList)
End If
i = 0
For Each varField In FieldList
OutputArray(iRow + i, 1) = CStr(varField)
i = i = 1
Next
End If 'IsEmpty(FieldList) Or IsMissing(FieldList)
ArrayToRange rngTarget.Cells(1, 0), OutputArray
Else
ReDim OutputArray(1 To 1, LBound(OutputArray, 2) To UBound(OutputArray, 2))
iCol = LBound(OutputArray, 2)
If IsEmpty(FieldList) Or IsMissing(FieldList) Then
For i = 0 To objRecordset.Fields.Count - 1
If i > UBound(OutputArray, 2) Then
Exit For
End If
OutputArray(1, iCol + i) = objRecordset.Fields(i).Name
Next i
Else
If InStr(TypeName(FieldList), "(") < 1 Then
FieldList = Array(FieldList)
End If
i = 0
For Each varField In FieldList
OutputArray(1, iCol + i) = CStr(varField)
i = i = 1
Next
End If ' IsEmpty(FieldList) Or IsMissing(FieldList)
ArrayToRange rngTarget.Cells(0, 1), OutputArray
End If ' Orientation = xlColumns
End If 'ShowFieldNames
Erase OutputArray
End Sub
Public Function ArrayTranspose(InputArray As Variant) As Variant
' Transpose InputArray.
' Returns InputArray unchanged if it is not a 2-Dimensional Variant(x,y)
Dim iRow As Long
Dim iCol As Long
Dim iRowCount As Long
Dim iColCount As Long
Dim boolNoRows As Boolean
Dim BoolNoCols As Boolean
Dim OutputArray As Variant
If IsEmpty(InputArray) Then
ArrayTranspose = InputArray
Exit Function
End If
If InStr(1, TypeName(InputArray), "(") < 1 Then
ArrayTranspose = InputArray
Exit Function
End If
' Check that we can read the array's dimensions:
On Error Resume Next
Err.Clear
iRowCount = 0
iRowCount = UBound(InputArray, 1)
If Err.Number <> 0 Then
boolNoRows = True
End If
Err.Clear
Err.Clear
iColCount = 0
iColCount = UBound(InputArray, 2)
If Err.Number <> 0 Then
BoolNoCols = True
End If
Err.Clear
If boolNoRows Then
' ALL arrays have a defined Ubound(MyArray, 1)!
' This variant's dimensions cannot be determined
OutputArray = InputArray
ElseIf BoolNoCols Then
' It's a vector. Strictly speaking, a vector cannot be 'transposed', as
' calling the ordinal a 'row' or a 'column' is arbitrary or meaningless.
' But... By convention, Excel users regard a vector as an array of 1 to n
' rows and 1 column. So we'll 'transpose' it into a Variant(1 to 1, 1 to n)
ReDim OutputArray(1 To 1, LBound(InputArray, 1) To UBound(InputArray, 1))
For iRow = LBound(InputArray, 1) To UBound(InputArray, 1)
OutputArray(1, iRow) = InputArray(iRow)
Next iRow
Else
ReDim OutputArray(LBound(InputArray, 2) To UBound(InputArray, 2), LBound(InputArray, 1) To UBound(InputArray, 1))
If IsEmpty(OutputArray) Then
ArrayTranspose = InputArray
Exit Function
End If
If InStr(1, TypeName(OutputArray), "(") < 1 Then
ArrayTranspose = InputArray
Exit Function
End If
For iRow = LBound(InputArray, 1) To UBound(InputArray, 1)
For iCol = LBound(InputArray, 2) To UBound(InputArray, 2)
OutputArray(iCol, iRow) = InputArray(iRow, iCol)
Next iCol
Next iRow
End If
ExitFunction:
ArrayTranspose = OutputArray
Erase OutputArray
End Function
Postscript: Running SQL on Excel 'Table' Objects
For completeness, here's the code for a barebones 'read Excel Table objects with SQL' function that handles all the text-file hacking in the background.
I'm posting it now, a while after my original answer went up, because everyone's using the rich 'table' object for tabulated data in Excel:
' Run a JOIN query on your tables, and write the field names and data to Sheet1:
SaveTable "Table1"
SaveTable "Table2"
SQL= SQL & "SELECT * "
SQL= SQL & " FROM Table1 "
SQL= SQL & " LEFT JOIN Table2 "
SQL= SQL & " ON Table1.Client = Table2.Client"
RunSQL SQL, Sheet1.Range("A1")
...And the full listing (give or take a couple of functions in the previous code dump) is:
Public Function RunSQL(SQL As String, TargetRange As Excel.Range, Optional DataSetName As String)
' Run SQL against table files in the local ExcelSQL folder and write the results to a target range
' The full implementation of ExcelSQL provides a fully-featured UI on a control sheet
' This is a cut-down version which runs everything automatically, without audit & error-reporting
' SQL can be read from ranges using the ReadRangeSQL function
' If no target range object is passed in, and a Data set name is specified, the recordset will be
' saved as [DataSetName].csv in the local Excel SQL folder for subsequent SQL queries
' If no target range is specified and no Data set name specified, returns the recordet object
Dim rst As ADODB.Recordset
If Left(SQL, 4) = "SQL_" Then
SQL = ReadRangeSQL(ThisWorkbook.Names(SQL).RefersToRange)
End If
Set rst = FetchTextRecordset(SQL)
If TargetRange Is Nothing Then
If DataSetName = "" Then
Set RunSQL = rst
Else
RecordsetToCSV rst, DataSetName, , , , , , , False
Set rst = Nothing
End If
Else
RecordsetToRange rst, TargetRange, True
Set rst = Nothing
End If
End Function
Public Function FetchTextRecordset(SQL As String) As ADODB.Recordset
' Fetch records from the saved text files in the Temp SQL Folder:
On Error Resume Next
Dim i As Integer
Dim iFrom As Integer
If InStr(1, connText, "IMEX=1", vbTextCompare) > 0 Then SetSchema
Set FetchTextRecordset = New ADODB.Recordset
With FetchTextRecordset
.CacheSize = 8
Set .ActiveConnection = connText
On Error GoTo ERR_ADO
.Open SQL, , adOpenStatic, , adCmdText + adAsyncFetch
i = 0
Do While .State > 1
i = (i + 1) Mod 3
Application.StatusBar = "Waiting for data" & String(i, ".")
Application.Wait Now + (0.25 / 24 / 3600)
Loop
End With
Application.StatusBar = False
ExitSub:
Exit Function
ERR_ADO:
Dim strMsg
strMsg = vbCrLf & vbCrLf & "If this is a 'file' error, someone's got one of the source data files open: try again in a few minutes." & vbCrLf & vbCrLf & "Otherwise, please make a note of this error message and contact the developer, or " & SUPPORT & "."
If Verbose Then
MsgBox "Error &H" & Hex(Err.Number) & ": " & Err.Description & strMsg, vbCritical + vbMsgBoxHelpButton, "Data retrieval error:", Err.HelpFile, Err.HelpContext
End If
Resume ExitSub
Exit Function
' Try this if SQL is too big to debug in the immediate window:
' FSO.OpenTextFile("C:\Temp\SQL.txt",ForWriting,True).Write SQL
' Shell "Notepad.exe C:\Temp\SQL.txt", vbNormalFocus
'Resume
End Function
Private Property Get connText() As ADODB.Connection
On Error GoTo ErrSub
Dim strTempFolder
If m_objConnText Is Nothing Then
Set m_objConnText = New ADODB.Connection
strTempFolder = TempSQLFolder ' this will test whether the folder permits SQL READ operations
Application.DisplayAlerts = False
' MS-Access ACE OLEDB Provider
m_strConnText = "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" & Chr(34) & strTempFolder & Chr(34) & ";Persist Security Info=True;"
m_strConnText = m_strConnText & "Extended Properties=" & Chr(34) & "text;CharacterSet=UNICODE;HDR=Yes;HDR=Yes;IMEX=1;MaxScanRows=1" & Chr(34) & ";"
End If
If Not m_objConnText Is Nothing Then
With m_objConnText
If .State = adStateClosed Then
Application.StatusBar = "Connecting to the local Excel tables"
.CursorLocation = adUseClient
.CommandTimeout = 90
.ConnectionString = m_strConnText
.Mode = adModeRead
.Open
End If
End With
If m_objConnText.State = adStateClosed Then
Set m_objConnText = Nothing
End If
End If
Set connText = m_objConnText
ExitSub:
Application.StatusBar = False
Exit Property
ErrSub:
MsgBox "Error connecting to the Excel local data. Please contact " & SUPPORT & ".", vbCritical + vbApplicationModal, "Database connection failure!", 10
Resume ErrEnd
' Resume ExitSub
ErrEnd:
End ' Terminal error. Halt.
End Property
Public Sub CloseConnections()
On Error Resume Next
Set m_objConnText = Nothing
End Sub
Public Function TempSQLFolder() As String
Application.Volatile False
' Location of temporary table files used by the SQL text data functions
' Also runs a background process to clear out files over 7 days old
' The best location is a named subfolder in the user's temp folder. The
' user local 'temp' folder is discoverable on all Windows systems using
' GetObject("Scripting.FileSystemObject").GetSpecialFolder(2).ShortPath
' and will usually be C:\Users\[User Name]\AppData\Local\Temp
' Dependencies:
' Object Property FSO (Returns Scripting.FilesystemObject)
'
Dim strCMD As String
Dim strMsg As String
Dim strNamedFolder As String
Static strTempFolder As String ' Cache it
Dim iRetry As Integer
Dim i As Long
' If we've already found a usable temp folder, use the static value
' without querying the file system and testing write privileges again:
If strTempFolder <> "" Then
TempSQLFolder = strTempFolder
Exit Function
End If
On Error Resume Next
strTempFolder = GetObject("Scripting.FileSystemObject").GetSpecialFolder(2).ShortPath
If Right(strTempFolder, 1) <> "\" Then
strTempFolder = strTempFolder & "\"
End If
strTempFolder = strTempFolder & "XLSQL"
If Not FSO.FolderExists(strTempFolder) Then
FSO.CreateFolder strTempFolder
End If
i = 1
Do Until FSO.FolderExists(strTempFolder) Or i > 6
Sleep i * 250
Application.StatusBar = "Waiting for SQL cache folder" & String(i Mod 4, ".")
Loop
If Not FSO.FolderExists(strTempFolder) Then
GoTo Retry
End If
If Right(strTempFolder, 1) <> "\" Then
strTempFolder = strTempFolder & "\"
End If
TempSQLFolder = strTempFolder
Application.StatusBar = False
End Function
Public Property Get FSO() As Scripting.FileSystemObject '
' Return a File System Object
On Error Resume Next
If m_objFSO Is Nothing Then
Set m_objFSO = CreateObject("Scripting.FileSystemObject") ' New Scripting.FileSystemObject
End If
If m_objFSO Is Nothing Then
Shell "Regsvr32.exe /s scrrun.dll", vbHide
Set m_objFSO = CreateObject("Scripting.FileSystemObject")
End If
Set FSO = m_objFSO
End Property
Public Sub SaveTable(Optional TableName As String = "*")
' Export a Table object to the local SQL Folder as a csv file
' If no name is specified, all tables are exported asynchronously
' This step is essential for running SQL on the tables
Dim wks As Excel.Worksheet
Dim oList As Excel.ListObject
Dim sFile As String
Dim bAsync As Boolean
If TableName = "*" Then
bAsync = True
Else
bAsync = False
End If
For Each wks In ThisWorkbook.Worksheets
For Each oList In wks.ListObjects
If oList.Name Like TableName Then
sFile = oList.Name
ArrayToCSV oList.Range.Value2, sFile, , , , , , , , bAsync
'Debug.Print "[" & sFile & ".csv] "
End If
Next oList
Next wks
SetSchema
End Sub
Public Sub RemoveTable(Optional TableName As String = "*")
On Error Resume Next
' Clear up the temporary 'Table' files in the user local temp folder:
Dim wks As Excel.Worksheet
Dim oList As Excel.ListObject
Dim sFile As String
Dim sFolder As String
sFolder = TempSQLFolder
For Each wks In ThisWorkbook.Worksheets
For Each oList In wks.ListObjects
If oList.Name Like TableName Then
sFile = oList.Name & ".csv"
If Len(Dir(sFile)) > 0 Then
Shell "CMD /c DEL " & Chr(34) & sFolder & sFile & Chr(34), vbHide ' asynchronous deletion
End If
End If
Next oList
Next wks
End Sub
Share and enjoy: this is all a horrible hack, but it gives you a stable SQL platform.
And we still don't have a stable 'native' platform for SQL on Excel: the Microsoft.ACE.OLEDB.14.0 Excel data provider still has the same memory leak as Microsoft.Jet.OLEDB.4.0 and the Excel ODBC driver that preceded it, twenty years ago.
Some notes:
sFullName = ActiveWorkbook.FullName
sSheet = ActiveSheet.Name
Set cn = CreateObject("adodb.connection")
scn = "Provider=Microsoft.Jet.OLEDB.4.0;Data Source=" _
& sFullName _
& ";Extended Properties=""Excel 8.0;HDR=Yes;IMEX=1"";"
cn.Open scn
Set rs = CreateObject("adodb.recordset")
For Each c In Sheet4.UsedRange
sSQL = sSQL & c.Value & " "
Next
rs.Open sSQL, cn
Sheet5.Range("a10").CopyFromRecordset rs
There is an ODBC driver for Excel.
See: http://support.microsoft.com/kb/178717
And: http://msdn.microsoft.com/en-us/library/ms711711%28v=vs.85%29.aspx
In order to get data out of a database and into Excel you do the following steps.
Record a macro
Import external data, choose a new source, select DSN ODBC as the type of source.
Now choose Excel-file as the type of ODBC source.
Pick the Excel sheet you want to query.
Every table needs to be in a named range, leave the option select a table checked, Excel will not allow us to insert a query just yet.
Follow the wizard and save the .odc file. Open it again and choose edit query. Now you can insert your select statement.
Stop recording and edit the recorded macro to suit your needs.
It looks like source and target are odbc queries. You need to parse the table name out of those queries and replace SoureTable and TargetTable in your query with the right table names.
Sub ExecuteSQL()
Dim sSql As String
Dim rCell As Range
Dim adConn As ADODB.Connection
Dim adRs As ADODB.Recordset
Dim lWherePos As Long
Const sSOURCE As String = "SourceTable"
Const sTARGET As String = "TargetTable"
Const sODBC As String = "ODBC;"
'Buld the sql statement
For Each rCell In Intersect(wshSql.UsedRange, wshSql.Columns(1)).Cells
If Not IsEmpty(rCell.Value) Then
sSql = sSql & rCell.Value & Space(1)
End If
Next rCell
'replace the table names
sSql = Replace(sSql, sSOURCE, GetTableName(wshSource.QueryTables(1).CommandText), 1, 1)
sSql = Replace(sSql, sTARGET, GetTableName(wshTarget.QueryTables(1).CommandText), 1, 1)
'execute the query
Set adConn = New ADODB.Connection
adConn.Open Replace(wshSource.QueryTables(1).Connection, sODBC, "")
Set adRs = adConn.Execute(sSql)
'copy the results
wshResults.Range("A1").CopyFromRecordset adRs
adRs.Close
adConn.Close
Set adRs = Nothing
Set adConn = Nothing
End Sub
Function GetTableName(sSql As String) As String
Dim lFromStart As Long
Dim lFromEnd As Long
Dim sReturn As String
Const sFROM As String = "FROM "
Const sWHERE As String = "WHERE "
'find where FROM starts and ends
'I'm looking for WHERE as the end, but you'll need to look for everything possible, like ORDER BY etc.
lFromStart = InStr(1, sSql, sFROM)
lFromEnd = InStr(lFromStart, sSql, sWHERE)
If lFromEnd = 0 Then
sReturn = Mid$(sSql, lFromStart + Len(sFROM), Len(sSql))
Else
sReturn = Mid$(sSql, lFromStart + Len(sFROM), lFromEnd - lFromStart - Len(sFROM) - 1)
End If
GetTableName = sReturn
End Function
Another problem that you might run into is the way Excel (or MSQuery) constructs the SQL statements in an external data query. If you leave it as the default, you'll likely get something like this
SELECT * FROM `C:\somepath\myfile.mdb`.tblTable1 tblTable1 WHERE ...
I have no idea why it does it that way, but you can change it to
SELECT * FROM tblTable1 WHERE ...
and the above code should work. Parsing SQL statements sucks, so don't expect this to be easy. Once you think you have all the possibilities, another will pop up.
Finally, you should get the error "Too few parameters, expected 1" or something similar. In SourceTable, the first field is emp_no, but you have emp_id in your SQL. Make sure your SQL in the SQL sheet is correct. It can be frustrating trying to track down those errors.
I'm using very simple code which helps me to query worksheet range :
Sub hello_jet()
Dim rs As ADODB.Recordset
Dim cn As ADODB.Connection
Dim strQuery As String
Set cn = New ADODB.Connection
With cn
.Provider = "Microsoft.ACE.OLEDB.12.0"
.ConnectionString = "Data Source=C:\yourPath\ADO_test.xls " & _
";Extended Properties=""Excel 8.0;HDR=Yes;"""
.Open
End With
'Microsoft.ACE.OLEDB.12.0 for database engine built in Windows 7 64
strQuery = "SELECT a,sum(c) FROM [Sheet1$A1:C6] GROUP BY a;"
''if range [Sheet1$A1:C6] is named as namedRange you can you its name directly in query:
'strQuery = "SELECT a,sum(c) FROM namedRange GROUP BY a;"
Set rs = cn.Execute(strQuery)
ActiveCell.CopyFromRecordset rs 'useful method
rs.Close
End Sub
Related
Why does VBA doesn't reach data after Row number 65,000 when using DAO with SQL language?
I have a VBA module that receives a database object, worksheet name, and two column field names as parameters to make a SQL query into another Excel table that has over 1,000,000 rows with information. But when I was debugging I noticed that my VBA code does not return the info after the row number 65,000 (approximately). This is returning wrong info and not acting properly as expected. So, how can I handle it in my existing code? Here is my code: Functions Const diretorioSA = "C:\Users\Bosch-PC\Desktop\dbLEGENDAS_ELETROPAR\" Const BaseEletro = "dbClientesEletropar.xlsb" Const dbClientes = "CLIENTESLDA" Public Function Number2Letter(ByVal ColNum As Long) As String Dim ColumnNumber As Long Dim ColumnLetter As String ColumnNumber = ColNum ColumnLetter = Split(Cells(1, ColumnNumber).Address, "$")(1) Number2Letter = ColumnLetter End Function Public Function GetWorkbook(ByVal sFullName As String) As Workbook Dim sFile As String Dim wbReturn As Workbook sFile = DIR(sFullName) On Error Resume Next Set wbReturn = Workbooks(sFile) If wbReturn Is Nothing Then Set wbReturn = Workbooks.Open(sFullName) End If On Error GoTo 0 Set GetWorkbook = wbReturn End Function Public Function ReplaceChars(ByVal str As String, ByVal Lista As String) As String Dim buff(), buffChars() As String ReDim buff(Len(str) - 1): ReDim buffChars(Len(Lista) - 1) For i = 1 To Len(str): buff(i - 1) = Mid$(str, i, 1): Next For i = 1 To Len(Lista): buffChars(i - 1) = Mid$(Lista, i, 1): Next For strEle = 0 To UBound(buff) For listaEle = 0 To UBound(buffChars) If buff(strEle) = buffChars(listaEle) Then buff(strEle) = "" End If Next listaEle novoTexto = novoTexto & buff(strEle) Next strEle ReplaceChars = novoTexto End Function Function ConsultaBaseDeDadosELETRO(ByVal CAMPO_PESQUISA As String, _ ByVal CAMPO_RETORNO As String, _ ByVal NOME_PLANILHA As String, _ ByRef BASES As Object, _ ByVal ARGUMENTO As String) As String On Error GoTo ERRO: Debug.Print BASES.Name Dim RSt22 As Recordset Set RSt22 = BASES.OpenRecordset("SELECT [" & CAMPO_RETORNO & "] FROM [" & NOME_PLANILHA & "$] WHERE [" & CAMPO_PESQUISA & "] IN ('" & ARGUMENTO & "') ;", dbOpenForwardOnly, dbReadOnly) Debug.Print RSt22.CacheSize & " | CONTAGEM: " & RSt22.RecordCount ConsultaBaseDeDadosELETRO = RSt22(CAMPO_RETORNO) Exit Function ERRO: Debug.Print VBA.Err.Description & " | Error number: " & VBA.Err.Number & " | " & VBA.Err.HelpFile ConsultaBaseDeDadosELETRO = "Sem registros" End Function Main Subroutine Sub ProcurarBaseEletro(ByVal PASTA As String, ByVal ARQUIVO As String, ByVal NOME_PLANILHA As String, ByVal CAMPO As String) If ActiveCell.value = "CGC" Or ActiveCell.value = "CNPJ" Or ActiveCell.value = "cgc" Or ActiveCell.value = "cnpj" Then Application.ScreenUpdating = False Dim wks As Worksheet: Set wks = ActiveSheet Dim db2 As database Dim CellRow As Single Dim Cellcol_info, CellCol As String Dim DiretorioBase As String: DiretorioBase = diretorioSA & BaseEletro Dim wb As Workbook: Set wb = GetWorkbook(DiretorioBase) If wb Is Nothing Then MsgBox "Base de dados não localizada!" & vbNewLine & "EM: " & DiretorioBase, vbCritical, "Atenção" Set wb = Nothing Set wks = Nothing Application.ScreenUpdating = True Exit Sub Else wks.Activate CellRow = ActiveCell.row CellCol = Number2Letter(ActiveCell.Column) Cellcol_info = Number2Letter(ActiveCell.Column + 1) CELLCOL_LROW = ActiveSheet.Cells(ActiveSheet.Rows.Count, CellCol).End(xlUp).row Set db2 = OpenDatabase(DiretorioBase, False, False, "Excel 8.0") Columns(Cellcol_info & ":" & Cellcol_info).Insert Shift:=xlToRight, CopyOrigin:=xlFormatFromLeftOrAbove Range(Cellcol_info & CellRow).value = CAMPO Dim Query As String Dim CelAtivaValue As String For i = CellRow + 1 To CELLCOL_LROW CelAtivaValue = UCase(Cells(i, CellCol).value) Query = ReplaceChars(CelAtivaValue, "/.- ") If Left(Query, 6) < 132714 Then Cells(i, Cellcol_info).value = ConsultaBaseDeDadosELETRO("CGC", CAMPO, NOME_PLANILHA, db2, Query) Else Cells(i, Cellcol_info).value = ConsultaBaseDeDadosELETRO("CGC", CAMPO, NOME_PLANILHA & 2, db2, Query) End If Next i wb.Close End If Else MsgBox "Texto da Célula ativa não é CGC/CNPJ, impossível fazer pesquisa", vbCritical, "Valor célula ativa: " & ActiveCell.value Application.ScreenUpdating = True Exit Sub End If Cells.EntireColumn.AutoFit MsgBox "Processo concluído com sucesso.", vbOKOnly, "Informativo do sistema" Application.ScreenUpdating = True End Sub
Older Excel formats (.xls) maintains a worksheet limit of 2^16 (65536) rows. Current Excel formats (.xlsx) maintains a worksheet limit of 2^20 (1,048,576) rows. Likely, you have a more recent version of MS Office (2007+) (given the .xlsb in BaseEletro) but your DAO code was not updated. Consider adjusting the DAO.OpenDatabase option to the newer current format. From Set db2 = OpenDatabase(DiretorioBase, False, False, "Excel 8.0") To Set db2 = OpenDatabase(DiretorioBase, False, False, "Excel 12.0 Xml")
Replace square brackets + contents with the contents as a mergefield
I am trying to change the contents of square brackets into a merge field. I've got 80-ish documents to go through some with none square brackets and some with a few (none nested). I have managed to run my code and it has worked for some files. Others (majority) have given an overflow error. When I examined what was happening in one of the files, the code picks up the contents correctly, it just puts the merge field in the wrong place which in turn causes it to keep finding the same set of square brackets. Public Function searchFiles(fFile As Variant, rootFolderStr2 As String, rootFolderStr As String) Dim strTemp As String, mfc As String, msg As String Dim startStr As Integer, endStr As Integer Dim objWord As New Word.Application Dim objDoc As Word.Document Dim aField As Field, fFolder As String Dim rng As Variant, myField As Field, oldField As Variant On Error GoTo ErrorHandler 'open file 'Open fFile For Input As #1 Set objDoc = objWord.Documents.Open(fFile) objDoc.TrackRevisions = False strTemp = objDoc.Range(0, objDoc.Range.End) startStr = InStrRev(strTemp, "[") endStr = InStrRev(strTemp, "]") Do While startStr <> 0 'Merge field contents mfc = Right(Left(strTemp, endStr - 1), endStr - startStr - 1) Set rng = objDoc.Range(startStr - 1, endStr) Set myField = objDoc.Fields.Add(Range:=rng, Type:=wdFieldMergeField, Text:=mfc) strTemp = objDoc.Range(0, objDoc.Range.End) 'Find next merge field startStr = InStrRev(strTemp, "[") endStr = InStrRev(strTemp, "]") If endStr < startStr And endStr <> -1 Then msg = "Error occured in " & fileName & " " & startStr & " " & endStr Debug.Print (msg) startStr = 0 endStr = 0 End If Loop 'put in right folder fFolder = Right(objDoc.FullName, Len(objDoc.FullName) - Len(rootFolderStr)) objDoc.SaveAs fileName:=rootFolderStr2 & "\" & fFolder objDoc.Close objWord.Quit ErrorHandler: If Err.Number <> 0 Then Debug.Print ("Error occured in file: " & fileName & " " & Err.Description) Exit Function End If End Function I'm struggling to understand how the objects in word work so forgive, please. Any answers as to what's causing this problem would be appreciated or any help with methods to do this in a better manner.
Try: Sub UpdateDocuments() Application.ScreenUpdating = False Dim strFolder As String, strFile As String, strDocNm As String, wdDoc As Document strDocNm = ActiveDocument.FullName strFolder = GetFolder If strFolder = "" Then Exit Sub strFile = Dir(strFolder & "\*.doc", vbNormal) While strFile <> "" If strFolder & "\" & strFile <> strDocNm Then Set wdDoc = Documents.Open(FileName:=strFolder & "\" & strFile, AddToRecentFiles:=False, Visible:=False) Call MakeFields(wdDoc) wdDoc.Close SaveChanges:=True End If strFile = Dir() Wend Set wdDoc = Nothing Application.ScreenUpdating = True End Sub Function GetFolder() As String Dim oFolder As Object GetFolder = "" Set oFolder = CreateObject("Shell.Application").BrowseForFolder(0, "Choose a folder", 0) If (Not oFolder Is Nothing) Then GetFolder = oFolder.Items.Item.Path Set oFolder = Nothing End Function Sub MakeFields(wdDoc As Document) With wdDoc.Range With .Find .ClearFormatting .Replacement.ClearFormatting .Forward = True .Format = False .Wrap = wdFindStop .MatchWildcards = True .Text = "\[*\]" .Execute End With Do While .Find.Found .Characters.First.Text = vbNullString .Characters.Last.Text = vbNullString .Fields.Add Range:=.Duplicate, Type:=wdFieldEmpty, Text:="MERGEFIELD " & .Text, Preserveformatting:=False .Collapse wdCollapseEnd .Find.Execute Loop End With End Sub The above code processes all documents in the selected folder.
OK. The generic advice is to always, always, always put option explicit as the start of your module or class. This helps highlights errors in your code related misuse of syntax and undeclared variables etc. In your posted code there is one undeclared variable 'Filename'. When working with Word it is always better to try to find a way of working with the word object model rather than extracting text. You can modify your existing code by replacing the instrrev with the .MoveStart/EndUntil methods. I've updated your code to use these move methods. If you don't understand what a keyword does then place your cursor on it and press F1. This will take you to the MS help page. For the Word object model the help pages need careful reading. Option Explicit ' Changed to sub as you are not returning any values Public Sub searchFiles(fFile As Variant, rootFolderStr2 As String, rootFolderStr As String) Const FieldOpen As String = "[" Const FieldClose As String = "]" Dim strTemp As String, mfc As String, msg As String Dim objWord As New Word.Application Dim objDoc As Word.Document ' Dim aField As FieldDim Dim fFolder As String ' Dim rng As Variant ' Dim myField As Field ' Dim oldField As Variant ' Not previously declared Dim Filename As String Dim SearchRng As Word.Range Dim FieldRng As Word.Range Dim Moved As Long 'open file 'Open fFile For Input As #1 On Error GoTo ErrorHandler Set objDoc = objWord.Documents.Open(fFile) objDoc.TrackRevisions = False 'strTemp = objDoc.Range(0, objDoc.Range.End) Set SearchRng = ActiveDocument.Content 'startStr = InStrRev(strTemp, "[") Moved = SearchRng.MoveStartUntil(cset:=FieldOpen) 'Do While startStr <> 0 Do Until Moved = 0 'Merge field contents 'mfc = Right(Left(strTemp, endStr - 1), endStr - startStr - 1) FieldRng.Start = SearchRng.Start + 1 'endStr = InStrRev(strTemp, "]") ' exit if we don't find a closing field marker ' The side effect (which we want) is that the end is also moved If SearchRng.MoveEndUntil(cset:=FieldClose) = 0 Then GoTo ErrorHandler FieldRng.End = SearchRng.End + 1 ' reduce the FieldRng to just the text FieldRng.Characters.First.Delete FieldRng.Characters.Last.Delete 'Set rng = objDoc.Range(startStr - 1, endStr 'Set myField = objDoc.Fields.Add(Range:=rng, Type:=wdFieldMergeField, Text:=mfc) objDoc.Fields.Add Range:=FieldRng, Type:=wdFieldMergeField, Text:=FieldRng.Text 'strTemp = objDoc.Range(0, objDoc.Range.End) ' We now need to move the start of the search range to after the mergefield SearchRng.Start = FieldRng.End + 1 'Find next merge field 'startStr = InStrRev(strTemp, "[") 'endStr = InStrRev(strTemp, "]") Moved = SearchRng.MoveStartUntil(cset:=FieldOpen) ' If endStr < startStr And endStr <> -1 Then ' msg = "Error occured in " & Filename & " " & startStr & " " & endStr ' Debug.Print (msg) ' startStr = 0 ' endStr = 0 ' End If Loop 'put in right folder fFolder = Right(objDoc.FullName, Len(objDoc.FullName) - Len(rootFolderStr)) objDoc.SaveAs Filename:=rootFolderStr2 & "\" & fFolder objDoc.Close objWord.Quit ErrorHandler: If Err.Number <> 0 Then Debug.Print ("Error occured in file: " & Filename & " " & Err.Description) Exit Sub End If End Sub The code above compiles without error but I haven't testing the logic. I'll leave that as 'an exercise for the reader'
Can't union two tables even though ADODB confirms equal field counts
I'm not able to union two csvs even though ADODB confirms via .Fields.Count that they both have the same number of columns. Here's the query that's failing: select * from csv1.csv union select * from csv2.csv with the error message: The number of columns in the two selected tables or queries of a union query do not match However, when I do select * from csv1.csv and select * from csv2.csv separately, ADODB confirms that .Fields.Count = 8 for both. Possible key to the problem: Do I need to create two separate connections? I'm only creating one connection (to the first csv) even though there are two csvs in the query. I was trying to figure out how to do two separate connections for the same query and it seemed like people weren't finding that necessary - I couldn't find two connections mentioned in equivalent queries people were running against csvs. Per #Parfait's request to see more of the code: GetDataFromCSV Public Function GetDataFromCSV(ByVal fileReport As Scripting.File, ByVal strQuery As String, ByVal arrSourceReports As Variant) As Boolean Dim strRevisedQuery As String strRevisedQuery = GetRevisedQueryWithFileAliasesReplacedWithTrueFileNames(strQuery, arrSourceReports) Dim cnn As ADODB.Connection Set cnn = OpenConnectionToCSV(fileReport) If cnn Is Nothing Then GetDataFromCSV = False Exit Function End If GetDataFromCSV = QueryDataFromCSV(cnn, strRevisedQuery, fileReport.Name, fileReport.Name) End Function OpenConnectionToCSV Private Function OpenConnectionToCSV(ByVal fileCSV As Scripting.File, Optional boolHeadersPresent As Boolean = True) As ADODB.Connection Dim cnn As ADODB.Connection Set cnn = New ADODB.Connection cnn.ConnectionTimeout = 0 Dim strfileCSVParentFolderPath As String strfileCSVParentFolderPath = fileCSV.ParentFolder If Right(strfileCSVParentFolderPath, 1) <> Application.PathSeparator Then strfileCSVParentFolderPath = strfileCSVParentFolderPath & Application.PathSeparator Dim strConn As String If boolHeadersPresent = False Then strConn = "Provider=Microsoft.Jet.OLEDB.4.0; Data Source=" & strfileCSVParentFolderPath & ";Extended Properties=""text;HDR=NO;FMT=Delimited""" Else strConn = "Provider=Microsoft.Jet.OLEDB.4.0; Data Source=" & strfileCSVParentFolderPath & ";Extended Properties=""text;HDR=YES;FMT=Delimited""" End If If strConn <> vbNullString Then On Error GoTo ErrorHandler Dim lngRetryCount As Long lngRetryCount = 0 cnn.Open strConn On Error GoTo 0 Set OpenConnectionToCSV = cnn End If Exit Function ErrorHandler: Select Case True Case InStr(1, Err.Description, "Connect timeout occurred", vbTextCompare) > 0 If lngRetryCount < 30 Then Application.Wait DateAdd("s", 1, Now) lngRetryCount = lngRetryCount + 1 Resume Else MsgBox "Can't connect to " & fileCSV.Path & ". Reading this file will be skipped." Exit Function End If Case Else MsgBox "Getting data from " & fileCSV.Name & " has failed with the following error message: " & Err.Number & ": " & Err.Description On Error GoTo 0 Resume End Select End Function QueryDataFromCSV Private Function QueryDataFromCSV(ByVal cnn As ADODB.Connection, ByVal strQuery As String, ByVal strCSVName As String, ByVal strFinalReportTitle As String) As Boolean QueryDataFromCSV = True Dim cmd As ADODB.Command Set cmd = PrepareQueryCommand(cnn, strQuery) CreateQueryDebugLog cmd.CommandText Dim rst As ADODB.Recordset Set rst = New ADODB.Recordset rst.Open cmd Dim Loop1 As Long With rst For Loop1 = 1 To .Fields.Count If .Fields(Loop1 - 1).Name = "F" & Loop1 Then If Loop1 < 4 Then MsgBox "Can't retrieve data from " & strCSVName & " because it is formatted improperly." Else MsgBox "Can't retrieve data from " & strCSVName & " because it is delimited improperly. The file is most likely delimited with a comma even though it has addresses or other fields that contain commas. Ask Encounters IT to change this report's delimiter to another character, such as | (pipe), in the Tidal batch file." End If QueryDataFromCSV = False Exit Function End If Next Loop1 End With CopyThisCSVRecordsetToResultSheets rst, strFinalReportTitle cnn.Close Set rst = Nothing Set cmd = Nothing Set cnn = Nothing End Function The error is occurring at rst.Open cmd in the above function QueryDataFromCSV Illustrating schema.ini creation for #Comintern: GetRevisedQueryWithFileAliasesReplacedWithTrueFileNames Private Function GetRevisedQueryWithFileAliasesReplacedWithTrueFileNames(ByVal strQuery As String, ByVal arrSourceReports As Variant) As String Dim FSO As Scripting.FileSystemObject Set FSO = New Scripting.FileSystemObject Dim lngPosition As Long lngPosition = 0 Do Until lngPosition > Len(strQuery) Dim lngStartPosition As Long lngStartPosition = InStr(lngPosition + 1, strQuery, "from", vbTextCompare) + 5 If lngStartPosition > lngPosition Then Dim lngEndPosition As Long lngEndPosition = InStr(lngStartPosition + 1, strQuery, " ", vbTextCompare) If lngEndPosition = 0 Then lngEndPosition = Len(strQuery) + 1 Dim strSourceReportTitle As String strSourceReportTitle = Mid(strQuery, lngStartPosition, lngEndPosition - lngStartPosition) Dim Loop2 As Long For Loop2 = LBound(arrSourceReports, 1) To UBound(arrSourceReports, 1) If arrSourceReports(Loop2, 1) = strSourceReportTitle Then Exit For Next Loop2 Dim fileSource As Scripting.File Set fileSource = FSO.GetFile(arrSourceReports(Loop2, 3)) If arrSourceReports(Loop2, 2) = "TAB" Then arrSourceReports(Loop2, 2) = Chr(9) CreateSchemaIni fileSource, arrSourceReports(Loop2, 2) Dim strRevisedQuery As String If strRevisedQuery = vbNullString Then strRevisedQuery = Replace(strQuery, "from " & strSourceReportTitle, "from " & fileSource.Name) Else strRevisedQuery = Replace(strRevisedQuery, "from " & strSourceReportTitle, "from " & fileSource.Name) End If lngPosition = lngEndPosition Else lngPosition = Len(strQuery) + 1 End If Loop GetRevisedQueryWithFileAliasesReplacedWithTrueFileNames = strRevisedQuery End Function CreateSchemaIni Private Sub CreateSchemaIni(ByVal fileReport As Scripting.File, ByVal strDelimiter As String) Dim intSystemFileNumber As Integer intSystemFileNumber = FreeFile() On Error GoTo ErrorHandler Open fileReport.ParentFolder.Path & Application.PathSeparator & "Schema.ini" For Output As #intSystemFileNumber Print #intSystemFileNumber, "[" & fileReport.Name & "]" Print #intSystemFileNumber, "Format=Delimited(" & strDelimiter & ")" Close #intSystemFileNumber Exit Sub ErrorHandler: Select Case True Case InStr(1, Err.Description, "Path/File Access Error", vbTextCompare) > 0 Dim strStandardQueryDebugLogPath As String strStandardQueryDebugLogPath = fileReport.ParentFolder.Path & Application.PathSeparator & "strQuery.txt" MsgBox strStandardQueryDebugLogPath & " was inaccessible. Creating log in same folder where your copy of the Mass Queryer is saved instead." Open Left(ThisWorkbook.Path, InStrRev(ThisWorkbook.Path, Application.PathSeparator, , vbTextCompare)) & "strQuery.txt" For Output As #intSystemFileNumber Print #intSystemFileNumber, "[" & fileReport.Name & "]" Print #intSystemFileNumber, "Format=Delimited(" & strDelimiter & ")" Close #intSystemFileNumber Exit Sub Case Else MsgBox "Creating a query debug log has failed with the following error message: " & Err.Number & ": " & Err.Description On Error GoTo 0 Resume End Select End Sub
With #Comintern's help, I was able to see that I made a silly mistake having nothing to do with the question title in actuality. You can see above that my CreateSchemaIni method was creating and then overwriting the Schema.ini file for each csv I was querying rather than creating and then appending to it. By changing that method to use Open For Append instead of Open For Output, the problem was solved.
Exporting powerpivot data to csv
I have an Excel workbook with powerpivot data in Excel data model. I don't have the file used to import the data into powerpivot. My goal is to get the data out of powerpivot to a csv so I can use it in some other software. I can't find any direct export options in powerpivot and since the data is larger than 1.1M rows it can't be pushed into Excel. I found this VBA which appears to work for smaller files but for larger ones I get a timeout error. Option Explicit Public Sub ExportToCsv() Dim wbTarget As Workbook Dim ws As Worksheet Dim rs As Object Dim sQuery As String 'Suppress alerts and screen updates With Application .ScreenUpdating = False .DisplayAlerts = False End With 'Bind to active workbook Set wbTarget = ActiveWorkbook Err.Clear On Error GoTo ErrHandler 'Make sure the model is loaded wbTarget.Model.Initialize 'Send query to the model sQuery = "EVALUATE 'combine 2010 - Q2 2015'" Set rs = CreateObject("ADODB.Recordset") rs.Open sQuery, wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection Dim CSVData As String CSVData = RecordsetToCSV(rs, True) 'Write to file Open "D:\tempMyFileName.csv" For Binary Access Write As #1 Put #1, , CSVData Close #1 rs.Close Set rs = Nothing ExitPoint: With Application .ScreenUpdating = True .DisplayAlerts = True End With Set rs = Nothing Exit Sub ErrHandler: MsgBox "An error occured - " & Err.Description, vbOKOnly Resume ExitPoint End Sub Public Function RecordsetToCSV(rsData As ADODB.Recordset, _ Optional ShowColumnNames As Boolean = True, _ Optional NULLStr As String = "") As String 'Function returns a string to be saved as .CSV file 'Option: save column titles Dim K As Long, RetStr As String If ShowColumnNames Then For K = 0 To rsData.Fields.Count - 1 RetStr = RetStr & ",""" & rsData.Fields(K).Name & """" Next K RetStr = Mid(RetStr, 2) & vbNewLine End If RetStr = RetStr & """" & rsData.GetString(adClipString, -1, """,""", """" & vbNewLine & """", NULLStr) RetStr = Left(RetStr, Len(RetStr) - 3) RecordsetToCSV = RetStr End Function
This seems to work without export file size restrictions by doing 1k rows at a time and using FileSystemObject. You'll need to add Microsoft ActiveX Data Objects Library and Microsoft Scripting Runtime as references. Option Explicit Public FSO As New FileSystemObject Public Sub ExportToCsv() Dim wbTarget As Workbook Dim ws As Worksheet Dim rs As Object Dim sQuery As String 'Suppress alerts and screen updates With Application .ScreenUpdating = False .DisplayAlerts = False End With 'Bind to active workbook Set wbTarget = ActiveWorkbook Err.Clear On Error GoTo ErrHandler 'Make sure the model is loaded wbTarget.Model.Initialize 'Send query to the model sQuery = "EVALUATE <Query>" Set rs = CreateObject("ADODB.Recordset") rs.Open sQuery, wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection Dim CSVData As String Call WriteRecordsetToCSV(rs, "<ExportPath>", True) rs.Close Set rs = Nothing ExitPoint: With Application .ScreenUpdating = True .DisplayAlerts = True End With Set rs = Nothing Exit Sub ErrHandler: MsgBox "An error occured - " & Err.Description, vbOKOnly Resume ExitPoint End Sub Public Sub WriteRecordsetToCSV(rsData As ADODB.Recordset, _ FileName As String, _ Optional ShowColumnNames As Boolean = True, _ Optional NULLStr As String = "") 'Function returns a string to be saved as .CSV file 'Option: save column titles Dim TxtStr As TextStream Dim K As Long, CSVData As String 'Open file Set TxtStr = FSO.CreateTextFile(FileName, True, True) If ShowColumnNames Then For K = 0 To rsData.Fields.Count - 1 CSVData = CSVData & ",""" & rsData.Fields(K).Name & """" Next K CSVData = Mid(CSVData, 2) & vbNewLine TxtStr.Write CSVData End If Do While rsData.EOF = False CSVData = """" & rsData.GetString(adClipString, 1000, """,""", """" & vbNewLine & """", NULLStr) CSVData = Left(CSVData, Len(CSVData) - IIf(rsData.EOF, 3, 2)) TxtStr.Write CSVData Loop TxtStr.Close End Sub
Rip 20 million rows from Power Pivot ("Item.data")
I received a workbook which contains two tables in power-pivot (one around one million rows, another 20 mill rows). I would like to rip this out (as anything really - but let's say a CSV) so that I can use it in R + PostGreSQL. I can't export to an Excel table as there are more than 1 million rows; and copy-pasting the data only works when I select around 200,000 rows. I tried converting the xlsx into a zip and opening the "item.data" file in notepad++, however it was encrypted. I put together some VBA which works for around 0.5 mill rows: Public Sub CreatePowerPivotDmvInventory() Dim conn As ADODB.Connection Dim sheet As Excel.Worksheet Dim wbTarget As Workbook On Error GoTo FailureOutput Set wbTarget = ActiveWorkbook wbTarget.Model.Initialize Set conn = wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection ' Call function by passing the DMV name ' E.g. Partners WriteDmvContent "Partners", conn MsgBox "Finished" Exit Sub FailureOutput: MsgBox Err.Description End Sub Private Sub WriteDmvContent(ByVal dmvName As String, ByRef conn As ADODB.Connection) Dim rs As ADODB.Recordset Dim mdx As String Dim i As Integer mdx = "EVALUATE " & dmvName Set rs = New ADODB.Recordset rs.ActiveConnection = conn rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic ' Setup CSV file (improve this code) Dim myFile As String myFile = "H:\output_table_" & dmvName & ".csv" Open myFile For Output As #1 ' Output column names For i = 0 To rs.Fields.count - 1 If i = rs.Fields.count - 1 Then Write #1, rs.Fields(i).Name Else Write #1, rs.Fields(i).Name, End If Next i ' Output of the query results Do Until rs.EOF For i = 0 To rs.Fields.count - 1 If i = rs.Fields.count - 1 Then Write #1, rs.Fields(i) Else Write #1, rs.Fields(i), End If Next i rs.MoveNext Loop Close #1 rs.Close Set rs = Nothing Exit Sub FailureOutput: MsgBox Err.Description End Sub
DAX Studio will allow you to query the data model in an Excel workbook and output to various formats, including flat files. The query you'll need is just: EVALUATE <table name>
I have found a working (VBA) solution [but greggy's also works for me too!] -> my table was too big to export in one chunk so I loop over and filter by 'month'. This seems to work and produces a 1.2 gb CSV after I append all together: Function YYYYMM(aDate As Date) YYYYMM = year(aDate) * 100 + month(aDate) End Function Function NextYYYYMM(YYYYMM As Long) If YYYYMM Mod 100 = 12 Then NextYYYYMM = YYYYMM + 100 - 11 Else NextYYYYMM = YYYYMM + 1 End If End Function Public Sub CreatePowerPivotDmvInventory() Dim conn As ADODB.Connection Dim tblname As String Dim wbTarget As Workbook On Error GoTo FailureOutput Set wbTarget = ActiveWorkbook wbTarget.Model.Initialize Set conn = wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection ' Call function by passing the DMV name tblname = "table1" WriteDmvContent tblname, conn MsgBox "Finished" Exit Sub FailureOutput: MsgBox Err.Description End Sub Private Sub WriteDmvContent(ByVal dmvName As String, ByRef conn As ADODB.Connection) Dim rs As ADODB.Recordset Dim mdx As String Dim i As Integer 'If table small enough: 'mdx = "EVALUATE " & dmvName 'Other-wise filter: Dim eval_field As String Dim eval_val As Variant 'Loop through year_month Dim CurrYM As Long, LimYM As Long Dim String_Date As String CurrYM = YYYYMM(#12/1/2000#) LimYM = YYYYMM(#12/1/2015#) Do While CurrYM <= LimYM String_Date = CStr(Left(CurrYM, 4)) + "-" + CStr(Right(CurrYM, 2)) Debug.Print String_Date eval_field = "yearmonth" eval_val = String_Date mdx = "EVALUATE(CALCULATETABLE(" & dmvName & ", " & dmvName & "[" & eval_field & "] = """ & eval_val & """))" Debug.Print (mdx) Set rs = New ADODB.Recordset rs.ActiveConnection = conn rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic ' Setup CSV file (improve this code) Dim myFile As String myFile = "H:\vba_tbl_" & dmvName & "_" & eval_val & ".csv" Debug.Print (myFile) Open myFile For Output As #1 ' Output column names For i = 0 To rs.Fields.count - 1 If i = rs.Fields.count - 1 Then Write #1, """" & rs.Fields(i).Name & """" Else Write #1, """" & rs.Fields(i).Name & """", End If Next i ' Output of the query results Do Until rs.EOF For i = 0 To rs.Fields.count - 1 If i = rs.Fields.count - 1 Then Write #1, """" & rs.Fields(i) & """" Else Write #1, """" & rs.Fields(i) & """", End If Next i rs.MoveNext Loop CurrYM = NextYYYYMM(CurrYM) i = i + 1 Close #1 rs.Close Set rs = Nothing Loop Exit Sub FailureOutput: MsgBox Err.Description End Sub
I've modified mptevsion script - now it saves data from the table to separate csv every n rows (100k rows by default, could be changed by changing chunk_size). The advantage of this script is that it doesn't rely on any field in the table to separate the data, to accomplish that it is using TOPNSKIP (https://dax.guide/topnskip/). Public Sub CreatePowerPivotDmvInventory() ActiveWorkbook.Model.Initialize Dim save_path As String Dim chunk_size As Long save_path = "H:\power pivot\csv" tblName = "data" chunk_size = 100000 Dim rs As ADODB.Recordset Dim mdx As String Dim i As Long Dim rows_limit As Long Dim rows_left As Long Dim conn As ADODB.Connection Set conn = ActiveWorkbook.Model.DataModelConnection.ModelConnection.ADOConnection ' calculating number of rows in a table mdx = "evaluate {COUNTROWS('" & tblName & "')}" Set rs = New ADODB.Recordset rs.ActiveConnection = conn rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic rows_limit = rs.Fields(0) rows_left = rows_limit chunk_id = 1 Do While rows_left > 0 If rows_left < chunk_size Then chunk_size = rows_left End If mdx = "define var data_table = '" & tblName & "'" & Chr(10) & _ "EVALUATE(" & Chr(10) & _ " TOPNSKIP(" & chunk_size & ", " & rows_limit - rows_left & ", data_table)" & Chr(10) & _ ");" Debug.Print (mdx) Set rs = New ADODB.Recordset rs.ActiveConnection = conn rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic ' Setup CSV file (improve this code) Dim myFile As String myFile = save_path & "\vba_tbl_" & tblName & "_" & chunk_id & ".csv" Debug.Print (myFile) Open myFile For Output As #1 ' Output column names For i = 0 To rs.Fields.Count - 1 If i = rs.Fields.Count - 1 Then Write #1, """" & rs.Fields(i).Name & """" Else Write #1, """" & rs.Fields(i).Name & """", End If Next i ' Output of the query results Do Until rs.EOF For i = 0 To rs.Fields.Count - 1 If i = rs.Fields.Count - 1 Then Write #1, """" & rs.Fields(i) & """" Else Write #1, """" & rs.Fields(i) & """", End If Next i rs.MoveNext Loop rows_left = rows_left - chunk_size chunk_id = chunk_id + 1 Close #1 rs.Close Set rs = Nothing Loop MsgBox "Finished" Exit Sub FailureOutput: MsgBox Err.Description End Sub