Speeding up a ms access sql query from excel - sql

I have the code below and it seems to be taking a while to open the recordset and run the query attached (62 seconds to be exact). While 1 minute is fine, when I need to do this 13 times, it begins to take a long time to run the code.
I've debugged the code down to just the opening of the recordset taking the longest time.
My question is: Is there a method to run this faster? (i'm connecting to MS Access 2013 from Excel 2013)
Thanks in advance,
Rich
Sub GetUnits2()
'Declaring the necessary variables.
Dim con As Object
Dim rs As Object
Dim AccessFile As String
Dim strTable As String
Dim SQL As String
Dim myValues() As Variant
Dim i As Long
Dim k As Long
Dim j As Integer
Dim SheetName As String
Dim WeekNumber As Long
Dim year As Long
Dim Model1 As String
Dim Model2 As String
Dim xlrow As Integer
Dim xlcol As Integer
SheetName = "Sheet2"
Sheets(SheetName).Select
Model1 = Sheets(SheetName).Cells(3, 2).Value
Model2 = Sheets(SheetName).Cells(4, 2).Value
'Disable screen flickering.
Application.ScreenUpdating = False
'Specify the file path of the accdb file. You can also use the full path of the file like:
AccessFile = "C:\Users\rich.wolff\Desktop\2014POSDatabase\HMKPOSDatabase2014.accdb"
On Error Resume Next
'Create the ADODB connection object.
Set con = CreateObject("ADODB.connection")
'Check if the object was created.
If Err.Number <> 0 Then
MsgBox "Connection was not created!", vbCritical, "Connection error"
Exit Sub
End If
On Error GoTo 0
'Open the connection.
con.Open "Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" & AccessFile
'Set Current Week, Year, & Starting Cell
WeekNumber = Sheets(SheetName).Cells(8, 14).Value
year = Sheets(SheetName).Cells(9, 14).Value
xlcol = 14 'Starting Column
xlrow = 11 'Starting Row
'Open Query Loop
For k = 1 To 1
SQL = "SELECT Sum(StoreSalesData.QTY) AS Units"
SQL = SQL & " FROM VSNConversionData INNER JOIN ([Sleepys Store List] INNER JOIN StoreSalesData ON [Sleepys Store List].[Store Code] = StoreSalesData.STR) ON VSNConversionData.VSN = StoreSalesData.VSN"
SQL = SQL & " WHERE (((VSNConversionData.VSNStyle)='" & Model2 & "') AND ((StoreSalesData.WeekNum)=" & WeekNumber & ") AND ((StoreSalesData.Year)=" & year & ") AND ((StoreSalesData.STR) In (SELECT FloorModels2.[Source Org]"
SQL = SQL & " FROM FloorModels2"
SQL = SQL & " WHERE (((FloorModels2.[Source Org]) In (SELECT FloorModels2.[Source Org]"
SQL = SQL & " FROM FloorModels2"
SQL = SQL & " WHERE (((FloorModels2.WeekNumber)=" & WeekNumber & ") AND ((FloorModels2.Year)=" & year & ") AND ((FloorModels2.VSNStyle)='" & Model1 & "')))) AND ((FloorModels2.WeekNumber)=" & WeekNumber & ") AND ((FloorModels2.Year)=" & year & ") AND ((FloorModels2.VSNStyle)='" & Model2 & "')))));"
On Error Resume Next
'Create the ADODB recordset object.
Set rs = CreateObject("ADODB.recordset")
'Check if the object was created.
If Err.Number <> 0 Then
Set rs = Nothing
Set con = Nothing
MsgBox "Connection was not created!", vbCritical, "Connection error"
Exit Sub
End If
On Error GoTo 0
'Set thee cursor location.
rs.CursorLocation = 3 'adUseClient on early binding
rs.CursorType = 1 'adOpenKeyset on early binding
'Open the recordset.
rs.Open SQL, con
'Redim the table that will contain the filtered data.
ReDim myValues(rs.RecordCount)
If Not (rs.EOF And rs.BOF) Then
rs.MoveFirst
Dim dbcol As Integer
dbcol = 0
Worksheets(SheetName).Cells(xlrow, xlcol).ClearContents
Worksheets(SheetName).Cells(xlrow, xlcol).Value = rs(dbcol).Value
Else
rs.Close
con.Close
Set rs = Nothing
Set con = Nothing
Application.ScreenUpdating = True
MsgBox "There are no records in the recordset!", vbCritical, "No Records"
Exit Sub
End If
'Close the recordet
rs.Close
Set rs = Nothing
If WeekNumber = 1 Then
year = year - 1
WeekNumber = 52
Else
year = year
WeekNumber = WeekNumber - 1
End If
' Next Column
xlcol = xlcol - 1
Next
'End Query Loop
con.Close
Set rs = Nothing
Set con = Nothing
Application.ScreenUpdating = True
End Sub

Have I wandered accidentally into a PHP forum?
Declare the ADODB libraries using tools:references - they will run faster, you get intellisense and a listing of all the available properties and options in the Object Browser, and you gain the ability to run the query asynchronously.
That's Early-Binding, an improvement on Late-Binding.
Next, open the Recordset object with dbForwardOnly (slightly faster) and dump it into a VBA array variant with the Recordset.GetRows method: transpose the array in your code, and write it to the range.
I can see that you've made progress on optmising the SQL: try saving it as a parameter query in the database. The ADODB.Command object can open a named query, populate the parameters, and return a recordset - the query itself may or may not run faster, but the lead time to parse the SQL will be significantly faster.

You could try:
Sub M_snb()
c00 = "C:\Users\rich.wolff\Desktop\2014POSDatabase\HMKPOSDatabase2014.accdb"
With Sheets("sheet2")
sn = Array(.Cells(3, 2), .Cells(4, 2), .Cells(8, 14), .Cells(9, 14)) ' model 1, model 2, weeknumber, year
End With
For j = 1 To 13
c01 = "SELECT Sum(StoreSalesData.QTY) AS Units"
c01 = c01 & " FROM VSNConversionData INNER JOIN ([Sleepys Store List] INNER JOIN StoreSalesData ON [Sleepys Store List].[Store Code] = StoreSalesData.STR) ON VSNConversionData.VSN = StoreSalesData.VSN"
c01 = c01 & " WHERE (((VSNConversionData.VSNStyle)='" & sn(1) & "') AND ((StoreSalesData.WeekNum)=" & sn(2) & ") AND ((StoreSalesData.Year)=" & sn(3) & ") AND ((StoreSalesData.STR) In (SELECT FloorModels2.[Source Org]"
c01 = c01 & " FROM FloorModels2"
c01 = c01 & " WHERE (((FloorModels2.[Source Org]) In (SELECT FloorModels2.[Source Org]"
c01 = c01 & " FROM FloorModels2"
c01 = c01 & " WHERE (((FloorModels2.WeekNumber)=" & sn(2) & ") AND ((FloorModels2.Year)=" & sn(3) & ") AND ((FloorModels2.VSNStyle)='" & sn(0) & "')))) AND ((FloorModels2.WeekNumber)=" & sn(2) & ") AND ((FloorModels2.Year)=" & sn(3) & ") AND ((FloorModels2.VSNStyle)='" & sn(1) & "')))));"
With CreateObject("ADODB.recordset")
.Open c01, "Provider=Microsoft.Jet.OLEDB.12.0;Data Source=" & c00
Sheets("sheets2").Cells(11, 14 + j).CopyFromRecordset .DataSource
End With
Next
End Sub

Related

VBA - Executing ADODB.CommandText

I promised myself I would not post this because I have this delusional thought that I am too good of a programmer, yet here we are.
I have altered what I posted earlier last week trying to figure out how to write a VBA function that would write data from an Excel Range to an MS SQL Table. That worked.
Towards the end of the program, I do not know how to construct the final execution of the code; I have tried everything from using the Command.Text in the upper levels, setting it to a Recordset, then executing the recordset, but nothing will make the little VBA troll happy. Here is what I currently have written:
Sub Connection()
Dim Tbl As String
Dim InsertQuery As New ADODB.Command
InsertQuery.CommandType = adCmdText
Dim xlRow As Long, xlCol As Integer
Dim DBconnection As New ADODB.Connection
Dim ConnString As String
Dim rst As New ADODB.Recordset
Dim a As Integer, sFieldName As String
Dim db As DAO.Database
Dim CurrentDb As Database
Dim ConnectionStr
ConnectionStr = "Provider=sqloledb;Server="";Inital Catalog="";Integrated Security=SSPI;User ID="";Password="""
DBconnection.Open ConnectionStr
xlRow = 1 'only one row being used *as of now*, and that is the top row in the excel sheet
xlCol = 119 'First column of misc. data
While Cells(xlRow, xlCol) <> ""
If LH = True Then
Tbl = "Info.CaseBLH"
InsertQuery.CommandText = "INSERT INTO " & Tbl & " VALUES('"
ElseIf RH = True Then
Tbl = "Info.CaseBRH"
InsertQuery.CommandText = "INSERT INTO " & Tbl & " VALUES('"
Else
MsgBox ("No available sheets")
'Application.Quit
End If
NK21Data.TableDefs(Tbl).Fields.Count
For a = 1 To Fields.Count - 1
'For xlCol = 119 To 230 'columns DO1 to HV1
Fields.Item(a) = Replace(Cells(xlRow, xlCol), "'", "''") & "', '" 'Includes mitigation for apostrophes in the data
If Cells(xlRow, xlCol) = "" Then
rst.Fields.Item(a) = "NULL"
End If
xlCol = xlCol + 1
Next a
a = a + 1
Fields.Item(a) = (Format(Now(), "M/D/YYYY") & "')" & vbCrLf)
Wend
'On Error GoTo ErrorHandler
DBconnection.Execute (InsertQuery.CommandText)
DBconnection.Close
Set DBconnection = Nothing
ErrorHandler:
If Err.Number <> 0 Then
Msg = "Error # " & Str(Err.Number) & " was generated by " _
& Err.Source & Chr(13) & "Error Line: " & Erl & Chr(13) & Err.Description
MsgBox Msg, , "Error", Err.HelpFile, Err.HelpContext
End If
End Sub
The error I get is:
Command text was not set for the command object.
This error occurs at:
DBconnection.Execute (InsertQuery.CommandText)
If I try using the following:
InsertQuery = DBconnection.Execute
I will get the following error:
Argument not optional
I've been at this for about (give or take) three days and I'm now having nightmares about it so if someone can help me figure out what to do for this I would greatly appreciate it.
I fixed up and cleaned the code from my earlier answer, tested it to work:
Here's the code:
Option Explicit
Sub DoItThen()
Dim i As Integer, sqlIns As String, sqlVals As String
Dim InsertQuery As New ADODB.Command
Dim firstRow As Long, firstCol As Integer, lastCol As Integer, currRow As Integer
Dim DBconnection As New ADODB.Connection
Dim ConnString As String
ConnString = "Provider=SQLOLEDB.1;Integrated Security=SSPI;Persist Security Info=False;Initial Catalog=Example;Data Source=MYMACHINENAME"
DBconnection.Open ConnString
InsertQuery.ActiveConnection = DBconnection
InsertQuery.CommandType = adCmdText
''build the command text side by side, named columns and values with param placeholders
sqlIns = "INSERT INTO person("
sqlVals = " VALUES("
''i could work these out by scanning the sheet i suppose. hardcoded for now
firstRow = 2
firstCol = 3
lastCol = 5
''generate the SQL - its this that lets the column names come in any order in the sheet
For i = firstCol To lastCol
sqlIns = sqlIns & Cells(firstRow, i) & ","
sqlVals = sqlVals & "?,"
InsertQuery.Parameters.Append InsertQuery.CreateParameter("p" & i - firstCol, adVarChar, adParamInput, 255)
Next i
''chop off the extra trailing commas and form a syntax correct command
InsertQuery.CommandText = Left$(sqlIns, Len(sqlIns) - 1) & ")" & Left$(sqlVals, Len(sqlVals) - 1) & ")"
''iterate the data part of the sheet and execute the query repeatedlty
currRow = firstRow + 1
While Cells(currRow, firstCol) <> ""
For i = firstCol To lastCol
InsertQuery.Parameters("p" & i - firstCol).Value = Cells(currRow, i)
Next i
InsertQuery.Execute , , adExecuteNoRecords ''dont return a resultset
currRow = currRow + 1
Wend
DBconnection.Close
Set DBconnection = Nothing
ErrorHandler:
If Err.Number <> 0 Then
MsgBox Err.Description
End If
End Sub
It takes the first row as the names of the columns in the db table - any order is fine
It builds a command and populates the parameters
It repeatedly fills the values and executes the query, populating the table
OK; don't shoot me - I'm no VBA whizz but I'm saying you should strive to make your code more like this:
Sub DoItThen()
Dim a As Integer, sql as String
Dim InsertQuery As New ADODB.Command
Dim xlRow As Long, xlCol As Integer
Dim DBconnection As New ADODB.Connection
Dim ConnString As String
ConnString = "Provider=sqloledb;Server="";Inital Catalog="";Integrated Security=SSPI;User ID="";Password="""
DBconnection.Open ConnString
InsertQuery.ActiveConnection = conn
InsertQuery.CommandType = adCmdText
If LH = True Then
sql = "INSERT INTO Info.CaseBLH VALUES(#p1"
ElseIf RH = True Then
sql = "INSERT INTO Info.CaseBRH VALUES(#p1"
Else
MsgBox ("No available sheets")
'Application.Quit
End If
''does this do anything? I don't know
NK21Data.TableDefs(Tbl).Fields.Count
''let us add some placeholders to the command: we add count-2 because we already have one ? in the command
''ps; don't know where you got fields.count from
For a = 2 To Fields.Count - 1
sql = sql & ",#p" & a
Next a
''finish off our command
InsertQuery.CommandText = sql & ")"
''now we have a command like INSERT INTO tbl VALUES(#p1, #p2, #p3.."
''and setting the command text might pre-populate the parameters collection
''with the same number of parameters as are in the command, so let's clear it and
''add the parameters again ourselves so we can control the type
InsertQuery.Parameters.Clear
''create a load of parameters
For a = 1 To Fields.Count - 1
InsertQuery.Parameters.Append InsertQuery.CreateParameter("#p" & a, adVarChar, adParamInput, 255) 'adjust if you have strings longer than 255
Next a
''Now all the parameters are set etc, we just go through all the rows,
''and all the columns and set the values, then execute the command, then change the values and execute again
''--> set the command up once and repeatedly execute it
xlRow = 1 'only one row being used *as of now*, and that is the top row in the excel sheet
xlCol = 119 'First column of misc. data
While Cells(xlRow, xlCol) <> ""
For a = 1 To Fields.Count - 1
InsertQuery.Parameters("#p" & a).Value = Cells(xlRow, xlCol + a)
Next a
InsertQuery.Execute , , adExecuteNoRecords ''dont return a resultset
Wend
DBconnection.Close
Set DBconnection = Nothing
ErrorHandler:
If Err.Number <> 0 Then
Msg = "Error # " & Str(Err.Number) & " was generated by " _
& Err.Source & Chr(13) & "Error Line: " & Erl & Chr(13) & Err.Description
MsgBox Msg, , "Error", Err.HelpFile, Err.HelpContext
End If
End Sub
You have 100 columns and 1000 rows to insert from an excel sheet:
You set up the command, INSERT INTO ... VALUES (... 100 #parameter placeholders...)
You clear the parameters collection (in case VBA has decided to 'help' by creating them) and add a load of parameters that represent your strings in your sheet
You then iterate the sheet, row by row, setting each column value on the row, into the relevant parameter and after you set all the columns (100 times), you execute the command then move to the next row, set the values again and execute again (1000 times)
I've got absolutely no way of testing this, sorry - it's my best guess but I fully expect it still has some error because I don't really know where you got Fields from. The answer with 8 votes from here was helpful: VBA, ADO.Connection and query parameters - I distinctly recall from when I was using VB6 about 20 years ago, that ADODB would try and prepopulate the parameters collection in certain circumstances, with its guesses at the parameter types; we routinely cleared it and added our own, but you might have some success proceeding with the default parameters it makes
The names of the parameters are not relevant; only the position. There's no requirement that #p1 from the query string matches the #p1 name given for the parameter - if the first parameter in the string were called #bob and you then cleared and added a parameter named #alice, whatever #alice's value was would be assigned to #bob because #bob is first in the query and #alice is first in the parameters collection. I used #pXXX as a parameter name for ease of reference in both cases
Here is my basic ADODB Execute template. This isn't meant to be an answer but more a helpful post. It should assist in showing you what you're doing incorrectly, which appears to be simple syntax issues as well as being really new to this (formatting and other pieces of code suggest that maybe you've gotten yourself "googled into a corner.").
Private Sub ADODBExample()
Dim vbSql As String, cnnstr as string
Dim cnn As ADODB.Connection
vbSql = "sql statement ;"
Set cnn = New Connection
cnnstr = "Provider=SQLOLEDB;Data Source=SERVERNAME;Initial Catalog=DBNAME;User ID=USERID;Password=PASSWORD; Trusted_Connection=No"
cnn.Open cnnstr
cnn.Execute vbSql
cnn.Close
Set cnn = Nothing
End Sub
More helpful tips -
Stop looping through cells, ranges and other worksheet/book objects. Learn to use arrays - itll make processing way better.
Simplicity is best. You appear to doing what I consider alot of unnecessary things, but then again I dont know all the requirements.
So I amended the code to the following:
Sub Connection()
Dim i As Integer, sqlIns As String, sqlVals As String
Dim InsertQuery As New ADODB.Command
Dim firstRow As Long, firstCol As Integer, lastCol As Integer, currRow As Integer
Dim DBconnection As New ADODB.Connection
Dim ConnString As String
Dim Tbl As String
ConnString = "Provider=sqloledb;Server=SERVER;Inital Catalog=DB;Integrated Security=SSPI;User ID=ID;Password=PW;"
DBconnection.Open ConnString
InsertQuery.ActiveConnection = DBconnection
InsertQuery.CommandType = adCmdText
If LH = True Then
Tbl = "Info.CaseBLH"
sqlIns = "INSERT INTO Info.CaseBLH("
ElseIf RH = True Then
Tbl = "Info.CaseBRH"
sqlIns = "INSERT INTO Info.CaseBRH("
Else
MsgBox ("No available sheets")
'Application.Quit
End If
''build the command text side by side, named columns and values with param placeholders
sqlVals = " VALUES("
''i could work these out by scanning the sheet i suppose. hardcoded for now
firstRow = 1
firstCol = 119
lastCol = 231
''generate the SQL - its this that lets the column names come in any order in the sheet
For i = firstCol To lastCol
sqlIns = sqlIns & Cells(firstRow, i) & ","
sqlVals = sqlVals & "?,"
InsertQuery.Parameters.Append InsertQuery.CreateParameter("p" & i - firstCol, adVarChar, adParamInput, 255)
Next i
''chop off the extra trailing commas and form a syntax correct command
InsertQuery.CommandText = Left$(sqlIns, Len(sqlIns) - 1) & ")" & Left$(sqlVals, Len(sqlVals) - 1) & ")"
''iterate the data part of the sheet and execute the query repeatedlty
currRow = firstRow ' - not needed as the data is automatically replaced with the code above
While Cells(currRow, firstCol) <> ""
For i = firstCol To lastCol - 1
InsertQuery.Parameters("p" & i - firstCol).Value = Cells(currRow, i)
Next i
InsertQuery.Execute , , adExecuteNoRecords ''dont return a resultset
Wend
DBconnection.Close
Set DBconnection = Nothing
ErrorHandler:
If Err.Number <> 0 Then
MsgBox Err.Description
End If
End Sub
Right at the
InsertQuery.Execute , , adExecuteNoRecords
Line I'm getting a error telling me there is a syntax error around the ':' which doesn't make any sense to me. If I append my code to send the error to the error handler, every single row it cycles through throws me an error saying there is a syntax error around '-' or '/'. I think it has something to do with the parameter.value line.

"Automation Error" "Unspecified Error" When Using Excel-VBA and ADODB Objects

I am creating a macro that looks through a folder, performs a SQL operation on each of the documents, and copies the results into another workbook; however, every time I debug and hit the Set rs = cn.Execute(sql) line I get an
"Automation Error" "Unspecified Error" .
The even weirder thing is that when I just run the code, I get an
"Execute of object _Connection failed" error.
I have tested the SQL code in microsoft SQL server already and each of these statements are almost verbatim with code I have previously got to work.
Option Explicit
Sub hardnessTests()
On Error GoTo ErrorHandling
Dim filename As Variant, n As Long
n = 0
Call turnOff
filename = Dir("T:\Marketing\Data Analytics\GIS Data\Water Quality Portal Data\County Summary Results\*")
While filename <> ""
Dim file As String, cn As Object, rs As Object, sql As String, hardField As ADODB.Field
file = "T:\Marketing\Data Analytics\GIS Data\Water Quality Portal Data\County Summary Results\" & filename
Set cn = CreateObject("ADODB.Connection")
With cn
.Provider = "Microsoft.ACE.OLEDB.12.0"
.connectionstring = "Data Source=" & file & ";Extended Properties=""Excel 12.0 Xml;HDR=YES;Readonly=false;IMEX=0"";"
.Open
End With
sql = "SELECT c.STATEFP, c.COUNTYFP, AVG(CAST(a.ResultMeasureValue AS NUMERIC)) AS Average_Surface_Water_Hardness, COUNT(a.ResultMeasureValue) AS Number_Of_Surface_Water_Tests, " & _
"AVG(CAST(b.ResultMeasureValue AS NUMERIC)) AS Average_Groundwater_Hardness, COUNT(b.ResultMeasureValue) AS Number_Of_Groundwater_Tests " & _
"FROM (SELECT * FROM [Sheet1$] WHERE CAST(LEFT(ActivityStartDate, 4) AS NUMERIC) >= 2010 and ActivityMediaSubdivisionName = 'Surface Water') a, " & _
"(SELECT * FROM [Sheet1$] WHERE CAST(LEFT(ActivityStartDate, 4) AS NUMERIC) >= 2010 AND ActivityMediaSubdivisionName = 'Groundwater') b, " & _
"(SELECT * FROM [Sheet1$] WHERE CAST(LEFT(ActivityStartDate, 4) AS NUMERIC) >= 2010 ) c GROUP BY c.STATEFP, c.COUNTYFP ORDER BY c.STATEFP, c.COUNTYFP"
Set rs = cn.Execute(sql)
Dim wb As Workbook, fieldCount As Long
fieldCount = 0
Set wb = Workbooks.Add
wb.SaveAs "T:\Marketing\Data Analytics\GIS Data\Water Quality Portal Data\County Summary Results\ALL_HARDNESS.xlsx"
If n = 0 Then
wb.Worksheets("Sheet1").Range("A2").CopyFromRecordset rs
Else:
wb.Worksheets("Sheet1").Range("A2").End(xlDown).CopyFromRecordset rs
End If
filename = Dir
n = n + 1
Wend
For Each hardField In rs.Fields
wb.Worksheets("Sheet1").Range("A1").Offset(0, fieldCount) = hardField.Name
fieldCount = fieldCount + 1
Next hardField
Call turnOn
Exit Sub
ErrorHandling:
MsgBox ("Source: " & Err.Source & vbNewLine & "Number: " & Err.Number & vbNewLine & "Description: " & Err.Description & vbNewLine & "Help Context: " & Err.HelpContext)
Done:
End Sub
Private Sub turnOff()
Application.Calculation = xlCalculationManual
Application.ScreenUpdating = False
End Sub
Private Sub turnOn()
Application.Calculation = xlCalculationManual
Application.ScreenUpdating = True
End Sub
Thanks for any help!

Using an existing external data connection to create a recordset

I have a macro that I use to get data from an Access database, pass it into a recordset and then drop it into a worksheet in a crosstab format. Currently all my data starts in a SQL Server, gets pulled into Access, and then my macro takes it from there.
I’m trying to cut Access out of the process. What I need is the code to point at an external data source rather than to an Access mdb, which results in me getting the same recordset for the rest of the macro to process. My whole code is below; I’ve marked the part I’m looking to change.
' Gets the prior incurred claims estimates data from the Access database
' "RestatedIncurredClaims.mdb" in the same folder as the model, and sets up
' the tables on the Prior_Claims sheet to contain the data.
Public Sub GetPriorClaimsData()
If [MODEL_NAME] = "" Then
Dim modelName As String
modelName = Replace(ThisWorkbook.Name, "ReserveModel_", "")
modelName = Left(modelName, InStr(modelName, ".") - 1)
[MODEL_NAME] = modelName
End If
' WANT TO CHANGE THIS PART
Dim dbPath As String
dbPath = ThisWorkbook.Path & "\RestatedIncurredClaims.mdb"
Application.Calculation = xlCalculationManual
On Error GoTo priorClaimsErr
Application.StatusBar = "Opening prior claims database..."
' Open the database
' Options:=False means non-exclusive, see:
' http://msdn.microsoft.com/en-us/library/office/ff835343.aspx
Dim db As Database
Set db = Workspaces(0).OpenDatabase(Name:=dbPath, _
Options:=False, ReadOnly:=True)
Application.StatusBar = "Getting prior claims data..."
' Execute query to get prior incurred claim estimates for this model only
Dim rs As Recordset
Set rs = db.OpenRecordset( _
"SELECT * FROM [Restated incurred claims] WHERE [model_name] = """ _
& [MODEL_NAME] & """")
' WANT TO LEAVE EVERYTHING ELSE THE SAME
Dim i As Long, numCellsFound As Long
Dim iLOB As Long, iTOS As Long, iReported As Long, iIncurred As Long
numCellsFound = 0
' Create the array that will hold the prior claims data during processing
Dim priorClaimsData() As Variant
ReDim priorClaimsData( _
0 To [PRIOR_CLAIMS_TABLES].Rows.Count - 1, _
0 To [PRIOR_CLAIMS_TABLES].Columns.Count - 1)
If rs.RecordCount > 0 Then
Application.StatusBar = "Clearing prior claims data..."
[PRIOR_CLAIMS_TABLES].ClearContents
Dim lookupLOB As New Dictionary
For i = 1 To [LST_LINES].Cells.Count
lookupLOB([LST_LINES].Cells(i).Value) = i
Next
Dim lookupTOS As New Dictionary
For i = 1 To [LST_TYPES_SHORT].Cells.Count
lookupTOS([LST_TYPES_SHORT].Cells(i).Value) = i
Next
Dim lookupDate As New Dictionary
For i = 1 To [PRIOR_CLAIMS_DATES].Cells.Count
lookupDate([PRIOR_CLAIMS_DATES].Cells(i).Value) = i
Next
rs.MoveFirst
Do Until rs.EOF
If rs.AbsolutePosition Mod 1000 = 0 Then
Application.StatusBar = "Processing prior claims data, row " _
& Format(rs.AbsolutePosition, "#,0") & "..."
End If
iLOB = lookupLOB(CStr(rs!model_lob))
iTOS = lookupTOS(CStr(rs!fnc_ben_typ_cd))
iReported = lookupDate(CStr(rs!acct_perd_yr_mo))
iIncurred = lookupDate(CStr(rs!clm_incr_yr_mo))
If iLOB <> 0 And iTOS <> 0 _
And iReported <> 0 And iIncurred <> 0 Then
iLOB = iLOB - 1
iTOS = iTOS - 1
iReported = iReported - 1
iIncurred = iIncurred - 1
priorClaimsData( _
iLOB * ROWS_PER_LOB + iIncurred, _
iTOS * COLS_PER_TOS + iReported) = rs!rst_incur_clm
numCellsFound = numCellsFound + 1
End If
rs.MoveNext
Loop
[PRIOR_CLAIMS_TABLES].Value = priorClaimsData
End If
If numCellsFound = 0 Then
MsgBox Prompt:="No prior estimates data found for this model (" _
& [MODEL_NAME] & ").", _
Title:="Warning", _
Buttons:=vbExclamation + vbOKOnly
End If
GoTo closeDb
priorClaimsErr:
MsgBox Prompt:="Failed to update the prior claim estimates data:" _
& vbCrLf & vbCrLf & Err.Description, _
Title:="Warning", _
Buttons:=vbExclamation + vbOKOnly
closeDb:
Application.StatusBar = "Closing prior claims database..."
If Not rs Is Nothing Then
rs.Close
Set rs = Nothing
End If
If Not db Is Nothing Then
db.Close
Set db = Nothing
End If
Application.StatusBar = "Recalculating..."
Application.Calculation = xlCalculationAutomatic
Application.StatusBar = False
End Sub
I initially thought that if I established the data connection and had it saved in an .odc file, that referencing that file in vba would be simple. But all I’ve been able to find is code for establishing new data connections directly in vba with a connection string. Is this what I have to do? If so is there a way to do it so that the code will work regardless of the user running it?
I'm using Excel 2010
Thanks
This is an ADO code sample you can use to connect to SQL Server:
You must add a reference to 'Microsoft ActiveX Data Objects 6.1' first
SQLSERVER_CONN_STRING = "Provider=SQLOLEDB.1;Data Source=<server name or IP address>;User ID=<User_id>;Password=<pwd>;Initial Catalog=<initial cat>;"
Dim oConn As ADODB.Connection
Dim rs as ADODB.Recorset
Dim sSQL as String
Set oConn = New ADODB.Connection
oConn.CommandTimeout = 60
oConn.ConnectionTimeout = 30
oConn.Open SQLSERVER_CONN_STRING
Set rs = New ADODB.Recordset
'note that SQL Server query syntax is different!
sSql = "SELECT * FROM [Restated incurred claims] WHERE [model_name] = '" & [MODEL_NAME] & "'")
rs.Open sSQL, oConn, adOpenStatic, adLockOptimistic, adCmdText
If Not rs Is Nothing Then
If rs.State = 1 Then
If rs.RecordCount > 0 Then
<your code here>
end if
End If
End If
If Not rs Is Nothing Then
If rs.State = 1 Then rs.Close
End if
If Not oConn Is Nothing Then
If oConn.State = 1 Then oConn.Close
End if

dbf files in Excel with SQL

Every day i need to make a report for salesman, we have 2 dbf files from witch i automatic want to make the report in Excel. Report from 1 dbf file works perfect, but i don't know how to join 2 dbf files in VBA.
I have to following script:
Option Explicit
Sub ReadDBF()
Dim con As Object
Dim rs As Object
Dim DBFFolder As String
Dim FileName As String
Dim FileName1 As String
Dim sql As String
Dim myValues() As String
Dim i As Integer
Dim j As Integer
Application.ScreenUpdating = False
DBFFolder = ThisWorkbook.Path & "\"
FileName = "project1.dbf"
FileName1 = "project2.dbf"
On Error Resume Next
Set con = CreateObject("ADODB.connection")
If Err.Number <> 0 Then
MsgBox "Connection was not created!", vbCritical, "Connection error"
Exit Sub
End If
On Error GoTo 0
con.Open "Provider=Microsoft.Jet.OLEDB.4.0;Data Source=" & DBFFolder & ";Extended Properties=dBASE IV;"
sql = "SELECT project_id, COUNT(*) AS total, salesman, MAX(date) AS max_date, projectname FROM " & FileName & FileName1 & " where DateValue(datumtijd) = Date() and FileName.project_id = FileName1.project_id " & "group by project_id, salesman"
On Error Resume Next
Set rs = CreateObject("ADODB.recordset")
If Err.Number <> 0 Then
MsgBox "Connection was not created!", vbCritical, "Connection error"
Exit Sub
End If
On Error GoTo 0
rs.CursorLocation = 3
rs.CursorType = 1
rs.Open sql, con
ReDim myValues(rs.RecordCount, 20)
i = 1
If Not (rs.EOF And rs.BOF) Then
'Go to the first record.
rs.MoveFirst
Do Until rs.EOF = True
myValues(i, 1) = rs!project_id
myValues(i, 2) = rs!salesman
myValues(i, 3) = rs!Total
myValues(i, 4) = rs!max_date
myValues(i, 5) = rs!project
rs.MoveNext
i = i + 1
Loop
Else
rs.Close
con.Close
Set rs = Nothing
Set con = Nothing
Application.ScreenUpdating = True
MsgBox "There are no records in the recordset!", vbCritical, "No Records"
Exit Sub
End If
Sheet1.Activate
For i = 1 To UBound(myValues)
For j = 1 To 4
Cells(i + 1, j) = myValues(i, j)
Next j
Next i
rs.Close
con.Close
Set rs = Nothing
Set con = Nothing
Columns("A:D").EntireColumn.AutoFit
Application.ScreenUpdating = True
MsgBox "The values were read from recordset successfully!", vbInformation, "Done"
End Sub
The query doesn't work just by adding the two tables together as the from source. What is it you are trying to actually get? Also, you should never concatenate strings to build a query. They should always be parameterized.
Now, that said, your syntax is simply incorrect. Your variables are referred to as "FileName" and "FileName1", but the underlying TABLES you are querying from are "Project1" and "Project2" respectively. You should learn about aliases to help simplify queries too, and learn proper JOIN syntax.
By listing the tables one after the other with no comma will cause it to fail. Here is a more accurate syntax and formatted for readability. Then I have edited your original query to match the context. You should also always qualify the field names from the respective table so others trying to help know where things are coming from. In the sample below, I have only GUESSED at which table has which columns by using the alias "P1." and "P2." respectively. You will probably need to change them. Also, since "Date" could be interpreted as a reserved word, I have wrapped it in [], but may need to be changed to tick characters (next to number 1) ``
select
P1.Project_ID,
COUNT(*) as Total,
P1.SalesMan,
MAX( P2.[Date] ) as Max_Date,
P1.ProjectName
from
Project1 P1
JOIN Project2 P2
on P1.Project_ID = P2.Project_ID
where
DateValue( P2.datumtijd ) = date()
group by
P1.Project_ID,
P1.SalesMan
The JOIN clause identifies the relation BETWEEN the two tables on respective columns. The WHERE clause is additional criteria you are looking for.
sql = "SELECT project_id, COUNT(*) AS total, salesman, " & _
"MAX(date) AS max_date, projectname " & _
" FROM " & FileName & " P1 " & _
" JOIN " & FileName1 & " P2 ON P1.Project_ID = P2.Project_ID" & _
" where DateValue(datumtijd) = Date() " & _
" group by project_id, salesman"

Rip 20 million rows from Power Pivot ("Item.data")

I received a workbook which contains two tables in power-pivot (one around one million rows, another 20 mill rows). I would like to rip this out (as anything really - but let's say a CSV) so that I can use it in R + PostGreSQL.
I can't export to an Excel table as there are more than 1 million rows; and copy-pasting the data only works when I select around 200,000 rows.
I tried converting the xlsx into a zip and opening the "item.data" file in notepad++, however it was encrypted.
I put together some VBA which works for around 0.5 mill rows:
Public Sub CreatePowerPivotDmvInventory()
Dim conn As ADODB.Connection
Dim sheet As Excel.Worksheet
Dim wbTarget As Workbook
On Error GoTo FailureOutput
Set wbTarget = ActiveWorkbook
wbTarget.Model.Initialize
Set conn = wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection
' Call function by passing the DMV name
' E.g. Partners
WriteDmvContent "Partners", conn
MsgBox "Finished"
Exit Sub
FailureOutput:
MsgBox Err.Description
End Sub
Private Sub WriteDmvContent(ByVal dmvName As String, ByRef conn As ADODB.Connection)
Dim rs As ADODB.Recordset
Dim mdx As String
Dim i As Integer
mdx = "EVALUATE " & dmvName
Set rs = New ADODB.Recordset
rs.ActiveConnection = conn
rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic
' Setup CSV file (improve this code)
Dim myFile As String
myFile = "H:\output_table_" & dmvName & ".csv"
Open myFile For Output As #1
' Output column names
For i = 0 To rs.Fields.count - 1
If i = rs.Fields.count - 1 Then
Write #1, rs.Fields(i).Name
Else
Write #1, rs.Fields(i).Name,
End If
Next i
' Output of the query results
Do Until rs.EOF
For i = 0 To rs.Fields.count - 1
If i = rs.Fields.count - 1 Then
Write #1, rs.Fields(i)
Else
Write #1, rs.Fields(i),
End If
Next i
rs.MoveNext
Loop
Close #1
rs.Close
Set rs = Nothing
Exit Sub
FailureOutput:
MsgBox Err.Description
End Sub
DAX Studio will allow you to query the data model in an Excel workbook and output to various formats, including flat files.
The query you'll need is just:
EVALUATE
<table name>
I have found a working (VBA) solution [but greggy's also works for me too!] -> my table was too big to export in one chunk so I loop over and filter by 'month'. This seems to work and produces a 1.2 gb CSV after I append all together:
Function YYYYMM(aDate As Date)
YYYYMM = year(aDate) * 100 + month(aDate)
End Function
Function NextYYYYMM(YYYYMM As Long)
If YYYYMM Mod 100 = 12 Then
NextYYYYMM = YYYYMM + 100 - 11
Else
NextYYYYMM = YYYYMM + 1
End If
End Function
Public Sub CreatePowerPivotDmvInventory()
Dim conn As ADODB.Connection
Dim tblname As String
Dim wbTarget As Workbook
On Error GoTo FailureOutput
Set wbTarget = ActiveWorkbook
wbTarget.Model.Initialize
Set conn = wbTarget.Model.DataModelConnection.ModelConnection.ADOConnection
' Call function by passing the DMV name
tblname = "table1"
WriteDmvContent tblname, conn
MsgBox "Finished"
Exit Sub
FailureOutput:
MsgBox Err.Description
End Sub
Private Sub WriteDmvContent(ByVal dmvName As String, ByRef conn As ADODB.Connection)
Dim rs As ADODB.Recordset
Dim mdx As String
Dim i As Integer
'If table small enough:
'mdx = "EVALUATE " & dmvName
'Other-wise filter:
Dim eval_field As String
Dim eval_val As Variant
'Loop through year_month
Dim CurrYM As Long, LimYM As Long
Dim String_Date As String
CurrYM = YYYYMM(#12/1/2000#)
LimYM = YYYYMM(#12/1/2015#)
Do While CurrYM <= LimYM
String_Date = CStr(Left(CurrYM, 4)) + "-" + CStr(Right(CurrYM, 2))
Debug.Print String_Date
eval_field = "yearmonth"
eval_val = String_Date
mdx = "EVALUATE(CALCULATETABLE(" & dmvName & ", " & dmvName & "[" & eval_field & "] = """ & eval_val & """))"
Debug.Print (mdx)
Set rs = New ADODB.Recordset
rs.ActiveConnection = conn
rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic
' Setup CSV file (improve this code)
Dim myFile As String
myFile = "H:\vba_tbl_" & dmvName & "_" & eval_val & ".csv"
Debug.Print (myFile)
Open myFile For Output As #1
' Output column names
For i = 0 To rs.Fields.count - 1
If i = rs.Fields.count - 1 Then
Write #1, """" & rs.Fields(i).Name & """"
Else
Write #1, """" & rs.Fields(i).Name & """",
End If
Next i
' Output of the query results
Do Until rs.EOF
For i = 0 To rs.Fields.count - 1
If i = rs.Fields.count - 1 Then
Write #1, """" & rs.Fields(i) & """"
Else
Write #1, """" & rs.Fields(i) & """",
End If
Next i
rs.MoveNext
Loop
CurrYM = NextYYYYMM(CurrYM)
i = i + 1
Close #1
rs.Close
Set rs = Nothing
Loop
Exit Sub
FailureOutput:
MsgBox Err.Description
End Sub
I've modified mptevsion script - now it saves data from the table to separate csv every n rows (100k rows by default, could be changed by changing chunk_size).
The advantage of this script is that it doesn't rely on any field in the table to separate the data, to accomplish that it is using TOPNSKIP (https://dax.guide/topnskip/).
Public Sub CreatePowerPivotDmvInventory()
ActiveWorkbook.Model.Initialize
Dim save_path As String
Dim chunk_size As Long
save_path = "H:\power pivot\csv"
tblName = "data"
chunk_size = 100000
Dim rs As ADODB.Recordset
Dim mdx As String
Dim i As Long
Dim rows_limit As Long
Dim rows_left As Long
Dim conn As ADODB.Connection
Set conn = ActiveWorkbook.Model.DataModelConnection.ModelConnection.ADOConnection
' calculating number of rows in a table
mdx = "evaluate {COUNTROWS('" & tblName & "')}"
Set rs = New ADODB.Recordset
rs.ActiveConnection = conn
rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic
rows_limit = rs.Fields(0)
rows_left = rows_limit
chunk_id = 1
Do While rows_left > 0
If rows_left < chunk_size Then
chunk_size = rows_left
End If
mdx = "define var data_table = '" & tblName & "'" & Chr(10) & _
"EVALUATE(" & Chr(10) & _
" TOPNSKIP(" & chunk_size & ", " & rows_limit - rows_left & ", data_table)" & Chr(10) & _
");"
Debug.Print (mdx)
Set rs = New ADODB.Recordset
rs.ActiveConnection = conn
rs.Open mdx, conn, adOpenForwardOnly, adLockOptimistic
' Setup CSV file (improve this code)
Dim myFile As String
myFile = save_path & "\vba_tbl_" & tblName & "_" & chunk_id & ".csv"
Debug.Print (myFile)
Open myFile For Output As #1
' Output column names
For i = 0 To rs.Fields.Count - 1
If i = rs.Fields.Count - 1 Then
Write #1, """" & rs.Fields(i).Name & """"
Else
Write #1, """" & rs.Fields(i).Name & """",
End If
Next i
' Output of the query results
Do Until rs.EOF
For i = 0 To rs.Fields.Count - 1
If i = rs.Fields.Count - 1 Then
Write #1, """" & rs.Fields(i) & """"
Else
Write #1, """" & rs.Fields(i) & """",
End If
Next i
rs.MoveNext
Loop
rows_left = rows_left - chunk_size
chunk_id = chunk_id + 1
Close #1
rs.Close
Set rs = Nothing
Loop
MsgBox "Finished"
Exit Sub
FailureOutput:
MsgBox Err.Description
End Sub