Read the Response and stored to database using Microsoft SMO (Server Management Objects) - vb.net

Im trying to call to an URL
And than i will get response into XML format and based what Response i will get it will create Tables successfully into database using Microsoft SMO (Server Management Objects) which Allowing me to create and generate databases and tables. but for some reason it coludnt insert any data to tables and they are NULL Screenshot
XML:
<ICECAT-interface xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://data.icecat.biz/xsd/files.index.xsd">
<files.index Generated="20200219011050">
<file path="export/freexml.int/EN/1424.xml" Product_ID="1424" Updated="20200218012414" Quality="ICECAT" Supplier_id="1" Prod_ID="C4811A" Catid="846" On_Market="1" Model_Name="11" Product_View="513730" HighPic="http://images.icecat.biz/img/gallery/1424_6912543175.jpg" HighPicSize="2478427" HighPicWidth="2598" HighPicHeight="3276" Date_Added="20051023000000" Limited="No" >
<M_Prod_ID Supplier_id="11" Supplier_name="Cisco">C4811A</M_Prod_ID>
<M_Prod_ID>Tempcomp3109</M_Prod_ID>
<Country_Markets>
<Country_Market Value="NL"/>
<Country_Market Value="BE"/>
<Country_Market Value="FR"/>
<Country_Market Value="UA"/>
<Country_Market Value="GB"/>
<Country_Market Value="DE"/>
<Country_Market Value="BG"/>
</Country_Markets>
</file>
</files.index>
</ICECAT-interface>
Module:
Class Module1
Public Shared Sub Main()
Dim url As String = "http://data.Icecat.biz/export/freexml/EN/daily.index.xml"
ProcessXMLFeedURL(url)
End Sub
Public Shared Function ProcessXMLFeedURL(MyURL As String) As Boolean
Dim OK As Boolean = False
Try
Dim rssReq As WebRequest = WebRequest.Create(MyURL)
Dim username As String = ""
Dim password As String = ""
Dim encoded As String = System.Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(username + ":" + password))
rssReq.Headers.Add("Authorization", "Basic " + encoded)
'//Get the WebResponse
Dim rep As WebResponse = rssReq.GetResponse()
'//Read the Response in a XMLTextReader
Dim xtr As XmlTextReader = New XmlTextReader(rep.GetResponseStream())
'// Set up the connection to the SQL server
Dim MyConnectionString As String = "Data Source=......"
Dim Connection As SqlConnection = New SqlConnection(MyConnectionString)
Dim MyServer As Server = New Server(New ServerConnection(Connection))
Dim db As Database = New Database(MyServer, "xxxxx")
db.Create()
'//Create a new DataSet
Dim ds As DataSet = New DataSet()
'//Read the Response into the DataSet
ds.ReadXml(xtr)
'// Parse tables
For i As Integer = 0 To ds.Tables.Count - 1
Dim Mytable As Table
Dim MyTableName As String = ds.Tables(i).TableName
If Not HaveTable(MyConnectionString, MyTableName) Then
'// Create the table
Try
Mytable = New Table(db, MyTableName)
Catch ex As Exception
Dim ii As Integer = 0
End Try
'// Create the columns
Dim Mycolumn As Column = New Column()
For Each dc As DataColumn In ds.Tables(i).Columns
Mycolumn = New Column(Mytable, dc.ColumnName)
Mycolumn.DataType = getdatatype(dc.DataType.ToString)
Mytable.Columns.Add(Mycolumn)
Next
Mytable.Create()
Dim PrimaryKeys() As DataColumn = ds.Tables(i).PrimaryKey
Dim PrimaryKey As DataColumn
For Each PrimaryKey In PrimaryKeys
Dim Myindex As Index = New Index(Mytable, PrimaryKey.ColumnName)
Myindex.IndexKeyType = IndexKeyType.DriPrimaryKey
Myindex.IndexedColumns.Add(New IndexedColumn(Myindex, PrimaryKey.ColumnName))
Mytable.Indexes.Add(Myindex)
Next
End If
Using MyConnection As SqlConnection = New SqlConnection(MyConnectionString)
MyConnection.Open()
Using bulkcopy As SqlBulkCopy = New SqlBulkCopy(MyConnection)
bulkcopy.DestinationTableName = "[" & MyTableName & "]"
Try
bulkcopy.WriteToServer(ds.Tables(i))
Catch ex As Exception
Dim iw As Integer = 0
End Try
End Using
MyConnection.Close()
End Using
Next
Catch ex As Exception
Throw ex '// Do errorhanddling here
End Try
Return OK
End Function
Shared Function getdatatype(Mydatatype As String) As DataType
Dim dty As DataType = Nothing
Select Case Mydatatype
Case Is = "System.Decimal"
dty = DataType.Decimal(2, 18)
Case Is = "System.String"
dty = DataType.VarChar(500)
Case Is = "System.Int32"
dty = DataType.Int
End Select
Return dty
End Function
Shared Function HaveTable(MyConnectionString As String, TableName As String) As Boolean
Dim OK As Boolean = False
Try
Dim dbConn As New SqlConnection(MyConnectionString)
dbConn.Open()
Dim restrictions(3) As String
restrictions(2) = TableName
Dim dbTbl As DataTable = dbConn.GetSchema("Tables", restrictions)
If dbTbl.Rows.Count > 0 Then
OK = True
End If
dbTbl.Dispose()
dbConn.Close()
dbConn.Dispose()
Catch ex As Exception
Dim ss As Integer = 0
End Try
Return OK
End Function
End Class
Can anyone please help me!

Related

oracle bulkcopy from vb.net databale

I am loading the CSV file to vb.net datatable .
then I am using Bulk copy command to write to database.
The problem is if one date cell in the data table is empty, i am receiving
ORA:-01840 input value not long enough for date format.
How to resolve this.
Public Class Form1
Dim cn As New OracleConnection("Data Source =(DESCRIPTION =(ADDRESS = (PROTOCOL = TCP)(HOST = ipaddressofserver)(PORT = portofserver))(CONNECT_DATA =(SERVER = DEDICATED)(SERVICE_NAME = oracleservicename)) ) ; User Id=useridofdb;Password=passwordofdb")
cn.Open()
MsgBox("connection opened")
Dim dt As DataTable = ReadCSV("D:\test.csv")
DataGridView1.DataSource = dt
Try
Dim _bulkCopy As New OracleBulkCopy(cn)
_bulkCopy.DestinationTableName = "TEST_TABLE"
_bulkCopy.BulkCopyTimeout = 10800
_bulkCopy.WriteToServer(dt)
cn.Close()
cn.Dispose()
cn = Nothing
MsgBox("Finished")
Catch ex As Exception
MsgBox(ex.Message)
End Try
End Sub
Function ReadCSV(ByVal path As String) As System.Data.DataTable
Try
Dim sr As New StreamReader(path)
Dim fullFileStr As String = sr.ReadToEnd()
sr.Close()
sr.Dispose()
Dim lines As String() = fullFileStr.Split(ControlChars.Lf)
Dim recs As New DataTable()
Dim sArr As String() = lines(0).Split(","c)
For Each s As String In sArr
recs.Columns.Add(New DataColumn())
Next
Dim row As DataRow
Dim finalLine As String = ""
For Each line As String In lines
row = recs.NewRow()
finalLine = line.Replace(Convert.ToString(ControlChars.Cr), "")
row.ItemArray = finalLine.Split(","c)
recs.Rows.Add(row)
Next
Return recs
Catch ex As Exception
Throw ex
End Try
End Function
End Class

DBNull value for parameter is not supported. Table-valued parameters cannot be DBNull

I am currently trying to pass a two table value parameters from a .net application to a database.
The error that I am getting is
DBNull value for parameter '#StageNotes' is not supported. Table-valued parameters cannot be DBNull.
I have checked that the parameter value is not null when I pass it, and the error that I get is not a sqlException, rather just a general Exception that passes this error back
The parameters are declared like so
Dim stageNotesparamTable2SQL As New SqlParameter("#StageNotes", SqlDbType.Structured)
Dim responsesParamTable2SQL As New SqlParameter("#Responses", SqlDbType.Structured)
and the values are assigned like this
Dim responseTable = ResponsesToDt()
responsesParamTable2SQL.Value = responseTable
Dim stageTable = StageNotesToDt()
stageNotesparamTable2SQL.Value = stageTable
The parameters are declared in the stored proc like this
#StageNotes [App].[StageNotesTableType] READONLY,
#Responses [app].QuestionResponseTableType READONLY,
When debugging I see that the stageNotesparamTable2SQL.Value is showing that there is a datatable in there and has the data so it is definitely not null.
I would be grateful if anyone could help me with this.
thanks
-- edit ---
Protected Function ResponsesToDt() As DataTable
Dim dt As New DataTable()
dt.Columns.Add("CheckId", GetType(Integer))
dt.Columns.Add("QuestionId", GetType(Integer))
dt.Columns.Add("AnswerId", GetType(Integer))
dt.Columns.Add("StaffNumber", GetType(String)).MaxLength = 15
Dim dictValues = _dicOfControlsAndValues.Where(Function(x) x.Key.Contains("ddl_Stage_"))
For Each item In dictValues
Dim ddlIdBreakDown = item.Key.ToString().Split("_").ToList
Dim checkid As Integer = item.Key.ToString().Split("_").Last
Dim questionId As Integer = Convert.ToInt16(ddlIdBreakDown(4))
Dim answerId As Integer = Convert.ToInt16(item.Value)
Dim staffNumber As String = GetLoggedOnStaffNumber()
dt.Rows.Add(checkid, questionId, answerId, staffNumber)
Next
Return dt
End Function
Protected Function StageNotesToDt() As DataTable
Dim dt As New DataTable
dt.Columns.Add("CheckId", GetType(Integer))
dt.Columns.Add("StageId", GetType(Integer))
dt.Columns.Add("StageNotes", GetType(String))
dt.Columns.Add("ModifiedDate", GetType(String))
Dim dictValues = _dicOfControlsAndValues.Where(Function(x) x.Key.Contains("textArea_Stage"))
For Each item In dictValues
Dim stageNote As String = String.Empty
Dim ddlIdBreakDown = item.Key.ToString().Split("_").ToList
If String.IsNullOrEmpty(item.Value.ToString()) Then
stageNote = "."
Else
stageNote = item.Value.ToString()
End If
Dim checkid As Integer = item.Key.ToString().Split("_").Last
Dim stageId As Integer = Convert.ToInt16(ddlIdBreakDown(2))
Dim stageNotes As String = stageNote
Dim modifiedDate As DateTime = DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss")
dt.Rows.Add(checkid, stageId, stageNotes, modifiedDate)
Next
Return dt
End Function
-- edit 2 --
DAL.ExecCommand("[App].[usp_SaveCheckDetails]", "",
checkIdParam,
userIdParam,
caseNoteParam,
checkNoteParam,
checkCompletedParam,
CheckFeedbackSentToUserId,
stageNotesparamTable2SQL,
responsesParamTable2SQL)
Then the DAL im having to call.
The DAL is a shared repo that Im not able to edit at the moment:
Public Function ExecCommand(ByVal spName As String, ByVal connStringName As String, ByVal ParamArray SPParms() As SqlClient.SqlParameter) As Boolean
Dim SqlConnection As New SqlConnection
Dim sqlCommand As New SqlClient.SqlCommand
Dim returnCode As Boolean = False
Try
If connStringName <> "" Then
SqlConnection.ConnectionString = ConfigurationManager.ConnectionStrings(connStringName).ConnectionString
Else
SqlConnection.ConnectionString = ConfigurationManager.ConnectionStrings("ConnectionString").ConnectionString
End If
sqlCommand.CommandType = CommandType.StoredProcedure
sqlCommand.CommandText = spName
For Each p As SqlParameter In SPParms
If p.Direction <> ParameterDirection.Output Then
If Not p.Value.GetType() = GetType(Integer) Then
If p.Value.ToString() = "" OrElse p.Value = Nothing Then
p.Value = DBNull.Value
End If
End If
End If
sqlCommand.Parameters.Add(p)
Next
sqlCommand.Connection = SqlConnection
SqlConnection.Open()
sqlCommand.ExecuteNonQuery()
returnCode = True
Catch sqlEx As SqlException
HelpersErrorHandling.WriteAppLogEntry(Me.GetType().Name, System.Reflection.MethodBase.GetCurrentMethod().ToString, sqlEx)
Catch ex As Exception
If Not ex.Message.ToString().Contains("Thread was being aborted") Then
HelpersErrorHandling.WriteAppLogEntry(Me.GetType().Name, System.Reflection.MethodBase.GetCurrentMethod().ToString, ex)
End If
Finally
Try
sqlCommand.Parameters.Clear()
sqlCommand.Dispose()
SqlConnection.Close()
SqlConnection.Dispose()
Catch ex As Exception
HelpersErrorHandling.WriteAppLogEntry(Me.GetType().Name, System.Reflection.MethodBase.GetCurrentMethod().ToString, ex)
End Try
End Try
Return returnCode
End Function

Performance improvement on vb.net code

I need to write 50 million records with 72 columns into text file, the file size is growing as 9.7gb .
I need to check each and every column length need to format as according to the length as defined in XML file.
Reading records from oracle one by one and checking the format and writing into text file.
To write 5 crores records it is taking more than 24 hours. how to increase the performance in the below code.
Dim valString As String = Nothing
Dim valName As String = Nothing
Dim valLength As String = Nothing
Dim valDataType As String = Nothing
Dim validationsArray As ArrayList = GetValidations(Directory.GetCurrentDirectory() + "\ReportFormat.xml")
Console.WriteLine("passed xml")
Dim k As Integer = 1
Try
Console.WriteLine(System.DateTime.Now())
Dim selectSql As String = "select * from table where
" record_date >= To_Date('01-01-2014','DD-MM-YYYY') and record_date <= To_Date('31-12-2014','DD-MM-YYYY')"
Dim dataTable As New DataTable
Dim oracleAccess As New OracleConnection(System.Configuration.ConfigurationManager.AppSettings("OracleConnection"))
Dim cmd As New OracleCommand()
cmd.Connection = oracleAccess
cmd.CommandType = CommandType.Text
cmd.CommandText = selectSql
oracleAccess.Open()
Dim Tablecolumns As New DataTable()
Using oracleAccess
Using writer = New StreamWriter(Directory.GetCurrentDirectory() + "\FileName.txt")
Using odr As OracleDataReader = cmd.ExecuteReader()
Dim sbHeaderData As New StringBuilder
For i As Integer = 0 To odr.FieldCount - 1
sbHeaderData.Append(odr.GetName(i))
sbHeaderData.Append("|")
Next
writer.WriteLine(sbHeaderData)
While odr.Read()
Dim sbColumnData As New StringBuilder
Dim values(odr.FieldCount - 1) As Object
Dim fieldCount As Integer = odr.GetValues(values)
For i As Integer = 0 To fieldCount - 1
Dim vals As Array = validationsArray(i).ToString.ToUpper.Split("|")
valName = vals(0).trim
valDataType = vals(1).trim
valLength = vals(2).trim
Select Case valDataType
Case "VARCHAR2"
If values(i).ToString().Length = valLength Then
sbColumnData.Append(values(i).ToString())
'sbColumnData.Append("|")
ElseIf values(i).ToString().Length > valLength Then
sbColumnData.Append(values(i).ToString().Substring(0, valLength))
'sbColumnData.Append("|")
Else
sbColumnData.Append(values(i).ToString().PadRight(valLength))
'sbColumnData.Append("|")
End If
Case "NUMERIC"
valLength = valLength.Substring(0, valLength.IndexOf(","))
If values(i).ToString().Length = valLength Then
sbColumnData.Append(values(i).ToString())
'sbColumnData.Append("|")
Else
sbColumnData.Append(values(i).ToString().PadLeft(valLength, "0"c))
'sbColumnData.Append("|")
End If
'sbColumnData.Append((values(i).ToString()))
End Select
Next
writer.WriteLine(sbColumnData)
k = k + 1
Console.WriteLine(k)
End While
End Using
writer.WriteLine(System.DateTime.Now())
End Using
End Using
Console.WriteLine(System.DateTime.Now())
'Dim Adpt As New OracleDataAdapter(selectSql, oracleAccess)
'Adpt.Fill(dataTable)
Return Tablecolumns
Catch ex As Exception
Console.WriteLine(System.DateTime.Now())
Console.WriteLine("Error: " & ex.Message)
Console.ReadLine()
Return Nothing
End Try

Deserialize json string and insert in SQL DB

I have a function which gets json string from jsonblob, deserializers it and returns in dataview. I want to know how can i insert the data from the dataview into my sql database using vb.net
My function is as follows
Public Function GetJsonString() As DataView
Dim dsContactMeta As DataSet = Nothing
Dim dv As DataView = Nothing
'url declaration for REST_ContactMeta
Dim REST_ContactMetaURL As Uri = New Uri("http://jsonblob.com/api/jsonBlob/5420df11e4b00ad1f05ed29b")
Dim strJSONData As String = ""
Dim response As HttpWebResponse
Dim reader As StreamReader
Try
Dim httpWebRequest As HttpWebRequest = WebRequest.Create(REST_ContactMetaURL)
httpWebRequest.Method = WebRequestMethods.Http.Get
httpWebRequest.ContentType = "application/json"
response = httpWebRequest.GetResponse()
reader = New StreamReader(response.GetResponseStream)
strJSONData = reader.ReadToEnd
strJSONData = "[" & strJSONData & "]" 'had to wrap the raw JSON data with brackets for parsing purposes
Dim myXmlNode As System.Xml.XmlNode = JsonConvert.DeserializeXmlNode("{""root"":" + strJSONData + "}", "root") 'error is thrown if the JSON does not have a root element
' Dim myXmlNode As System.Xml.XmlNode = JsonConvert.DeserializeXmlNode(strJSONData)
dsContactMeta = New DataSet("Test")
Try
dsContactMeta.ReadXml(New XmlNodeReader(myXmlNode))
If dsContactMeta.Tables("root") Is Nothing Or dsContactMeta.Tables("root").Rows.Count = 0 Then
Return Nothing 'in case of error return nothing
Else
dsContactMeta.Tables("root").Columns("job_id").SetOrdinal(0) 'setting the field name as the first column
dv = New DataView
dv.Table = dsContactMeta.Tables("root")
Return dv
End If
Catch ex As Exception
Return Nothing
End Try
Catch ex As Exception
Return Nothing 'if for any reason the API has failed connecting to the server or other unknown error then return nothing
End Try
End Function
Insert record into database from DataView, for that you need to read DataView:
For Each rowView As DataRowView In dataView
Debug.Write(item & " ")
Dim row as DataRow = rowView.Row;
// Do something //
//insert into tablename(col1,col2,..) values(row['col1'],row['col2'],..)
Next

Datatable columns and rows are in a wrong order

I'm trying to place some information from a spreadsheet using EPPLUS into a databale which will be exported to sql server afterwards.
The Problem I am having is that when I create the datatable columns, and then start inserting the rows, the end result shows a misalignment between row information and column headings.
Here is the code:
Sub ConvertPackage2DT()
Dim FILENAME As String = "C:\Data.xlsx"
Dim flInfo As New FileInfo(FILENAME)
_package = New ExcelPackage(flInfo) 'load the excel file
Dim strSheetName As String = "Radiology"
Dim Conn As SqlConnection = New SqlConnection("Data Source=dwsqlserver1\instance3;Initial Catalog=LocalDatasets_Dev;Integrated Security=True") 'connection to server end
Dim DTExport As New DataTable
If Not IsNothing(_package) Then
Dim _workbook As ExcelWorkbook = _package.Workbook
Dim _sheets As ExcelWorksheets = _workbook.Worksheets
Dim sh As ExcelWorksheet = _sheets.Item(strSheetName)
If _package.Workbook.Worksheets.Count <> 0 Then
For Each _sheet In _sheets
Dim SHEET_NAME As String = _sheet.Name
Dim tbl As New DataTable
Dim hasHeader = True ' adjust accordingly '
For ColNum = 1 To sh.Dimension.End.Column
tbl.Columns.Add(sh.Cells(1, ColNum).Value)
Next
System.Diagnostics.Debug.Print(tbl.Columns.Count)
For rowNum = 2 To sh.Dimension.End.Row
Dim v(sh.Dimension.End.Column - 1) As Object
For X = 0 To sh.Dimension.End.Column - 2
v(X) = sh.Cells(rowNum, X + 1).Value
Next
tbl.Rows.Add(v)
'tbl.Rows.Add(wsRow.Select(Function(cell) cell.Text).ToArray)
Next
Conn.Open()
Using bulkcopy As SqlBulkCopy = New SqlBulkCopy(Conn)
bulkcopy.DestinationTableName = "[LocalDatasets_Dev].[dbo].[" & GetNewTableName(6) & "]"
bulkcopy.WriteToServer(tbl)
End Using
Conn.Close()
MsgBox("Complete")
Exit Sub
Next _sheet
End If
End If
End Sub
End Module
Can anyone help? Thank you kindly