Is it possible to return an array of dictionaries? - asp.net-core

I wrote a function that will return a single sql record as a dictionary. Is it possible to return an array of dictionaries so I can return multiple records in this way?
public static async Task<Dictionary<string, string>> SQLMultiRecordToDictionary(string TableName, string SearchField, string SearchValue)
{
Dictionary<string, string> QueryResult = new Dictionary<string, string>();
// is TableName sane
if (!IsTextSane(TableName)) { return QueryResult; }
//
await using (var connection = new SqliteConnection("Data Source=" + dbFullPathName))
{
connection.Open();
SqliteCommand sqlcmd = connection.CreateCommand();
sqlcmd.CommandText = "SELECT * FROM " + TableName + " WHERE " + SearchField + "=#SearchValue";
sqlcmd.Parameters.AddWithValue("#SearchValue", SearchValue);
SqliteDataReader sqlreader = sqlcmd.ExecuteReader();
// generate dictionary keys with blank values
// this prevents key not existing issues when no record is returned
// i prefer no/blank values in the keys when no record returned for this project
for (int i = 0; i < sqlreader.FieldCount; i++)
{
QueryResult.Add(sqlreader.GetName(i), ""); // blank value
}
// add the values to the keys
while (sqlreader.Read())
{
for (int i = 0; i <= sqlreader.FieldCount - 1; i++)
{
QueryResult[sqlreader.GetName(i)] = sqlreader.GetString(i);
}
}
return QueryResult;
}
}

The working end result thanks to Tisa:
public static async Task<List<Dictionary<string, string>>> SQLMultiRecordToDictionaryList(string TableName, string SearchField, string SearchValue)
{
List<Dictionary<string, string>> QueryResult = new List<Dictionary<string, string>>();
Dictionary<string, string> SQLRecord = new Dictionary<string, string>();
//
// is TableName sane, if not return nothing
if (!IsTextSane(TableName)) { return QueryResult; }
//
await using (var connection = new SqliteConnection("Data Source=" + dbFullPathName))
{
connection.Open();
SqliteCommand sqlcmd = connection.CreateCommand();
sqlcmd.CommandText = "SELECT * FROM " + TableName + " WHERE " + SearchField + "=#SearchValue";
sqlcmd.Parameters.AddWithValue("#SearchValue", SearchValue);
SqliteDataReader sqlreader = sqlcmd.ExecuteReader();
// generate dictionary keys with blank values if no rows
// this prevents key not existing issues when no record is returned
// i prefer no/blank values in the keys when no record returned for this project
if (!sqlreader.HasRows)
{
for (int i = 0; i < sqlreader.FieldCount; i++)
{
SQLRecord.Add(sqlreader.GetName(i), ""); // blank value
}
QueryResult.Add(SQLRecord);
}
//
// add the values to the keys if there are rows (this doesn't run if no rows returned)
while (sqlreader.Read())
{
SQLRecord = new Dictionary<string, string>();
for (int i = 0; i <= sqlreader.FieldCount - 1; i++)
{
SQLRecord.Add(sqlreader.GetName(i), sqlreader.GetString(i));
}
QueryResult.Add(SQLRecord);
}
return QueryResult;
}
}

Related

Automating Import of large csv files(~3gb) with C#

I am a bit new to this but my goal is to import the data from a csv file into a sql table and include additional values for each row being the file name and date. I was able to accomplish this using entity frame work and iterating through each row of the file but with the size of the files it will take too long too actually complete.
I am looking for a method to accomplish this import faster. I was looking into potentially using csvhelper with sqlbulkcopy to accomplish this but was not sure if there was a way to pass in the additional values needed for each row.
public void Process(string filePath)
{
InputFilePath = filePath;
DateTime fileDate = DateTime.Today;
string[] fPath = Directory.GetFiles(InputFilePath);
foreach (var file in fPath)
{
string fileName = Path.GetFileName(file);
char[] delimiter = new char[] { '\t' };
try
{
using (var db = new DatabaseName())
{
using (var reader = new StreamReader(file))
{
string line;
int count = 0;
int sCount = 0;
reader.ReadLine();
reader.ReadLine();
while ((line = reader.ReadLine()) != null)
{
count++;
string[] row = line.Split(delimiter);
var rowload = new ImportDestinationTable()
{
ImportCol0 = row[0],
ImportCol1 = row[1],
ImportCol2 = TryParseNullable(row[2]),
ImportCol3 = row[3],
ImportCol4 = row[4],
ImportCol5 = row[5],
IMPORT_FILE_NM = fileName,
IMPORT_DT = fileDate
};
db.ImportDestinationTable.Add(rowload);
if (count > 100)
{
db.SaveChanges();
count = 0;
}
}
db.SaveChanges();
//ReadLine();
}
}
}
static int? TryParseNullable(string val)
{
int outValue;
return int.TryParse(val, out outValue) ? (int?)outValue : null;
}
}

Entity Framework in stock market

public List<StockMarket> ReadAllRecords(string TxtFilePath, string TxtFileName)
{
List<StockMarket> Stock = new List<StockMarket>();
String[] a = File.ReadAllLines(TxtFilePath + TxtFileName);
foreach (var b in a)
{
String[] d = b.Split(',');
StockMarket S = new StockMarket();
S.ProductId = d[0];
S.ProductName = d[1];
S.StockId = d[2];
S.StockName = d[3];
S.StockPrice = d[4];
S.NumberofStocks = d[5];
S.Currency = d[6];
Stock.Add(S);
}
return Stock;
}
public List<StockMarket> GetValidRecords(List<StockMarket> Stock, string ErrorFilePath, string ErrorFileName)
{
List<StockMarket> Valid = new List<StockMarket>();
List<StockMarket> InValid = new List<StockMarket>();
foreach (var s in Stock)
{
bool ValidRecord = true;
if (String.IsNullOrEmpty(s.ProductId) || !s.ProductId.All(Char.IsDigit))
{
ValidRecord = false;
}
if (!s.ProductName.StartsWith("ABC") || s.ProductName.Length != 6)
{
ValidRecord = false;
}
if (String.IsNullOrEmpty(s.StockId) || !s.StockId.All(Char.IsDigit))
{
ValidRecord = false;
}
if (!s.StockName.StartsWith("SBC") || s.StockName.Length != 7)
{
ValidRecord = false;
}
if (string.IsNullOrEmpty(s.StockPrice))
{
ValidRecord = false;
}
if (string.IsNullOrEmpty(s.NumberofStocks) || !s.NumberofStocks.All(char.IsDigit))
{
ValidRecord = false;
}
if (!(s.Currency.Equals("INR") || s.Currency.Equals("USD") || s.Currency.Equals("EUR")))
{
ValidRecord = false;
}
if (ValidRecord)
{
Valid.Add(s);
}
else
{
InValid.Add(s);
}
}
LogErrorRecord(InValid, ErrorFilePath, ErrorFileName);
return Valid;
}
public List<StockMarket> CalculateTotalPrice(List<StockMarket> Stock)
{
foreach (var s in Stock)
{
if (s.Currency.Equals("INR"))
{
s.TotalPrice = (Convert.ToDouble(s.StockPrice) * Convert.ToDouble(s.NumberofStocks) * 1).ToString();
}
else if (s.Currency.Equals("USD"))
{
s.TotalPrice = (Convert.ToDouble(s.StockPrice) * Convert.ToDouble(s.NumberofStocks) * 0.5).ToString();
}
else if (s.Currency.Equals("EUR"))
{
s.TotalPrice = (Convert.ToDouble(s.StockPrice) * Convert.ToDouble(s.NumberofStocks) * 0.75).ToString();
}
}
return Stock;
}
public void LogErrorRecord(List<StockMarket> InvalidStock, string ErrorFilePath, string ErrorFileName)
{
List<String> InvalidItems = new List<string>();
foreach (var I in InvalidStock)
{
InvalidItems.Add(I.ProductId + " " + I.ProductName + " " + I.StockId + " " + I.StockName + " " + I.StockPrice + " " + I.NumberofStocks + " " + I.Currency);
}
File.AppendAllLines(ErrorFilePath + ErrorFileName, InvalidItems);
}
public void SavetoDB(List<StockMarket> Stock, SqlConnection connection)
{
String Query = "insert into StockMarket(ProductId,Productname,StockId,StockName,StockPrice,NumberofStocks,Currency,TotalPrice) Values(#ProductId,#ProductName,#StockId,#StockName,#StockPrice,#NumberofStocks,#Currency,#TotalPrice)";
connection.Open();
foreach (var a in Stock)
{
SqlCommand cmd = new SqlCommand(Query, connection);
cmd.Parameters.Add("#ProductId", a.ProductId);
cmd.Parameters.Add("#ProductName", a.ProductName);
cmd.Parameters.Add("#StockId", a.StockId);
cmd.Parameters.Add("#StockName", a.StockName);
cmd.Parameters.Add("#StockPrice", a.StockPrice);
cmd.Parameters.Add("#NumberofStocks", a.NumberofStocks);
cmd.Parameters.Add("#Currency", a.Currency);
cmd.Parameters.Add("#TotalPrice", a.TotalPrice);
int b = cmd.ExecuteNonQuery();
}
connection.Close();
}
public void SaveDistinctProductName(List<StockMarket> Stock, SqlConnection connection)
{
String Query = "if not exists( select * from Product where ProductId = #ProductId) begin insert into Product (ProductId,ProductName) Values(#ProductId,#ProductName)end";
connection.Open();
foreach (var a in Stock)
{
SqlCommand cmd = new SqlCommand(Query, connection);
cmd.Parameters.Add("#ProductId", a.ProductId);
cmd.Parameters.Add("#ProductName", a.ProductName);
int b = cmd.ExecuteNonQuery();
}
connection.Close();
}
public void SaveDistinctStockName(List<StockMarket> Stock, SqlConnection connection)
{
String Query = "if not exists( select * from Stock where StockId = #StockId) begin insert into Stock (StockId,StockName) Values(#StockId,#StockName)end";
connection.Open();
foreach (var a in Stock)
{
SqlCommand cmd = new SqlCommand(Query, connection);
cmd.Parameters.Add("#StockId", a.StockId);
cmd.Parameters.Add("#StockName", a.StockName);
int b = cmd.ExecuteNonQuery();
}
connection.Close();
}
Entity Framework allows you to create a model by writing code or using boxes and lines in the EF Designer. Both of these approaches can be used to target an existing database or create a new database. This short video explains the differences and how to find the one that is right for you.
Please let me know the overview of the code
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using System.IO;
using System.Linq;
using System.Text;
namespace EmployeeValidation
{
public class Program
{
public static void Main()
{
/*
* Pass the file path, file names and connection string if any in this method alone.
* Do not hardcode in any other methods
*/
SqlConnection connection = new SqlConnection(#"Data Source=NA03OSDVP00746\SQLEXPRESS;Initial Catalog=DBEmployeeValidation;Integrated Security=True");
EmployeeValidator empValidator = new EmployeeValidator();
empValidator.ProcessData(#"D:\Employee_Validator\Input File\", "Emp_122014.xml", #"D:\Employee_Validator\Error File\", "Emp_122014.xml", connection);
}
}
}
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using System.Data;
using System.Collections;
using System.Xml.Linq;
using System.Linq;
using System.Text.RegularExpressions;
using System.Data.Linq;
using System.Globalization;
using System.Xml.Serialization;
using System.IO;
namespace EmployeeValidation
{
public class EmployeeValidator
{
/*
* Do not remove the attached TestProject. It is meant for auto evaluation of your source code.
* Do not attach any test classess to the attached test project.
* Do not attach any new test projects.
* You are not required to write any automated test cases. You are supposed to write only the code.
*/
public void ProcessData(string xmlFilePath, string xmlFileName,string errorFilePath, string errorFileName, SqlConnection connection)
{
//Do your logic here
//Step 1
//ReadAllEmployeesFromXmlFile
List<Employee> lstemp = new List<Employee>();
List<Employee> validemp = new List<Employee>();
lstemp = ReadAllEmployeesFromXmlFile(xmlFilePath,xmlFileName);
validemp = PickValidEmployees(lstemp);
SaveValidEmployeesToDB(validemp, connection);
ReadfromDBtoTxt(connection);
//Step 2
//PickValidEmployees
//Step 3
//SaveValidEmployeesToDataBase
}
public List<Employee> ReadAllEmployeesFromXmlFile(string xmlFilePath, string xmlFileName)
{
//Read the employee details from the xml file and return it in List collection
//Do not hardcode the filename and the file path here
//Do not return the date with time appended to it.
string employeefile = xmlFilePath + xmlFileName;
List<Employee> empdetail = new List<Employee>();
XElement getelementfile = XElement.Load(employeefile);
IEnumerable<XElement> items = getelementfile.Elements();
foreach (var item in items)
{
string _EmployeeId = item.Element("EmployeeId").Value;
string _EmployeeName = item.Element("EmployeeName").Value;
string _EmailId = item.Element("EmailId").Value;
string _DateOfJoining = item.Element("DateOfJoining").Value;
empdetail.Add(new Employee(){ EmployeeId= _EmployeeId,
EmployeeName= _EmployeeName,
EmailId=_EmailId,
DateOfJoining=_DateOfJoining
});
}
return empdetail;
}
public List<Employee> PickValidEmployees(List<Employee> employees)
{
//Pick the valid employees from the List collection
//Return the valid employees in a List
List<Employee> valid = new List<Employee>();
List<Employee> Invalid = new List<Employee>();
List<string> empnum = new List<string>();
bool isvalid = true;
foreach(Employee em in employees)
{
Regex rgxisnumeric = new Regex(#"^\d$");
Regex rgxisalphanumeric=new Regex( #"^\d*[a-zA-Z]{1,}\d*");
Regex rgxemail = new Regex(#"^([\w\.\-]+)#([\w\-]+)((\.(\w){2,3})+)$");
Regex rgxDate= new Regex(#"^((0[1-9]|1[0-2])\/((0|1)[0-9]|2[0-9]|3[0-1])\/((19|20)\d\d))$");
if (!empnum.Contains(em.EmployeeId))
{
empnum.Add(em.EmployeeId);
isvalid = true;
}
else
{
isvalid = false;
}
int empname;
isvalid= isvalid && (!string.IsNullOrEmpty(em.EmployeeId)) && (rgxisnumeric.IsMatch(em.EmployeeId));
isvalid= isvalid && (int.TryParse(em.EmployeeName, out empname)== false);
isvalid= isvalid && (!string.IsNullOrEmpty(em.EmployeeName)) && (rgxisalphanumeric.IsMatch(em.EmployeeName));
isvalid= isvalid && (!string.IsNullOrEmpty(em.EmailId)) && (rgxemail.IsMatch(em.EmailId));
isvalid= isvalid && (!string.IsNullOrEmpty(em.DateOfJoining)) && (rgxDate.IsMatch(em.DateOfJoining));
if(isvalid)
{
DateTime dt;
isvalid= isvalid && DateTime.TryParseExact(em.DateOfJoining,"MM/dd/yyyy",new CultureInfo("en-US"),DateTimeStyles.None, out dt);
}
if(isvalid)
{
valid.Add(em);
}
else
{
Invalid.Add(em);
}
}
SaveInValidEmployeesTotxt(Invalid);
return valid;//Return only valid employees in List
}
public void SaveValidEmployeesToDB(List<Employee> employees, SqlConnection connection)
{
//Do not Prefix Database name in the SQL Query. Query should be "Insert into SBA.TableName"
//Should not be "Insert into DatabaseName.SBA.TableName"
//Do not hardcode the connection string here
SqlConnection conn = connection;
foreach(Employee emp in employees)
{
string command = "Insert into SBA.Employees (EmployeeId,EmployeeName,EmailId,DateOfJoining) values (#EmployeeId,#EmployeeName,#EmailId,#DateOfJoining)";
SqlCommand cmd = new SqlCommand(command, conn);
conn.Open();
cmd.Parameters.AddWithValue("#EmployeeId",emp.EmployeeId);
cmd.Parameters.AddWithValue("#EmployeeName",emp.EmployeeName);
cmd.Parameters.AddWithValue("#EmailId",emp.EmailId);
cmd.Parameters.AddWithValue("#DateOfJoining",DateTime.Parse(emp.DateOfJoining).ToString("MM/dd/yyyy"));
cmd.ExecuteNonQuery();
conn.Close();
}
}
public void SaveInValidEmployeesTotxt(List<Employee> Invalid)
{
string invalidpath = #"D:\Employee_Validator\Error File\Emp_122014.xml";
XmlSerializer serialise = new XmlSerializer(typeof(List<Employee>));
TextWriter writeinvalid = new StreamWriter(invalidpath);
serialise.Serialize(writeinvalid,Invalid);
}
public void ReadfromDBtoTxt(SqlConnection connection)
{
string newfilepath = #"D:\Employee_Validator\DBtoTXT\EmpoValid_" + DateTime.Now.ToString("MMyyyy") + ".txt";
List<Employee> dbtotextlist = new List<Employee>();
if (!File.Exists(newfilepath))
{
var g= File.Create(newfilepath);
g.Close();
}
SqlCommand cmd = new SqlCommand("Select * from SBA.Employees",connection);
connection.Open();
SqlDataReader readdata = cmd.ExecuteReader();
while (readdata.Read())
{
dbtotextlist.Add(new Employee
{
EmployeeId = readdata["EmployeeId"].ToString(),
EmployeeName = readdata["EmployeeName"].ToString(),
EmailId = readdata["EmailId"].ToString(),
DateOfJoining = readdata["DateOfJoining"].ToString()
});
}
sconnection.Close();
StreamWriter sw = File.AppendText(newfilepath);
foreach(Employee s in dbtotextlist)
{
sw.WriteLine(s.EmployeeId+","+s.EmployeeName+","+s.EmailId+","+ DateTime.Parse(s.DateOfJoining).ToString("MM/dd/yyyy"));
}
sw.Flush();
sw.Close();
}
}
}
static void Main(string[] args)
{
SqlConnection connectionObject = new SqlConnection(#"Data Source=NA03OSDVP00746\SQLEXPRESS;Initial Catalog= DBFXCalculation;Integrated Security=True");
Main fxcalculatorobj = new Main();
fxcalculatorobj.ProcessData(#"D:\frameworksample\Input File\", "TradeOrders_032013.txt",
#"D:\frameworksample\ErrorLog\", "InvalidRecords_032014.txt", connectionObject,
#"D:\frameworksample\Archive\", "TradeOrders_032013_Processed.txt");
/*
* Pass the file path, file names and connection string in this method alone.
* Do not hardcode in any other methods
*/
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Data.SqlClient;
using System.Globalization;
using System.Text.RegularExpressions;
using System.Reflection;
namespace frameworksample
{
class Main
{
public void ProcessData(string sourceFolder, string fileName, string errorLogFilePath,
string errorLogFileName, SqlConnection connectionObject,
string archiveFilePath, string archiveFileName)
{
//Step 1:ReadAllDataFromInputFile
List<Trade> trades = ReadAllDataFromInputFile(sourceFolder, fileName);
//Step 2:PickValidTradeDetails;
List<Trade> validateTrade = PickValidTradeDetails(trades, errorLogFilePath, errorLogFileName);
//Step 3: SaveValidRecordsToDB
SaveValidRecordsToDB(validateTrade, connectionObject);
//Step 4:CalculateFXRate
List<FXRate> fxRates = CalculateFXRate(connectionObject);
//Step 5:SaveFXRate
//List<FXRate> fxRates = new List<FXRate>();
SaveFXRate(fxRates, connectionObject);
//Step 6:CopyToArchive
CopyToArchive(archiveFilePath, archiveFileName);
//Validate data
}
public List<Trade> ReadAllDataFromInputFile(string sourceFolder, string fileName)
{
List<Trade> trades = new List<Trade>();
//Do your logic to read file and storing it into list of trades ..here..
//Do not hardcode the filename and the file path here
using(StreamReader sr= File.OpenText(sourceFolder+fileName))
{
string s="";
while((s=sr.ReadLine())!=null)
{
if(s.Contains(','))
{
string[] splited= s.Split(',');
Trade tradeitem= new Trade();
tradeitem.TradeId= splited[0];
tradeitem.ISIN= splited[1];
tradeitem.TradeDate= splited[2];
tradeitem.MaturityDate= splited[3];
tradeitem.SchemeName= splited[4];
tradeitem.TradeType= splited[5];
tradeitem.Currency= splited[6];
tradeitem.Amount= splited[7];
trades.Add(tradeitem);
}
}
}
return trades;
}
public List<Trade> PickValidTradeDetails(List<Trade> trades, string errorLogFilePath, string errorLogFileName)
{
//email: \w+([-+.']\w+)*#\w+([-.]\w+)*\.\w+([-.]\w+)* ,,,, \b[A-Z0-9._%+-]+#[A-Z0-9.-]+\.[A-Z]{2,4}\b
// Step 1 : filter the valid trades and invalid trades, save the invalid
List<Trade> validTrades = new List<Trade>(); //identify all the valid trades and assign.
//Do not hardcode the filename and the file path here
List<Trade> invalidTrade = new List<Trade>();
bool isValid = true;
foreach (Trade x in trades)
{
isValid=(!string.IsNullOrEmpty(x.TradeId));
Regex rgxtradeid = new Regex(#"\bTR\d{3}\b");
isValid = isValid && rgxtradeid.IsMatch(x.TradeId);
isValid=(isValid && (!string.IsNullOrEmpty(x.ISIN)));
Regex rgxisin = new Regex(#"\bISIN\d{3}\b");
isValid = (rgxisin.IsMatch(x.ISIN));
Regex rgxdate = new Regex("((0[1-9]|1[0-2])\\/((0|1)[0-9]|2[0-9]|3[0-1])\\/((19|20)\\d\\d))$");
DateTime dt;
isValid = (isValid && (!string.IsNullOrEmpty(x.TradeDate)) && (rgxdate.IsMatch(x.TradeDate)));
isValid = (isValid && (!string.IsNullOrEmpty(x.MaturityDate)) && (rgxdate.IsMatch(x.MaturityDate)));
if (isValid)
{
DateTime MD = DateTime.Parse(x.MaturityDate);
DateTime TD = DateTime.Parse(x.TradeDate);
int year = MD.Year - TD.Year;
isValid = isValid && (year > 5);
isValid = isValid && (!string.IsNullOrEmpty(x.TradeType));
isValid = isValid && (x.Currency.Equals("GBP") || x.Currency.Equals("EUR") || x.Currency.Equals("USD") || x.Currency.Equals("INR"));
isValid = isValid && (!string.IsNullOrEmpty(x.Amount));
int i;
bool isnumeric = int.TryParse(x.Amount, out i);
isValid = isValid && (isnumeric);
}
if(isValid)
{
Trade val= new Trade();
val.TradeId= x.TradeId;
val.ISIN= x.ISIN;
val.TradeDate= x.TradeDate;
val.MaturityDate= x.MaturityDate;
val.SchemeName= x.SchemeName;
val.TradeType= x.TradeType;
val.Currency= x.Currency;
val.Amount= x.Amount;
validTrades.Add(val);
}
else
{
Trade valerror= new Trade();
valerror.TradeId= x.TradeId;
valerror.ISIN= x.ISIN;
valerror.TradeDate= x.TradeDate;
valerror.MaturityDate= x.MaturityDate;
valerror.SchemeName= x.SchemeName;
valerror.TradeType= x.TradeType;
valerror.Currency= x.Currency;
valerror.Amount= x.Amount;
invalidTrade.Add(valerror);
}
}
SaveInvalidRecordsToLogFile(invalidTrade, errorLogFilePath, errorLogFileName);
// SaveInvalidRecordsToLogFile(List<Trades>); // pass all the invalid trades to log...
return validTrades;
}
public bool SaveInvalidRecordsToLogFile(List<Trade> invalidTrades, string errorLogFilePath, string errorLogFileName)
{
//Do your logic here
//Do not hardcode the filename and the file path here
if (invalidTrades != null && invalidTrades.Count > 0)
{
string errorLogfile = errorLogFilePath + errorLogFileName;
try
{
if (!File.Exists(errorLogfile))
{
var invalidfile = File.Create(errorLogfile);
invalidfile.Close();
}
using (StreamWriter swinvalid = File.AppendText(errorLogfile))
{
swinvalid.WriteLine("TradeId|ISIN|TradeDate|MaturityDate|Tradetype|Currency|Amount");
foreach (Trade ivt in invalidTrades)
{
swinvalid.WriteLine(ivt.TradeId + "," + ivt.ISIN + "," + ivt.TradeDate + "," + ivt.MaturityDate + "'" + ivt.TradeType + "," + ivt.Currency + "," + ivt.Amount);
}
}
}
catch (Exception ex)
{
throw new FXCalculatorException(ex.Message);
}
}
return true;
}
public bool SaveValidRecordsToDB(List<Trade> validTrades, SqlConnection sqlConnectionObject)
{
//Do your logic here to upload to DB table
//Do not hardcode the connection string here
//Do not create the redundant connection Object for SqlConnection, use the conncetionObject given in the method parameter.
//Do not Prefix Database name in the SQL Query. Query should be "Insert into SBA.TableName"
//Should not be "Insert into DatabaseName.SBA.TableName"
//var ConnectionString = sqlConnectionObject.ConnectionString;
if (validTrades.Count > 0 && validTrades != null)
{
SqlConnection conn = sqlConnectionObject;
conn.Open();
foreach (Trade valid in validTrades)
{
SqlCommand cmd = new SqlCommand(("Insert into SBA.Trade_Details (TradeID,ISIN,TradeDate,MaturityDate,SchemeName,TradeType,Currency,Amount) values (#TradeID,#ISIN,#TradeDate,#MaturityDate,#SchemeName,#TradeType,#Currency,#Amount)"),conn);
cmd.Parameters.Add("#TradeID",valid.TradeId);
cmd.Parameters.Add("#ISIN", valid.ISIN);
cmd.Parameters.Add("#TradeDate", valid.TradeDate);
cmd.Parameters.Add("#MaturityDate", valid.MaturityDate);
cmd.Parameters.Add("#SchemeName", valid.SchemeName);
cmd.Parameters.Add("#TradeType", valid.TradeType);
cmd.Parameters.Add("#Currency", valid.Currency);
cmd.Parameters.Add("#Amount", valid.Amount);
cmd.ExecuteNonQuery();
}
conn.Close();
}
return true;
}
public List<FXRate> CalculateFXRate(SqlConnection sqlConnectionObject)
{
// TODO :Read the Trade details for TradeType FX from database and calculate the rates.
// Calculate the rate for each trade and add in a list of FXRates.
//Do not Prefix Database name in the SQL Query. Query should be "Insert into SBA.TableName"
//Should not be "Insert into DatabaseName.SBA.TableName"
//List<FXRate> FxRates = null; // assign list of FXRates;
//Do not hardcode the connection string here
//Do not create the redundant connection Object for SqlConnection, use the conncetionObject given in the method parameter.
List<FXRate> FxRates = new List<FXRate>();
List<Trade> trades = new List<Trade>();
try
{
SqlConnection conne = sqlConnectionObject;
string queryString = "Select * from SBA.Trade_Details";
SqlCommand cmd = new SqlCommand(queryString, conne);
conne.Open();
SqlDataReader datareader = cmd.ExecuteReader();
while (datareader.Read())
{
Trade validfx = new Trade{TradeId = datareader["TradeId"].ToString(),ISIN = datareader["ISIN"].ToString(),TradeDate = datareader["TradeDate"].ToString(),
MaturityDate = datareader["MaturityDate"].ToString(),SchemeName = datareader["SchemeName"].ToString(),TradeType = datareader["TradeType"].ToString(),
Currency = datareader["Currency"].ToString(), Amount = datareader["Amount"].ToString()};
trades.Add(validfx);
}
conne.Close();
foreach (Trade trad_para_to_calc_fx in trades)
{
FXRate fx = new FXRate();
fx.TradeId = trad_para_to_calc_fx.TradeId;
fx.Currency = trad_para_to_calc_fx.Currency;
fx.Amount = trad_para_to_calc_fx.Amount;
float amount = float.Parse(fx.Amount, CultureInfo.InvariantCulture.NumberFormat);
if (trad_para_to_calc_fx.Currency == "USD")
{
fx.AppliedFXRate = float.Parse("0.5",CultureInfo.InvariantCulture.NumberFormat).ToString();
float app_fx_rate = float.Parse("0.5",CultureInfo.InvariantCulture.NumberFormat);
fx.CalculatedFXRate = ((app_fx_rate) * (amount)).ToString();
}
if (trad_para_to_calc_fx.Currency == "GBP")
{
fx.AppliedFXRate = float.Parse("0.6", CultureInfo.InvariantCulture.NumberFormat).ToString();
float app_fx_rate = float.Parse("0.7",CultureInfo.InvariantCulture.NumberFormat);
fx.CalculatedFXRate = ((app_fx_rate) * (amount)).ToString();
}
if (trad_para_to_calc_fx.Currency == "EUR")
{
fx.AppliedFXRate = float.Parse("0.7", CultureInfo.InvariantCulture.NumberFormat).ToString();
float app_fx_rate = float.Parse("0.7",CultureInfo.InvariantCulture.NumberFormat);
fx.CalculatedFXRate = ((app_fx_rate) * (amount)).ToString();
}
if (trad_para_to_calc_fx.Currency == "INR")
{
fx.AppliedFXRate = float.Parse("1", CultureInfo.InvariantCulture.NumberFormat).ToString();
float app_fx_rate = float.Parse("1",CultureInfo.InvariantCulture.NumberFormat);
fx.CalculatedFXRate = ((app_fx_rate) * (amount)).ToString();
}
FxRates.Add(fx);
}
}
catch (Exception ex)
{
throw new FXCalculatorException(ex.Message);
}
return FxRates;
}
public bool SaveFXRate(List<FXRate> fxRates, SqlConnection sqlConnectionObject)
{
//Do your logic here to upload to DB table
//Do not hardcode the connection string here
//Do not create the redundant connection Object for SqlConnection, use the conncetionObject given in the method parameter.
//Do not Prefix Database name in the SQL Query. Query should be "Insert into SBA.TableName"
//Should not be "Insert into DatabaseName.SBA.TableName"
try
{
if (fxRates.Count > 0 && fxRates != null)
{
SqlConnection conne = sqlConnectionObject;
conne.Open();
foreach(FXRate calculated in fxRates)
{
SqlCommand cmd = new SqlCommand("Insert into SBA.FX_Rate (TradeId,Currency,Amount,AppliedFXRate,CalculatedFXRate) values (#TradeId,#Currency,#Amount,#AppliedFXRate,#CalculatedFXRate)", conne);
cmd.Parameters.AddWithValue("#TradeId", calculated.TradeId);
cmd.Parameters.AddWithValue("#Currency",calculated.Currency);
cmd.Parameters.AddWithValue("#Amount",calculated.Amount);
cmd.Parameters.AddWithValue("#AppliedFXRate",calculated.AppliedFXRate);
cmd.Parameters.AddWithValue("#CalculatedFXRate", calculated.CalculatedFXRate);
cmd.ExecuteNonQuery();
}
conne.Close();
}
}
catch (Exception ex)
{
throw new FXCalculatorException(ex.Message);
}
return true;
}
public bool CopyToArchive(string sourcePathWithFileName, string targetPathWithFileName)
{
//Do your logic here
//Do not hardcode the filename and the file path here
try
{
string inputpath="";
string input="";
FileInfo[] files;
DirectoryInfo Di;
string targetFile = sourcePathWithFileName + targetPathWithFileName;
Di = new DirectoryInfo(#"D:\frameworksample\");
files = Di.GetFiles("*.txt", SearchOption.AllDirectories);
foreach (FileInfo di1 in files)
{
if (di1.Name == "TradeOrders_032013.txt")
{
inputpath = di1.DirectoryName.ToString();
input = inputpath+"\\" + di1.Name.ToString();
}
}
if (!Directory.Exists(sourcePathWithFileName))
{
Directory.CreateDirectory(sourcePathWithFileName);
var targetfilecreation = File.Create(targetFile);
targetfilecreation.Close();
}
else
{
File.Delete(targetFile);
Directory.Delete(sourcePathWithFileName, true);
Directory.CreateDirectory(sourcePathWithFileName);
var targetfilecreation =File.Create(targetFile);
targetfilecreation.Close();
}
System.IO.File.Copy(input, targetFile, true);
}
catch (Exception ex)
{
throw new FXCalculatorException(ex.Message);
}
//File.Copy(sourcePathWithFileName + "\\" + targetPathWithFileName, true);
return true;
}
private void ProcessData(Main main)
{
throw new NotImplementedException();
}
//internal void ProcessData(string p, string p_2, string p_3, string p_4, System.Data.SqlClient.SqlConnection connectionObject, string p_5, string p_6)
//{
// throw new NotImplementedException();
//}
}
}

Using C# remove unnecessary “TABLE_NAME” from Excel worksheets

Can anyone tell me, I am going to upload excel file, this file has unnecessary table like "_xlnm#Print_Titles" that I need to remove or delete that field. This a my method. But it is does not work for remove or delete.
static string[] GetExcelSheetNames(string connectionString)
{
OleDbConnection con = null;
DataTable dt = null;
con = new OleDbConnection(connectionString);
con.Open();
dt = con.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null);
if ((dt == null) )
{
return null;
}
String[] excelSheetNames = new String[dt.Rows.Count];
int i = 0;
foreach (DataRow row in dt.Rows)
{
excelSheetNames[i] = row["TABLE_NAME"].ToString();
if ((excelSheetNames[i].Contains("_xlnm#Print_Titles") || (excelSheetNames[i].Contains("Print_Titles"))))
{
if (true)
{
row.Table.Rows.Remove(row);
dt.AcceptChanges();
}
}
i++;
}
return excelSheetNames;
}
Instead of removing items in the foreach loop, we'll find them and add them to a list, then we'll go through that list and remove them from your data table.
static string[] GetExcelSheetNames(string connectionString)
{
OleDbConnection con = null;
DataTable dt = null;
con = new OleDbConnection(connectionString);
con.Open();
dt = con.GetOleDbSchemaTable(OleDbSchemaGuid.Tables, null);
if ((dt == null))
{
return null;
}
String[] excelSheetNames = new String[dt.Rows.Count];
var rowsToRemove = new List<DataRow>();
for (int i = 0; i < dt.Rows.Count; i++)
{
var row = dt.Rows[i];
excelSheetNames[i] = row["TABLE_NAME"].ToString();
if ((excelSheetNames[i].Contains("_xlnm#Print_Titles") || (excelSheetNames[i].Contains("Print_Titles"))))
{
rowsToRemove.Add(dt.Rows[i]);
}
i++;
}
foreach (var dataRow in rowsToRemove)
{
dt.Rows.Remove(dataRow);
}
return excelSheetNames;
}
Those _xlnm and "$" are sheets that, turns out, shouldn't be normally accessed by the users.
You can solve this in 2 ways.
Ignore them
Drop them
The former is highly recommended.
To do this you need to use the following code:
if (!dt.Rows[i]["Table_Name"].ToString().Contains("FilterDatabase") && !dt.Rows[i]["Table_Name"].ToString().EndsWith("$'"))
{
}
You can either use .Contains() and/or .EndsWith() to filter out those sheets.

Topic views do not show up in jTable

I try to code a forum using java swing. Firstly, on click for the jTable, it will lead to eForumContent.class which pass in the id together.
jTable.addMouseListener(new java.awt.event.MouseAdapter() {
public void mouseClicked(java.awt.event.MouseEvent e) {
int id = 0;
eForumTopics topics = new eForumTopics(id);
topics.retrieveDiscussion();
eForumThreadContent myWindow = new eForumThreadContent(id);
myWindow.getJFrame().setVisible(true);
}
});
I initialize the id first. But I set my id in database table to auto number. Then I call the retrieveDiscussion method to get the id from database and pass it to eForumThreadContent. Here are my codes for retrieveDiscussion method.
public boolean retrieveDiscussion(){
boolean success = false;
ResultSet rs = null;
DBController db = new DBController();
db.setUp("IT Innovation Project");
String dbQuery = "SELECT * FROM forumTopics WHERE topic_id = '" + id
+ "'";
rs = db.readRequest(dbQuery);
db.terminate();
return success;
}
}
Then, inside the eForumThreadContent, I want to display the content of chosen thread using a table. So I use the id which I pass in just now and insert into my sql statement. Here are my codes.
public eForumThreadContent(int id) {
this.id = id;
}
if (jTable == null) {
Vector columnNames = new Vector(); // Vector class allows dynamic
// array of objects
Vector data = new Vector();
try {
DBController db = new DBController();
db.setUp("IT Innovation Project");
Class.forName("sun.jdbc.odbc.JdbcOdbcDriver").newInstance();
String dsn = "IT Innovation Project";
String s = "jdbc:odbc:" + dsn;
Connection con = DriverManager.getConnection(s, "", "");
String sql = "Select topic_title,topic_description,topic_by from forumTopics WHERE topic_id = '"+id+"'";
java.sql.Statement statement = con.createStatement();
ResultSet resultSet = statement.executeQuery(sql);
ResultSetMetaData metaData = resultSet.getMetaData();
int columns = metaData.getColumnCount();
for (int i = 1; i <= columns; i++) {
columnNames.addElement(metaData.getColumnName(i));
}
while (resultSet.next()) {
Vector row = new Vector(columns);
for (int i = 1; i <= columns; i++) {
row.addElement(resultSet.getObject(i));
}
data.addElement(row);
}
resultSet.close();
((Connection) statement).close();
} catch (Exception e) {
System.out.println(e);
}
jTable = new JTable(data, columnNames);
TableColumn column;
for (int i = 0; i < jTable.getColumnCount(); i++) {
column = jTable.getColumnModel().getColumn(i);
if (i == 1) {
column.setPreferredWidth(400); // second column is bigger
}else {
column.setPreferredWidth(200);
}
}
String header[] = { "Title", "Description", "Posted by" };
for (int i = 0; i < jTable.getColumnCount(); i++) {
TableColumn column1 = jTable.getTableHeader().getColumnModel()
.getColumn(i);
column1.setHeaderValue(header[i]);
}
jTable.getTableHeader().setFont( new Font( "Dialog" , Font.PLAIN, 20 ));
jTable.getTableHeader().setForeground(Color.white);
jTable.getTableHeader().setBackground(new Color(102, 102, 102));
jTable.setEnabled(false);
jTable.setRowHeight(100);
jTable.getRowHeight();
jTable.setFont( new Font( "Dialog" , Font.PLAIN, 18 ));
}
return jTable;
}
However, the table inside eForumThreadContent is empty even when I clicked on certain thread. It also gives me an error message.
[Microsoft][ODBC Microsoft Access Driver] Data type mismatch in criteria expression.
at sun.jdbc.odbc.JdbcOdbc.createSQLException(Unknown Source)
at sun.jdbc.odbc.JdbcOdbc.standardError(Unknown Source)
at sun.jdbc.odbc.JdbcOdbc.SQLExecDirect(Unknown Source)
at sun.jdbc.odbc.JdbcOdbcStatement.execute(Unknown Source)
at sun.jdbc.odbc.JdbcOdbcStatement.executeQuery(Unknown Source)
at DBController.database.DBController.readRequest(DBController.java:27)
at kioskeForum.entity.eForumTopics.retrieveDiscussion(eForumTopics.java:67)
at kioskeForum.ui.eForumDiscussion$3.mouseClicked(eForumDiscussion.java:296)
I research online to get an idea for topic views using id. But my table does not show up anything. Can somebody enlighten me how to fix it? Or any other ways to display topic views in java swing? Thanks in advance.

"Out Of Memory" exception while exporting from SQL Server to Excel, using EPPlus library

I'm using the EPPlus .NET library in order to export data from SQL Server to an Excel file.
I'm using the SqlConnection class to read the data. For every row of the SqlDataReader cursor, I iterate through all the excel cells for the corresponding row, and enter the data from the reader.
The issue is that I'm getting an "out of memory" error when im using this function for large tables. I need a method to create some kind of a buffer inside the Read CURSOR.
A concise code example:
Dim sqlConnection As SqlConnection = New SqlConnection()
sqlConnection.ConnectionString = sqlConnectionString.ConnectionString 'connectionstring built before
Dim query As SqlCommand = New SqlCommand(query...)
Dim newFileStream As New FileStream("c:\junk\test.xlsx", System.IO.FileMode.Create,System.IO.FileAccess.ReadWrite)
Using excelApp As New ExcelPackage(newFileStream)
sqlConnection.Open()
Dim sqlReader As SqlDataReader = query.ExecuteReader()
Dim numOfColumns As Byte = sqlReader.FieldCount()
Dim rowNumber As Integer = 1
While sqlReader.Read()
Dim currentColumn As Byte
For currentColumn = 1 To numOfColumns
ws.Cells(rowNumber,currentColumn).Value = sqlReader.Item(currentColumn - 1)
Next
rowNumber += 1
End While
excelApp.Save()
End Using
newFileStream.Close()
Since you will have to split the files anyway when you hit Excel's limits, here's some code that reads from the database in chunks into multiple Excel files:
static class Program
{
private static string _dataSource;
private static string _database;
private static string _table;
private static string _outputPath;
private static int _batchSize;
public static void Main()
{
try
{
_dataSource = ConfigurationManager.AppSettings["DataSource"];
_database = ConfigurationManager.AppSettings["Database"];
_table = ConfigurationManager.AppSettings["Table"];
_outputPath = ConfigurationManager.AppSettings["OutputPath"];
_batchSize = int.Parse(ConfigurationManager.AppSettings["BatchSize"]);
CreateExcel(_dataSource, _database, _table, _outputPath, "SELECT * FROM " + _table);
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.WriteLine("All done!");
}
public static void CreateExcel(string dataSource, string databaseName, string tableName, string outputFilePath, string queryNoParameters)
{
var sqlConnectionString = new SqlConnectionStringBuilder
{
DataSource = dataSource,
InitialCatalog = databaseName,
IntegratedSecurity = true
};
using (var connection = new SqlConnection(sqlConnectionString.ConnectionString))
{
connection.Open();
using (var command = new SqlCommand { Connection = connection, CommandType = CommandType.Text, CommandText = queryNoParameters })
using (var sqlReader = command.ExecuteReader())
{
int i = 0;
while (WriteExcelFile(tableName, GetFileInfo(databaseName, tableName, outputFilePath, i++),
sqlReader, sqlReader.FieldCount, _batchSize))
{
Console.WriteLine("Reading next batch...");
}
}
}
}
private static bool WriteExcelFile(string tableName, FileInfo fileInfo, IDataReader sqlReader, int numOfColumns, int count)
{
using (var excelPackage = new ExcelPackage(fileInfo))
{
ExcelWorksheet worksheet = excelPackage.Workbook.Worksheets.Add(tableName);
Console.WriteLine("Populating header row...");
for (var currentColumn = 1; currentColumn <= numOfColumns; currentColumn++)
{
worksheet.Cells[1, currentColumn].Value = sqlReader.GetName(currentColumn - 1);
worksheet.Column(currentColumn).Style.Numberformat.Format =
TranslateSystemtypeToExceltype(sqlReader.GetFieldType(currentColumn - 1));
}
Console.WriteLine("Reading data rows...");
int rowNumber = 2;
while (rowNumber <= count + 1 && sqlReader.Read())
{
for (var currentColumn = 1; currentColumn <= numOfColumns; currentColumn++)
worksheet.Cells[rowNumber, currentColumn].Value = sqlReader[currentColumn - 1];
rowNumber++;
}
if (rowNumber == 2) //nothing read
{
Console.WriteLine("Nothing to read, reached end of table!");
return false;
}
Console.WriteLine("Saving Excel file...");
excelPackage.Save();
return rowNumber == count + 2; //in which case we want to read more
}
}
private static FileInfo GetFileInfo(string databaseName, string tableName, string outputFilePath, int i)
{
return new FileInfo(Path.Combine(outputFilePath,
Path.ChangeExtension(
string.Format("{0}_{1}_{2}", databaseName, tableName.Replace('.', '-'), i), "xlsx")));
}
public static string TranslateSystemtypeToExceltype(Type sysType)
{
if (sysType == typeof(string))
return "#";
if (sysType == typeof(DateTime))
return "dd/MM/YYYY";
if (sysType == typeof(Decimal))
return "0.000";
if (sysType == typeof(bool))
return "#";
if (sysType == typeof(int))
return "0";
if (sysType == typeof(short))
return "0";
if (sysType == typeof(double))
return "0.000";
return "General";
}
}