Java txt to sqlite database data transfer how can be faster - optimization

I wrote a java program about transfering a txt file to a sqlite db but it takes really more time there are about 83000 data (200 data takes about 1 minute).
how can i increase transfering speed. İ tried adding arraylist than get but its not change
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.Statement;
public class NewMain {
public static Connection connect() {
Connection conn = null;
try {
String url1 = "jdbc:sqlite:c:/Users/sozdemir/Desktop/sozluk.db";
conn = DriverManager.getConnection(url1);
String sql1 = "CREATE TABLE IF NOT EXISTS KELIMELER (Kelime PRIMARYKEY NOT NULL);";
Statement s = conn.createStatement();
s.execute(sql1);
} catch (Exception e) {
System.out.println(e.getMessage());
}
return conn;
}
public void Verigir(String kelime){
String sql = "INSERT INTO KELIMELER (kelime) VALUES(?)";
try (Connection conn = this.connect();
PreparedStatement statement = conn.prepareStatement(sql)
){
statement.setString(1, kelime);
statement.executeUpdate();
} catch (Exception e) {
}
}
public static void main(String[] args) throws IOException {
/* connect();*/
NewMain app = new NewMain();
String kelime = null;
BufferedReader in = null;
int adet;
adet= 0;
in = new BufferedReader(new FileReader("C://Users//sozdemir//Desktop//ozluk.txt"));
while ((kelime=in.readLine()) !=null) {
app.Verigir(kelime);
adet +=1;
System.out.println(81742 - adet);
}
}
}

Some hints:
Use a single prepareStatement
Put everything inside a transaction
Use PRAGMA synchronous = OFF and PRAGMA journal_mode = OFF
This will give you very fast inserts.

Related

Read data from database using UDF in pig

I have requirement to read data from a database and analyse the data using pig.
I have written a UDF in java Referring following link
register /tmp/UDFJars/CassandraUDF_1-0.0.1-SNAPSHOT-jar-with-dependencies.jar;
A = Load '/user/sampleFile.txt' using udf.DBLoader('10.xx.xxx.4','username','password','select * from customer limit 10') as (f1 : chararray);
DUMP A;
package udf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import com.data.ConnectionCassandra;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
public class DBLoader extends LoadFunc {
private final Log log = LogFactory.getLog(getClass());
Session session;
private ArrayList mProtoTuple = null;
private String jdbcURL;
private String user;
private String pass;
private int count = 0;
private String query;
ResultSet result;
List<Row> rows;
int colSize;
protected TupleFactory mTupleFactory = TupleFactory.getInstance();
public DBLoader() {
}
public DBLoader(String jdbcURL, String user, String pass, String query) {
this.jdbcURL = jdbcURL;
this.user = user;
this.pass = pass;
this.query = query;
}
#Override
public InputFormat getInputFormat() throws IOException {
log.info("Inside InputFormat");
// TODO Auto-generated method stub
try {
return new TextInputFormat();
} catch (Exception exception) {
log.error(exception.getMessage());
log.error(exception.fillInStackTrace());
throw new IOException();
}
}
#Override
public Tuple getNext() throws IOException {
log.info("Inside get Next");
Row row = rows.get(count);
if (row != null) {
mProtoTuple = new ArrayList<Object>();
for (int colNum = 0; colNum < colSize; colNum++) {
mProtoTuple.add(row.getObject(colNum));
}
} else {
return null;
}
Tuple t = mTupleFactory.newTuple(mProtoTuple);
mProtoTuple.clear();
return t;
}
#Override
public void prepareToRead(RecordReader arg0, PigSplit arg1) throws IOException {
log.info("Inside Prepare to Read");
session = null;
if (query == null) {
throw new IOException("SQL Insert command not specified");
}
if (user == null || pass == null) {
log.info("Creating Session with user name and password as: " + user + " : " + pass);
session = ConnectionCassandra.connectToCassandra1(jdbcURL, user, pass);
log.info("Session Created");
} else {
session = ConnectionCassandra.connectToCassandra1(jdbcURL, user, pass);
}
log.info("Executing Query " + query);
result = session.execute(query);
log.info("Query Executed :" + query);
rows = result.all();
count = 0;
colSize = result.getColumnDefinitions().asList().size();
}
#Override
public void setLocation(String location, Job job) throws IOException {
log.info("Inside Set Location");
try {
FileInputFormat.setInputPaths(job, location);
} catch (Exception exception) {
log.info("Some thing went wrong : " + exception.getMessage());
log.debug(exception);
}
}
}
Above is my pig script and java code.
Here /user/sampleFile.txt is a dummy file with no data.
I am getting following exception:
Pig Stack Trace
ERROR 1066: Unable to open iterator for alias A
org.apache.pig.impl.logicalLayer.FrontendException: ERROR 1066: Unable to open iterator for alias A
at org.apache.pig.PigServer.openIterator(PigServer.java:892)
at org.apache.pig.tools.grunt.GruntParser.processDump(GruntParser.java:774)
at org.apache.pig.tools.pigscript.parser.PigScriptParser.parse(PigScriptParser.java:372)
at org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:198)
at org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:173)
at org.apache.pig.tools.grunt.Grunt.exec(Grunt.java:84)
at org.apache.pig.Main.run(Main.java:484)
at org.apache.pig.Main.main(Main.java:158)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Caused by: java.io.IOException: Job terminated with anomalous status FAILED
at org.apache.pig.PigServer.openIterator(PigServer.java:884)
... 13 more
Vivek! Do you even get in prepareToRead? (I see you did some logging, so it would be nice to know what you actually have in log) Also it would be really great to provide full stacktrace as I see you don't have full underlying exception.
Just some thoughts - I never tried writing a LoadFunc without implementing my own InputFormat and RecordReader - TextInputFormat checks for file existence and it's size (and creates a number of InputSplits based on file size(s)), so if your dummy file is empty there is a big possibility that no InputSplits are produced or zero-length InputSplit is produced. As it has zero-length it may cause pig to throw that exception. So the good suggestion is to implement own InputFormat (it's actually pretty easy). Also just as a fast try - try
set pig.splitCombination false
Probably it won't help, but it's easy to try.

Write object as blob via BinaryStream in Java SQLite update - NOT NULL constraint violation

I'm trying to write an object into a database as a blob, but I'm getting java.sql.SQLException: [SQLITE_CONSTRAINT] Abort due to constraint violation (NOT NULL constraint failed: table_name.blob).
Initially, I tried creating the blob as a SerialBlob and setting it on a PreparedStatement with setBlob, but I got java.sql.SQLException: not implemented by SQLite JDBC driver. So, guided by this answer, I tried to create an inputStream for the object, and set that as the binary stream.
I don't know why the database believes that the constraint is being violated - in the tests below, testStreamContentNotNull passes, but testDatabaseWrite throws the aforementioned SQLException.
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.Statement;
import org.junit.Test;
public class StackOverflowTest {
private static final String testStringToBeWrittenAsBlob = "abcdef";
#Test
public void testStreamContentNotNull() {
try {
try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
try (ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(testStringToBeWrittenAsBlob);
oos.flush();
oos.close();
try (ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray())) {
try (ObjectInputStream ois = new ObjectInputStream(bis)) {
assertThat((String)ois.readObject()).isEqualTo(testStringToBeWrittenAsBlob);
}
}
}
}
} catch (Exception e) {
fail();
}
}
#Test
public void testDatabaseWrite() {
Connection c = null;
Statement stmt = null;
String sql = null;
try {
// Initialize the connection
Class.forName("org.sqlite.JDBC");
c = DriverManager.getConnection("jdbc:sqlite::memory:");
c.setAutoCommit(false);
// Create the table
stmt = c.createStatement();
sql = "CREATE TABLE table_name " +
"(id INT PRIMARY KEY NOT NULL," +
"blob BLOB NOT NULL)";
stmt.executeUpdate(sql);
c.commit();
stmt.close();
// Table has been created - now write to it
sql = "INSERT INTO table_name (id, blob) VALUES (?, ?)";
PreparedStatement p_stmt = c.prepareStatement(sql);
try(ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
try(ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(testStringToBeWrittenAsBlob);
oos.flush();
oos.close();
try (ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray())) {
try (ObjectInputStream ois = new ObjectInputStream(bis)) {
p_stmt.setString(1, "test-id");
p_stmt.setBinaryStream(2, ois);
p_stmt.execute(); // <--- This is where the exception is thrown
c.commit();
}
}
}
}
c.close();
} catch (Exception e) {
fail();
}
}
}
For anyone who finds this question, thanks to this post I got it to work by just reading from the ByteArrayOutputStream::toBytes - i.e. the following test passes:
#Test
public void testDatabaseWriteAsBytes() {
Connection c = null;
Statement stmt = null;
String sql = null;
try {
// Initialize the connection
Class.forName("org.sqlite.JDBC");
c = DriverManager.getConnection("jdbc:sqlite::memory:");
c.setAutoCommit(false);
// Create the table
stmt = c.createStatement();
sql = "CREATE TABLE table_name " +
"(id INT PRIMARY KEY NOT NULL," +
"blob BLOB NOT NULL)";
stmt.executeUpdate(sql);
c.commit();
stmt.close();
// Table has been created - now write to it
sql = "INSERT INTO table_name (id, blob) VALUES (?, ?)";
PreparedStatement p_stmt = c.prepareStatement(sql);
try(ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
try(ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(testStringToBeWrittenAsBlob);
oos.flush();
oos.close();
p_stmt.setString(1, "test-id");
p_stmt.setBytes(2, bos.toByteArray());
p_stmt.execute(); // <--- No exception here now!
c.commit();
}
}
String selectSql = "SELECT blob FROM table_name WHERE id='test-id'";
Statement selectStatement = c.createStatement();
ResultSet resultSet = selectStatement.executeQuery(selectSql);
if (resultSet.next()) {
try (ObjectInputStream ois = new ObjectInputStream(resultSet.getBinaryStream(1))) {
assertThat((String)ois.readObject()).isEqualTo(testStringToBeWrittenAsBlob);
}
} else {
fail("Nothing in result set");
}
c.close();
} catch (Exception e) {
e.printStackTrace();
fail();
}
}
but I'm keeping this question open because I still don't know why this failed in the first place.

How should we handle SQL exceptions in the DAO layer?

How should we handle SQLExceptions in the DAO layer using Hibernate?
Create the class EmployeeDaoImpl.java
package com.mypro.module.emp.dao;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import org.apache.commons.dbutils.DbUtils;
import com.mypro.module.emp.exception.EmployeeException;
import com.mypro.module.emp.model.Employee;
public class EmployeeDaoImpl extends JdbcDaoSupportImpl implements EmployeeDao {
#Override
public int save(Employee employee) throws EmployeeException {
// TODO Auto-generated method stub
String sql = "INSERT INTO EMPLOYEE VALUES(?,?,?)";
Connection conToUse = null;
PreparedStatement ps = null;
int status = 0;
try {
conToUse = getConnection();
ps = conToUse.prepareStatement(sql);
ps.setInt(1, employee.getEmpNo());
ps.setString(2, employee.getEmpName());
ps.setLong(3, employee.getEmpSal());
status = ps.executeUpdate();
} catch (SQLException e) {
// TODO Auto-generated catch block
throw new EmployeeException(
"%%% Exception occured in EmployeeDao save() %%% " + e);
} finally {
DbUtils.closeQuietly(ps);
}
return status;
}
}

SQL Query to JTable NullPointerException

This runs perfectly:
package sledmonitor;
import java.io.File;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
public class GetList3 {
private Connection conn;
private String ConnectURL = "jdbc:odbc:Driver={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=." + File.separator + "db" + File.separator + "sledprod.mdb";
private String user = "";
private String pw = "";
String offer;
GetList3(String sql){
try{
Class.forName("sun.jdbc.odbc.JdbcOdbcDriver").newInstance();
conn = DriverManager.getConnection(ConnectURL, user, pw);
Statement stmt = conn.createStatement();
System.out.println(sql);
ResultSet rs = stmt.executeQuery(sql);
while(rs.next()){
offer = rs.getString("offernumber");
System.out.println(offer);
}
rs.close();
stmt.close();
}
catch(Exception e){
e.printStackTrace();
}
finally{
if(conn!=null){
try{
conn.close();
}
catch(Exception e){
e.printStackTrace();
}
}
}
}
}
Now I would like to show the Results in a JTable. So I make this, what normaly should work:
while(rs.next()){
offer = rs.getString("offernumber");
List.tablemodel.addRow(new String[]{offer});
}
But this somehow throws following NullPointerException:
SELECT o.offernumber, o.TotalMPL
FROM sled_offers o left join sled_changes c on o.offerguid = c.objectid
WHERE (c.nextsequencenr is null or c.nextsequencenr=99999) AND TotalMPL='0'
AND (o.offerstate=2 or o.offerstate=4 or o.offerstate=5)
java.lang.NullPointerException
at sledmonitor.GetList3.<init>(GetList3.java:28)
at sledmonitor.GetList.<init>(GetList.java:45)
at sledmonitor.Monitor$4.mouseClicked(Monitor.java:99)
at java.awt.AWTEventMulticaster.mouseClicked(AWTEventMulticaster.java:270)
at java.awt.Component.processMouseEvent(Component.java:6508)
at javax.swing.JComponent.processMouseEvent(JComponent.java:3321)
at java.awt.Component.processEvent(Component.java:6270)
at java.awt.Container.processEvent(Container.java:2229)
at java.awt.Component.dispatchEventImpl(Component.java:4861)
at java.awt.Container.dispatchEventImpl(Container.java:2287)
at java.awt.Component.dispatchEvent(Component.java:4687)
at java.awt.LightweightDispatcher.retargetMouseEvent(Container.java:4832)
at java.awt.LightweightDispatcher.processMouseEvent(Container.java:4501)
at java.awt.LightweightDispatcher.dispatchEvent(Container.java:4422)
at java.awt.Container.dispatchEventImpl(Container.java:2273)
at java.awt.Window.dispatchEventImpl(Window.java:2719)
at java.awt.Component.dispatchEvent(Component.java:4687)
at java.awt.EventQueue.dispatchEventImpl(EventQueue.java:703)
at java.awt.EventQueue.access$000(EventQueue.java:102)
at java.awt.EventQueue$3.run(EventQueue.java:662)
at java.awt.EventQueue$3.run(EventQueue.java:660)
at java.security.AccessController.doPrivileged(Native Method)
at java.security.ProtectionDomain$1.doIntersectionPrivilege(ProtectionDomain.java:76)
at java.security.ProtectionDomain$1.doIntersectionPrivilege(ProtectionDomain.java:87)
at java.awt.EventQueue$4.run(EventQueue.java:676)
at java.awt.EventQueue$4.run(EventQueue.java:674)
at java.security.AccessController.doPrivileged(Native Method)
at java.security.ProtectionDomain$1.doIntersectionPrivilege(ProtectionDomain.java:76)
at java.awt.EventQueue.dispatchEvent(EventQueue.java:673)
at java.awt.EventDispatchThread.pumpOneEventForFilters(EventDispatchThread.java:244)
at java.awt.EventDispatchThread.pumpEventsForFilter(EventDispatchThread.java:163)
at java.awt.EventDispatchThread.pumpEventsForHierarchy(EventDispatchThread.java:151)
at java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:147)
at java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:139)
at java.awt.EventDispatchThread.run(EventDispatchThread.java:97)
The class List looks like this:
package sledmonitor;
import java.awt.BorderLayout;
import java.awt.EventQueue;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.border.EmptyBorder;
import javax.swing.table.DefaultTableModel;
import javax.swing.JScrollPane;
import javax.swing.JTable;
public class List extends JFrame {
private JPanel contentPane;
private JTable table;
static DefaultTableModel tablemodel;
/**
* Launch the application.
*/
public static void main(String[] args) {
EventQueue.invokeLater(new Runnable() {
public void run() {
try {
List frame = new List();
frame.setVisible(true);
} catch (Exception e) {
e.printStackTrace();
}
}
});
}
/**
* Create the frame.
*/
public List() {
setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
setBounds(100, 100, 450, 300);
contentPane = new JPanel();
contentPane.setBorder(new EmptyBorder(5, 5, 5, 5));
contentPane.setLayout(new BorderLayout(0, 0));
setContentPane(contentPane);
JScrollPane scrollPane = new JScrollPane();
contentPane.add(scrollPane, BorderLayout.CENTER);
tablemodel = new DefaultTableModel();
tablemodel.addColumn("OfferNumber");
table = new JTable(tablemodel);
scrollPane.setViewportView(table);
}
}
Any idea?
You seem to be accessing the static field List.tablemodel without instanciating first the List object, so you aren't calling List() which doesn't instanciate your tablemodel object.
You should then use a static constructor:
static {
tablemodel = new DefaultTableModel();
tablemodel.addColumn("OfferNumber");
}
And remove those 2 lines of code from the List() constructor.
use this
DefaultTableModel myData = new DefaultTableModel();
JTable table = new JTable(myData);
// initialize the table with model type constructor
while(rs.next()){
String offer = rs.getString("offernumber");
Object [] rowData=new Object[]{offer,null};
myData.addRow(rowData);
}
//use addRow(Object[] rowData) method of defaulttablemodel class

J2ME connect localhost nullpointerexception 0

I am trying to connect localhost and insert data into database through j2me application.but when I am connecting the server it shows there is a nullpointerexception 0 error.
this is midlet code
import java.io.DataOutputStream;
import java.io.InputStream;
import javax.microedition.io.HttpConnection;
import javax.microedition.lcdui.Command;
import javax.microedition.lcdui.CommandListener;
import javax.microedition.lcdui.Display;
import javax.microedition.lcdui.Displayable;
import javax.microedition.lcdui.Form;
import javax.microedition.lcdui.StringItem;
import javax.microedition.midlet.*;
public class Midlet_1 extends MIDlet implements CommandListener {
Display mdDisplay;
Form mForm;
StringItem messageitem;
Command exit, connectCommand;
public Midlet_1() {
mForm = new Form("My Counter midlet");
messageitem = new StringItem(null, "");
exit = new Command("Exit", Command.EXIT, 0);
connectCommand = new Command("Connect", Command.SCREEN, 0);
mForm.append(messageitem);
mForm.addCommand(exit);
mForm.addCommand(connectCommand);
mForm.setCommandListener(this);
}
public void startApp() {
mdDisplay = Display.getDisplay(this);
mdDisplay.setCurrent(mForm);
}
public void pauseApp() {
}
public void destroyApp(boolean unconditional) {
}
public void commandAction(Command c, Displayable d) {
if (c == exit) {
notifyDestroyed();
} else if (c == connectCommand) {
Form waitform = new Form("Waiting");
mdDisplay.setCurrent(waitform);
Thread t = new Thread() {
public void run() {
connect();
}
};
t.start();
}
}
private void connect() {
try {
HttpConnection hs = null;
InputStream in = null;
String url = "localhost:8080/testweb/src/save";
hs.setRequestProperty("User-Agent", "Profile/MIDP-2.0,Configuration/CLDC-2.0");
hs.setRequestProperty("Content-Language", "en-US");
hs.setRequestMethod(HttpConnection.POST);
DataOutputStream ds = hs.openDataOutputStream();
ds.writeUTF("nam56");
ds.writeUTF("67");
ds.writeUTF("0716522549");
ds.flush();
ds.close();
in = hs.openInputStream();
int connectlength = (int) hs.getLength();
byte[] raw = new byte[connectlength];
int length = in.read(raw);
// int ch;
// StringBuffer sb=new StringBuffer();
// while((ch=in.read())!=-1){
// sb.append((char)ch);
// }
in.close();
hs.close();
String s = new String(raw, 0, length);
messageitem.setText(s);
} catch (Exception e) {
messageitem.setText(e.toString());
System.out.println(e);
}
mdDisplay.setCurrent(mForm);
}
}
and this is servlet code
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException, ClassNotFoundException, SQLException {
response.setContentType("text/html;charset=UTF-8");
PrintWriter out = response.getWriter();
try {
DataInputStream in=new DataInputStream(request.getInputStream());
String name=in.readUTF();
String id=in.readUTF();
String contt=in.readUTF();
Connection c=DBcon.setconConnection();
Statement s=c.createStatement();
s.executeUpdate("insert into details values('"+id+"','"+name+"''"+contt+"')");
out.print("successfullllll");
} finally {
out.close();
}
}
please check this out.....
This might work only if you are running an emulator on the same machine as the server. Try to replace locahost by 127.0.0.1.
In your connect() method, I can see that you initialized hs as null then you called setRequestProperty. Try to initialize hs properly before calling its methods.