Move large data between tables in oracle with bulk insert - sql

I want to move 1 million rows of data to another table. Im using query:
insert into Table1
select * from Table2;
in my PL/SQL function. But this way is too slow.
How can I do this with Bulk Insert method?
Source and destination table has same structure.
Tables have hash partition and 1 index.

Forget about bulk insert. Because the insert into select is the best bulk you can load.
The fastest would be to disable the indexes (mark them unusable) and do this in a SINGLE
insert:
insert /*+ append */ into TARGET
select COLS
from SOURCE;
commit;
and rebuild the indexes using UNRECOVERABLE (and maybe even parallel).
PS: If the table is partitioned (Both source and target, you can even use parallel inserts)
FOLLOW UP:
Check the performance of the below select
SELECT /*+ PARALLEL(A 4)
USE_HASH(A) ORDERED */
YOUR_COLS
FROM
YOUR_TABLE A
WHERE
ALL_CONDITIONS;
If faster then
INSERT /*+ APPEND */
INTO
TARGET
SELECT /*+ PARALLEL(A 4)
USE_HASH(A) ORDERED */
YOUR_COLS
FROM
YOUR_TABLE A
WHERE
ALL_CONDITIONS;

USING Bulk Collect
Converting to collections and bulk processing can increase the volume and complexity of your code. If you need a serious boost in performance, however, that increase is well-justified.
Collections, an evolution of PL/SQL tables that allows us to manipulate many variables at once, as a unit. Collections, coupled with two new features introduced with Oracle 8i, BULK_COLLECT and FORALL, can dramatically increase the performance of data manipulation code within PL/SQL.
CREATE OR REPLACE PROCEDURE test_proc (p_array_size IN PLS_INTEGER DEFAULT 100)
IS
TYPE ARRAY IS TABLE OF all_objects%ROWTYPE;
l_data ARRAY;
CURSOR c IS SELECT * FROM all_objects;
BEGIN
OPEN c;
LOOP
FETCH c BULK COLLECT INTO l_data LIMIT p_array_size;
FORALL i IN 1..l_data.COUNT
INSERT INTO t1 VALUES l_data(i);
EXIT WHEN c%NOTFOUND;
END LOOP;
CLOSE c;
END test_proc;

this procedure created by rohit sahani......
using bulk collect i create this procedure
in this procedure i also did update and insert in table and
in this procedure i did collect data in bulk and then update in salary with the use of forall statement .
CREATE OR REPLACE PROCEDURE sam_proc_1(l_salary NUMBER)
IS
l_address VARCHAR(100) := 'karnatka';
a_address VARCHAR(100) := 'bihar';
c_limit PLS_INTEGER := 100;
TYPE employee_ids_t IS TABLE OF emp.emp_id%TYPE;
l_employee_ids employee_ids_t;
CURSOR employees_cur IS SELECT emp_id FROM emp;
-- CURSOR CUR1 IS SELECT EMP_ID, FNAME, ADDRESS FROM EMP;
BEGIN
OPEN employees_cur;
LOOP
FETCH employees_cur BULK COLLECT INTO l_employee_ids LIMIT c_limit;
EXIT WHEN l_employee_ids.COUNT = 0;
-----here start updating
FORALL indx IN 1 .. l_employee_ids.COUNT SAVE EXCEPTIONS
UPDATE emp a
SET a.salary = a.salary + l_salary
WHERE a.emp_id = l_employee_ids(indx);
commit;
END LOOP;
-----------
BEGIN
UPDATE emp v
SET v.address = l_address
WHERE v.address = a_address;
COMMIT;
EXCEPTION WHEN
OTHERS THEN
RAISE_APPLICATION_ERROR(-20001,'An error encountered while updating in address - '
||SQLCODE||' -ERROR- '||SQLERRM);
END;
---------
BEGIN
FOR I IN ( SELECT EMP_ID, FNAME, ADDRESS FROM EMP)
LOOP
INSERT INTO students
( stu_id, stu_name, stu_address)
VALUES
(i.emp_id, i.fname, i.address);
END LOOP;
COMMIT;
EXCEPTION WHEN
OTHERS THEN
RAISE_APPLICATION_ERROR(-20001,'An error encountered while inserting - '
||SQLCODE||' -ERROR- '||SQLERRM);
END;
------------
EXCEPTION
WHEN OTHERS
THEN
IF SQLCODE = -24381
THEN
FOR indx IN 1 .. SQL%BULK_EXCEPTIONS.COUNT
LOOP
-- Caputring errors occured during update
DBMS_OUTPUT.put_line( SQL%BULK_EXCEPTIONS (indx).ERROR_INDEX
||','||
SQL%BULK_EXCEPTIONS (indx).ERROR_CODE);
--<You can inset the error records to a table here>
END LOOP;
ELSE
RAISE;
END IF;
END sam_proc_1;
-----END PROCEDURE;c

Related

Create a generic PL SQL procedure to log Bulk Collect errors dynamically

I recently learned about the use of BULK COLLECT in SQL. I found a way to handle exceptions generated by DML statements :
SET SERVEROUTPUT ON SIZE 99999;
--
DECLARE
--
bulk_errors exception;
PRAGMA exception_init(bulk_errors, -24381);
--
--
CURSOR cursEmployee IS
SELECT EMPLOYEE_ID, EMPLOYEE_NAME
FROM EMPLOYEE;
TYPE employee_table IS TABLE OF cursEmployee%rowtype;
employee_rec employee_table;
--
BEGIN
--
OPEN cursEmployee;
FETCH cursEmployee BULK COLLECT INTO employee_rec LIMIT 10000;
--
WHILE employee_rec.COUNT != 0 LOOP
--
FORALL indx IN INDICES OF employee_rec save exceptions
--
INSERT INTO EMPLOYEE (
EMPLOYEE_ID,
EMPLOYEE_NAME
)
VALUES (
employee_rec.EMPLOYEE_ID,
employee_rec.EMPLOYEE_NAME
);
--
COMMIT;
--
FETCH cursEmployee BULK COLLECT INTO employee_rec LIMIT 10000;
--
END LOOP;
exception when bulk_errors then
for i in 1 .. sql%bulk_exceptions.COUNT loop
dbms_output.put_line('Employee Id : ' || sql%bulk_exceptions(i).EMPLOYEE_ID);
dbms_output.put_line('Employee Id : ' || sql%bulk_exceptions(i).EMPLOYEE_NAME);
dbms_output.put_line('Error Message: '||sqlerrm(-sql%bulk_exceptions(i).error_code));
end loop;
CLOSE cursEmployee;
END;
/
I then created a generic procedure to log the exceptions :
CREATE OR REPLACE PROCEDURE LOG_BULK_EXCEPTIONS IS
BEGIN
IF SQL%BULK_EXCEPTIONS.COUNT > 0 THEN
DBMS_OUTPUT.PUT_LINE(' Errors occured during a BULK COLLECT statement : ');
DBMS_OUTPUT.PUT_LINE(' Number of exceptions : ' || SQL%BULK_EXCEPTIONS.COUNT );
--
FOR i IN 1 .. SQL%BULK_EXCEPTIONS.COUNT LOOP
DBMS_OUTPUT.PUT_LINE(' Error : ' || SQLERRM(-SQL%BULK_EXCEPTIONS(i).ERROR_CODE));
DBMS_OUTPUT.PUT_LINE('Backtrace : ' || DBMS_UTILITY.FORMAT_ERROR_BACKTRACE);
END LOOP;
--
END IF;
END;
/
I find this way of logging exceptions a bit limited : we just get the error (numeric value, cannot insert null into, etc...). I'm looking for a way to add information about the data / specific elements in the cursor that raised the error.
To do this, I need to pass a column name as parameter to my procedure, and concatenate it to obtain this sort of statement :
dbms_output.put_line(' Internal Id : ' || sql%bulk_exceptions(i).MY_COLUMN_PARAMETER);
This way, I could use this logging procedure everywhere in the Database, which would be great.
Does anyone know how to concatenate a string parameter to this " sql%bulk_exceptions(i). " and execute it correctly ?
Yes, you can get what you are looking for provided what you want to add in in the original collection. The sql%bulk_exceptions collection has another column, ERROR_INDEX. It contains the index of the row in the original collection. This allows you to reference values from the that collection via
employee_rec(sql%bulk_exceptions(i).error_index).id;
employee_rec(sql%bulk_exceptions(i).error_index).name;
Your procedure has another issue. Your exception block is outside the you loop processing the bulk collection. As a consequence your bulk buffers will be processed only until the first buffer contains an error; no subsequent buffers will be processed. You can avoid this by creating a block inside the processing loop and handling exceptions within the inner block. Also, nice to see you went to the effort to actually close the cursor. However, it is in the exception block so it only executes if there is a exception. See here for example of each. Since I did not want to create over 10000 rows for demo I reduced the Bulk Limit to 3.

Novice to DB - Oracle

I'm pretty new to Oracle and Database.
I'm trying to write a stored procedure with cursors. How do I write a select statement inside the cursor loop, and how do I loop through the result set that I get from the select inside that cursor loop?
For example:
Open Curs1;
Exit When Curs1%NotFound;
Loop
Select column1,column2 from table -- Returns multiple records. How to loop through this record set and perform CRUD operations.
End loop;
Close Curs1;
Use a FOR-loop cursor - they are faster and simpler than the open/fetch/close syntax.
begin
for results1 in
(
select ...
) loop
--Do something here
for results2 in
(
select ...
where some_table.some_column = results1.some_column
) loop
--Do something here
end loop;
end loop;
end;
/
Although this literally answers the question, you generally do not want to have loops inside loops like this. If possible, it would be better to combine the two SQL statements with a join and then loop through the results.
Try to use the following example that fetches rows one at a time from the cursor variable emp_cv into the user-defined record emp_rec:
declare
TYPE YourType IS ref cursor return YourTable%rowtype;
tab_cv YourType;
tab_rec YourTable%rowtype;
begin
loop
fetch tab_cv into emp_rec;
exit when tab_cv%notfound;
...
end loop;
end;
The BULK COLLECT clause lets you fetch entire columns from the result set, or the entire result set at once. The following example, retrieves columns from a cursor into a collection:
declare
type NameList IS table of emp.ename%type;
names NameList;
cursor c1 is select ename from emp where job = 'CLERK';
begin
open c1;
fetch c1 bulk collect into names;
...
close c1;
end;
The following example uses the LIMIT clause. With each iteration of the loop, the FETCH statement fetches 100 rows (or less) into index-by table acct_ids. The previous values are overwritten.
declare
type NumList is table of number index by binary_integer;
cursor c1 is select acct_id from accounts;
acct_ids NumList;
rows natural := 100; -- set limit
begin
open c1;
loop
/* The following statement fetches 100 rows (or less). */
fetch c1 bulk collect into acct_ids limit rows;
exit when c1%notfound;
...
end loop;
close c1;
end;
You need to declare the CURSOR and FETCH the records in the loop.
DECLARE
CURSOR curs1
IS
SELECT column1,
column2
FROM yourtable;
v_column1 yourtable.column1%TYPE;
v_column2 yourtable.column2%TYPE;
BEGIN
OPEN curs1;
LOOP
FETCH curs1
INTO v_column1,
v_column2;
EXIT
WHEN curs1%NOTFOUND;
INSERT INTO yourtable2(col1)VALUES( '000'||v_column1 );
-- A sample DML operation.
--Do other operations on individual records here.
END LOOP;
CLOSE curs1;
END;

How to execute results of dbms_output.put_line

There is a table contains this kind of data: select to_char(sysdate,'day') from dual in a column. I want to get results of the every query that the table keeps.
My result set should be the result of select to_char(sysdate,'day') from dual query. So in this case it is a tuesday.
SO_SQL_BODY is Varchar2.
I wrote this code but it returns only table data.
CREATE or replace PROCEDURE a_proc
AS
CURSOR var_cur IS
select SO_SQL_BODY FROM SO_SUB_VARS group by SO_SQL_BODY;
var_t var_cur%ROWTYPE;
TYPE var_ntt IS TABLE OF var_t%TYPE;
var_names var_ntt;
BEGIN
OPEN var_cur;
FETCH var_cur BULK COLLECT INTO var_names;
CLOSE var_cur;
FOR indx IN 1..var_names.COUNT LOOP
DBMS_OUTPUT.PUT_LINE(var_names(indx).SO_SQL_BODY);
END LOOP;
END a_proc;
DECLARE
res varchar2(4000);
sql_str varchar2(1000);
BEGIN
FOR r IN
(select SO_SQL_BODY FROM SO_SUB_VARS WHERE SO_SQL_BODY IS NOT NULL
)
LOOP
sql_str := r.SO_SQL_BODY;
EXECUTE immediate sql_str INTO res;
dbms_output.put_line(sql_str);
dbms_output.put_line('***********************');
dbms_output.put_line(res);
dbms_output.put_line('***********************');
END LOOP;
END;
/
Try this - iterate to not null records - execute them and print the result.This script works supposing the fact that SO_SQL_BODY contains a query which projects only one column.Also if the projection is with more than two columns then try to use a refcursor and dbms_sql package
İf var_names(indx).SO_SQL_BODY output is a runnable sql text;
CREATE or replace PROCEDURE a_proc
AS
CURSOR var_cur IS
select SO_SQL_BODY FROM SO_SUB_VARS group by SO_SQL_BODY;
var_t var_cur%ROWTYPE;
TYPE var_ntt IS TABLE OF var_t%TYPE;
var_names var_ntt;
BEGIN
OPEN var_cur;
FETCH var_cur BULK COLLECT INTO var_names;
CLOSE var_cur;
FOR indx IN 1..var_names.COUNT LOOP
DBMS_OUTPUT.PUT_LINE(var_names(indx).SO_SQL_BODY);
EXECUTE IMMEDIATE var_names(indx).SO_SQL_BODY;
END LOOP;
END a_proc;
You don't need a full cursor for this example. An implicit one would make it a lot shorter.
create or replace procedure a_proc is
lReturnValue varchar2(250);
begin
for q in (select so_sql_body from so_sub_vars group by so_sql_body)
loop
execute immediate q.so_sql_body into lReturnValue;
dbms_output.put_line(lReturnValue);
end loop;
end a_proc;
You should add an exception handler that will care for cases where there is a bad SQL query in your table. Also note that executing querys saved in a database table is your entry point to SQL injection.

PLSQL Performance Issue

This is more of a doubt than a problem. I have a requirement of just selecting some of the common fields from one table and insert it into a different table. I have done the code with 2 different styles but both with BULK COLLECT. Which is the better option to go for or is there any other way apart from this?
Please find the necessary details below.
Procedure (Way 1):
create or replace procedure a2 is
cursor c1 is select id,address from emp1;
type t is table of c1%rowtype;
c t;
begin
open c1;
loop
fetch c1 bulk collect into c;
exit when c.count=0;
forall j in 1..c.count save exceptions
insert into empl1(id,address) values (c(j).id,c(j).address);
end loop;
commit;
exception when others then
for j in 1..sql%bulk_exceptions.count loop
dbms_output.put_line('The sql error is error occured');
end loop;
end a2;
/
Running the above procedure and output:
declare
a number;
begin
dbms_output.put_line ('before procedure: ' || to_char(sysdate, 'HH24:MI:SS'));
a2;
dbms_output.put_line ('after procedure: ' || to_char(sysdate, 'HH24:MI:SS'));
end;
Output :
before procedure: 23:44:48
after procedure: 23:45:47
PL/SQL procedure successfully completed.
So the above procedure took 59 seconds to insert 34801020 records.
Now please find the second procedure.
Procedure (Way 2):
create or replace procedure a3 is
cursor c1 is select id,address from emp1;
type t is table of c1%rowtype;
c t;
begin
select id,address bulk collect into c from emp1;
forall j in 1..c.count save exceptions
insert into empl1(id,address) values (c(j).id,c(j).address);
exception when others then
for j in 1..sql%bulk_exceptions.count loop
dbms_output.put_line('The sql error is error occured');
end loop;
end a3;
/
Running the above procedure with output .
declare
a number;
begin
dbms_output.put_line ('before procedure: ' || to_char(sysdate, 'HH24:MI:SS'));
a3;
dbms_output.put_line ('after procedure: ' || to_char(sysdate, 'HH24:MI:SS'));
end;
Output :
before procedure: 23:47:57
after procedure: 23:48:53
PL/SQL procedure successfully completed.
This Procedure took 56 seconds to insert 34801020 records.
Total records in emp1 table.
SQL> select count(1) from emp1;
COUNT(1)
----------
34801020
Hence My question :
Which of the above two methods is the best way to insert 3 million records into the table and please suggest me if there is any other better way to do the above process of insertion.
I rans a test using a similar piece of code with a similar dataset size
Using a cursor loop
create or replace procedure a2 is
cursor c1 is select empno,ename from bigemp;
type t is table of c1%rowtype;
c t;
begin
open c1;
loop
fetch c1 bulk collect into c;
exit when c.count=0;
forall j in 1..c.count save exceptions
insert into bigemp2(empno,ename) values (c(j).empno,c(j).ename);
end loop;
commit;
exception when others then
for j in 1..sql%bulk_exceptions.count loop
dbms_output.put_line('The sql error is error occured');
end loop;
end a2;
SQL> exec a2
PL/SQL procedure successfully completed.
Elapsed: 00:00:56.93
Doing a regular insert statement, not using a cursor-for loop
SQL> insert into bigemp2( empno, ename )
select empno, ename from bigemp t2
29360128 rows created.
Elapsed: 00:00:11.30
Now do a direct path insert
SQL> insert /*+ append */ into bigemp2( empno, ename )
select empno, ename from bigemp t2
;
29360128 rows created.
Elapsed: 00:00:06.01
Add some parallelism
SQL> alter session enable parallel dml;
Session altered
SQL> insert /*+ append parallel(2 ) */ into bigemp2( empno, ename )
select /* parallel( t2, 2 ) */ empno, ename from bigemp t2
;
29360128 rows created.
Elapsed: 00:00:03.52
So in summary, just by using the appropriate technique, we can make the process go and order of magnitude faster (about 16x faster)

If table have 0 rows exit procedure

I have a procedure inside a package and I want to implement a logic, wich will not insert the temp table into the main table if the temp table have 0 rows, and then go to the next procedure of the package.
IF (not exists(select 1 from temp)) THEN
RETURN;
ELSE
EXECUTE IMMEDIATE 'TRUNCATE TABLE main';
INSERT --+APPEND
INTO main
Select * from temp;
EXECUTE IMMEDIATE 'TRUNCATE TABLE temp';
END IF;
With this solution, the package is compiled with error.
Can anyone give me some tips?
you can use loop, without any variables, just first iteration, something like this
FOR a in (select 1 from temp where rownum = 1) LOOP
EXECUTE IMMEDIATE 'TRUNCATE TABLE main';
INSERT --+APPEND
INTO main
Select * from temp;
EXECUTE IMMEDIATE 'TRUNCATE TABLE temp';
END LOOP;
Just count one row and then test whether the result is 0 or 1:
declare
l_row_check integer := 0;
begin
select count(*) into l_row_check from main
where rownum = 1;
if l_row_check = 0 then
execute immediate 'truncate table main';
insert --+ append
into main
select * from temp;
execute immediate 'truncate table temp';
end if;
end;
The easiest is to use a variable to check:
--- suggested edit: add condition to select 1 row at most and avoid
-- counting big table.
select count(1) into v_count from temp where rownum <=1;
IF (v_count=0) THEN
RETURN;
ELSE
EXECUTE IMMEDIATE 'TRUNCATE TABLE main';
INSERT --+APPEND
INTO main
Select * from temp;
EXECUTE IMMEDIATE 'TRUNCATE TABLE temp';
END IF;
Some answer on here use a SELECT INTO method, but I find those a bit tricky.
Since if for example SELECT ColumnA INTO vcColumnA FROM Temp will not have any records, you will end up with the error ORA-01403: no data found.
And those can be hard to find if you have a bigger database.
To loop through a table and do something with values I think cursors and records are more safe.
For example:
DECLARE
CURSOR cTemp IS
SELECT ColumnA, ColumnB
FROM Temp;
rTemp cTemp%ROWTYPE;
BEGIN
OPEN cTemp;
LOOP
FETCH cTemp INTO rTemp;
-- Exit when we read all lines in the Temp table.
EXIT WHEN cTemp%NOTFOUND;
--Do something with every row.
--For example, print ColumnB.
DBMS_OUTPUT.PUT_LINE(rTemp.ColumnB);
END LOOP;
CLOSE cTemp;
END;
/