PostgreSQL 9.5 Insert/Update while partitioning table - sql

I need to achieve updating (via ON CONFLICT()) row in a partitioned tables.
So far, my tries:
Table creation:
CREATE TABLE public.my_tbl
(
goid character varying(255) NOT NULL,
timestamps timestamp without time zone[],
somenumber numeric[],
CONSTRAINT my_tbl_pkey PRIMARY KEY (goid)
)
WITH (
OIDS=FALSE
);
ALTER TABLE public.my_tbl
OWNER TO postgres;
Table Sequence:
CREATE SEQUENCE public.fixations_data_pkey_seq
INCREMENT 1
MINVALUE 1
MAXVALUE 9223372036854775807
START 1
CACHE 1;
ALTER TABLE public.fixations_data_pkey_seq
OWNER TO postgres;
Table partition trigger, which creates new table with name "table_YYYY_MM_DD", where "YYYY_MM_DD" - current date (query execution date):
CREATE OR REPLACE FUNCTION public.my_tbl_insert_trigger()
RETURNS trigger AS
$BODY$
DECLARE
table_master varchar(255) := 'my_tbl';
table_part varchar(255) := '';
BEGIN
-- Partition table name --------------------------------------------------
table_part := table_master
|| '_' || DATE_PART( 'year', NOW() )::TEXT
|| '_' || DATE_PART( 'month', NOW() )::TEXT
|| '_' || DATE_PART( 'day', NOW() )::TEXT;
-- Check if partition exists --------------------------------
PERFORM
1
FROM
pg_class
WHERE
relname = table_part
LIMIT
1;
-- If not exist, create new one --------------------------------------------
IF NOT FOUND
THEN
-- Create parition, which inherits master table --------------------------
EXECUTE '
CREATE TABLE ' || table_part || '
(
goid character varying(255) NOT NULL DEFAULT nextval(''' || table_master || '_pkey_seq''::regclass),
CONSTRAINT ' || table_part || '_pkey PRIMARY KEY (goid)
)
INHERITS ( ' || table_master || ' )
WITH ( OIDS=FALSE )';
-- Create indices for current table-------------------------------
EXECUTE '
CREATE INDEX ' || table_part || '_adid_date_index
ON ' || table_part || '
USING btree
(goid)';
END IF;
-- Insert row into table (without ON CONFLICT)--------------------------------------------
EXECUTE '
INSERT INTO ' || table_part || '
SELECT ( (' || QUOTE_LITERAL(NEW) || ')::' || TG_RELNAME || ' ).*';
RETURN NULL;
END;
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION public.my_tbl_insert_trigger()
OWNER TO postgres;
CREATE TRIGGER my_tbl_insert_trigger
BEFORE INSERT
ON my_tbl
FOR EACH ROW
EXECUTE PROCEDURE my_tbl_insert_trigger();
After this I can insert new rows into table:
INSERT INTO my_tbl (goid, timestamps, somenumber)
VALUES ('qwe123SSsssa3', '{"2016-11-16 00:00:00", "2016-11-16 01:00:00"}', '{3, 12333}')
But when I'm trying to do UPSERT:
INSERT INTO my_tbl (goid, timestamps, somenumber)
VALUES ('qwe123SSsssa3', '{"2016-11-16 02:00:00"}', '{999}')
ON CONFLICT (goid)
DO UPDATE
SET timestamps=array_append(my_tbl.timestamps::timestamp[], '2016-11-16 02:00:00'),
somenumber=array_append(my_tbl.somenumber,'999');
I'm geting DUPLICATE PKEY error.
I guess, that I have to add ON CONFLICT to third EXECUTE in trigger function. But how should I do this?

Well , I've changed my third EXECUTE to :
-- Insert row into table (with ON CONFLICT)--------------------------------------------
EXECUTE '
INSERT INTO ' || table_part || '
SELECT ( (' || QUOTE_LITERAL(NEW) || ')::' || TG_RELNAME || ' ).*
ON CONFLICT (goid)
DO UPDATE
SET timestamps=' || table_part || '.timestamps::timestamp[] || ' || QUOTE_LITERAL(NEW.timestamps) || ',
somenumber=' || table_part || '.somenumber::numeric[] || ' || QUOTE_LITERAL(NEW.somenumber) || '
';
RETURN NULL;
Now, when I execute query:
INSERT INTO my_tbl (goid, timestamps, somenumber)
VALUES ('potato_1', ARRAY['2016-11-16 12:00:00', '2016-11-16 15:00:00']::timestamp[], ARRAY[223, 211]::numeric[]);
there are no any errors, and it extends array-type columns as I expected
I can admit that this is a dirty solution, but it seems that it works.
If someone has a better solution, I'll glad to look at it.

Related

How Can I Format and Print Column Data Types For Oracle SQL?

I am trying to generate a Create Table Statement. To do this, I have created 2 procedures to Extract Tables and Columns from my Schema. The Output of these procedures is then spooled to a SQL file and is supposed to look like this:
The Formatted Output
Right now, My output Looks like this:
----
---- Run on October 04, 2020 at 22:00
----
-- Start Extracting table IMAGE
CREATE TABLE IMAGE (
MFR CHAR(3,)
, PRODUCT CHAR(5,)
, IMAGE BLOB(4000,)
, TECHSPECS BFILE(530,)
);-- END of Table IMAGE creation
--
--
-- Start Extracting table ORDERS
CREATE TABLE ORDERS (
ORDERNUM NUMBER(227,0)
, ORDERDATE DATE(7,)
, CUST NUMBER(223,0)
, REP NUMBER(223,0)
, MANUF CHAR(3,)
, PROD CHAR(5,)
, QTY NUMBER(225,0)
, AMOUNT NUMBER(225,2)
);-- END of Table ORDERS creation
--
--
-- Start Extracting table PRODUCTS
CREATE TABLE PRODUCTS (
MFR CHAR(3,)
, PRODUCT CHAR(5,)
, DESCRIPTION VARCHAR2(100,)
, PRICE NUMBER(225,2)
, QTYONHAND NUMBER(225,0)
);-- END of Table PRODUCTS creation
--
--
---- Oracle Catalog Extract Utility V1.0 ----
---- Run on October 04, 2020 at 22:00
As you can see, the Data types, along with their Data_length, Data_scale, and DAta_precision are not perfectly formatted as per the requirements. I also haven't been able to make a column that displays the null status of each column. The Date, BFILE, and BLOB as shown in the Output are especially troublesome for me to format properly.
This is my code as of now:
SET ECHO OFF
SET FEEDBACK ON
SET WRAP OFF
--The Procedure to Extract The Columns
SET SERVEROUTPUT ON
CREATE OR REPLACE PROCEDURE Extract_Columns (
--Creating Variables of passed values
wSee IN OUT varchar2,
wTable IN USER_TABLES.table_name%type
)
AS
--The Cursor to run through the Columns
CURSOR Extract_C IS
SELECT COLUMN_NAME, DATA_TYPE, DATA_PRECISION, DATA_SCALE, DATA_LENGTH
FROM USER_TAB_COLUMNS
WHERE TABLE_NAME = wTable
ORDER BY Column_ID;
CurrentRow Extract_C%ROWTYPE;
--Creating variables
wItterations NUMBER(1):=0;
BEGIN
FOR CurrentRow IN Extract_C LOOP
IF wItterations = 0 THEN
wsee := wsee || CHR(10) || ' ' || RPAD(currentrow.column_name, 15) || currentrow.data_type || '(' || currentrow.DATA_LENGTH || currentrow.DATA_SCALE || ')';
ELSE
wsee := wsee || CHR(10) || ', ' || RPAD(currentrow.column_name, 15) || currentrow.data_type || '(' || currentrow.DATA_LENGTH || currentrow.DATA_PRECISION || ',' || currentRow.DATA_SCALE || ')';
END IF;
wItterations:= wItterations +1;
END LOOP;
END;
/
SHOW ERRORS;
SET SERVEROUTPUT ON
CREATE OR REPLACE PROCEDURE Extract_Tables
AS
l_crt varchar2(356);
CURSOR Extract_T IS
SELECT TABLE_NAME
FROM USER_TABLES
ORDER BY TABLE_NAME;
CurrentRow Extract_T%ROWTYPE;
--Creating Variables
wvers VARCHAR2(256) := 'V1.0';
wcur_Tim VARCHAR2(200) := '' || TO_CHAR( CURRENT_DATE, 'Month DD, YYYY') || ' at ' || To_CHAR(CURRENT_DATE, 'HH24:MI');
wheader VARCHAR2(45) := 'CREATE TABLE ' || CurrentRow.table_name ||' (';
wspacing NUMBER := length(wheader);
BEGIN
DBMS_OUTPUT.PUT_LINE('---- Oracle Catalog Extract Utility ' || wvers || ' ----');
DBMS_OUTPUT.PUT_LINE('----');
DBMS_OUTPUT.PUT_LINE('---- Run on ' || wcur_Tim);
DBMS_OUTPUT.PUT_LINE('----');
FOR CurrentRow IN Extract_T LOOP
DBMS_OUTPUT.PUT_LINE('-- Start Extracting table ' || CurrentRow.table_name || '');
DBMS_OUTPUT.PUT_LINE('CREATE TABLE ' || CurrentRow.table_name ||' (');
--This is where i should be calling the other procedure
Extract_Columns(l_crt, CurrentRow.table_name);
--This is where create tables procedure begins again
DBMS_OUTPUT.PUT_LINE(l_crt || chr(10) || LPAD(' ', 15 + length(currentrow.Table_name)) || ');' || '-- END of Table ' || currentrow.Table_name || ' creation');
DBMS_OUTPUT.PUT_LINE('--' || CHR(10) || '--' );
l_crt := NULL;
END LOOP;
--Ending Statement
DBMS_OUTPUT.PUT_LINE('---- Oracle Catalog Extract Utility ' || wvers || ' ----');
DBMS_OUTPUT.PUT_LINE('---- Run on ' || wcur_Tim);
END;
/
SHOW ERRORS;
To execute, I have to compile the 'Extract_column' procedure and then 'Extract_tables' procedure, then I run the tables procedure to generate my output.
How can I format my current output in a way that it exactly matches the picture above?

PLSQL procedure for automatic column type changing

Let's assume following table:
drop table test1;
create table test1(
A number(10)
);
insert into test1 values (1);
insert into test1 values (10);
So as you can see table TEST1 is already populated. What i need to do is to change types of column A to varchar2. Since this column has values we can't just use following code:
alter table test1 modify A varchar2(10);
So i have wrote stored procedure which:
Renames column A to A1 ->
Then adds new column called A of type varchar2 ->
Then updates column A with values from column A1 ->
And ultimately drops old column A1.
Code which runs this process is following:
create or replace procedure change_col_type_to_varchar2(p_tab in varchar2, p_col in varchar2)
is
v_string clob;
cursor cur is
select column_name
from all_tab_columns
where table_name = upper(p_tab) and
column_name in (select regexp_substr(p_col,'[^,]+', 1, level) from dual
connect by regexp_substr(p_col, '[^,]+', 1, level) is not null);
begin
for i in cur loop
v_string := 'alter table ' || p_tab || ' rename column ' || i.column_name || ' to ' || i.column_name || '1' || ';';
dbms_lob.append(v_string,''||chr(10)||'');
dbms_lob.append(v_string, 'alter table ' || p_tab || ' add ' || i.column_name || ' varchar2(10);');
dbms_lob.append(v_string,''||chr(10)||'');
dbms_lob.append(v_string, 'update ' || p_tab || ' set ' || i.column_name || ' = ' || i.column_name || '1' || ';');
dbms_lob.append(v_string,''||chr(10)||'');
dbms_lob.append(v_string, 'alter table ' || p_tab || ' drop column ' || i.column_name || '1' || ';');
EXECUTE IMMEDIATE v_string;
DBMS_OUTPUT.PUT_LINE(v_string);
v_string := NULL;
end loop;
end;
I'am trying to apply this procedure to TEST1:
begin
change_col_type_to_varchar2('TEST1', 'A');
end;
And get error:
Error report -
ORA-23290: This operation may not be combined with any other operation
ORA-06512: at "YAVORSKYIY_DM.CHANGE_COL_TYPE_TO_VARCHAR2", line 19
ORA-06512: at "YAVORSKYIY_DM.CHANGE_COL_TYPE_TO_VARCHAR2", line 19
ORA-06512: at line 2
23290. 00000 - "This operation may not be combined with any other operation"
*Cause: ALTER TABLE RENAME COLUMN/CONSTRAINT operation was given in
conjunction with another ALTER TBALE Operation. This is not
allowed.
*Action: Ensure that RENAME COLUMN/CONSTRAINT is the only operation
specified in the ALTER TABLE.
But just typing :
alter table test1 rename column A to A1;
alter table test1 add A varchar2(100);
update test1 set A = A1;
alter table test1 drop column A1;
Works perfect.
Does anybody have any ideas about how to overcome this problem?
Appreciate your help.
the below will do what you asked for.
declare
procedure change_col_type_to_varchar2(p_tab in varchar2, p_col in varchar2)
is
v_string clob;
cursor cur is
select column_name
from all_tab_columns
where table_name = upper(p_tab) and
column_name in (select regexp_substr(p_col,'[^,]+', 1, level)
from dual
connect by regexp_substr(p_col, '[^,]+', 1, level) is not null);
begin
for i in cur loop
v_string := 'alter table ' || p_tab || ' rename column ' || i.column_name || ' to ' || i.column_name || '1';
execute immediate v_string;
v_string := 'alter table ' || p_tab || ' add ' || i.column_name || ' varchar2(10)';
execute immediate v_string;
v_string := 'update ' || p_tab || ' set ' || i.column_name || ' = ' || i.column_name || '1' ;
execute immediate v_string;
v_string := 'alter table ' || p_tab || ' drop column ' || i.column_name || '1' ;
execute immediate v_string;
v_string := NULL;
end loop;
end;
begin
DBMS_OUTPUT.PUT_LINE('Before calling');
change_col_type_to_varchar2('TEST1','A');
DBMS_OUTPUT.PUT_LINE('After calling');
end;
Well, execute each statement alone, instead of concatenating them. And you don't need LOBs, varchar2 for each one should be enough.
I propose other algorithm.
create new table 'table2' with varchar column;
select all values from table1.A and insert to table2 with to_char() conversion;
drop table1;
rename table2 to table1;
profit!

Query in PostgreSQL with large quantity of squid access requests

Hello people, I'm using a log daemon (https://github.com/paranormal/blooper) in Squid Proxy to put access log into PostreSQL and I make a Trigger Function:
DECLARE
newtime varchar := EXTRACT (MONTH FROM NEW."time")::varchar;
newyear varchar := EXTRACT (YEAR FROM NEW."time")::varchar;
user_name varchar := REPLACE (NEW.user_name, '.', '_');
partname varchar := newtime || '_' || newyear;
tablename varchar := user_name || '.accesses_' || partname;
BEGIN
IF NEW.user_name IS NOT NULL THEN
EXECUTE 'CREATE SCHEMA IF NOT EXISTS ' || user_name;
EXECUTE 'CREATE TABLE IF NOT EXISTS '
|| tablename
|| '('
|| 'CHECK (user_name = ''' || NEW.user_name || ''' AND EXTRACT(MONTH FROM "time") = ' || newtime || ' AND EXTRACT (YEAR FROM "time") = ' || newyear || ')'
|| ') INHERITS (public.accesses)';
EXECUTE 'CREATE INDEX IF NOT EXISTS access_index_' || partname || '_user_name ON ' || tablename || ' (user_name)';
EXECUTE 'CREATE INDEX IF NOT EXISTS access_index_' || partname || '_time ON ' || tablename || ' ("time")';
EXECUTE 'INSERT INTO ' || tablename || ' SELECT $1.*' USING NEW;
END IF;
RETURN NULL;
END;
The main function of it is make a table partition by user_name and by month-year of the access, inhering from a master clean table:
CREATE TABLE public.accesses
(
id integer NOT NULL DEFAULT nextval('accesses_id_seq'::regclass),
"time" timestamp with time zone NOT NULL,
time_response integer,
mac_source macaddr,
ip_source inet NOT NULL,
ip_destination inet,
user_name character varying(40),
http_status_code numeric(3,0) NOT NULL,
http_reply_size bigint NOT NULL,
http_request_method character varying(15) NOT NULL,
http_request_url character varying(4166) NOT NULL,
http_content_type character varying(100),
squid_hier_code character varying(20),
squid_request_status character varying(50),
user_id integer,
CONSTRAINT accesses_http_request_method_fkey FOREIGN KEY (http_request_method)
REFERENCES public.http_requests (method) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION,
CONSTRAINT accesses_http_status_code_fkey FOREIGN KEY (http_status_code)
REFERENCES public.http_statuses (code) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION,
CONSTRAINT accesses_user_id_fkey FOREIGN KEY (user_id)
REFERENCES public.users (id) MATCH SIMPLE
ON UPDATE NO ACTION ON DELETE NO ACTION
)
The main problem is get the sum of http_reply_size grouping by user_name and time, my query is:
SELECT
"time",
user_name,
sum(http_reply_size)
FROM
accesses
WHERE
extract(epoch from "time") BETWEEN 1516975122 AND 1516996722
GROUP BY
"time",
user_name
But this query is very slow in the server (3'237'976 rows currently in 2 days only). So, PostgreSQL has something to optimize a query with that need, or I need to use another SQL or NoSQL system.
Try to include a CHECK condition on each partition so doesn't have to scan all tables.
In my case is like this:
CREATE TABLE IF NOT EXISTS ' || table_name || '(
CONSTRAINT ' || pk || ' PRIMARY KEY (avl_id),
CHECK ( event_time >= ''' || begin_time || ''' AND event_time < ''' || end_time || ''' )
) INHERITS (avl_db.avl);
Also don't use extract(epoch from "time") that will need to calculate the value for each row and can't use the index you create for "time"
so use like this to get advantage of the index.
WHERE "time" >= '2018-01-01'::timestamp with time zone
and "time" < '2018-02-01'::timestamp with time zone

Create tables whose information stored in another table using stored procedure

I need to write a stored procedure to create table(s) whose information like table_name, column_name, data_type are stored in another table as below image...
No need to worry about Primary n Foreign Key specifications.
Is it possible to do that in ORACLE?
Thanks in Advance.
Yes. Construct the DDL string in a loop and execute it with execute immediate.
For example:
begin
for r in (
select 'create table ' || td.table_name || chr(10)||'( ' ||
listagg(rpad(td.column_name,31) || td.data_type, chr(10)||', ') within group (order by id) ||
' )' as create_table
from table_definitions td
group by td.id, td.table_name
order by td.id
)
loop
dbms_output.put_line(r.create_table || ';' || chr(10));
execute immediate r.create_table;
end loop;
end;
Demo setup:
create table table_definitions
( id integer not null
, table_name varchar2(30) not null
, column_name varchar2(30) not null, data_type varchar2(30) not null
, constraint tabdef_uk unique (table_name, column_name) );
insert all
into table_definitions values (1, 'EMP', 'EMP_ID', 'NUMBER')
into table_definitions values (1, 'EMP', 'EMP_NAME', 'VARCHAR2(30)')
into table_definitions values (1, 'EMP', 'SALARY', 'NUMBER')
into table_definitions values (1, 'EMP', 'DEPT_ID', 'NUMBER')
into table_definitions values (2, 'DEPT', 'DEPT_ID', 'NUMBER')
into table_definitions values (2, 'DEPT', 'DEPT_NAME', 'VARCHAR2(30)')
into table_definitions values (2, 'DEPT', 'LOCATION', 'VARCHAR2(30)')
select * from dual;
However, there are some problems with this whole approach.
I had to add lengths to your VARCHAR2 columns. You'd need to do the same for the others if you didn't want plain NUMBER for every numeric column. Also, there is no column ordering, so they could be generated in any order. I couldn't see the purpose of the id column - was it meant to be unique, or unique within table_name? And there is no provision for NOT NULL, defaults etc.
I would be rather nervous if I came across this in a system had to work on. What is it for?
I'm sure there will be a better approach, but meanwhile you can work on the below :
Your proc code should be something like this:
select distinct id, table_name
bulk collect into ip_id, ip_tab_name
from tab_details;
for x in ip_id.first .. ip_id.last loop
select column_name, data_type bulk collect into v_col_name,
v_data_type from tab_Details where id= ip_id(x);
v_sql := 'create table ' || ip_tab_name(x) || ' ( col1 number) ';
execute immediate v_sql;
for i in v_col_name.first .. v_col_name.last loop
v_sql1 := 'alter table ' || ip_tab_name(x) || ' add ' || v_col_name(i)
|| ' ' || v_data_type(i);
execute immediate v_sql1;
end loop; -- (for i loop)
v_sql2 := 'alter table ' || ip_tab_name(x) || ' drop column col1 ';
execute immediate v_sql2;
end loop;-- (for x loop)

Get Aggregate Values of Each Column In a Table

I am new to Oracle; I want to be able to generate a report that gives me the aggregate values for each column on a table. I haven't found anything that does this automatically, so I created a table where I upload the columns names, then I am using the data in that table to build individual SQL Statements:
Here is my table:
CREATE TABLE CARIAS.TEST_CA_1
(
DATA_BASE_NAME VARCHAR2(100 BYTE),
SCHEMA_NAME VARCHAR2(100 BYTE),
TABLE_NAME VARCHAR2(100 BYTE),
FIELD_NAME VARCHAR2(20 BYTE)
);
INSERT INTO TEST_CA_1 (DATA_BASE_NAME, SCHEMA_NAME, TABLE_NAME, FIELD_NAME) VALUES ('FDWP', 'FDW', 'D_CLAIM', 'FIELD_CLAIM_OFFICE');
INSERT INTO TEST_CA_1 (DATA_BASE_NAME, SCHEMA_NAME, TABLE_NAME, FIELD_NAME) VALUES ('FDWP', 'FDW', 'D_CLAIM', 'CLAIM_SYMBOL_CODE');
INSERT INTO TEST_CA_1 (DATA_BASE_NAME, SCHEMA_NAME, TABLE_NAME, FIELD_NAME) VALUES ('FDWP', 'FDW', 'D_CLAIM', 'HANDLING_DIVISION');
Here is the query that builds the statements:
SELECT
FIELD_NAME,
SQL_STATEMENT
FROM
(
--Creates Count Statements
SELECT 'SELECT ' || FIELD_NAME ||', COUNT(*) AS CNT FROM ' || SCHEMA_NAME || '.' || TABLE_NAME ||' GROUP BY ' || FIELD_NAME || ';' AS SQL_STATEMENT, FIELD_NAME FROM TEST_CA_1
UNION ALL
--Creates MIN and MAX Values Statement
SELECT 'SELECT MIN(' || FIELD_NAME || ') AS MIN_VALUE, MAX(' || FIELD_NAME || ') AS MAX_VALUE FROM ' || SCHEMA_NAME || '.' || TABLE_NAME || ';' AS SQL_STATEMENT
,FIELD_NAME
FROM TEST_CA_1
)
ORDER BY FIELD_NAME;
Then I use those "built" statements to get what I want. Is there an easier way to do this?
A first approach could be this:
create or replace function exec_on_table_field(i_table in varchar2,
i_field in varchar2,
i_function in varchar2)
return clob is
l_result clob;
begin
execute immediate 'select to_clob(' || i_function || '(' || i_field ||
')) from ' || i_table
into l_result;
return l_result;
end;
call this function with
select u.TABLE_NAME, u.COLUMN_NAME,
to_char(exec_on_table_field(u.TABLE_NAME, u.COLUMN_NAME, 'min')),
to_char(exec_on_table_field(u.TABLE_NAME, u.COLUMN_NAME, 'max')) from
user_tab_cols u where u.TABLE_NAME like 'PS_CS%' and rownum < 5;
hope it helps
The answer provided by Frank Ockenfuss worked great. I also found some of the information I was looking for by using the following Oracle View:
SELECT *
FROM ALL_TAB_COL_STATISTICS U
WHERE U.TABLE_NAME LIKE 'MY_TABLE_NAME';