Attached picture is my source data in a Snowflake table. I need to copy this into S3 as 2 individual files.
EXECUTE IMMEDIATE
$$
DECLARE
load_dt_tss timestamp;
file_name varchar2(30);
c1 CURSOR FOR SELECT DISTINCT RECORD_CONTENT, LOAD_DT_TS FROM mytable where LOAD_DT_TS >= '2022-02-09 00:00:00';
BEGIN
for record in c1 do
load_dt_tss:=RECORD.LOAD_DT_TS;
file_name:=load_dt_tss||'.csv';
EXECUTE IMMEDIATE 'copy into #STG_SF_S3/'||:file_name|| 'from (select record_content from mytable where LOAD_DT_TS >= ''2022-02-09 00:00:00'') FILE_FORMAT = (TYPE=JSON,COMPRESSION = NONE) single = FALSE overwrite=FALSE';
end for;
RETURN 0;
END;
$$ ;
Above given is the Query I'm using. When running this query, I get Error as:
"Uncaught exception of type 'STATEMENT_ERROR' on line 10 at position 3 : SQL compilation error: syntax error line 1 at position 40 unexpected '21' ".
Can someone please help.
CREATE database AAP_LDS_DEV_DB;
create schema AAP_LDS_DEV_DB.PURCHASE_ORDER;
create table AAP_LDS_DEV_DB.PURCHASE_ORDER.T_PO_STG(RECORD_CONTENT variant, LOAD_DT_TS timestamp_ntz );
insert into AAP_LDS_DEV_DB.PURCHASE_ORDER.T_PO_STG (SELECT parse_json(column1), to_timestamp(column2) from values
('{"blar":"blar1"}','2022-02-10'),
('{"blar":"blar2"}','2022-02-11')
);
EXECUTE IMMEDIATE
$$
DECLARE
sql text;
file_name text;
c1 CURSOR FOR SELECT DISTINCT RECORD_CONTENT, LOAD_DT_TS FROM AAP_LDS_DEV_DB.PURCHASE_ORDER.T_PO_STG where LOAD_DT_TS >= '2022-02-09 00:00:00';
BEGIN
for record in c1 do
file_name := to_char(RECORD.LOAD_DT_TS, 'yyyymmdd_hhmmss') || '.csv';
sql := 'copy into #STG_SF_S3_DEV_JJC/' || :file_name || ' from (select record_content from AAP_LDS_DEV_DB.PURCHASE_ORDER.T_PO_STG where LOAD_DT_TS = ''' || RECORD.LOAD_DT_TS || ''') FILE_FORMAT = (TYPE=JSON,COMPRESSION = NONE) single = FALSE overwrite=FALSE';
EXECUTE IMMEDIATE sql;
end for;
RETURN 0;
END;
$$ ;
EXECUTE IMMEDIATE
$$
DECLARE
file_name varchar2(30);
records variant;
c1 CURSOR FOR SELECT DISTINCT RECORD_CONTENT, LOAD_DT_TS FROM mytable where LOAD_DT_TS >= '2022-02-09 00:00:00';
BEGIN
for record in c1 do
file_name := 'PO'||'_'||to_date(RECORD.LOAD_DT_TS)||'_'||TO_TIME(RECORD.LOAD_DT_TS)||'.json';
records := RECORD.record_content;
create or replace temporary table temp_test_pvt(records variant);
insert into temp_test_pvt select parse_json(:records) ;
EXECUTE IMMEDIATE 'copy into #STG_SF_S3/' || :file_name ||
' from (select distinct records from temp_test_pvt)
FILE_FORMAT = (TYPE=JSON, COMPRESSION = NONE) single = true overwrite=FALSE detailed_output = TRUE ';
end for;
RETURN 0;
END;
$$ ;
This is my final query. With this I am able to COPY one record as one file from Snowflake table into S3.
Related
postgres 12
I am trying to loop through a table which has schema , table_names and columns
I want to do various things like finding nulls ,row count etc. I failed at the first hurdle trying to update the col records.
table i am using
CREATE TABLE test.table_study (
table_schema text,
table_name text,
column_name text,
records int,
No_Nulls int,
No_Blanks int,
per_pop int
);
I populate the table with some schema names ,tables and columns from information_schema.columns
insert into test.table_study select table_schema, table_name, column_name
from information_schema.columns
where table_schema like '%white'
order by table_schema, table_name, ordinal_position;
I want to populate the rest with a function
function :-
CREATE OR REPLACE PROCEDURE test.insert_data_population()
as $$
declare s record;
declare t record;
declare c record;
BEGIN
FOR s IN SELECT distinct table_schema FROM test.table_study
LOOP
FOR t IN SELECT distinct table_name FROM test.table_study where table_schema = s.table_schema
loop
FOR c IN SELECT column_name FROM test.table_study where table_name = t.table_name
LOOP
execute 'update test.table_study set records = (select count(*) from ' || s.table_schema || '.' || t.table_name || ') where table_study.table_name = '|| t.table_name ||';';
END LOOP;
END LOOP;
END LOOP;
END;
$$
LANGUAGE plpgsql;
I get this error SQL Error [42703]: ERROR: column "age" does not exist. the table age does exist.
when I take out the where clause
execute 'update referralunion.testinsert ti set records = (select count(*) from ' || s.table_schema || '.' || t.table_name || ') ;';
it works, I just cant figure out whats wrong?
Your procedure is structured entirely wrong. What it results in is an attempt to get every column name for every table name in every schema. I would guess results in your column does not exist error. Further is shows procedural thinking. SQL requires think in terms of sets. Below I use basically your query to demonstrate then a revised version which uses a single loop.
-- setup (dropping schema references)
create table table_study (
table_schema text,
table_name text,
column_name text,
records int,
no_nulls int,
no_blanks int,
per_pop int
);
insert into table_study(table_schema, table_name, column_name)
values ('s1','t1','age')
, ('s2','t1','xyz');
-- procedure replacing EXECUTE with Raise Notice.
create or replace procedure insert_data_population()
as $$
declare
s record;
t record;
c record;
line int = 0;
begin
for s in select distinct table_schema from table_study
loop
for t in select distinct table_name from table_study where table_schema = s.table_schema
loop
for c in select column_name from table_study where table_name = t.table_name
loop
line = line+1;
raise notice '%: update table_study set records = (select count(*) from %.% where table_study.table_name = %;'
, line, s.table_schema, t.table_name, c.column_name;
end loop;
end loop;
end loop;
end;
$$
language plpgsql;
Run procedure
do $$
begin
call insert_data_population();
end;
$$;
RESULTS
1: update table_study set records = (select count(*) from s2.t1 where table_study.table_name = age; 2: update table_study set records = (select count(*) from s2.t1 where table_study.table_name = xyz; 3: update table_study set records = (select count(*) from s1.t1 where table_study.table_name = age; 4: update table_study set records = (select count(*) from s1.t1 where table_study.table_name = xyz;
Notice lines 2 and 3. Each references a column name that does not exist in the table. This results from the FOR structure with the same table name in different schema.
Revision for Single Select statement with Single For loop.
create or replace
procedure insert_data_population()
language plpgsql
as $$
declare
s record;
line int = 0;
begin
for s in select distinct table_schema, table_name, column_name from table_study
loop
line = line+1;
raise notice '%: update table_study set records = (select count(*) from %.% where table_study.table_name = %;'
, line, s.table_schema, s.table_name, s.column_name;
end loop;
end;
$$;
do $$
begin
call insert_data_population();
end;
$$;
RESULTS
1: update table_study set records = (select count(*) from s2.t1 where table_study.table_name = xyz;
2: update table_study set records = (select count(*) from s1.t1 where table_study.table_name = age;
Note: In Postgres DECLARE begins a block. It is not necessary to declared each variable. I would actually consider it bad practice. In theory it could require an end for each declare as each could be considered a nested block. Fortunately Postgres does not require this.
Morning,
I'm trying to write a script that will convert Unload tables (UNLD to HDL files) creating a flat file using PLSQL. I keep getting syntax errors trying to run it and would appreciate some help from an expert out there!
Here are the errors:
Error(53,21): PLS-00330: invalid use of type name or subtype name
Error(57,32): PLS-00222: no function with name 'UNLDTABLE' exists in this scope
Our guess is that the unldTable variable is being treated as a String, rather than a database table object (Not really expereinced in PLSQL)
CREATE OR REPLACE PROCEDURE UNLD_TO_HDL (processComponent IN VARCHAR2)
IS
fHandle UTL_FILE.FILE_TYPE;
concatData VARCHAR2(240);
concatHDLMetaTags VARCHAR2(240);
outputFileName VARCHAR2(240);
TYPE rowArrayType IS TABLE OF VARCHAR2(240);
rowArray rowArrayType;
emptyArray rowArrayType;
valExtractArray rowArrayType;
hdlFileName VARCHAR2(240);
unldTable VARCHAR2(240);
countUNLDRows Number;
dataType VARCHAR2(240);
current_table VARCHAR2(30);
value_to_char VARCHAR2(240);
BEGIN
SELECT HDL_FILE_NAME
INTO hdlFileName
FROM GNC_HDL_CREATION_PARAMS
WHERE PROCESS_COMPONENT = processComponent;
SELECT UNLD_TABLE
INTO unldTable
FROM GNC_HDL_CREATION_PARAMS
WHERE PROCESS_COMPONENT = processComponent
FETCH NEXT 1 ROWS ONLY;
SELECT LISTAGG(HDL_META_TAG,'|')
WITHIN GROUP(ORDER BY HDL_META_TAG)
INTO concatHDLMetaTags
FROM GNC_MIG_CONTROL
WHERE HDL_COMP = processComponent;
SELECT DB_FIELD
BULK COLLECT INTO valExtractArray
FROM GNC_MIG_CONTROL
WHERE HDL_COMP = processComponent
ORDER BY HDL_META_TAG;
fHandle := UTL_FILE.FOPEN('./', hdlFileName, 'W');
UTL_FILE.PUTF(fHandle, concatHDLMetaTags + '\n');
SELECT num_rows INTO countUNLDRows FROM user_tables where table_name = unldTable;
FOR row in 1..countUNLDRows LOOP
rowArray := emptyArrayType;
FOR value in 1..valExtractArray.COUNT LOOP
rowArray.extend();
SELECT data_type INTO dataType FROM all_tab_columns where table_name = unldTable AND column_name = valExtractArray(value);
IF dataType = 'VARCHAR2' THEN (SELECT valExtractArray(value) INTO value_to_char FROM current_table WHERE ROWNUM = row);
ELSIF dataType = 'DATE' THEN (SELECT TO_CHAR(valExtractArray(value),'YYYY/MM/DD') INTO value_to_char FROM current_table WHERE ROWNUM = row);
ELSIF dataType = 'NUMBER' THEN (SELECT TO_CHAR(valExtractArray(value)) INTO value_to_char FROM current_table WHERE ROWNUM = row);
ENDIF;
rowArray(value) := value_to_char;
END LOOP;
concatData := NULL;
FOR item in 1..rowArray.COUNT LOOP
IF item = rowArray.COUNT
THEN concatData := (COALESCE(concatData,'') || rowArray(item));
ELSE concatData := (COALESCE(concatData,'') || rowArray(item) || '|');
END IF;
END LOOP;
UTL_FILE.PUTF(fHandle, concatData + '/n');
END LOOP;
UTL_FILE.FCLOSE(fHandle);
END;
Thanks,
Adam
I believe it is just an overlook in your code. You define unldTable as a varchar, which is used correctly until you try to access it as if it were a varray on line 51
rowArray(value) := unldTable(row).valExtractArray(value);
Given that you have not defined it as a varray, unldTable(row) is making the interpreter believe that you are referring to a function.
EDIT
Now that you have moved on, you should resolve the problem of invoking SELECT statements on tables that are unknown at runtime. To do so you need to make use of Dynamic SQL; you can do it in several way, the most direct being an Execute immediate statement in your case:
mystatement := 'SELECT valExtractArray(value) INTO :value_to_char FROM ' || current_table || ' WHERE ROWNUM = ' || row;
execute immediate mystatement USING OUT value_to_char;
It looks like you need to generate a cursor as
select [list of columns from GNC_MIG_CONTROL.DB_FIELD]
from [table name from GNC_HDL_CREATION_PARAMS.UNLD_TABLE]
Assuming setup like this:
create table my_table (business_date date, id integer, dummy1 varchar2(1), dummy2 varchar2(20));
create table gnc_hdl_creation_params (unld_table varchar2(30), process_component varchar2(30));
create table gnc_mig_control (db_field varchar2(30), hdl_comp varchar2(30), hdl_meta_tag integer);
insert into my_table(business_date, id, dummy1, dummy2) values (date '2018-01-01', 123, 'X','Some more text');
insert into gnc_hdl_creation_params (unld_table, process_component) values ('MY_TABLE', 'XYZ');
insert into gnc_mig_control (db_field, hdl_comp, hdl_meta_tag) values ('BUSINESS_DATE', 'XYZ', '1');
insert into gnc_mig_control (db_field, hdl_comp, hdl_meta_tag) values ('ID', 'XYZ', '2');
insert into gnc_mig_control (db_field, hdl_comp, hdl_meta_tag) values ('DUMMY1', 'XYZ', '3');
insert into gnc_mig_control (db_field, hdl_comp, hdl_meta_tag) values ('DUMMY2', 'XYZ', '4');
You could build a query like this:
select unld_table, listagg(expr, q'[||'|'||]') within group (order by hdl_meta_tag) as expr_list
from ( select t.unld_table
, case tc.data_type
when 'DATE' then 'to_char('||c.db_field||',''YYYY-MM-DD'')'
else c.db_field
end as expr
, c.hdl_meta_tag
from gnc_hdl_creation_params t
join gnc_mig_control c
on c.hdl_comp = t.process_component
left join user_tab_columns tc
on tc.table_name = t.unld_table
and tc.column_name = c.db_field
where t.process_component = 'XYZ'
)
group by unld_table;
Output:
UNLD_TABLE EXPR_LIST
----------- --------------------------------------------------------------------------------
MY_TABLE to_char(BUSINESS_DATE,'YYYY-MM-DD')||'|'||ID||'|'||DUMMY1||'|'||DUMMY2
Now if you plug that logic into a PL/SQL procedure you could have something like this:
declare
processComponent constant gnc_hdl_creation_params.process_component%type := 'XYZ';
unloadSQL long;
unloadCur sys_refcursor;
text long;
begin
select 'select ' || listagg(expr, q'[||'|'||]') within group (order by hdl_meta_tag) || ' as text from ' || unld_table
into unloadSQL
from ( select t.unld_table
, case tc.data_type
when 'DATE' then 'to_char('||c.db_field||',''YYYY/MM/DD'')'
else c.db_field
end as expr
, c.hdl_meta_tag
from gnc_hdl_creation_params t
join gnc_mig_control c
on c.hdl_comp = t.process_component
left join user_tab_columns tc
on tc.table_name = t.unld_table
and tc.column_name = c.db_field
where t.process_component = processComponent
)
group by unld_table;
open unloadCur for unloadSQL;
loop
fetch unloadCur into text;
dbms_output.put_line(text);
exit when unloadCur%notfound;
end loop;
close unloadCur;
end;
Output:
2018/01/01|123|X|Some more text
2018/01/01|123|X|Some more text
Now you just have to make that into a procedure, change dbms_output to utl_file and add your meta tags etc and you're there.
I've assumed there is only one distinct unld_table per process component. If there are more you'll need a loop to work through each one.
For a slightly more generic approach, you could build a cursor-to-csv generator which could encapsulate the datatype handling, and then you'd only need to build the SQL as select [columns] from [table]. You might then write a generic cursor to file processor, where you pass in the filename and a cursor and it does the lot.
Edit: I've updated my cursor-to-csv generator to provide file output, so you just need to pass it a cursor and the file details.
I have the following procedure, when I run it displays all the insertions in console, the field ind_id is increased but does not save any record.
create or replace function renameDescripcionTramite() returns varchar as $$
declare
aguja record;
pajar record;
last_id integer;
begin
for aguja in
select trim(t.tra_descripcion) tra_descripcion from tab_tramite t
join tab_serietramite st on st.tra_id = t.tra_id
group by t.tra_descripcion
order by t.tra_descripcion loop
for pajar in
select u.uni_id,s.ser_id,st.sts_id,t.tra_id,trim(t.tra_descripcion) tra_descripcion
from tab_unidad u join tab_series s on u.uni_id = s.uni_id
join tab_serietramite st on st.ser_id = s.ser_id
join tab_tramite t on t.tra_id = st.tra_id
order by u.uni_id,s.ser_id,t.tra_id loop
if aguja.tra_descripcion = pajar.tra_descripcion then
insert into tab_indexpediente (uni_id,ser_id,tra_id) values (pajar.uni_id,pajar.ser_id,pajar.tra_id);-- returning ind_id into last_id;
--update tab_serietramite set wid = last_id where sts_id = pajar.sts_id;
RAISE NOTICE 'insert into tab_indexpediente (uni_id,ser_id,tra_id) values (%',cast(pajar.uni_id as text)
|| ',' || cast(pajar.ser_id as text) || ',' || cast(pajar.tra_id as text) || ')';
end if;
end loop;
--RAISE NOTICE ' ANTERIOR ES %',aguja;
end loop;
return tra_descripcion;
end;
$$ LANGUAGE plpgsql;
Can someone help me please?
Yes, I did not know that this would affect the procedure.
Thank's laurenz Albe
I have the following procedure, which does not compile correctly, because it refers to non existing objects (table does not exist)
Here is only a section of the code (i used generic names for tables and columns):
DECLARE
C INTEGER := 0;
BEGIN
SELECT COUNT(1) INTO C FROM USER_TABLES WHERE TABLE_NAME = 'MY_TABLE';
IF C > 0 THEN
DECLARE
CURSOR c_maps IS SELECT COLUM_NAME1, COLUM_NAME2 FROM MY_TABLE WHERE ACTIVE = 1;
BEGIN
FOR prec IN c_maps LOOP
some code...;
END LOOP;
EXECUTE IMMEDIATE 'some code..';
END;
END IF;
END;
/
I don't know how to write this statement dynamically, since the table "MY_TABLE" does not exist:
CURSOR c_maps IS SELECT COLUM_NAME1, COLUM_NAME2 FROM MY_TABLE WHERE ACTIVE =1;
I also tried to write it like:
CURSOR c_maps IS SELECT COLUM_NAME1, COLUM_NAME2 FROM (Select 'MY_TABLE' from dual) WHERE ACTIVE = 1;
However, than it refers to the column "ACTIVE" which also does not exist at compile time...It is possible to write the whole procedure inside "execute immediate" - block? I have tried different variants, however without success
You may need to open the cursor in a different way, so that the non existing table is only referred in dynamic SQL; for example:
declare
c integer := 0;
curs sys_refcursor;
v1 number;
v2 number;
begin
select count(1)
into c
from user_tables
where table_name = 'MY_TABLE';
if c > 0
then
open curs for 'select column_name1, column_name2 from my_table where active = 1';
loop
fetch curs into v1, v2;
exit when curs%NOTFOUND;
dbms_output.put_line(v1 || ' - ' || v2);
end loop;
else
dbms_output.put_line('The table does not exist');
end if;
end;
/
I have a PostgreSQL database and I need to do an update over values of specific Columns. The number of columns is so big and I need to do the same operation to different table So better to extract them dynamically.
More specifically I want to extract from the table all the columns whose names ends with "_suffix" and do an update on their values.
I started trying to make a script but I don't know if it is the right road!
SELECT columns.column_name
FROM information_schema.columns
WHERE columns.table_name = 'myInitialTable' AND columns.column_name like '%\_suffix%' AND columns.table_schema = 'public';
I created a view of this query and I used it in the following function :
CREATE OR REPLACE FUNCTION updatetable() RETURNS int4 AS
$BODY$
DECLARE r RECORD;
BEGIN
FOR r IN SELECT * from v_reduced_table LOOP
update myInitialTable
set r.column_name = case
when r.column_name = '' then NULL
when r.column_name = 'value1' or r.column_name = 'value2' then 'xxxxx'
else r.column_name end;
END LOOP;
return 1;
END;
$BODY$
LANGUAGE plpgsql;
SELECT updatetable() as output;
this query do a loop on every column ending with suffix and updates its values. but when I run it I get
ERROR: syntax error at or near "$1"
LINE 1: update myInitialTable set $1 = case when $2 = '' then NULL when ...
Any help is appreciated :)
In your function you need to use dynamic commands.
The funcion format() is often very helpful.
Example data:
create table my_table(col1_suffix text, col2_suffix text, col3_suffix text);
insert into my_table values ('a', 'b', 'c');
Example function:
CREATE OR REPLACE FUNCTION update_my_table() RETURNS void AS
$BODY$
DECLARE r RECORD;
BEGIN
FOR r IN
SELECT columns.column_name
FROM information_schema.columns
WHERE columns.table_name = 'my_table'
AND columns.column_name like '%\_suffix%'
AND columns.table_schema = 'public'
LOOP
EXECUTE(FORMAT($f$
UPDATE my_table
SET %s = CASE
WHEN '%s' = 'col1_suffix' THEN 'col1'
WHEN '%s' = 'col2_suffix' OR '%s' = 'col3_suffix' THEN 'xxxxx'
END;$f$, r.column_name, r.column_name, r.column_name, r.column_name));
END LOOP;
END;
$BODY$
LANGUAGE plpgsql;
Usage:
select update_my_table();
select * from my_table;
col1_suffix | col2_suffix | col3_suffix
-------------+-------------+-------------
col1 | xxxxx | xxxxx
(1 row)