Need to increase performance-bulk collect in cursor with limit and in the for loop inserting into th

938190 · Oct 28, 2013 2:11PM

Hi all,

I have a performance issue in the below code,where i am trying to insert the data from table_stg into target_tab and in parent_tab tables and then to child tables via cursor with bulk collect .the target_tab and parent_tab are huge tables and have a row wise trigger enabled on it .the trigger is mandatory . This timetaken for this block to execute is 5000 seconds.Now my requirement is to reduce it to 5 to 10 mins.

can someone please guide me here.Its bit urgent .Awaiting for your response.

declare

vmax_Value NUMBER(5);

vcnt number(10);

id_val number(20);

pc_id number(15);

vtable_nm VARCHAR2(100);

vstep_no VARCHAR2(10);

vsql_code VARCHAR2(10);

vsql_errm varchar2(200);

vtarget_starttime timestamp;

limit_in number :=10000;

idx number(10);

cursor stg_cursor is

select

DESCRIPTION,

SORT_CODE,

ACCOUNT_NUMBER,

to_number(to_char(CORRESPONDENCE_DATE,'DD')) crr_day,

to_char(CORRESPONDENCE_DATE,'MONTH') crr_month,

to_number(substr(to_char(CORRESPONDENCE_DATE,'DD-MON-YYYY'),8,4)) crr_year,

PARTY_ID,

GUID,

PAPERLESS_REF_IND,

PRODUCT_TYPE,

PRODUCT_BRAND,

PRODUCT_HELD_ID,

NOTIFICATION_PREF,

UNREAD_CORRES_PERIOD,

EMAIL_ID,

MOBILE_NUMBER,

TITLE,

SURNAME,

POSTCODE,

EVENT_TYPE,

PRIORITY_IND,

SUBJECT,

EXT_PRD_ID_TX,

EXT_PRD_HLD_ID_TX,

EXT_SYS_ID,

EXT_PTY_ID_TX,

ACCOUNT_TYPE_CD,

COM_PFR_TYP_TX,

COM_PFR_OPT_TX,

COM_PFR_RSN_CD

from table_stg;

type rec_type is table of stg_rec_type index by pls_integer;

v_rt_all_cols rec_type;

BEGIN

vstep_no := '0';

vmax_value := 0;

vtarget_starttime := systimestamp;

id_val := 0;

pc_id := 0;

success_flag := 0;

vstep_no := '1';

vtable_nm := 'before cursor';

OPEN stg_cursor;

vstep_no := '2';

vtable_nm := 'After cursor';

LOOP

vstep_no := '3';

vtable_nm := 'before fetch';

--loop

FETCH stg_cursor BULK COLLECT INTO v_rt_all_cols LIMIT limit_in;

vstep_no := '4';

vtable_nm := 'after fetch';

--EXIT WHEN v_rt_all_cols.COUNT = 0;

EXIT WHEN stg_cursor%NOTFOUND;

FOR i IN 1 .. v_rt_all_cols.COUNT

LOOP

dbms_output.put_line(upper(v_rt_all_cols(i).event_type));

if (upper(v_rt_all_cols(i).event_type) = upper('System_enforced')) then

vstep_no := '4.1';

vtable_nm := 'before seq sel';

select PC_SEQ.nextval into pc_id from dual;

vstep_no := '4.2';

vtable_nm := 'before insert corres';

INSERT INTO target1_tab

(ID,

PARTY_ID,

PRODUCT_BRAND,

SORT_CODE,

ACCOUNT_NUMBER,

EXT_PRD_ID_TX,

EXT_PRD_HLD_ID_TX,

EXT_SYS_ID,

EXT_PTY_ID_TX,

ACCOUNT_TYPE_CD,

COM_PFR_TYP_TX,

COM_PFR_OPT_TX,

COM_PFR_RSN_CD,

status)

VALUES

(pc_id,

v_rt_all_cols(i).party_id,

decode(v_rt_all_cols(i).product_brand,'LTB',2,'HLX',1,'HAL',1,'BOS',3,'VER',4,0),

v_rt_all_cols(i).sort_code,

'XXXX'||substr(trim(v_rt_all_cols(i).ACCOUNT_NUMBER),length(trim(v_rt_all_cols(i).ACCOUNT_NUMBER))-3,4),

v_rt_all_cols(i).EXT_PRD_ID_TX,

v_rt_all_cols(i).EXT_PRD_HLD_ID_TX,

v_rt_all_cols(i).EXT_SYS_ID,

v_rt_all_cols(i).EXT_PTY_ID_TX,

v_rt_all_cols(i).ACCOUNT_TYPE_CD,

v_rt_all_cols(i).COM_PFR_TYP_TX,

v_rt_all_cols(i).COM_PFR_OPT_TX,

v_rt_all_cols(i).COM_PFR_RSN_CD,

NULL);

vstep_no := '4.3';

vtable_nm := 'after insert corres';

else

select COM_SEQ.nextval into id_val from dual;

vstep_no := '6';

vtable_nm := 'before insertcomm';

if (upper(v_rt_all_cols(i).event_type) = upper('REMINDER')) then

vstep_no := '6.01';

vtable_nm := 'after if insertcomm';

insert into parent_tab

(ID ,

CTEM_CODE,

CHA_CODE,

CT_CODE,

CONTACT_POINT_ID,

SOURCE,

RECEIVED_DATE,

SEND_DATE,

RETRY_COUNT)

values

(id_val,

lower(v_rt_all_cols(i).event_type),

decode(v_rt_all_cols(i).product_brand,'LTB',2,'HLX',1,'HAL',1,'BOS',3,'VER',4,0),

'Email',

v_rt_all_cols(i).email_id,

'IADAREMINDER',

systimestamp,

0);

else

vstep_no := '6.02';

vtable_nm := 'after else insertcomm';

insert into parent_tab

(ID ,

CTEM_CODE,

CHA_CODE,

CT_CODE,

CONTACT_POINT_ID,

SOURCE,

RECEIVED_DATE,

SEND_DATE,

RETRY_COUNT)

values

(id_val,

lower(v_rt_all_cols(i).event_type),

decode(v_rt_all_cols(i).product_brand,'LTB',2,'HLX',1,'HAL',1,'BOS',3,'VER',4,0),

'Email',

v_rt_all_cols(i).email_id,

'CORRESPONDENCE',

systimestamp,

0);

END if;

vstep_no := '6.11';

vtable_nm := 'before chop';

if (v_rt_all_cols(i).ACCOUNT_NUMBER is not null) then

v_rt_all_cols(i).ACCOUNT_NUMBER := 'XXXX'||substr(trim(v_rt_all_cols(i).ACCOUNT_NUMBER),length(trim(v_rt_all_cols(i).ACCOUNT_NUMBER))-3,4);

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Correspondence.AccountNumberMasked',

v_rt_all_cols(i).ACCOUNT_NUMBER);

end if;

vstep_no := '6.1';

vtable_nm := 'before stateday';

if (v_rt_all_cols(i).crr_day is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

--'IB.Correspondence.Date.Day',

'IB.Crsp.Date.Day',

v_rt_all_cols(i).crr_day);

end if;

vstep_no := '6.2';

vtable_nm := 'before statemth';

if (v_rt_all_cols(i).crr_month is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

--'IB.Correspondence.Date.Month',

'IB.Crsp.Date.Month',

v_rt_all_cols(i).crr_month);

end if;

vstep_no := '6.3';

vtable_nm := 'before stateyear';

if (v_rt_all_cols(i).crr_year is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

--'IB.Correspondence.Date.Year',

'IB.Crsp.Date.Year',

v_rt_all_cols(i).crr_year);

end if;

vstep_no := '7';

vtable_nm := 'before type';

if (v_rt_all_cols(i).product_type is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Product.ProductName',

v_rt_all_cols(i).product_type);

end if;

vstep_no := '9';

vtable_nm := 'before title';

if (trim(v_rt_all_cols(i).title) is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE )

values

(id_val,

'IB.Customer.Title',

trim(v_rt_all_cols(i).title));

end if;

vstep_no := '10';

vtable_nm := 'before surname';

if (v_rt_all_cols(i).surname is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Customer.LastName',

v_rt_all_cols(i).surname);

end if;

vstep_no := '12';

vtable_nm := 'before postcd';

if (trim(v_rt_all_cols(i).POSTCODE) is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Customer.Addr.PostCodeMasked',

substr(replace(v_rt_all_cols(i).POSTCODE,' ',''),length(replace(v_rt_all_cols(i).POSTCODE,' ',''))-2,3));

end if;

vstep_no := '13';

vtable_nm := 'before subject';

if (trim(v_rt_all_cols(i).SUBJECT) is not null) then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Correspondence.Subject',

v_rt_all_cols(i).subject);

end if;

vstep_no := '14';

vtable_nm := 'before inactivity';

if (trim(v_rt_all_cols(i).UNREAD_CORRES_PERIOD) is null or

trim(v_rt_all_cols(i).UNREAD_CORRES_PERIOD) = '3' or

trim(v_rt_all_cols(i).UNREAD_CORRES_PERIOD) = '6' or

trim(v_rt_all_cols(i).UNREAD_CORRES_PERIOD) = '9') then

insert into child_tab

(COM_ID,

KEY,

VALUE)

values

(id_val,

'IB.Correspondence.Inactivity',

v_rt_all_cols(i).UNREAD_CORRES_PERIOD);

end if;

vstep_no := '14.1';

vtable_nm := 'after notfound';

end if;

vstep_no := '15';

vtable_nm := 'after notfound';

END LOOP;

end loop;

vstep_no := '16';

vtable_nm := 'before closecur';

CLOSE stg_cursor;

vstep_no := '17';

vtable_nm := 'before commit';

DELETE FROM table_stg;

COMMIT;

vstep_no := '18';

vtable_nm := 'after commit';

EXCEPTION

WHEN OTHERS THEN

ROLLBACK;

success_flag := 1;

vsql_code := SQLCODE;

vsql_errm := SUBSTR(sqlerrm,1,200);

error_logging_pkg.inserterrorlog('samp',vsql_code,vsql_errm, vtable_nm,vstep_no);

RAISE_APPLICATION_ERROR (-20011, 'samp '||vstep_no||' SQLERRM:'||SQLERRM);

end;

Thanks

Nikolay Savvinov · Oct 28, 2013 2:17PM

Hi,

what's your excuse for not using INSERT SELECT here?

Best regards,

Nikolay

938190 · Oct 28, 2013 2:25PM

Hi Nikolay,

Thanks for your quick response.

Insert select can be used for inserting into the target_tab but due to the row wise trigger in target_tab ,the performance is still low.

And for inserting into the parent_tab which holds 3 millions records and in this table also a row wise trigger is there -which hinders the performance .

We need a for loop to insert the data into the child_tab for each and every record that is getting inserted in the parent_tab .so here we cant use the insert select ..

Please correct me if i am wrong and can you please suggest the ways to tune this code.

938190 · Oct 28, 2013 2:28PM

Hi Nikolay,

Thanks for your quick response.

Insert select can be used for inserting into the target_tab but due to the row wise trigger in target_tab ,the performance is still low.

And for inserting into the parent_tab which holds 3 millions records and in this table also a row wise trigger is there -which hinders the performance .

We need a for loop to insert the data into the child_tab for each and every record that is getting inserted in the parent_tab .so here we cant use the insert select ..

Please correct me if i am wrong and can you please suggest the ways to tune this code.

Mike Kutz · Oct 28, 2013 2:35PM

I've done 10M rows across parent, child, grandchild in <10s on a 28 disk SAN built in 2000 and a server that is so old that is now has the power of an iPhone 5.

If you want this to go faster, you need to convince the powers-that-be that TRIGGERS ARE EVIL!!

BTW - you should look to see if multi-table INSERT statements can work.

This way it is an "INSERT ALL into Parent .. Child ... SELECT..."

At the very least, use DBMS_PARALLEL_EXECUTE to break down the job instead of FOR LOOPs....

Nikolay Savvinov · Oct 28, 2013 2:36PM

Hi,

I still don't see a valid excuse. If triggers make the insert slow, why make it yet slower by doing row-by-row processing? And why not disable triggers prior to inserting?

If you need to insert into child tables, why not do it as INSERT...SELECT as well?

Forgive my being blunt, but you cannot take a piece of garbage and tune it into a piece of cake. You need to use a different approach. In the long-term, if you want a successful career you need to develop a different mindset (think in sets, not arrays).

Best regards,

Nikolay

938190 · Oct 28, 2013 2:56PM

Hi Nikolay,

The trigger is mandatory as i need to compare the new value to old value and so it cant be disabled.

Thanks ,For inserting into child tables ,i will use the insert..select

Nikolay Savvinov · Oct 28, 2013 3:04PM

Hi,

I'm not quite sure what you mean by comparing the old and new value (it's an INSERT, there is no old value), but whatever or business requirement is, chances are you can implement it without triggers.

Best regards,

Nikolay

Unknown · Oct 28, 2013 5:51PM

Its bit urgent

NO - it is NOT urgent. Not to us.

If you have an urgent problem you need to hire a consultant.

I have a performance issue in the below code,

Maybe you do and maybe you don't. How are we to really know? You haven't posted ANYTHING indicating that a performance issue exists. Please read the FAQ for how to post a tuning request and the info you need to provide. First and foremost you have to post SOMETHING that actually shows that a performance issue exists. Troubleshooting requires FACTS not just a subjective opinion.

where i am trying to insert the data from table_stg into target_tab and in parent_tab tables and then to child tables via cursor with bulk collect .the target_tab and parent_tab are huge tables and have a row wise trigger enabled on it .the trigger is mandatory . This timetaken for this block to execute is 5000 seconds.Now my requirement is to reduce it to 5 to 10 mins.

Personally I think 5000 seconds (about 1 hr 20 minutes) is very fast for processing 800 trillion rows of data into parent and child tables. Why do you think that is slow?

Your code has several major flaws that need to be corrected before you can even determine what, if anything, needs to be tuned.

This code has the EXIT statement at the beginning of the loop instead of at the end

    FETCH stg_cursor BULK COLLECT INTO v_rt_all_cols LIMIT limit_in;
              vstep_no  := '4';
              vtable_nm := 'after fetch';
--EXIT WHEN v_rt_all_cols.COUNT = 0;
    EXIT WHEN stg_cursor%NOTFOUND;

The correct place for the %NOTFOUND test when using BULK COLLECT is at the END of the loop; that is, the last statement in the loop.

You can use a COUNT test at the start of the loop but ironically you have commented it out and have now done it wrong. Either move the NOTFOUND test to the end of the loop or remove it and uncomment the COUNT test.

WHEN OTHERS THEN
  ROLLBACK;

That basically says you don't even care what problem occurs or whether the problem is for a single record of your 10,000 in the collection. You pretty much just throw away any stack trace and substitute your own message.

Your code also has NO exception handling for any of the individual steps or blocks of code.

The code you posted also begs the question of why you are using NAME=VALUE pairs for child data rows? Why aren't you using a standard relational table for this data?

As others have noted you are using slow-by-slow (row by row processing). Let's assume that PL/SQL, the bulk collect and row-by-row is actually necessary.

Then you should be constructing the parent and child records into collections and then inserting them in BULK using FORALL.

1. Create a collection for the new parent rows

2. Create a collection for the new child rows

3. For each set of LIMIT source row data

a. empty the parent and child collections

b. populate those collections with new parent/child data

c. bulk insert the parent collection into the parent table

d. bulk insert the child collection into the child table

And unless you really want to either load EVERYTHING or abandon everything you should use bulk exception handling so that the clean data gets processed and only the dirty data gets rejected.

Forum Stats

Quick Links

Categories

Need to increase performance-bulk collect in cursor with limit and in the for loop inserting into th

Answers

Data

Databases

Development

Development Tools

Open Source

Programs

Forum Stats

Categories

Need to increase performance-bulk collect in cursor with limit and in the for loop inserting into th

Answers