1 2 Previous Next 23 Replies Latest reply: Oct 4, 2010 12:20 AM by OraDBA02 RSS

    db_file_multiblock_read_count not in effect

    OraDBA02
      10G EBS on RHEL 5.3
      
      select * from v$version;
      
      BANNER
      ----------------------------------------------------------------
      Oracle Database 10g Enterprise Edition Release 10.2.0.4.0 - 64bi
      PL/SQL Release 10.2.0.4.0 - Production
      CORE    10.2.0.4.0      Production
      TNS for Linux: Version 10.2.0.4.0 - Production
      NLSRTL Version 10.2.0.4.0 - Production
      I am running a simple join (HASH) between two tables (one 26 Gb and other 550 MB) with MBRC set to 128. Default MBRC set at instance level is 32. Query is taking ~30 min and from other session when i observe p1,p2,p3 values. i am getting 'db files scattered read' with p3=16 all the time.

      I dont know why Oracle is not picking 128 blocks at a time from disk?
      Here are session parameter from where i am executing my SQL.
      select SID,NAME,VALUE,ISDEFAULT from V$SES_OPTIMIZER_ENV where SID=482;
      
             SID NAME                                                                   VALUE                                                                            ISD
      ---------- ---------------------------------------------------------------------- -------------------------------------------------------------------------------- ---
             482 parallel_execution_enabled                                             true                                                                             YES
             482 optimizer_features_enable                                              10.2.0.4                                                                         YES
             482 cpu_count                                                              8                                                                                YES
             482 active_instance_count                                                  1                                                                                YES
             482 parallel_threads_per_cpu                                               2                                                                                YES
             482 hash_area_size                                                         104857600                                                                        YES
             482 bitmap_merge_area_size                                                 1048576                                                                          YES
             482 sort_area_size                                                         52428800                                                                         NO
             482 sort_area_retained_size                                                0                                                                                YES
             482 _sort_elimination_cost_ratio                                           5                                                                                NO
             482 _db_file_optimizer_read_count                                          128                                                                              NO
             482 pga_aggregate_target                                                   3145728 KB                                                                       YES
             482 _pga_max_size                                                          629140 KB                                                                        NO
             482 parallel_query_mode                                                    enabled                                                                          YES
             482 parallel_dml_mode                                                      disabled                                                                         YES
             482 parallel_ddl_mode                                                      enabled                                                                          YES
             482 optimizer_mode                                                         all_rows                                                                         YES
             482 cursor_sharing                                                         exact                                                                            YES
             482 _b_tree_bitmap_plans                                                   false                                                                            NO
             482 star_transformation_enabled                                            false                                                                            YES
             482 _fast_full_scan_enabled                                                false                                                                            NO
             482 optimizer_index_cost_adj                                               100                                                                              YES
             482 optimizer_index_caching                                                0                                                                                YES
             482 query_rewrite_enabled                                                  true                                                                             YES
             482 query_rewrite_integrity                                                enforced                                                                         YES
             482 _like_with_bind_as_equality                                            true                                                                             NO
             482 workarea_size_policy                                                   auto                                                                             YES
             482 optimizer_dynamic_sampling                                             2                                                                                YES
             482 statistics_level                                                       typical                                                                          YES
             482 skip_unusable_indexes                                                  true                                                                             YES
             482 _optimizer_cost_based_transformation                                   off                                                                              NO
             482 optimizer_secure_view_merging                                          true                                                                             YES
      
      optimizer_features_enable                                              10.2.0.4
      optimizer_mode                                                         ALL_ROWS
      
             SID EVENT                                            P1         P2         P3 MODULE                                           SQL_HASH_VALUE SQL_ID
      ---------- ---------------------------------------- ---------- ---------- ---------- ------------------------------------------------ -------------- -------------
             482 db file scattered read                          700     574297         16 SQL*Plus                                             3880486416 cp2fnrvmnr1hh
      
      select pname, pval1 from sys.aux_stats$ ;
      
      PNAME                               PVAL1
      ------------------------------ ----------
      STATUS
      DSTART
      DSTOP
      FLAGS                                   1
      CPUSPEEDNW                     1061.34876
      IOSEEKTIM                              10
      IOTFRSPEED                           4096
      SREADTIM
      MREADTIM
      CPUSPEED
      MBRC
      MAXTHR
      SLAVETHR
      I knew there are certain case, where MBRC is not respected like reading from cache etc....But even if i execute the query with Parallel slaves, i observed same p3 values at 16.
      Is that due to not having system statistics ?
      I am not sure whether i should gather system stats for an EBS db or not ? I would rather request if some EBS expert shed any light on this so that i can make a decision to have them or not.

      Many thanks in advance.
        • 1. Re: db_file_multiblock_read_count not in effect
          sb92075
          I dont know why Oracle is not picking 128 blocks at a time from disk?
          post SQL & EXPLAIN PLAN
          • 2. Re: db_file_multiblock_read_count not in effect
            635471
            What are the extent sizes on the data segments?
            • 3. Re: db_file_multiblock_read_count not in effect
              Santosh Kumar
              14.4.1.2 Multiblock Read Count
              
              In release 10.2, the optimizer uses the value of mbrc when performing full table scans (FTS). The value of db_file_multiblock_read_count is set to the maximum allowed by the operating system by default. However, the optimizer uses mbrc=8 for costing. The "real" mbrc is actually somewhere in between since serial multiblock read requests are processed by the buffer cache and split in two or more requests if some blocks are already pinned in the buffer cache, or when the segment size is smaller than the read size. The mbrc value gathered as part of workload statistics is thus useful for FTS estimation.
              
              During the gathering process of workload statistics, it is possible that mbrc and mreadtim will not be gathered if no table scans are performed during serial workloads, as is often the case with OLTP systems. On the other hand, FTS occur frequently on DSS systems but may run parallel and bypass the buffer cache. In such cases, sreadtim will still be gathered since index lookup are performed using the buffer cache. If Oracle cannot gather or validate gathered mbrc or mreadtim, but has gathered sreadtim and cpuspeed, then only sreadtim and cpuspeed will be used for costing. FTS cost will be computed using analytical algorithm implemented in previous releases. Another alternative to computing mbrc and mreadtim is to force FTS in serial mode to allow the optimizer to gather the data.
              Reference: http://download.oracle.com/docs/cd/B19306_01/server.102/b14211/stats.htm

              Regards,
              S.K.
              • 4. Re: db_file_multiblock_read_count not in effect
                OraDBA02
                select * from AP.AP_INVOICES_ALL
                
                Plan hash value: 3474762098
                
                -------------------------------------------------------------------------------------
                | Id  | Operation         | Name            | Rows  | Bytes | Cost (%CPU)| Time     |
                -------------------------------------------------------------------------------------
                |   0 | SELECT STATEMENT  |                 |       |       |   610K(100)|          |
                |   1 |  TABLE ACCESS FULL| AP_INVOICES_ALL |    65M|    19G|   610K  (3)| 02:02:04 |
                -------------------------------------------------------------------------------------
                TABLESPACE_NAME                BLOCK_SIZE INITIAL_EXTENT NEXT_EXTENT EXTENT_MAN ALLOCATIO SEGMEN
                ------------------------------ ---------- -------------- ----------- ---------- --------- ------
                APPS_TS_TX_DATA                      8192         131072      131072 LOCAL      UNIFORM   AUTO
                • 5. Re: db_file_multiblock_read_count not in effect
                  Charles Hooper
                  OraDBA02 wrote:
                  TABLESPACE_NAME                BLOCK_SIZE INITIAL_EXTENT NEXT_EXTENT EXTENT_MAN ALLOCATIO SEGMEN
                  ------------------------------ ---------- -------------- ----------- ---------- --------- ------
                  APPS_TS_TX_DATA                      8192         131072      131072 LOCAL      UNIFORM   AUTO
                  If this is the tablespace that contains the AP_INVOICES_ALL table, then David_Aldridge pinpointed the problem for you. Mulltiblock reads (ignoring db file parallel read) cannot cross extend boundaries. 131072 / 8192 = 16, so the maximum multiblock read size in this tablespace is 16 blocks (or less if one or more of those blocks is already in the buffer cache).

                  Charles Hooper
                  Co-author of "Expert Oracle Practices: Oracle Database Administration from the Oak Table"
                  http://hoopercharles.wordpress.com/
                  IT Manager/Oracle DBA
                  K&M Machine-Fabricating, Inc.
                  • 6. Re: db_file_multiblock_read_count not in effect
                    askraks
                    Hi,


                    one 26 Gb and other 550 MB

                    Here compare to hash join , nested loop join will perform better. hashing will take lot of time compare to nested loop join in this particular case.

                    Who told you to use hash join?

                    Please reply and will work towards other part of the question


                    Kind Regards,
                    Rakesh Jayappa
                    • 7. Re: db_file_multiblock_read_count not in effect
                      Charles Hooper
                      Rakesh jayappa wrote:
                      Hi,
                      one 26 Gb and other 550 MB

                      Here compare to hash join , nested loop join will perform better. hashing will take lot of time compare to nested loop join in this particular case.

                      Who told you to use hash join?

                      Please reply and will work towards other part of the question


                      Kind Regards,
                      Rakesh Jayappa
                      Rakesh,

                      Why do you suggest that a nested loops join will outperform a hash join when one table is 550MB and the other is 26GB?

                      If an index does not exist on the join columns, your advice is likely very bad (would you like to full scan the 26GB table many times to perform the nested loops join or the 550MB table many times to perform the nested loops join). If an index exists and only a small number of rows are returned, your advice might be good or might not be good, partially dependent on the clustering factor of the index.

                      http://download.oracle.com/docs/cd/B19306_01/server.102/b14211/optimops.htm
                      "Hash joins are used for joining large data sets. The optimizer uses the smaller of two tables or data sources to build a hash table on the join key in memory. It then scans the larger table, probing the hash table to find the joined rows.

                      The optimizer uses a hash join to join two tables if they are joined using an equijoin and if either of the following conditions are true:
                      •A large amount of data needs to be joined.
                      •A large fraction of a small table needs to be joined."

                      Charles Hooper
                      Co-author of "Expert Oracle Practices: Oracle Database Administration from the Oak Table"
                      http://hoopercharles.wordpress.com/
                      IT Manager/Oracle DBA
                      K&M Machine-Fabricating, Inc.
                      • 8. Re: db_file_multiblock_read_count not in effect
                        askraks
                        Hi,


                        Sorry i missed index part

                        Inner query block must have index on column

                        Kin d Regards,
                        Rakesh Jayappa
                        • 9. Re: db_file_multiblock_read_count not in effect
                          Jonathan Lewis
                          Rakesh jayappa wrote:

                          Sorry i missed index part
                          Inner query block must have index on column
                          But even if indexes existed in both directions, identifying exactly which rows should be joined, you still wouldn't have enough information to say that a nested loop would be better than a hash join. You have no information about how much data is actually going to be reported from each table, or how many rows from one table join to how many rows from the other table, or how the relevant data is scattered through the tables.

                          Out of curiosity I have to ask, when you made your suggestion which table were you thinking ought to be the outer table in the nested loop ?

                          Regards
                          Jonathan Lewis
                          • 10. Re: db_file_multiblock_read_count not in effect
                            Randolf Geist
                            OraDBA02 wrote:
                            I am running a simple join (HASH) between two tables (one 26 Gb and other 550 MB) with MBRC set to 128. Default MBRC set at instance level is 32. Query is taking ~30 min and from other session when i observe p1,p2,p3 values. i am getting 'db files scattered read' with p3=16 all the time.

                            I dont know why Oracle is not picking 128 blocks at a time from disk?
                            Here are session parameter from where i am executing my SQL.
                            select SID,NAME,VALUE,ISDEFAULT from V$SES_OPTIMIZER_ENV where SID=482;
                            
                            SID NAME                                                                   VALUE                                                                            ISD
                            ---------- ---------------------------------------------------------------------- -------------------------------------------------------------------------------- ---
                            482 _db_file_optimizer_read_count                                          128                                                                              NO
                            I think Charles and David have already pointed out the reason why you are only getting the 128K multi-block reads (due to the 128K extent size in the tablespace).

                            By the way, depending on your hardware and operating system version you might not even get a difference in runtime between 128K and 1M multi-block reads. So even if your process used those bigger reads, it is not said that your join operation would perform better. You might want to use ASH (if you have a diagnostic license) or extended SQL trace to understand where most of the time of your execution is spent (could be for instance reading / writing to TEMP). Running this join in parallel might be a reasonable option.

                            Please note however that none of your information provided shows a clear evidence that at runtime the execution will actually attempt to read 128 blocks for multi-block reads, since you show us neither the "db_file_multiblock_read_count" nor the "_db_file_exec_read_count" parameter. The latter ultimately determines what at runtime will be used for direct path or scattered reads (although I've recently observed a case in 11.2 where the runtime engine obviously ignored the setting and used larger reads in certain cases). The shown "_db_file_optimizer_read_count" parameter is only used by the optimizer for the cost calculation, not by the runtime engine. It suggests that the "_db_file_exec_read_count" parameter could be set to the same value, but it has not be the same if explicitly set to a different value.

                            You can check V$PARAMETER for the "_db_file_exec_read_count" parameter resp. the underlying X$KSPPI, X$KSPPCV and X$KSPPSV fixed tables to see your instance and session settings for the parameter.
                            I knew there are certain case, where MBRC is not respected like reading from cache etc....But even if i execute the query with Parallel slaves, i observed same p3 values at 16.
                            Is that due to not having system statistics ?
                            I am not sure whether i should gather system stats for an EBS db or not ? I would rather request if some EBS expert shed any light on this so that i can make a decision to have them or not.
                            You need again to differentiate: Even if you gather System Statistics this will not influence what the runtime engine at runtime will be using for multi-block reads. Gathering WORKLOAD System Statistics would merely measure what at actual runtime happens and set the MBRC value of the System Statistics to a corresponding value so that the optimizer uses for cost calculation a value that is derived from the actual runtime multi-block read count.

                            Regards,
                            Randolf

                            Oracle related stuff blog:
                            http://oracle-randolf.blogspot.com/

                            Co-author of the "OakTable Expert Oracle Practices" book:
                            http://www.apress.com/book/view/1430226684
                            http://www.amazon.com/Expert-Oracle-Practices-Database-Administration/dp/1430226684
                            • 11. Re: db_file_multiblock_read_count not in effect
                              OraDBA02
                              Thanks Randolf for highlighting important points...
                              By the way, depending on your hardware and operating system version you might not even get a difference in runtime between 128K and 1M multi-block reads. So even if your process used those 
                              bigger reads, it is not said that your join operation would perform better. You might want to use ASH (if you have a diagnostic license) or extended SQL trace to understand where most of the time of your execution is spent (could be for instance reading / writing to TEMP). Running this join in parallel might be a reasonable option.
                              I took 10046 trace and observed that all scattered reads were scanning 16 blocks from disk.Most of the time was spent on reading those blocks only.

                              Here is AUTOT TRACEONLY output (it took 3 hrs)
                              Statistics
                              ----------------------------------------------------------
                                        0  recursive calls
                                        0  db block gets
                                  3503945  consistent gets
                                  2994732  physical reads
                                        0  redo size
                              SP2-0642: SQL*Plus internal error state 1075, context 1:5:4294967295
                              Unsafe to proceed
                                   144680  bytes received via SQL*Net from client
                                    13124  SQL*Net roundtrips to/from client
                                        0  sorts (memory)
                                        0  sorts (disk)
                                 65614819  rows processed
                              I tried parallel option (parallel 4 hint) and it does improve elapsed time by a huge factor even though p3 values was 16 there as well. However, i cannot use px options for this table in my
                              EBS application for other business reasons.
                              Please note however that none of your information provided shows a clear evidence that at runtime the execution will actually attempt to read 128 blocks for multi-block reads, since you show us neither the "db_file_multiblock_read_count" nor the "_db_file_exec_read_count" parameter. The latter ultimately determines what at runtime will be used for direct path or scattered reads (although I've recently observed a case in 11.2 where the runtime engine obviously ignored the setting and used larger reads in certain cases). The shown "_db_file_optimizer_read_count" parameter is only used by the optimizer for the cost calculation, not by the runtime engine. It suggests that the "_db_file_exec_read_count" parameter could be set to the same value, but it has not be the same if explicitly set to a different value.
                              db_file_multiblock_read_count is set to 32 at instance level and 128 at session level.

                              dbfile_exec_read_count is also 32 at instance level.
                              ( Though i have not touched '_db_file_exec_read_count' parameter.I think,Oracle has changed it from 32 to 128 upon my change in MBRC at session level)
                              Do you see any harm in collecting system statistics for my EBS (10.2.0.4) ? Can that be beneficial ?
                              
                              Edited by: OraDBA02 on Oct 1, 2010 12:03 AM                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            
                              • 12. Re: db_file_multiblock_read_count not in effect
                                Randolf Geist
                                OraDBA02 wrote:
                                I took 10046 trace and observed that all scattered reads were scanning 16 blocks from disk.Most of the time was spent on reading those blocks only.
                                Can you share the trace profile analysis for the statement/session, so e.g. the TKPROF output or the output from some other trace profile analyzer like OraSRP or TVD$XTAT?
                                Do you see any harm in collecting system statistics for my EBS (10.2.0.4) ? Can that be beneficial ?
                                You probably want to follow the best practices that are recommended for EBS (which I don't know off the top of my head regarding collecting System Statistics).

                                Note that collecting System Statistics potentially affects every SQL statement since the cost calculation of the CBO will be based on those values gathered, so the impact is rather global and you would want to test such a change thoroughly.

                                Regards,
                                Randolf

                                Oracle related stuff blog:
                                http://oracle-randolf.blogspot.com/

                                Co-author of the "OakTable Expert Oracle Practices" book:
                                http://www.apress.com/book/view/1430226684
                                http://www.amazon.com/Expert-Oracle-Practices-Database-Administration/dp/1430226684
                                • 13. Re: db_file_multiblock_read_count not in effect
                                  askraks
                                  Dear Jonathan,

                                  Please correct me if i am wrong, If driving table must be small and driven table must conatin index on a column then optimzer will favor nested loop join.

                                  Let me know your answer

                                  Kind Regards,
                                  Rakesh
                                  • 14. Re: db_file_multiblock_read_count not in effect
                                    OraDBA02
                                    Hi Randolf,

                                    Many thanks for replying back.
                                    I again captured 10046 (level 12) trace and here is TKPROF and AUTOT outout:

                                    TKPROF
                                    ********************************************************************************
                                    
                                    select *
                                    from
                                     AP.AP_INVOICES_ALL
                                    
                                    
                                    call     count       cpu    elapsed       disk      query    current        rows
                                    ------- ------  -------- ---------- ---------- ---------- ----------  ----------
                                    Parse        1      0.00       0.00          0          0          0           0
                                    Execute      1      0.00       0.00          0          0          0           0
                                    Fetch      297     12.13      62.89      78181      79132          0     1480001
                                    ------- ------  -------- ---------- ---------- ---------- ----------  ----------
                                    total      299     12.13      62.89      78181      79132          0     1480001
                                    
                                    Misses in library cache during parse: 1
                                    Optimizer mode: ALL_ROWS
                                    Parsing user id: 353
                                    
                                    Elapsed times include waiting on following events:
                                      Event waited on                             Times   Max. Wait  Total Waited
                                      ----------------------------------------   Waited  ----------  ------------
                                      SQL*Net message to client                     298        0.00          0.00
                                      SQL*Net more data to client                 79549        0.34          2.41
                                      db file sequential read                        51        0.02          0.22
                                      db file scattered read                       5020        0.05         49.26
                                      SQL*Net message from client                   297        1.49         85.12
                                    ********************************************************************************
                                    AUTOT TRACEONLY
                                    BHAVIK_DBA.FIN_P>alter session set events '10046 trace name context forever,level 12';
                                    
                                    Session altered.
                                    
                                    BHAVIK_DBA.FIN_P>set arraysize 5000
                                    BHAVIK_DBA.FIN_P>set autot traceonly
                                    BHAVIK_DBA.FIN_P>set timing on
                                    BHAVIK_DBA.FIN_P>select * from AP.AP_INVOICES_ALL;
                                    
                                    65641138 rows selected.
                                    
                                    Elapsed: 01:53:01.18
                                    
                                    Execution Plan
                                    ----------------------------------------------------------
                                    Plan hash value: 3474762098
                                    
                                    -------------------------------------------------------------------------------------
                                    | Id  | Operation         | Name            | Rows  | Bytes | Cost (%CPU)| Time     |
                                    -------------------------------------------------------------------------------------
                                    |   0 | SELECT STATEMENT  |                 |    65M|    19G|   610K  (3)| 02:02:04 |
                                    |   1 |  TABLE ACCESS FULL| AP_INVOICES_ALL |    65M|    19G|   610K  (3)| 02:02:04 |
                                    -------------------------------------------------------------------------------------
                                    
                                    
                                    Statistics
                                    ----------------------------------------------------------
                                            153  recursive calls
                                              0  db block gets
                                        3550453  consistent gets
                                        3415889  physical reads
                                             72  redo size
                                    SP2-0642: SQL*Plus internal error state 1075, context 1:5:4294967295
                                    Unsafe to proceed
                                         144746  bytes received via SQL*Net from client
                                          13130  SQL*Net roundtrips to/from client
                                              0  sorts (memory)
                                              0  sorts (disk)
                                       65641138  rows processed
                                    TRACE FILE
                                    $ grep "db file scattered read" db_ora_11833.trc | head -100
                                    WAIT #31: nam='db file scattered read' ela= 9012 file#=658 block#=572941 blocks=12 obj#=30588 tim=1255782683983379
                                    WAIT #31: nam='db file scattered read' ela= 11076 file#=663 block#=572826 blocks=15 obj#=30588 tim=1255782684184941
                                    WAIT #31: nam='db file scattered read' ela= 9316 file#=665 block#=572890 blocks=15 obj#=30588 tim=1255782684206354
                                    WAIT #31: nam='db file scattered read' ela= 7284 file#=668 block#=573002 blocks=15 obj#=30588 tim=1255782684224624
                                    WAIT #31: nam='db file scattered read' ela= 9096 file#=671 block#=572906 blocks=15 obj#=30588 tim=1255782684240597
                                    WAIT #31: nam='db file scattered read' ela= 20502 file#=673 block#=572700 blocks=13 obj#=30588 tim=1255782684281482
                                    WAIT #31: nam='db file scattered read' ela= 8340 file#=700 block#=572858 blocks=15 obj#=30588 tim=1255782684303476
                                    WAIT #31: nam='db file scattered read' ela= 21312 file#=710 block#=572842 blocks=15 obj#=30588 tim=1255782684332570
                                    WAIT #31: nam='db file scattered read' ela= 6310 file#=711 block#=572953 blocks=6 obj#=30588 tim=1255782684347241
                                    WAIT #31: nam='db file scattered read' ela= 9236 file#=711 block#=572960 blocks=9 obj#=30588 tim=1255782684361140
                                    WAIT #31: nam='db file scattered read' ela= 9974 file#=712 block#=572921 blocks=16 obj#=30588 tim=1255782684378695
                                    WAIT #31: nam='db file scattered read' ela= 9259 file#=713 block#=573241 blocks=16 obj#=30588 tim=1255782684396755
                                    WAIT #31: nam='db file scattered read' ela= 8850 file#=714 block#=573002 blocks=15 obj#=30588 tim=1255782684414708
                                    WAIT #31: nam='db file scattered read' ela= 12039 file#=715 block#=573033 blocks=16 obj#=30588 tim=1255782684434008
                                    WAIT #31: nam='db file scattered read' ela= 9086 file#=658 block#=573033 blocks=16 obj#=30588 tim=1255782684450094
                                    WAIT #31: nam='db file scattered read' ela= 13060 file#=663 block#=572921 blocks=16 obj#=30588 tim=1255782684465360
                                    WAIT #31: nam='db file scattered read' ela= 9401 file#=665 block#=572986 blocks=15 obj#=30588 tim=1255782684479067
                                    WAIT #31: nam='db file scattered read' ela= 10590 file#=668 block#=573097 blocks=16 obj#=30588 tim=1255782684491916
                                    WAIT #31: nam='db file scattered read' ela= 11811 file#=671 block#=573001 blocks=16 obj#=30588 tim=1255782684754554
                                    WAIT #31: nam='db file scattered read' ela= 13112 file#=673 block#=572809 blocks=16 obj#=30588 tim=1255782684770442
                                    WAIT #31: nam='db file scattered read' ela= 8808 file#=700 block#=572954 blocks=15 obj#=30588 tim=1255782684782720
                                    WAIT #31: nam='db file scattered read' ela= 10236 file#=710 block#=572937 blocks=16 obj#=30588 tim=1255782684795988
                                    WAIT #31: nam='db file scattered read' ela= 8265 file#=711 block#=573065 blocks=2 obj#=30588 tim=1255782684808551
                                    WAIT #31: nam='db file scattered read' ela= 635 file#=711 block#=573068 blocks=13 obj#=30588 tim=1255782684809747
                                    WAIT #31: nam='db file scattered read' ela= 8513 file#=712 block#=573033 blocks=16 obj#=30588 tim=1255782684820261
                                    WAIT #31: nam='db file scattered read' ela= 8991 file#=713 block#=573338 blocks=15 obj#=30588 tim=1255782684831208
                                    WAIT #31: nam='db file scattered read' ela= 11160 file#=714 block#=573097 blocks=16 obj#=30588 tim=1255782684844654
                                    WAIT #31: nam='db file scattered read' ela= 8417 file#=715 block#=573131 blocks=14 obj#=30588 tim=1255782684863506
                                    WAIT #31: nam='db file scattered read' ela= 11405 file#=658 block#=573129 blocks=16 obj#=30588 tim=1255782684877096
                                    WAIT #31: nam='db file scattered read' ela= 9414 file#=663 block#=573018 blocks=15 obj#=30588 tim=1255782684888979
                                    WAIT #31: nam='db file scattered read' ela= 9073 file#=665 block#=573081 blocks=16 obj#=30588 tim=1255782684900070
                                    WAIT #31: nam='db file scattered read' ela= 6434 file#=668 block#=573193 blocks=16 obj#=30588 tim=1255782684908622
                                    WAIT #31: nam='db file scattered read' ela= 7977 file#=671 block#=573097 blocks=3 obj#=30588 tim=1255782684918663
                                    WAIT #31: nam='db file scattered read' ela= 13802 file#=671 block#=573101 blocks=2 obj#=30588 tim=1255782684933329
                                    WAIT #31: nam='db file scattered read' ela= 3371 file#=671 block#=573104 blocks=3 obj#=30588 tim=1255782684937367
                                    WAIT #31: nam='db file scattered read' ela= 7875 file#=671 block#=573108 blocks=5 obj#=30588 tim=1255782684945985
                                    WAIT #31: nam='db file scattered read' ela= 9199 file#=673 block#=572890 blocks=15 obj#=30588 tim=1255782684956428
                                    WAIT #31: nam='db file scattered read' ela= 10985 file#=700 block#=573033 blocks=16 obj#=30588 tim=1255782684969887
                                    1 2 Previous Next