Re: Large Binary Columns - Slow Query

On Jan 10, 2016 21:50, "Saulo Merlo" <smerlo50@outlook.com> wrote:
>
> The binary columns are large so I think that's why the query referencing them is slow.
>
> PostgreSQL 9.2
>
> Is there a way to speed it up, maybe compression on transfer? Some indexes has millions of rows...
>
> Maybe create an index to some specifics situations, and not general ones.

Can you show us the table definitions?

What are you trying to do?

>
>
> Thank you.
>
>
> QUERY:
>
> SELECT ni.segment_index AS note_id,
>        f.inode_id AS file_id,
>        f.node_full_path AS filename,
>        f.last_changed AS date_created,
>        f.file_data AS main_binary,
>        medium.inode_id AS medium_id,
>        medium.file_data AS medium_binary,
>        thumbnail.inode_id AS thumbnail_id,
>        thumbnail.file_data AS thumbnail_binary
> FROM gorfs.nodes AS f
> INNER JOIN gorfs.inode_segments AS fd ON fd.st_ino_target = f.inode_id
> INNER JOIN gorfs.inode_segments AS mv ON mv.st_ino_target = fd.st_ino
> AND mv.segment_index = 'main.with_name'
> INNER JOIN gorfs.inode_segments AS fi ON fi.st_ino_target = mv.st_ino
> INNER JOIN gorfs.inode_segments AS fn ON fn.st_ino_target = fi.st_ino
> INNER JOIN gorfs.inode_segments AS ni ON ni.st_ino_target = fn.st_ino
> LEFT JOIN
>   (SELECT f.inode_id,
>           f.file_data,
>           fi.st_ino
>    FROM gorfs.nodes AS f
>    INNER JOIN gorfs.inode_segments AS fd ON fd.st_ino_target = f.inode_id
>    INNER JOIN gorfs.inode_segments AS mv ON mv.st_ino_target = fd.st_ino
>    AND mv.segment_index = 'medium.with_name'
>    INNER JOIN gorfs.inode_segments AS fi ON fi.st_ino_target = mv.st_ino ) AS medium ON medium.st_ino = fn.st_ino_target
> LEFT JOIN
>   (SELECT f.inode_id,
>           f.file_data,
>           fi.st_ino
>    FROM gorfs.nodes AS f
>    INNER JOIN gorfs.inode_segments AS fd ON fd.st_ino_target = f.inode_id
>    INNER JOIN gorfs.inode_segments AS mv ON mv.st_ino_target = fd.st_ino
>    AND mv.segment_index = 'thumbnail.with_name'
>    INNER JOIN gorfs.inode_segments AS fi ON fi.st_ino_target = mv.st_ino ) AS thumbnail ON thumbnail.st_ino = fn.st_ino_target
> WHERE f.file_data IS NOT NULL LIMIT 500;
>
> EXPLAIN ANALYZE:
>
> "Limit  (cost=20243.45..1593866.18 rows=500 width=180) (actual time=210.036..44701.406 rows=500 loops=1)"
> "  ->  Nested Loop Left Join  (cost=20243.45..757747912.38 rows=240759 width=180) (actual time=210.036..44701.223 rows=500 loops=1)"
> "        ->  Nested Loop Left Join  (cost=19724.45..450144827.58 rows=240759 width=148) (actual time=196.881..44353.113 rows=490 loops=1)"
> "              ->  Nested Loop  (cost=19205.46..168833316.21 rows=240759 width=108) (actual time=196.239..43707.952 rows=486 loops=1)"
> "                    ->  Nested Loop  (cost=19205.46..164947484.34 rows=354030 width=107) (actual time=196.233..43706.095 rows=486 loops=1)"
> "                          ->  Nested Loop  (cost=19205.46..159233457.64 rows=520593 width=99) (actual time=196.223..43704.141 rows=486 loops=1)"
> "                                ->  Nested Loop  (cost=19205.46..153056028.27 rows=987163 width=107) (actual time=196.202..43700.535 rows=486 loops=1)"
> "                                      Join Filter: ((("t"."st_ino")::bigint = ("p"."st_ino_target")::bigint) AND (CASE WHEN ("f"."constant_name" = 'S_IFREG'::"gorfs"."mode_t_constant_name") THEN (SubPlan 4) ELSE NULL::"bytea" END IS NOT NULL))"
> "                                      ->  Nested Loop  (cost=19205.46..83831623.03 rows=1458893 width=36) (actual time=155.629..43307.564 rows=486 loops=1)"
> "                                            Join Filter: ((B'00000000000000001111000000000000'::"bit" & ("t"."st_mode")::"bit") = ("f"."bits")::"bit")"
> "                                            Rows Removed by Join Filter: 10673"
> "                                            ->  Nested Loop  (cost=18686.46..83243898.69 rows=1458893 width=41) (actual time=155.432..43302.768 rows=486 loops=1)"
> "                                                  ->  Nested Loop  (cost=18686.46..59543972.56 rows=2145267 width=41) (actual time=155.420..43300.521 rows=486 loops=1)"
> "                                                        ->  Merge Join  (cost=18686.46..29848042.37 rows=4751151 width=16) (actual time=155.397..43289.955 rows=486 loops=1)"
> "                                                              Merge Cond: (("fd"."st_ino")::bigint = ("mv"."st_ino_target")::bigint)"
> "                                                              ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fd"  (cost=0.00..24356396.96 rows=83232691 width=16) (actual time=0.080..43016.366 rows=2037019 loops=1)"
> "                                                              ->  Index Scan using "ix_inode_segments_climb_tree" on "inode_segments" "mv"  (cost=0.00..7695601.08 rows=6986452 width=16) (actual time=0.197..96.693 rows=487 loops=1)"
> "                                                                    Index Cond: (("segment_index")::"text" = 'main.with_name'::"text")"
> "                                                        ->  Index Scan using "pk_inodes" on "inodes" "t"  (cost=0.00..6.24 rows=1 width=29) (actual time=0.020..0.020 rows=1 loops=486)"
> "                                                              Index Cond: (("st_ino")::bigint = ("fd"."st_ino_target")::bigint)"
> "                                                  ->  Index Scan using "ix_inode_segments_st_ino_targets" on "inode_segments" "fi"  (cost=0.00..11.04 rows=1 width=16) (actual time=0.003..0.004 rows=1 loops=486)"
> "                                                        Index Cond: (("st_ino_target")::bigint = ("mv"."st_ino")::bigint)"
> "                                            ->  Materialize  (cost=519.00..519.97 rows=23 width=36) (actual time=0.001..0.003 rows=23 loops=486)"
> "                                                  ->  Subquery Scan on "f"  (cost=519.00..519.86 rows=23 width=36) (actual time=0.138..0.452 rows=23 loops=1)"
> "                                                        ->  CTE Scan on "stat_h_with_bits" "sb"  (cost=519.00..519.63 rows=23 width=72) (actual time=0.137..0.447 rows=23 loops=1)"
> "                                                              CTE stat_h"
> "                                                                ->  Values Scan on "*VALUES*"  (cost=0.00..0.29 rows=23 width=68) (actual time=0.007..0.030 rows=23 loops=1)"
> "                                                              CTE stat_h_with_bits"
> "                                                                ->  CTE Scan on "stat_h" "s"  (cost=0.00..518.71 rows=23 width=68) (actual time=0.120..0.407 rows=23 loops=1)"
> "                                                                      SubPlan 6"
> "                                                                        ->  Aggregate  (cost=22.51..22.52 rows=1 width=32) (actual time=0.013..0.013 rows=1 loops=23)"
> "                                                                              ->  Function Scan on "regexp_split_to_table" "digits"  (cost=0.01..10.01 rows=1000 width=32) (actual time=0.008..0.009 rows=3 loops=23)"
> "                                      ->  Index Scan using "ix_inode_segments_st_ino_targets" on "inode_segments" "p"  (cost=0.00..10.97 rows=1 width=79) (actual time=0.002..0.002 rows=1 loops=486)"
> "                                            Index Cond: (("st_ino_target")::bigint = ("fd"."st_ino_target")::bigint)"
> "                                      SubPlan 4"
> "                                        ->  Aggregate  (cost=36.46..36.47 rows=1 width=47) (actual time=0.712..0.712 rows=1 loops=486)"
> "                                              ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fs"  (cost=0.00..36.33 rows=52 width=47) (actual time=0.003..0.004 rows=1 loops=486)"
> "                                                    Index Cond: (("st_ino")::bigint = ("p"."st_ino_target")::bigint)"
> "                                ->  Index Scan using "pk_inodes" on "inodes" "i"  (cost=0.00..6.25 rows=1 width=8) (actual time=0.006..0.006 rows=1 loops=486)"
> "                                      Index Cond: (("st_ino")::bigint = ("p"."st_ino")::bigint)"
> "                                      Filter: ((("st_ino")::bigint = 2) OR ((B'00000000000000001111000000000000'::"bit" & ("st_mode")::"bit") = B'00000000000000000100000000000000'::"bit"))"
> "                          ->  Index Scan using "ix_inode_segments_st_ino_targets" on "inode_segments" "fn"  (cost=0.00..10.97 rows=1 width=16) (actual time=0.003..0.003 rows=1 loops=486)"
> "                                Index Cond: (("st_ino_target")::bigint = ("fi"."st_ino")::bigint)"
> "                    ->  Index Scan using "ix_inode_segments_st_ino_targets" on "inode_segments" "ni"  (cost=0.00..10.97 rows=1 width=17) (actual time=0.003..0.003 rows=1 loops=486)"
> "                          Index Cond: (("st_ino_target")::bigint = ("fn"."st_ino")::bigint)"
> "              ->  Nested Loop  (cost=519.00..1168.43 rows=1 width=48) (actual time=1.296..1.326 rows=1 loops=486)"
> "                    Join Filter: ((B'00000000000000001111000000000000'::"bit" & ("t"."st_mode")::"bit") = ("sb"."bits")::"bit")"
> "                    Rows Removed by Join Filter: 16"
> "                    ->  Nested Loop  (cost=0.00..648.22 rows=1 width=33) (actual time=1.079..1.080 rows=1 loops=486)"
> "                          ->  Nested Loop  (cost=0.00..641.97 rows=1 width=24) (actual time=1.076..1.077 rows=1 loops=486)"
> "                                ->  Nested Loop  (cost=0.00..635.72 rows=1 width=32) (actual time=1.026..1.027 rows=1 loops=486)"
> "                                      ->  Nested Loop  (cost=0.00..624.74 rows=1 width=16) (actual time=0.981..0.982 rows=1 loops=486)"
> "                                            ->  Nested Loop  (cost=0.00..602.15 rows=1 width=16) (actual time=0.964..0.965 rows=1 loops=486)"
> "                                                  ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fi"  (cost=0.00..21.51 rows=52 width=16) (actual time=0.003..0.003 rows=1 loops=486)"
> "                                                        Index Cond: (("st_ino")::bigint = ("fn"."st_ino_target")::bigint)"
> "                                                  ->  Index Scan using "pk_inode_segments" on "inode_segments" "mv"  (cost=0.00..11.16 rows=1 width=16) (actual time=0.947..0.947 rows=1 loops=493)"
> "                                                        Index Cond: ((("st_ino")::bigint = ("fi"."st_ino_target")::bigint) AND (("segment_index")::"text" = 'medium.with_name'::"text"))"
> "                                            ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fd"  (cost=0.00..22.07 rows=52 width=16) (actual time=0.023..0.023 rows=1 loops=345)"
> "                                                  Index Cond: (("st_ino")::bigint = ("mv"."st_ino_target")::bigint)"
> "                                      ->  Index Scan using "ix_inode_segments_st_ino_targets" on "inode_segments" "p"  (cost=0.00..10.97 rows=1 width=16) (actual time=0.062..0.062 rows=1 loops=345)"
> "                                            Index Cond: (("st_ino_target")::bigint = ("fd"."st_ino_target")::bigint)"
> "                                ->  Index Scan using "pk_inodes" on "inodes" "i"  (cost=0.00..6.25 rows=1 width=8) (actual time=0.069..0.069 rows=1 loops=345)"
> "                                      Index Cond: (("st_ino")::bigint = ("p"."st_ino")::bigint)"
> "                                      Filter: ((("st_ino")::bigint = 2) OR ((B'00000000000000001111000000000000'::"bit" & ("st_mode")::"bit") = B'00000000000000000100000000000000'::"bit"))"
> "                          ->  Index Scan using "pk_inodes" on "inodes" "t"  (cost=0.00..6.24 rows=1 width=21) (actual time=0.003..0.003 rows=1 loops=345)"
> "                                Index Cond: (("st_ino")::bigint = ("p"."st_ino_target")::bigint)"
> "                    ->  CTE Scan on "stat_h_with_bits" "sb"  (cost=519.00..519.63 rows=23 width=72) (actual time=0.002..0.010 rows=23 loops=345)"
> "                          CTE stat_h"
> "                            ->  Values Scan on "*VALUES*"  (cost=0.00..0.29 rows=23 width=68) (actual time=0.003..0.025 rows=23 loops=1)"
> "                          CTE stat_h_with_bits"
> "                            ->  CTE Scan on "stat_h" "s"  (cost=0.00..518.71 rows=23 width=68) (actual time=0.101..0.419 rows=23 loops=1)"
> "                                  SubPlan 9"
> "                                    ->  Aggregate  (cost=22.51..22.52 rows=1 width=32) (actual time=0.012..0.012 rows=1 loops=23)"
> "                                          ->  Function Scan on "regexp_split_to_table" "digits"  (cost=0.01..10.01 rows=1000 width=32) (actual time=0.008..0.008 rows=3 loops=23)"
> "                    SubPlan 2"
> "                      ->  Aggregate  (cost=36.46..36.47 rows=1 width=47) (actual time=0.314..0.314 rows=1 loops=345)"
> "                            ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fs"  (cost=0.00..36.33 rows=52 width=47) (actual time=0.012..0.034 rows=1 loops=345)"
> "                                  Index Cond: (("st_ino")::bigint = ("p"."st_ino_target")::bigint)"
> "        ->  Nested Loop  (cost=519.00..1168.21 rows=1 width=48) (actual time=0.236..0.304 rows=1 loops=490)"
> "              Join Filter: ((B'00000000000000001111000000000000'::"bit" & ("t"."st_mode")::"bit") = ("sb"."bits")::"bit")"
> "              Rows Removed by Join Filter: 16"
> "              ->  Nested Loop  (cost=0.00..648.01 rows=1 width=33) (actual time=0.174..0.206 rows=1 loops=490)"
> "                    ->  Nested Loop  (cost=0.00..641.76 rows=1 width=24) (actual time=0.145..0.176 rows=1 loops=490)"
> "                          ->  Nested Loop  (cost=0.00..635.50 rows=1 width=32) (actual time=0.099..0.125 rows=1 loops=490)"
> "                                ->  Nested Loop  (cost=0.00..624.53 rows=1 width=16) (actual time=0.096..0.107 rows=1 loops=490)"
> "                                      ->  Nested Loop  (cost=0.00..602.15 rows=1 width=16) (actual time=0.059..0.069 rows=1 loops=490)"
> "                                            ->  Index Scan using "ix_inode_segments_st_inos" on "inode_segments" "fi"  (cost=0.00..21.51 rows=52 width=16) (actual time=0.004..0.004 rows=1 loops=490)"
> "                                                  Index Cond: (("st_ino")::bigint = ("fn"."st_ino_target")::bigint)"
> "                                            ->  Index Scan using "pk_inode_segments" on "inode_segments" "mv"  (cost=0.00..11.16 rows=1 width=16) (actual time=0.053..0.062 rows=1 loops=503)"
> "                                                  Index Cond: ((("st_ino")::bigint = ("fi"."st_ino_target")::bigint) AND (("segment_index")::"text" = 'thumbnail.with_name'::"text
>
> ...