From a3cc5b99a8d98a4d9d8d61a2331f407162be8115 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 9 Nov 2022 15:21:06 +0100 Subject: [PATCH] store: Go back to the less clever but correct way of pruning The change in aea5b577 has a logic error in that we look at a fixed number of rows from the table, filter out the ones we don't need to copy, and then insert the remaining rows. The error is that the logic assumes that if we do not insert any rows that we are done with copying, but that's wrong, since not inserting rows now just means that no rows in the batch needed to be copied even though a subsequent batch might still have such rows. This reverts commit aea5b577db0fc5f08e8de17267c859186d917448. --- store/postgres/src/relational/prune.rs | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 76fdff3f731..27f0444d188 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -91,22 +91,9 @@ impl TablePair { loop { let start = Instant::now(); let LastVid { last_vid, rows } = conn.transaction(|| { - // The query limits the number of rows we look at in each - // batch, rather than the number of matching rows to make - // the timing for each batch independent of how many rows we - // actually need to copy. This has the downside that it will - // force a sequential scan, but limiting by the number of - // matching rows runs the risk of a batch taking - // unpredictably long if there are only very few matches. - // Strictly speaking, the `vid >= $3` and `limit` clauses on - // the outer query are redundant. sql_query(&format!( "with cp as (insert into {dst}({column_list}) \ - select {column_list} from ( \ - select {column_list} from {src} \ - where vid >= $3 \ - order by vid \ - limit $4) a \ + select {column_list} from {src} \ where lower(block_range) <= $2 \ and coalesce(upper(block_range), 2147483647) > $1 \ and coalesce(upper(block_range), 2147483647) <= $2 \ @@ -166,15 +153,9 @@ impl TablePair { loop { let start = Instant::now(); let LastVid { rows, last_vid } = conn.transaction(|| { - // See the comment on the query in `copy_final_entities` for - // why this query is written this way sql_query(&format!( "with cp as (insert into {dst}({column_list}) \ - select {column_list} from ( \ - select {column_list} from {src} \ - where vid >= $2 \ - order by vid \ - limit $3) a + select {column_list} from {src} \ where coalesce(upper(block_range), 2147483647) > $1 \ and block_range && int4range($1, null) \ and vid >= $2 \