v2-0001-Skip-index-vacuuming-dynamically.patch
application/octet-stream
Filename: v2-0001-Skip-index-vacuuming-dynamically.patch
Type: application/octet-stream
Part: 0
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: format-patch
Series: patch v2-0001
Subject: Skip index vacuuming dynamically.
| File | + | − |
|---|---|---|
| src/backend/access/gist/gistxlog.c | 4 | 4 |
| src/backend/access/hash/hash_xlog.c | 4 | 4 |
| src/backend/access/heap/heapam.c | 0 | 51 |
| src/backend/access/heap/pruneheap.c | 3 | 10 |
| src/backend/access/heap/vacuumlazy.c | 249 | 199 |
| src/backend/access/nbtree/nbtree.c | 3 | 3 |
| src/backend/access/rmgrdesc/heapdesc.c | 0 | 9 |
| src/backend/commands/vacuum.c | 11 | 4 |
| src/backend/replication/logical/decode.c | 0 | 1 |
| src/include/access/heapam.h | 1 | 1 |
| src/include/access/heapam_xlog.h | 1 | 3 |
From 1be226b363b72b2326c1648389502e1f3b12ba64 Mon Sep 17 00:00:00 2001
From: Peter Geoghegan <pg@bowt.ie>
Date: Sat, 13 Mar 2021 20:37:32 -0800
Subject: [PATCH v2] Skip index vacuuming dynamically.
Based on skip_index_vacuum.patch from Masahiko Sawada:
https://postgr.es/m/CAD21AoAtZb4+HJT_8RoOXvu4HM-Zd4HKS3YSMCH6+-W=bDyh-w@mail.gmail.com
Also remove tupgone special case to decouple index vacuuming from
initial heap scan's pruning. Unify dynamic index vacuum skipping with
the index_cleanup mechanism added to Postgres 12 by commits a96c41fe and
dd695979.
---
src/include/access/heapam.h | 2 +-
src/include/access/heapam_xlog.h | 4 +-
src/backend/access/gist/gistxlog.c | 8 +-
src/backend/access/hash/hash_xlog.c | 8 +-
src/backend/access/heap/heapam.c | 51 ---
src/backend/access/heap/pruneheap.c | 13 +-
src/backend/access/heap/vacuumlazy.c | 448 +++++++++++++----------
src/backend/access/nbtree/nbtree.c | 6 +-
src/backend/access/rmgrdesc/heapdesc.c | 9 -
src/backend/commands/vacuum.c | 15 +-
src/backend/replication/logical/decode.c | 1 -
11 files changed, 276 insertions(+), 289 deletions(-)
diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h
index bc0936bc2d..0bef090420 100644
--- a/src/include/access/heapam.h
+++ b/src/include/access/heapam.h
@@ -180,7 +180,7 @@ extern int heap_page_prune(Relation relation, Buffer buffer,
struct GlobalVisState *vistest,
TransactionId old_snap_xmin,
TimestampTz old_snap_ts_ts,
- bool report_stats, TransactionId *latestRemovedXid,
+ bool report_stats,
OffsetNumber *off_loc);
extern void heap_page_prune_execute(Buffer buffer,
OffsetNumber *redirected, int nredirected,
diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h
index 178d49710a..150c2fe384 100644
--- a/src/include/access/heapam_xlog.h
+++ b/src/include/access/heapam_xlog.h
@@ -53,7 +53,7 @@
#define XLOG_HEAP2_REWRITE 0x00
#define XLOG_HEAP2_CLEAN 0x10
#define XLOG_HEAP2_FREEZE_PAGE 0x20
-#define XLOG_HEAP2_CLEANUP_INFO 0x30
+/* 0x30 is reserved */
#define XLOG_HEAP2_VISIBLE 0x40
#define XLOG_HEAP2_MULTI_INSERT 0x50
#define XLOG_HEAP2_LOCK_UPDATED 0x60
@@ -397,8 +397,6 @@ extern void heap2_desc(StringInfo buf, XLogReaderState *record);
extern const char *heap2_identify(uint8 info);
extern void heap_xlog_logical_rewrite(XLogReaderState *r);
-extern XLogRecPtr log_heap_cleanup_info(RelFileNode rnode,
- TransactionId latestRemovedXid);
extern XLogRecPtr log_heap_clean(Relation reln, Buffer buffer,
OffsetNumber *redirected, int nredirected,
OffsetNumber *nowdead, int ndead,
diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c
index 1c80eae044..5da9805073 100644
--- a/src/backend/access/gist/gistxlog.c
+++ b/src/backend/access/gist/gistxlog.c
@@ -184,10 +184,10 @@ gistRedoDeleteRecord(XLogReaderState *record)
*
* GiST delete records can conflict with standby queries. You might think
* that vacuum records would conflict as well, but we've handled that
- * already. XLOG_HEAP2_CLEANUP_INFO records provide the highest xid
- * cleaned by the vacuum of the heap and so we can resolve any conflicts
- * just once when that arrives. After that we know that no conflicts
- * exist from individual gist vacuum records on that index.
+ * already. XLOG_HEAP2_CLEAN records provide the highest xid cleaned by
+ * the vacuum of the heap and so we can resolve any conflicts just once
+ * when that arrives. After that we know that no conflicts exist from
+ * individual gist vacuum records on that index.
*/
if (InHotStandby)
{
diff --git a/src/backend/access/hash/hash_xlog.c b/src/backend/access/hash/hash_xlog.c
index 02d9e6cdfd..7b8b8c8b74 100644
--- a/src/backend/access/hash/hash_xlog.c
+++ b/src/backend/access/hash/hash_xlog.c
@@ -992,10 +992,10 @@ hash_xlog_vacuum_one_page(XLogReaderState *record)
* Hash index records that are marked as LP_DEAD and being removed during
* hash index tuple insertion can conflict with standby queries. You might
* think that vacuum records would conflict as well, but we've handled
- * that already. XLOG_HEAP2_CLEANUP_INFO records provide the highest xid
- * cleaned by the vacuum of the heap and so we can resolve any conflicts
- * just once when that arrives. After that we know that no conflicts
- * exist from individual hash index vacuum records on that index.
+ * that already. XLOG_HEAP2_CLEAN records provide the highest xid cleaned
+ * by the vacuum of the heap and so we can resolve any conflicts just once
+ * when that arrives. After that we know that no conflicts exist from
+ * individual hash index vacuum records on that index.
*/
if (InHotStandby)
{
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 3b435c107d..8b80aa1b10 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -7945,29 +7945,6 @@ bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate)
return nblocksfavorable;
}
-/*
- * Perform XLogInsert to register a heap cleanup info message. These
- * messages are sent once per VACUUM and are required because
- * of the phasing of removal operations during a lazy VACUUM.
- * see comments for vacuum_log_cleanup_info().
- */
-XLogRecPtr
-log_heap_cleanup_info(RelFileNode rnode, TransactionId latestRemovedXid)
-{
- xl_heap_cleanup_info xlrec;
- XLogRecPtr recptr;
-
- xlrec.node = rnode;
- xlrec.latestRemovedXid = latestRemovedXid;
-
- XLogBeginInsert();
- XLogRegisterData((char *) &xlrec, SizeOfHeapCleanupInfo);
-
- recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_CLEANUP_INFO);
-
- return recptr;
-}
-
/*
* Perform XLogInsert for a heap-clean operation. Caller must already
* have modified the buffer and marked it dirty.
@@ -8497,27 +8474,6 @@ ExtractReplicaIdentity(Relation relation, HeapTuple tp, bool key_changed,
return key_tuple;
}
-/*
- * Handles CLEANUP_INFO
- */
-static void
-heap_xlog_cleanup_info(XLogReaderState *record)
-{
- xl_heap_cleanup_info *xlrec = (xl_heap_cleanup_info *) XLogRecGetData(record);
-
- if (InHotStandby)
- ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, xlrec->node);
-
- /*
- * Actual operation is a no-op. Record type exists to provide a means for
- * conflict processing to occur before we begin index vacuum actions. see
- * vacuumlazy.c and also comments in btvacuumpage()
- */
-
- /* Backup blocks are not used in cleanup_info records */
- Assert(!XLogRecHasAnyBlockRefs(record));
-}
-
/*
* Handles XLOG_HEAP2_CLEAN record type
*/
@@ -8536,10 +8492,6 @@ heap_xlog_clean(XLogReaderState *record)
/*
* We're about to remove tuples. In Hot Standby mode, ensure that there's
* no queries running for which the removed tuples are still visible.
- *
- * Not all HEAP2_CLEAN records remove tuples with xids, so we only want to
- * conflict on the records that cause MVCC failures for user queries. If
- * latestRemovedXid is invalid, skip conflict processing.
*/
if (InHotStandby && TransactionIdIsValid(xlrec->latestRemovedXid))
ResolveRecoveryConflictWithSnapshot(xlrec->latestRemovedXid, rnode);
@@ -9716,9 +9668,6 @@ heap2_redo(XLogReaderState *record)
case XLOG_HEAP2_FREEZE_PAGE:
heap_xlog_freeze_page(record);
break;
- case XLOG_HEAP2_CLEANUP_INFO:
- heap_xlog_cleanup_info(record);
- break;
case XLOG_HEAP2_VISIBLE:
heap_xlog_visible(record);
break;
diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c
index 8bb38d6406..ac7e540944 100644
--- a/src/backend/access/heap/pruneheap.c
+++ b/src/backend/access/heap/pruneheap.c
@@ -182,13 +182,10 @@ heap_page_prune_opt(Relation relation, Buffer buffer)
*/
if (PageIsFull(page) || PageGetHeapFreeSpace(page) < minfree)
{
- TransactionId ignore = InvalidTransactionId; /* return value not
- * needed */
-
/* OK to prune */
(void) heap_page_prune(relation, buffer, vistest,
limited_xmin, limited_ts,
- true, &ignore, NULL);
+ true, NULL);
}
/* And release buffer lock */
@@ -213,8 +210,6 @@ heap_page_prune_opt(Relation relation, Buffer buffer)
* send its own new total to pgstats, and we don't want this delta applied
* on top of that.)
*
- * Sets latestRemovedXid for caller on return.
- *
* off_loc is the offset location required by the caller to use in error
* callback.
*
@@ -225,7 +220,7 @@ heap_page_prune(Relation relation, Buffer buffer,
GlobalVisState *vistest,
TransactionId old_snap_xmin,
TimestampTz old_snap_ts,
- bool report_stats, TransactionId *latestRemovedXid,
+ bool report_stats,
OffsetNumber *off_loc)
{
int ndeleted = 0;
@@ -251,7 +246,7 @@ heap_page_prune(Relation relation, Buffer buffer,
prstate.old_snap_xmin = old_snap_xmin;
prstate.old_snap_ts = old_snap_ts;
prstate.old_snap_used = false;
- prstate.latestRemovedXid = *latestRemovedXid;
+ prstate.latestRemovedXid = InvalidTransactionId;
prstate.nredirected = prstate.ndead = prstate.nunused = 0;
memset(prstate.marked, 0, sizeof(prstate.marked));
@@ -363,8 +358,6 @@ heap_page_prune(Relation relation, Buffer buffer,
if (report_stats && ndeleted > prstate.ndead)
pgstat_update_heap_dead_tuples(relation, ndeleted - prstate.ndead);
- *latestRemovedXid = prstate.latestRemovedXid;
-
/*
* XXX Should we update the FSM information of this page ?
*
diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c
index 366c122bd1..4b2bf29d51 100644
--- a/src/backend/access/heap/vacuumlazy.c
+++ b/src/backend/access/heap/vacuumlazy.c
@@ -131,6 +131,12 @@
*/
#define PREFETCH_SIZE ((BlockNumber) 32)
+/*
+ * The threshold of the percentage of heap blocks having LP_DEAD line pointer
+ * above which index vacuuming goes ahead.
+ */
+#define SKIP_VACUUM_PAGES_RATIO 0.01
+
/*
* DSM keys for parallel vacuum. Unlike other parallel execution code, since
* we don't need to worry about DSM keys conflicting with plan_node_id we can
@@ -294,8 +300,12 @@ typedef struct LVRelStats
{
char *relnamespace;
char *relname;
- /* useindex = true means two-pass strategy; false means one-pass */
- bool useindex;
+ /* hasindex = true means two-pass strategy; false means one-pass */
+ bool hasindex;
+ /* mayskipindexes = true means we may decide to skip vacuum indexing */
+ bool mayskipindexes;
+ /* mustskipindexes = true means we must always skip vacuum indexing */
+ bool mustskipindexes;
/* Overall statistics about rel */
BlockNumber old_rel_pages; /* previous value of pg_class.relpages */
BlockNumber rel_pages; /* total number of pages */
@@ -312,7 +322,6 @@ typedef struct LVRelStats
BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */
LVDeadTuples *dead_tuples;
int num_index_scans;
- TransactionId latestRemovedXid;
bool lock_waiter_detected;
/* Used for error callback */
@@ -344,20 +353,20 @@ static BufferAccessStrategy vac_strategy;
static void lazy_scan_heap(Relation onerel, VacuumParams *params,
LVRelStats *vacrelstats, Relation *Irel, int nindexes,
bool aggressive);
-static void lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats);
+static void vacuum_indexes_mark_unused(Relation onerel, LVRelStats *vacrelstats,
+ Relation *Irel, IndexBulkDeleteResult **indstats,
+ int nindexes, LVParallelState *lps,
+ BlockNumber has_dead_items_pages);
static bool lazy_check_needs_freeze(Buffer buf, bool *hastup,
LVRelStats *vacrelstats);
-static void lazy_vacuum_all_indexes(Relation onerel, Relation *Irel,
- IndexBulkDeleteResult **stats,
- LVRelStats *vacrelstats, LVParallelState *lps,
- int nindexes);
static void lazy_vacuum_index(Relation indrel, IndexBulkDeleteResult **stats,
LVDeadTuples *dead_tuples, double reltuples, LVRelStats *vacrelstats);
static void lazy_cleanup_index(Relation indrel,
IndexBulkDeleteResult **stats,
double reltuples, bool estimated_count, LVRelStats *vacrelstats);
-static int lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer,
- int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer);
+static int mark_unused_page(Relation onerel, BlockNumber blkno, Buffer buffer,
+ int tupindex, LVRelStats *vacrelstats,
+ Buffer *vmbuffer);
static bool should_attempt_truncation(VacuumParams *params,
LVRelStats *vacrelstats);
static void lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats);
@@ -443,7 +452,6 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params,
ErrorContextCallback errcallback;
Assert(params != NULL);
- Assert(params->index_cleanup != VACOPT_TERNARY_DEFAULT);
Assert(params->truncate != VACOPT_TERNARY_DEFAULT);
/* not every AM requires these to be valid, but heap does */
@@ -502,8 +510,25 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params,
/* Open all indexes of the relation */
vac_open_indexes(onerel, RowExclusiveLock, &nindexes, &Irel);
- vacrelstats->useindex = (nindexes > 0 &&
- params->index_cleanup == VACOPT_TERNARY_ENABLED);
+
+ /*
+ * hasindex determines if we'll use one-pass strategy. Note that this is
+ * not the same thing as skipping index vacuuming.
+ *
+ * mayskipindexes tracks if we could in principle decide to skip index
+ * vacuuming. This can become false later.
+ *
+ * mustskipindexes tracks if we are obligated to skip index vacuuming
+ * because of index_cleanup reloption.
+ *
+ * FIXME: This is too duplicative. Fix this issue when you fix the
+ * closely related VACOPT_TERNARY_DEFAULT issue in vacuum.c.
+ */
+ vacrelstats->hasindex = (nindexes > 0);
+ vacrelstats->mayskipindexes =
+ (params->index_cleanup != VACOPT_TERNARY_ENABLED);
+ vacrelstats->mustskipindexes =
+ (params->index_cleanup == VACOPT_TERNARY_DISABLED);
/*
* Setup error traceback support for ereport(). The idea is to set up an
@@ -689,39 +714,6 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params,
}
}
-/*
- * For Hot Standby we need to know the highest transaction id that will
- * be removed by any change. VACUUM proceeds in a number of passes so
- * we need to consider how each pass operates. The first phase runs
- * heap_page_prune(), which can issue XLOG_HEAP2_CLEAN records as it
- * progresses - these will have a latestRemovedXid on each record.
- * In some cases this removes all of the tuples to be removed, though
- * often we have dead tuples with index pointers so we must remember them
- * for removal in phase 3. Index records for those rows are removed
- * in phase 2 and index blocks do not have MVCC information attached.
- * So before we can allow removal of any index tuples we need to issue
- * a WAL record containing the latestRemovedXid of rows that will be
- * removed in phase three. This allows recovery queries to block at the
- * correct place, i.e. before phase two, rather than during phase three
- * which would be after the rows have become inaccessible.
- */
-static void
-vacuum_log_cleanup_info(Relation rel, LVRelStats *vacrelstats)
-{
- /*
- * Skip this for relations for which no WAL is to be written, or if we're
- * not trying to support archive recovery.
- */
- if (!RelationNeedsWAL(rel) || !XLogIsNeeded())
- return;
-
- /*
- * No need to write the record at all unless it contains a valid value
- */
- if (TransactionIdIsValid(vacrelstats->latestRemovedXid))
- (void) log_heap_cleanup_info(rel->rd_node, vacrelstats->latestRemovedXid);
-}
-
/*
* lazy_scan_heap() -- scan an open heap relation
*
@@ -730,9 +722,9 @@ vacuum_log_cleanup_info(Relation rel, LVRelStats *vacrelstats)
* page, and set commit status bits (see heap_page_prune). It also builds
* lists of dead tuples and pages with free space, calculates statistics
* on the number of live tuples in the heap, and marks pages as
- * all-visible if appropriate. When done, or when we run low on space for
- * dead-tuple TIDs, invoke vacuuming of indexes and call lazy_vacuum_heap
- * to reclaim dead line pointers.
+ * all-visible if appropriate. When done, or when we run low on space
+ * for dead-tuple TIDs, invoke vacuum_indexes_mark_unused to vacuum
+ * indexes and mark dead line pointers for reuse via a second heap pass.
*
* If the table has at least two indexes, we execute both index vacuum
* and index cleanup with parallel workers unless parallel vacuum is
@@ -762,7 +754,8 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
TransactionId relfrozenxid = onerel->rd_rel->relfrozenxid;
TransactionId relminmxid = onerel->rd_rel->relminmxid;
BlockNumber empty_pages,
- vacuumed_pages,
+ reuse_marked_pages,
+ has_dead_items_pages,
next_fsm_block_to_vacuum;
double num_tuples, /* total number of nonremovable tuples */
live_tuples, /* live tuples (reltuples estimate) */
@@ -798,7 +791,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
vacrelstats->relnamespace,
vacrelstats->relname)));
- empty_pages = vacuumed_pages = 0;
+ empty_pages = reuse_marked_pages = has_dead_items_pages = 0;
next_fsm_block_to_vacuum = (BlockNumber) 0;
num_tuples = live_tuples = tups_vacuumed = nkeep = nunused = 0;
@@ -810,7 +803,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
vacrelstats->scanned_pages = 0;
vacrelstats->tupcount_pages = 0;
vacrelstats->nonempty_pages = 0;
- vacrelstats->latestRemovedXid = InvalidTransactionId;
vistest = GlobalVisTestFor(onerel);
@@ -819,8 +811,10 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
* be used for an index, so we invoke parallelism only if there are at
* least two indexes on a table.
*/
- if (params->nworkers >= 0 && vacrelstats->useindex && nindexes > 1)
+ if (params->nworkers >= 0 && nindexes > 1)
{
+ Assert(vacrelstats->hasindex);
+
/*
* Since parallel workers cannot access data in temporary tables, we
* can't perform parallel vacuum on them.
@@ -937,8 +931,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
Page page;
OffsetNumber offnum,
maxoff;
- bool tupgone,
- hastup;
+ bool hastup;
int prev_dead_count;
int nfrozen;
Size freespace;
@@ -947,6 +940,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
bool all_frozen = true; /* provided all_visible is also true */
bool has_dead_items; /* includes existing LP_DEAD items */
TransactionId visibility_cutoff_xid = InvalidTransactionId;
+ bool tuple_totally_frozen;
/* see note above about forcing scanning of last page */
#define FORCE_CHECK_PAGE() \
@@ -1051,23 +1045,22 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
vmbuffer = InvalidBuffer;
}
- /* Work on all the indexes, then the heap */
- lazy_vacuum_all_indexes(onerel, Irel, indstats,
- vacrelstats, lps, nindexes);
-
- /* Remove tuples from heap */
- lazy_vacuum_heap(onerel, vacrelstats);
-
/*
- * Forget the now-vacuumed tuples, and press on, but be careful
- * not to reset latestRemovedXid since we want that value to be
- * valid.
+ * Won't be skipping index vacuuming now, since that is only
+ * something vacuum_indexes_mark_unused() does when dead tuple
+ * space hasn't been overrun.
*/
- dead_tuples->num_tuples = 0;
+ vacrelstats->mayskipindexes = false;
+
+ /* Remove the collected garbage tuples from table and indexes */
+ vacuum_indexes_mark_unused(onerel, vacrelstats, Irel, indstats,
+ nindexes, lps, has_dead_items_pages);
/*
* Vacuum the Free Space Map to make newly-freed space visible on
* upper-level FSM pages. Note we have not yet processed blkno.
+ * Even if we skipped heap vacuum, FSM vacuuming could be worthwhile
+ * since we could have updated the freespace of empty pages.
*/
FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno);
next_fsm_block_to_vacuum = blkno;
@@ -1240,7 +1233,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
*/
tups_vacuumed += heap_page_prune(onerel, buf, vistest,
InvalidTransactionId, 0, false,
- &vacrelstats->latestRemovedXid,
&vacrelstats->offnum);
/*
@@ -1310,8 +1302,6 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
tuple.t_len = ItemIdGetLength(itemid);
tuple.t_tableOid = RelationGetRelid(onerel);
- tupgone = false;
-
/*
* The criteria for counting a tuple as live in this block need to
* match what analyze.c's acquire_sample_rows() does, otherwise
@@ -1337,14 +1327,11 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
*
* If the tuple is HOT-updated then it must only be
* removed by a prune operation; so we keep it just as if
- * it were RECENTLY_DEAD. Also, if it's a heap-only
- * tuple, we choose to keep it, because it'll be a lot
- * cheaper to get rid of it in the next pruning pass than
- * to treat it like an indexed tuple. Finally, if index
- * cleanup is disabled, the second heap pass will not
- * execute, and the tuple will not get removed, so we must
- * treat it like any other dead tuple that we choose to
- * keep.
+ * it were RECENTLY_DEAD. Actually we always keep it
+ * because it hardly seems worth introducing a special
+ * case. This allows us to delay committing to index
+ * vacuuming until the last moment -- no need to worry
+ * about making tuple LP_DEAD within mark_unused_page().
*
* If this were to happen for a tuple that actually needed
* to be deleted, we'd be in trouble, because it'd
@@ -1353,12 +1340,7 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
* to detect that case and abort the transaction,
* preventing corruption.
*/
- if (HeapTupleIsHotUpdated(&tuple) ||
- HeapTupleIsHeapOnly(&tuple) ||
- params->index_cleanup == VACOPT_TERNARY_DISABLED)
- nkeep += 1;
- else
- tupgone = true; /* we can delete the tuple */
+ nkeep += 1;
all_visible = false;
break;
case HEAPTUPLE_LIVE:
@@ -1443,35 +1425,22 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
break;
}
- if (tupgone)
- {
- lazy_record_dead_tuple(dead_tuples, &(tuple.t_self));
- HeapTupleHeaderAdvanceLatestRemovedXid(tuple.t_data,
- &vacrelstats->latestRemovedXid);
- tups_vacuumed += 1;
- has_dead_items = true;
- }
- else
- {
- bool tuple_totally_frozen;
+ num_tuples += 1;
+ hastup = true;
- num_tuples += 1;
- hastup = true;
+ /*
+ * Each non-removable tuple must be checked to see if it needs
+ * freezing. Note we already have exclusive buffer lock.
+ */
+ if (heap_prepare_freeze_tuple(tuple.t_data,
+ relfrozenxid, relminmxid,
+ FreezeLimit, MultiXactCutoff,
+ &frozen[nfrozen],
+ &tuple_totally_frozen))
+ frozen[nfrozen++].offset = offnum;
- /*
- * Each non-removable tuple must be checked to see if it needs
- * freezing. Note we already have exclusive buffer lock.
- */
- if (heap_prepare_freeze_tuple(tuple.t_data,
- relfrozenxid, relminmxid,
- FreezeLimit, MultiXactCutoff,
- &frozen[nfrozen],
- &tuple_totally_frozen))
- frozen[nfrozen++].offset = offnum;
-
- if (!tuple_totally_frozen)
- all_frozen = false;
- }
+ if (!tuple_totally_frozen)
+ all_frozen = false;
} /* scan along page */
/*
@@ -1518,38 +1487,22 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
/*
* If there are no indexes we can vacuum the page right now instead of
- * doing a second scan. Also we don't do that but forget dead tuples
- * when index cleanup is disabled.
+ * doing a second scan
*/
- if (!vacrelstats->useindex && dead_tuples->num_tuples > 0)
+ if (nindexes == 0 && dead_tuples->num_tuples > 0)
{
- if (nindexes == 0)
- {
- /* Remove tuples from heap if the table has no index */
- lazy_vacuum_page(onerel, blkno, buf, 0, vacrelstats, &vmbuffer);
- vacuumed_pages++;
- has_dead_items = false;
- }
- else
- {
- /*
- * Here, we have indexes but index cleanup is disabled.
- * Instead of vacuuming the dead tuples on the heap, we just
- * forget them.
- *
- * Note that vacrelstats->dead_tuples could have tuples which
- * became dead after HOT-pruning but are not marked dead yet.
- * We do not process them because it's a very rare condition,
- * and the next vacuum will process them anyway.
- */
- Assert(params->index_cleanup == VACOPT_TERNARY_DISABLED);
- }
+ Assert(!vacrelstats->hasindex);
/*
- * Forget the now-vacuumed tuples, and press on, but be careful
- * not to reset latestRemovedXid since we want that value to be
- * valid.
+ * Mark LP_DEAD item pointers for reuse now, in an incremental
+ * fashion. This is safe because the table has no indexes (and so
+ * vacuum_indexes_mark_unused() will never be called).
*/
+ mark_unused_page(onerel, blkno, buf, 0, vacrelstats, &vmbuffer);
+ reuse_marked_pages++;
+ has_dead_items = false;
+
+ /* Forget the now-vacuumed tuples */
dead_tuples->num_tuples = 0;
/*
@@ -1660,12 +1613,24 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
if (hastup)
vacrelstats->nonempty_pages = blkno + 1;
+ /*
+ * Remember the number of pages having at least one LP_DEAD line
+ * pointer. This could be from this VACUUM, a previous VACUUM, or
+ * even opportunistic pruning.
+ */
+ if (has_dead_items)
+ has_dead_items_pages++;
+
/*
* If we remembered any tuples for deletion, then the page will be
- * visited again by lazy_vacuum_heap, which will compute and record
- * its post-compaction free space. If not, then we're done with this
- * page, so remember its free space as-is. (This path will always be
- * taken if there are no indexes.)
+ * visited again by vacuum_indexes_mark_unused, which will compute and
+ * record its post-compaction free space. If not, then we're done
+ * with this page, so remember its free space as-is.
+ *
+ * This path will always be taken if there are no indexes. However,
+ * it might not be taken if INDEX_CLEANUP is off -- that works the
+ * same as the case where we decide to skip index vacuuming. See also
+ * vacuum_indexes_mark_unused(), where that is decided.
*/
if (dead_tuples->num_tuples == prev_dead_count)
RecordPageWithFreeSpace(onerel, blkno, freespace);
@@ -1706,20 +1671,14 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
}
/* If any tuples need to be deleted, perform final vacuum cycle */
- /* XXX put a threshold on min number of tuples here? */
+ Assert(vacrelstats->hasindex || dead_tuples->num_tuples == 0);
if (dead_tuples->num_tuples > 0)
- {
- /* Work on all the indexes, and then the heap */
- lazy_vacuum_all_indexes(onerel, Irel, indstats, vacrelstats,
- lps, nindexes);
-
- /* Remove tuples from heap */
- lazy_vacuum_heap(onerel, vacrelstats);
- }
+ vacuum_indexes_mark_unused(onerel, vacrelstats, Irel, indstats,
+ nindexes, lps, has_dead_items_pages);
/*
* Vacuum the remainder of the Free Space Map. We must do this whether or
- * not there were indexes.
+ * not there were indexes, and whether or not we skipped index vacuuming.
*/
if (blkno > next_fsm_block_to_vacuum)
FreeSpaceMapVacuumRange(onerel, next_fsm_block_to_vacuum, blkno);
@@ -1727,8 +1686,13 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
/* report all blocks vacuumed */
pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, blkno);
- /* Do post-vacuum cleanup */
- if (vacrelstats->useindex)
+ /*
+ * Do post-vacuum cleanup.
+ *
+ * Note that this take places when vacuum_indexes_mark_unused() decided to
+ * skip index vacuuming.
+ */
+ if (vacrelstats->hasindex)
lazy_cleanup_all_indexes(Irel, indstats, vacrelstats, lps, nindexes);
/*
@@ -1738,16 +1702,27 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
if (ParallelVacuumIsActive(lps))
end_parallel_vacuum(indstats, lps, nindexes);
- /* Update index statistics */
- if (vacrelstats->useindex)
+ /*
+ * Update index statistics.
+ *
+ * Note that this take places when vacuum_indexes_mark_unused() decided to
+ * skip index vacuuming.
+ */
+ if (vacrelstats->hasindex)
update_index_statistics(Irel, indstats, nindexes);
- /* If no indexes, make log report that lazy_vacuum_heap would've made */
- if (vacuumed_pages)
+ /*
+ * If no indexes, make log report that vacuum_indexes_mark_unused would've
+ * made when it skipped vacuuming.
+ *
+ * Note: We're distinguishing between "freed" (i.e. newly made LP_DEAD
+ * through pruning) and removed (i.e. mark_unused_page() marked LP_UNUSED).
+ */
+ if (!vacrelstats->hasindex)
ereport(elevel,
(errmsg("\"%s\": removed %.0f row versions in %u pages",
vacrelstats->relname,
- tups_vacuumed, vacuumed_pages)));
+ tups_vacuumed, reuse_marked_pages)));
initStringInfo(&buf);
appendStringInfo(&buf,
@@ -1779,21 +1754,96 @@ lazy_scan_heap(Relation onerel, VacuumParams *params, LVRelStats *vacrelstats,
}
/*
- * lazy_vacuum_all_indexes() -- vacuum all indexes of relation.
+ * Remove the collected garbage tuples from the table and its indexes.
*
- * We process the indexes serially unless we are doing parallel vacuum.
+ * We may be able to skip index vacuuming (we may even be required to do so by
+ * reloption)
*/
+#define DEBUGELOG LOG
static void
-lazy_vacuum_all_indexes(Relation onerel, Relation *Irel,
- IndexBulkDeleteResult **stats,
- LVRelStats *vacrelstats, LVParallelState *lps,
- int nindexes)
+vacuum_indexes_mark_unused(Relation onerel, LVRelStats *vacrelstats,
+ Relation *Irel, IndexBulkDeleteResult **indstats,
+ int nindexes, LVParallelState *lps,
+ BlockNumber has_dead_items_pages)
{
- Assert(!IsParallelWorker());
- Assert(nindexes > 0);
+ bool skipping = false;
+ int tupindex;
+ int npages;
+ PGRUsage ru0;
+ Buffer vmbuffer = InvalidBuffer;
+ LVSavedErrInfo saved_err_info;
- /* Log cleanup info before we touch indexes */
- vacuum_log_cleanup_info(onerel, vacrelstats);
+ /* Should not end up here with no indexes */
+ Assert(vacrelstats->hasindex);
+ Assert(nindexes > 0);
+ Assert(!IsParallelWorker());
+
+ /* In INDEX_CLEANUP off case we always skip index and heap vacuuming */
+ if (vacrelstats->mustskipindexes)
+ {
+ elog(DEBUGELOG, "must skip index vacuuming of %s", vacrelstats->relname);
+ skipping = true;
+ }
+
+ /*
+ * Check whether or not to do index vacuum and heap vacuum.
+ *
+ * We do both index vacuum and heap vacuum if more than
+ * SKIP_VACUUM_PAGES_RATIO of all heap pages have at least one LP_DEAD
+ * line pointer. This is normally a case where dead tuples on the heap
+ * are highly concentrated in relatively few heap blocks, where the
+ * index's enhanced deletion mechanism that is clever about heap block
+ * dead tuple concentrations including btree's bottom-up index deletion
+ * works well. Also, since we can clean only a few heap blocks, it would
+ * be a less negative impact in terms of visibility map update.
+ *
+ * If we skip vacuum, we just ignore the collected dead tuples. Note that
+ * vacrelstats->dead_tuples could have tuples which became dead after
+ * HOT-pruning but are not marked dead yet. We do not process them because
+ * it's a very rare condition, and the next vacuum will process them anyway.
+ */
+ else if (vacrelstats->mayskipindexes)
+ {
+ BlockNumber rel_pages_threshold;
+
+ rel_pages_threshold =
+ (double) vacrelstats->rel_pages * SKIP_VACUUM_PAGES_RATIO;
+
+ if (has_dead_items_pages < rel_pages_threshold)
+ {
+ elog(DEBUGELOG, "decided to skip index vacuuming of %s with %u LP_DEAD blocks and %u block threshold with %u blocks in total",
+ vacrelstats->relname, has_dead_items_pages, rel_pages_threshold, vacrelstats->rel_pages);
+ skipping = true;
+ }
+ else
+ {
+ elog(DEBUGELOG, "decided not to skip index vacuuming of %s with %u LP_DEAD blocks and %u block threshold with %u blocks in total",
+ vacrelstats->relname, has_dead_items_pages, rel_pages_threshold, vacrelstats->rel_pages);
+ }
+ }
+ else
+ {
+ elog(DEBUGELOG, "never had the choice to skip index vacuuming of %s", vacrelstats->relname);
+ }
+
+ if (skipping)
+ {
+ /*
+ * Note: We're distinguishing between "freed" (i.e. newly made LP_DEAD
+ * through pruning) and removed (i.e. mark_unused_page() marked
+ * LP_UNUSED).
+ */
+ ereport(elevel,
+ (errmsg("\"%s\": freed %d row versions in %u pages",
+ vacrelstats->relname,
+ vacrelstats->dead_tuples->num_tuples,
+ vacrelstats->rel_pages)));
+
+ vacrelstats->dead_tuples->num_tuples = 0;
+ return;
+ }
+
+ /* Okay, we're going to do index vacuuming */
/* Report that we are now vacuuming indexes */
pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
@@ -1813,14 +1863,16 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel,
lps->lvshared->reltuples = vacrelstats->old_live_tuples;
lps->lvshared->estimated_count = true;
- lazy_parallel_vacuum_indexes(Irel, stats, vacrelstats, lps, nindexes);
+ lazy_parallel_vacuum_indexes(Irel, indstats, vacrelstats, lps,
+ nindexes);
}
else
{
int idx;
for (idx = 0; idx < nindexes; idx++)
- lazy_vacuum_index(Irel[idx], &stats[idx], vacrelstats->dead_tuples,
+ lazy_vacuum_index(Irel[idx], &indstats[idx],
+ vacrelstats->dead_tuples,
vacrelstats->old_live_tuples, vacrelstats);
}
@@ -1828,28 +1880,13 @@ lazy_vacuum_all_indexes(Relation onerel, Relation *Irel,
vacrelstats->num_index_scans++;
pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS,
vacrelstats->num_index_scans);
-}
-
-/*
- * lazy_vacuum_heap() -- second pass over the heap
- *
- * This routine marks dead tuples as unused and compacts out free
- * space on their pages. Pages not having dead tuples recorded from
- * lazy_scan_heap are not visited at all.
- *
- * Note: the reason for doing this as a second pass is we cannot remove
- * the tuples until we've removed their index entries, and we want to
- * process index entry removal in batches as large as possible.
- */
-static void
-lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats)
-{
- int tupindex;
- int npages;
- PGRUsage ru0;
- Buffer vmbuffer = InvalidBuffer;
- LVSavedErrInfo saved_err_info;
+ /*
+ * Now mark LP_DEAD line pointers deleted from indexes as unused, and
+ * compact out free space on pages -- this is the second heap pass. Pages
+ * not having dead tuples recorded from lazy_scan_heap are not visited at
+ * all.
+ */
/* Report that we are now vacuuming the heap */
pgstat_progress_update_param(PROGRESS_VACUUM_PHASE,
@@ -1882,7 +1919,7 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats)
++tupindex;
continue;
}
- tupindex = lazy_vacuum_page(onerel, tblk, buf, tupindex, vacrelstats,
+ tupindex = mark_unused_page(onerel, tblk, buf, tupindex, vacrelstats,
&vmbuffer);
/* Now that we've compacted the page, record its available space */
@@ -1911,20 +1948,31 @@ lazy_vacuum_heap(Relation onerel, LVRelStats *vacrelstats)
/* Revert to the previous phase information for error traceback */
restore_vacuum_error_info(vacrelstats, &saved_err_info);
+
+ /* Forget the now-vacuumed tuples */
+ vacrelstats->dead_tuples->num_tuples = 0;
}
/*
- * lazy_vacuum_page() -- free dead tuples on a page
- * and repair its fragmentation.
+ * mark_unused_page() -- mark dead line pointers on page for reuse.
*
* Caller must hold pin and buffer cleanup lock on the buffer.
*
* tupindex is the index in vacrelstats->dead_tuples of the first dead
* tuple for this page. We assume the rest follow sequentially.
* The return value is the first tupindex after the tuples of this page.
+ *
+ * Prior to PostgreSQL 14 there were rare cases where this routine had to set
+ * tuples with storage to unused. These days it is strictly responsible for
+ * marking LP_DEAD stub line pointers from pruning that took place during
+ * lazy_scan_heap() (or from existing LP_DEAD line pointers encountered
+ * there). However, we still share infrastructure with heap pruning, and
+ * still require a super-exclusive lock -- this should now be unnecessary. In
+ * the future we should be able to optimize this -- it can work with only an
+ * exclusive lock.
*/
static int
-lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer,
+mark_unused_page(Relation onerel, BlockNumber blkno, Buffer buffer,
int tupindex, LVRelStats *vacrelstats, Buffer *vmbuffer)
{
LVDeadTuples *dead_tuples = vacrelstats->dead_tuples;
@@ -1954,6 +2002,8 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer,
break; /* past end of tuples for this block */
toff = ItemPointerGetOffsetNumber(&dead_tuples->itemptrs[tupindex]);
itemid = PageGetItemId(page, toff);
+
+ Assert(ItemIdIsDead(itemid));
ItemIdSetUnused(itemid);
unused[uncnt++] = toff;
}
@@ -1973,7 +2023,7 @@ lazy_vacuum_page(Relation onerel, BlockNumber blkno, Buffer buffer,
recptr = log_heap_clean(onerel, buffer,
NULL, 0, NULL, 0,
unused, uncnt,
- vacrelstats->latestRemovedXid);
+ InvalidTransactionId);
PageSetLSN(page, recptr);
}
@@ -2849,14 +2899,14 @@ count_nondeletable_pages(Relation onerel, LVRelStats *vacrelstats)
* Return the maximum number of dead tuples we can record.
*/
static long
-compute_max_dead_tuples(BlockNumber relblocks, bool useindex)
+compute_max_dead_tuples(BlockNumber relblocks, bool hasindex)
{
long maxtuples;
int vac_work_mem = IsAutoVacuumWorkerProcess() &&
autovacuum_work_mem != -1 ?
autovacuum_work_mem : maintenance_work_mem;
- if (useindex)
+ if (hasindex)
{
maxtuples = MAXDEADTUPLES(vac_work_mem * 1024L);
maxtuples = Min(maxtuples, INT_MAX);
@@ -2886,7 +2936,7 @@ lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks)
LVDeadTuples *dead_tuples = NULL;
long maxtuples;
- maxtuples = compute_max_dead_tuples(relblocks, vacrelstats->useindex);
+ maxtuples = compute_max_dead_tuples(relblocks, vacrelstats->hasindex);
dead_tuples = (LVDeadTuples *) palloc(SizeOfDeadTuples(maxtuples));
dead_tuples->num_tuples = 0;
diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c
index c02c4e7710..1810a2e6aa 100644
--- a/src/backend/access/nbtree/nbtree.c
+++ b/src/backend/access/nbtree/nbtree.c
@@ -1204,9 +1204,9 @@ backtrack:
* index tuple refers to pre-cutoff heap tuples that were
* certainly already pruned away during VACUUM's initial heap
* scan by the time we get here. (heapam's XLOG_HEAP2_CLEAN
- * and XLOG_HEAP2_CLEANUP_INFO records produce conflicts using
- * a latestRemovedXid value for the pointed-to heap tuples, so
- * there is no need to produce our own conflict now.)
+ * records produce conflicts using a latestRemovedXid value
+ * for the pointed-to heap tuples, so there is no need to
+ * produce our own conflict now.)
*
* Backends with snapshots acquired after a VACUUM starts but
* before it finishes could have visibility cutoff with a
diff --git a/src/backend/access/rmgrdesc/heapdesc.c b/src/backend/access/rmgrdesc/heapdesc.c
index e60e32b935..1018ed78be 100644
--- a/src/backend/access/rmgrdesc/heapdesc.c
+++ b/src/backend/access/rmgrdesc/heapdesc.c
@@ -134,12 +134,6 @@ heap2_desc(StringInfo buf, XLogReaderState *record)
appendStringInfo(buf, "cutoff xid %u ntuples %u",
xlrec->cutoff_xid, xlrec->ntuples);
}
- else if (info == XLOG_HEAP2_CLEANUP_INFO)
- {
- xl_heap_cleanup_info *xlrec = (xl_heap_cleanup_info *) rec;
-
- appendStringInfo(buf, "latestRemovedXid %u", xlrec->latestRemovedXid);
- }
else if (info == XLOG_HEAP2_VISIBLE)
{
xl_heap_visible *xlrec = (xl_heap_visible *) rec;
@@ -235,9 +229,6 @@ heap2_identify(uint8 info)
case XLOG_HEAP2_FREEZE_PAGE:
id = "FREEZE_PAGE";
break;
- case XLOG_HEAP2_CLEANUP_INFO:
- id = "CLEANUP_INFO";
- break;
case XLOG_HEAP2_VISIBLE:
id = "VISIBLE";
break;
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index c064352e23..6ab6d7a431 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -1880,11 +1880,18 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params)
onerelid = onerel->rd_lockInfo.lockRelId;
LockRelationIdForSession(&onerelid, lmode);
- /* Set index cleanup option based on reloptions if not yet */
- if (params->index_cleanup == VACOPT_TERNARY_DEFAULT)
+ /*
+ * Set index cleanup option based on reloptions if not yet, though only if
+ * set -- we want VACOPT_TERNARY_DEFAULT to mean "decide dynamically in
+ * vacuumlazy.c".
+ *
+ * FIXME: This doesn't work when some other reloption is set -- what we
+ * need is a new default, 'auto' or similar.
+ */
+ if (params->index_cleanup == VACOPT_TERNARY_DEFAULT &&
+ onerel->rd_options != NULL)
{
- if (onerel->rd_options == NULL ||
- ((StdRdOptions *) onerel->rd_options)->vacuum_index_cleanup)
+ if (((StdRdOptions *) onerel->rd_options)->vacuum_index_cleanup)
params->index_cleanup = VACOPT_TERNARY_ENABLED;
else
params->index_cleanup = VACOPT_TERNARY_DISABLED;
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 5f596135b1..11fcd861f7 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -481,7 +481,6 @@ DecodeHeap2Op(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
*/
case XLOG_HEAP2_FREEZE_PAGE:
case XLOG_HEAP2_CLEAN:
- case XLOG_HEAP2_CLEANUP_INFO:
case XLOG_HEAP2_VISIBLE:
case XLOG_HEAP2_LOCK_UPDATED:
break;
base-commit: 0ea71c93a06ddc38e0b72e48f1d512e5383a9c1b
--
2.27.0