From 5b9d1b8cfd11b8f7c8defc718c0e670375e1eed2 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Wed, 9 Oct 2024 14:54:24 -0400 Subject: [PATCH v24 09/12] Push Bitmap Table Scan prefetch code into heap AM In order to completely remove the layering violation in bitmap table scan code, we must avoid using the VM for skipping prefetches as well. To accomplish this, push prefetch code down into the heap implementation fo the bitmap table scan AM functions. This fixes another layering violation related to prefetching mentioned in tableam.h. This commit moves prefetch-related members from the BitmapHeapScanState to the BitmapHeapScanDesc and localizes the prefetch functions to heap AM code. heapam_scan_bitmap_next_block() no longer needs to take blockno as a parameter because it was only being used in generic bitmap table scan code to check if the prefetch block was falling behind the current block. --- src/backend/access/heap/heapam.c | 25 +++- src/backend/access/heap/heapam_handler.c | 136 +++++++++++++++------- src/backend/access/table/tableam.c | 4 +- src/backend/executor/nodeBitmapHeapscan.c | 130 ++++++--------------- src/include/access/heapam.h | 21 ++-- src/include/access/relscan.h | 9 +- src/include/access/tableam.h | 63 +++++----- src/include/nodes/execnodes.h | 10 -- 8 files changed, 201 insertions(+), 197 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 3452634040..4ea8933c44 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1029,7 +1029,8 @@ TableScanDesc heap_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, - uint32 flags) + uint32 flags, + int prefetch_maximum) { HeapScanDesc scan; @@ -1049,8 +1050,17 @@ heap_beginscan(Relation relation, Snapshot snapshot, { BitmapHeapScanDesc *bscan = palloc(sizeof(BitmapHeapScanDesc)); + bscan->rs_pfblockno = InvalidBlockNumber; bscan->rs_vmbuffer = InvalidBuffer; + bscan->rs_pvmbuffer = InvalidBuffer; bscan->rs_empty_tuples_pending = 0; + + /* Only used for serial BHS */ + bscan->rs_prefetch_pages = 0; + bscan->rs_prefetch_target = -1; + + bscan->rs_prefetch_maximum = prefetch_maximum; + scan = &bscan->rs_heap_base; } else @@ -1181,6 +1191,12 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, { BitmapHeapScanDesc *bscan = (BitmapHeapScanDesc *) scan; + bscan->rs_pfblockno = InvalidBlockNumber; + + /* Only used for serial BHS */ + bscan->rs_prefetch_pages = 0; + bscan->rs_prefetch_target = -1; + /* * Reset empty_tuples_pending, a field only used by bitmap heap scan, * to avoid incorrectly emitting NULL-filled tuples from a previous @@ -1193,6 +1209,11 @@ heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params, ReleaseBuffer(bscan->rs_vmbuffer); bscan->rs_vmbuffer = InvalidBuffer; } + if (BufferIsValid(bscan->rs_pvmbuffer)) + { + ReleaseBuffer(bscan->rs_pvmbuffer); + bscan->rs_pvmbuffer = InvalidBuffer; + } } /* @@ -1252,6 +1273,8 @@ heap_endscan(TableScanDesc sscan) bscan->rs_empty_tuples_pending = 0; if (BufferIsValid(bscan->rs_vmbuffer)) ReleaseBuffer(bscan->rs_vmbuffer); + if (BufferIsValid(bscan->rs_pvmbuffer)) + ReleaseBuffer(bscan->rs_pvmbuffer); } pfree(scan); diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 04f8adc71e..78193b1248 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -54,6 +54,10 @@ static bool SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, HeapTuple tuple, OffsetNumber tupoffset); +static void BitmapAdjustPrefetchIterator(BitmapHeapScanDesc *bscan); +static void BitmapAdjustPrefetchTarget(BitmapHeapScanDesc *bscan); +static void BitmapPrefetch(BitmapHeapScanDesc *bscan); + static BlockNumber heapam_scan_get_blocks_done(HeapScanDesc hscan); @@ -2116,7 +2120,7 @@ heapam_estimate_rel_size(Relation rel, int32 *attr_widths, static bool heapam_scan_bitmap_next_block(TableScanDesc scan, - BlockNumber *blockno, bool *recheck, + bool *recheck, uint64 *lossy_pages, uint64 *exact_pages) { BitmapHeapScanDesc *bscan = (BitmapHeapScanDesc *) scan; @@ -2130,14 +2134,15 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, hscan->rs_cindex = 0; hscan->rs_ntuples = 0; - *blockno = InvalidBlockNumber; *recheck = true; + BitmapAdjustPrefetchIterator(bscan); + do { CHECK_FOR_INTERRUPTS(); - tbmres = tbm_iterate(&scan->st.tbmiterator); + tbmres = tbm_iterate(&scan->st.bts.tbmiterator); if (tbmres == NULL) return false; @@ -2153,7 +2158,7 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, } while (!IsolationIsSerializable() && tbmres->blockno >= hscan->rs_nblocks); /* Got a valid block */ - *blockno = tbmres->blockno; + block = tbmres->blockno; *recheck = tbmres->recheck; /* @@ -2174,8 +2179,6 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, return true; } - block = tbmres->blockno; - /* * Acquire pin on the target heap page, trading in any pin we held before. */ @@ -2269,6 +2272,18 @@ heapam_scan_bitmap_next_block(TableScanDesc scan, else (*lossy_pages)++; + /* + * If private, we can error out if the the prefetch block doesn't stay + * ahead of the current block. + */ + if (scan->st.bts.pstate == NULL && + !tbm_exhausted(&scan->st.bts.prefetch_iterator) && + bscan->rs_pfblockno < block) + elog(ERROR, "prefetch and main iterators are out of sync. :fblockno: %d. block: %d", bscan->rs_pfblockno, block); + + /* Adjust the prefetch target */ + BitmapAdjustPrefetchTarget(bscan); + /* * Return true to indicate that a valid block was found and the bitmap is * not exhausted. If there are no visible tuples on this page, @@ -2288,6 +2303,9 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, OffsetNumber targoffset; Page page; ItemId lp; +#ifdef USE_PREFETCH + ParallelBitmapHeapState *pstate = scan->st.bts.pstate; +#endif /* USE_PREFETCH */ if (bscan->rs_empty_tuples_pending > 0) { @@ -2296,6 +2314,7 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, */ ExecStoreAllNullTuple(slot); bscan->rs_empty_tuples_pending--; + BitmapPrefetch(bscan); return true; } @@ -2305,6 +2324,36 @@ heapam_scan_bitmap_next_tuple(TableScanDesc scan, if (hscan->rs_cindex < 0 || hscan->rs_cindex >= hscan->rs_ntuples) return false; +#ifdef USE_PREFETCH + + /* + * Try to prefetch at least a few pages even before we get to the second + * page if we don't stop reading after the first tuple. + */ + if (!pstate) + { + if (bscan->rs_prefetch_target < bscan->rs_prefetch_maximum) + bscan->rs_prefetch_target++; + } + else if (pstate->prefetch_target < bscan->rs_prefetch_maximum) + { + /* take spinlock while updating shared state */ + SpinLockAcquire(&pstate->mutex); + if (pstate->prefetch_target < bscan->rs_prefetch_maximum) + pstate->prefetch_target++; + SpinLockRelease(&pstate->mutex); + } +#endif /* USE_PREFETCH */ + + /* + * We issue prefetch requests *after* fetching the current page to try to + * avoid having prefetching interfere with the main I/O. Also, this should + * happen only when we have determined there is still something to do on + * the current page, else we may uselessly prefetch the same page we are + * just about to request for real. + */ + BitmapPrefetch(bscan); + targoffset = hscan->rs_vistuples[hscan->rs_cindex]; page = BufferGetPage(hscan->rs_cbuf); lp = PageGetItemId(page, targoffset); @@ -2618,27 +2667,28 @@ SampleHeapTupleVisible(TableScanDesc scan, Buffer buffer, * iterator in prefetch_pages. For each block the main iterator returns, we * decrement prefetch_pages. */ -void -BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) +static void +BitmapAdjustPrefetchIterator(BitmapHeapScanDesc *bscan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; + TableScanDesc scan = &(&bscan->rs_heap_base)->rs_base; + ParallelBitmapHeapState *pstate = scan->st.bts.pstate; TBMIterateResult *tbmpre; if (pstate == NULL) { - TBMIterator *prefetch_iterator = &node->prefetch_iterator; + TBMIterator *prefetch_iterator = &scan->st.bts.prefetch_iterator; - if (node->prefetch_pages > 0) + if (bscan->rs_prefetch_pages > 0) { /* The main iterator has closed the distance by one page */ - node->prefetch_pages--; + bscan->rs_prefetch_pages--; } else if (!tbm_exhausted(prefetch_iterator)) { /* Do not let the prefetch iterator get behind the main one */ tbmpre = tbm_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + bscan->rs_pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } return; } @@ -2652,9 +2702,9 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * Note that moving the call site of BitmapAdjustPrefetchIterator() * exacerbates the effects of this bug. */ - if (node->prefetch_maximum > 0) + if (bscan->rs_prefetch_maximum > 0) { - TBMIterator *prefetch_iterator = &node->prefetch_iterator; + TBMIterator *prefetch_iterator = &scan->st.bts.prefetch_iterator; SpinLockAcquire(&pstate->mutex); if (pstate->prefetch_pages > 0) @@ -2678,7 +2728,7 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) if (!tbm_exhausted(prefetch_iterator)) { tbmpre = tbm_iterate(prefetch_iterator); - node->pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; + bscan->rs_pfblockno = tbmpre ? tbmpre->blockno : InvalidBlockNumber; } } } @@ -2693,33 +2743,34 @@ BitmapAdjustPrefetchIterator(BitmapHeapScanState *node) * page/tuple, then to one after the second tuple is fetched, then * it doubles as later pages are fetched. */ -void -BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) +static void +BitmapAdjustPrefetchTarget(BitmapHeapScanDesc *bscan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; + TableScanDesc scan = &(&bscan->rs_heap_base)->rs_base; + ParallelBitmapHeapState *pstate = scan->st.bts.pstate; if (pstate == NULL) { - if (node->prefetch_target >= node->prefetch_maximum) + if (bscan->rs_prefetch_target >= bscan->rs_prefetch_maximum) /* don't increase any further */ ; - else if (node->prefetch_target >= node->prefetch_maximum / 2) - node->prefetch_target = node->prefetch_maximum; - else if (node->prefetch_target > 0) - node->prefetch_target *= 2; + else if (bscan->rs_prefetch_target >= bscan->rs_prefetch_maximum / 2) + bscan->rs_prefetch_target = bscan->rs_prefetch_maximum; + else if (bscan->rs_prefetch_target > 0) + bscan->rs_prefetch_target *= 2; else - node->prefetch_target++; + bscan->rs_prefetch_target++; return; } /* Do an unlocked check first to save spinlock acquisitions. */ - if (pstate->prefetch_target < node->prefetch_maximum) + if (pstate->prefetch_target < bscan->rs_prefetch_maximum) { SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target >= node->prefetch_maximum) + if (pstate->prefetch_target >= bscan->rs_prefetch_maximum) /* don't increase any further */ ; - else if (pstate->prefetch_target >= node->prefetch_maximum / 2) - pstate->prefetch_target = node->prefetch_maximum; + else if (pstate->prefetch_target >= bscan->rs_prefetch_maximum / 2) + pstate->prefetch_target = bscan->rs_prefetch_maximum; else if (pstate->prefetch_target > 0) pstate->prefetch_target *= 2; else @@ -2732,19 +2783,20 @@ BitmapAdjustPrefetchTarget(BitmapHeapScanState *node) /* * BitmapPrefetch - Prefetch, if prefetch_pages are behind prefetch_target */ -void -BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) +static void +BitmapPrefetch(BitmapHeapScanDesc *bscan) { #ifdef USE_PREFETCH - ParallelBitmapHeapState *pstate = node->pstate; + TableScanDesc scan = &(&bscan->rs_heap_base)->rs_base; + ParallelBitmapHeapState *pstate = scan->st.bts.pstate; if (pstate == NULL) { - TBMIterator *prefetch_iterator = &node->prefetch_iterator; + TBMIterator *prefetch_iterator = &scan->st.bts.prefetch_iterator; if (!tbm_exhausted(prefetch_iterator)) { - while (node->prefetch_pages < node->prefetch_target) + while (bscan->rs_prefetch_pages < bscan->rs_prefetch_target) { TBMIterateResult *tbmpre = tbm_iterate(prefetch_iterator); bool skip_fetch; @@ -2755,8 +2807,8 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) tbm_end_iterate(prefetch_iterator); break; } - node->prefetch_pages++; - node->pfblockno = tbmpre->blockno; + bscan->rs_prefetch_pages++; + bscan->rs_pfblockno = tbmpre->blockno; /* * If we expect not to have to actually read this heap page, @@ -2766,9 +2818,9 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) */ skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLES) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(scan->rs_rd, tbmpre->blockno, - &node->pvmbuffer)); + &bscan->rs_pvmbuffer)); if (!skip_fetch) PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); @@ -2780,7 +2832,7 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) if (pstate->prefetch_pages < pstate->prefetch_target) { - TBMIterator *prefetch_iterator = &node->prefetch_iterator; + TBMIterator *prefetch_iterator = &scan->st.bts.prefetch_iterator; if (!tbm_exhausted(prefetch_iterator)) { @@ -2813,14 +2865,14 @@ BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan) break; } - node->pfblockno = tbmpre->blockno; + bscan->rs_pfblockno = tbmpre->blockno; /* As above, skip prefetch if we expect not to need page */ skip_fetch = (!(scan->rs_flags & SO_NEED_TUPLES) && !tbmpre->recheck && - VM_ALL_VISIBLE(node->ss.ss_currentRelation, + VM_ALL_VISIBLE(scan->rs_rd, tbmpre->blockno, - &node->pvmbuffer)); + &bscan->rs_pvmbuffer)); if (!skip_fetch) PrefetchBuffer(scan->rs_rd, MAIN_FORKNUM, tbmpre->blockno); diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index bd8715b679..3dafcea964 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -117,7 +117,7 @@ table_beginscan_catalog(Relation relation, int nkeys, struct ScanKeyData *key) Snapshot snapshot = RegisterSnapshot(GetCatalogSnapshot(relid)); return relation->rd_tableam->scan_begin(relation, snapshot, nkeys, key, - NULL, flags); + NULL, flags, 0); } @@ -184,7 +184,7 @@ table_beginscan_parallel(Relation relation, ParallelTableScanDesc pscan) } return relation->rd_tableam->scan_begin(relation, snapshot, 0, NULL, - pscan, flags); + pscan, flags, 0); } diff --git a/src/backend/executor/nodeBitmapHeapscan.c b/src/backend/executor/nodeBitmapHeapscan.c index 4eae377d91..21ceff5310 100644 --- a/src/backend/executor/nodeBitmapHeapscan.c +++ b/src/backend/executor/nodeBitmapHeapscan.c @@ -37,8 +37,6 @@ #include -/* XXX: temporary include so prefetch lift and shift works */ -#include "access/heapam.h" #include "access/relscan.h" #include "access/tableam.h" #include "access/visibilitymap.h" @@ -95,6 +93,17 @@ BitmapHeapNext(BitmapHeapScanState *node) if (!node->initialized) { TBMIterator tbmiterator; + int prefetch_maximum; + Relation rel = node->ss.ss_currentRelation; +#ifdef USE_PREFETCH + TBMIterator prefetch_iterator = {0}; +#endif /* USE_PREFETCH */ + + /* + * Maximum number of prefetches for the tablespace if configured, + * otherwise the current value of the effective_io_concurrency GUC. + */ + prefetch_maximum = get_tablespace_io_concurrency(rel->rd_rel->reltablespace); if (!pstate) { @@ -104,14 +113,6 @@ BitmapHeapNext(BitmapHeapScanState *node) elog(ERROR, "unrecognized result from subplan"); node->tbm = tbm; - -#ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - node->prefetch_iterator = tbm_begin_iterate(node->tbm, dsa, - pstate ? - pstate->prefetch_iterator : - InvalidDsaPointer); -#endif /* USE_PREFETCH */ } else if (BitmapShouldInitializeSharedState(pstate)) { @@ -134,7 +135,7 @@ BitmapHeapNext(BitmapHeapScanState *node) pstate->tbmiterator = tbm_prepare_shared_iterate(tbm); #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) + if (prefetch_maximum > 0) { pstate->prefetch_iterator = tbm_prepare_shared_iterate(tbm); @@ -149,13 +150,12 @@ BitmapHeapNext(BitmapHeapScanState *node) pstate ? pstate->tbmiterator : InvalidDsaPointer); - #ifdef USE_PREFETCH - if (node->prefetch_maximum > 0) - node->prefetch_iterator = tbm_begin_iterate(tbm, dsa, - pstate ? - pstate->prefetch_iterator : - InvalidDsaPointer); + if (prefetch_maximum > 0) + prefetch_iterator = tbm_begin_iterate(tbm, dsa, + pstate ? + pstate->prefetch_iterator : + InvalidDsaPointer); #endif /* USE_PREFETCH */ /* @@ -179,14 +179,17 @@ BitmapHeapNext(BitmapHeapScanState *node) scan = table_beginscan_bm(node->ss.ss_currentRelation, node->ss.ps.state->es_snapshot, + pstate, 0, NULL, - need_tuples); + need_tuples, + prefetch_maximum); node->ss.ss_currentScanDesc = scan; } - scan->st.tbmiterator = tbmiterator; + scan->st.bts.tbmiterator = tbmiterator; + scan->st.bts.prefetch_iterator = prefetch_iterator; node->initialized = true; goto new_page; @@ -202,37 +205,6 @@ BitmapHeapNext(BitmapHeapScanState *node) CHECK_FOR_INTERRUPTS(); -#ifdef USE_PREFETCH - - /* - * Try to prefetch at least a few pages even before we get to the - * second page if we don't stop reading after the first tuple. - */ - if (!pstate) - { - if (node->prefetch_target < node->prefetch_maximum) - node->prefetch_target++; - } - else if (pstate->prefetch_target < node->prefetch_maximum) - { - /* take spinlock while updating shared state */ - SpinLockAcquire(&pstate->mutex); - if (pstate->prefetch_target < node->prefetch_maximum) - pstate->prefetch_target++; - SpinLockRelease(&pstate->mutex); - } -#endif /* USE_PREFETCH */ - - /* - * We issue prefetch requests *after* fetching the current page to - * try to avoid having prefetching interfere with the main I/O. - * Also, this should happen only when we have determined there is - * still something to do on the current page, else we may - * uselessly prefetch the same page we are just about to request - * for real. - */ - BitmapPrefetch(node, scan); - /* * If we are using lossy info, we have to recheck the qual * conditions at every tuple. @@ -255,27 +227,13 @@ BitmapHeapNext(BitmapHeapScanState *node) new_page: - BitmapAdjustPrefetchIterator(node); - /* * Returns false if the bitmap is exhausted and there are no further * blocks we need to scan. */ - if (!table_scan_bitmap_next_block(scan, &node->blockno, &node->recheck, + if (!table_scan_bitmap_next_block(scan, &node->recheck, &node->stats.lossy_pages, &node->stats.exact_pages)) break; - - /* - * If private, we can error out if the the prefetch block doesn't stay - * ahead of the current block. - */ - if (node->pstate == NULL && - !tbm_exhausted(&node->prefetch_iterator) && - node->pfblockno < node->blockno) - elog(ERROR, "prefetch and main iterators are out of sync. pfblockno: %d. blockno: %d", node->pfblockno, node->blockno); - - /* Adjust the prefetch target */ - BitmapAdjustPrefetchTarget(node); } /* @@ -347,30 +305,22 @@ ExecReScanBitmapHeapScan(BitmapHeapScanState *node) /* * End iteration on iterators saved in scan descriptor. */ - tbm_end_iterate(&scan->st.tbmiterator); + tbm_end_iterate(&scan->st.bts.tbmiterator); + + /* If we did not already clean up the prefetch iterator, do so now. */ + if (!tbm_exhausted(&scan->st.bts.prefetch_iterator)) + tbm_end_iterate(&scan->st.bts.prefetch_iterator); /* rescan to release any page pin */ table_rescan(node->ss.ss_currentScanDesc, NULL); } - /* If we did not already clean up the prefetch iterator, do so now. */ - if (!tbm_exhausted(&node->prefetch_iterator)) - tbm_end_iterate(&node->prefetch_iterator); - /* release bitmaps and buffers if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); node->tbm = NULL; node->initialized = false; - node->pvmbuffer = InvalidBuffer; node->recheck = true; - node->blockno = InvalidBlockNumber; - node->pfblockno = InvalidBlockNumber; - /* Only used for serial BHS */ - node->prefetch_pages = 0; - node->prefetch_target = -1; ExecScanReScan(&node->ss); @@ -429,7 +379,11 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) /* * End iteration on iterators saved in scan descriptor. */ - tbm_end_iterate(&scanDesc->st.tbmiterator); + tbm_end_iterate(&scanDesc->st.bts.tbmiterator); + + /* If we did not already clean up the prefetch iterator, do so now. */ + if (!tbm_exhausted(&scanDesc->st.bts.prefetch_iterator)) + tbm_end_iterate(&scanDesc->st.bts.prefetch_iterator); /* * close table scan @@ -437,17 +391,11 @@ ExecEndBitmapHeapScan(BitmapHeapScanState *node) table_endscan(scanDesc); } - /* If we did not already clean up the prefetch iterator, do so now. */ - if (!tbm_exhausted(&node->prefetch_iterator)) - tbm_end_iterate(&node->prefetch_iterator); - /* * release bitmaps and buffers if any */ if (node->tbm) tbm_free(node->tbm); - if (node->pvmbuffer != InvalidBuffer) - ReleaseBuffer(node->pvmbuffer); } /* ---------------------------------------------------------------- @@ -480,18 +428,13 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->ss.ps.ExecProcNode = ExecBitmapHeapScan; scanstate->tbm = NULL; - scanstate->pvmbuffer = InvalidBuffer; /* Zero the statistics counters */ memset(&scanstate->stats, 0, sizeof(BitmapHeapScanInstrumentation)); - scanstate->prefetch_pages = 0; - scanstate->prefetch_target = -1; scanstate->initialized = false; scanstate->pstate = NULL; scanstate->recheck = true; - scanstate->blockno = InvalidBlockNumber; - scanstate->pfblockno = InvalidBlockNumber; /* * Miscellaneous initialization @@ -531,13 +474,6 @@ ExecInitBitmapHeapScan(BitmapHeapScan *node, EState *estate, int eflags) scanstate->bitmapqualorig = ExecInitQual(node->bitmapqualorig, (PlanState *) scanstate); - /* - * Maximum number of prefetches for the tablespace if configured, - * otherwise the current value of the effective_io_concurrency GUC. - */ - scanstate->prefetch_maximum = - get_tablespace_io_concurrency(currentRelation->rd_rel->reltablespace); - scanstate->ss.ss_currentRelation = currentRelation; /* diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index d5f8cd5eef..0ddef490c7 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -22,8 +22,6 @@ #include "access/table.h" /* for backward compatibility */ #include "access/tableam.h" #include "nodes/lockoptions.h" -/* XXX: temporary include so prefetch lift and shift works */ -#include "nodes/execnodes.h" #include "nodes/primnodes.h" #include "storage/bufpage.h" #include "storage/dsm.h" @@ -105,6 +103,15 @@ typedef struct BitmapHeapScanDesc { HeapScanDescData rs_heap_base; + /* used to validate pf stays ahead of current block */ + BlockNumber rs_pfblockno; + /* maximum value for prefetch_target */ + int rs_prefetch_maximum; + /* Current target for prefetch distance */ + int rs_prefetch_target; + /* # pages prefetch iterator is ahead of current */ + int rs_prefetch_pages; + /* * These fields are only used for bitmap scans for the "skip fetch" * optimization. Bitmap scans needing no fields from the heap may skip @@ -115,6 +122,8 @@ typedef struct BitmapHeapScanDesc /* page of VM containing info for current block */ Buffer rs_vmbuffer; + /* page of VM containing info for prefetch block */ + Buffer rs_pvmbuffer; int rs_empty_tuples_pending; } BitmapHeapScanDesc; @@ -299,7 +308,8 @@ typedef enum extern TableScanDesc heap_beginscan(Relation relation, Snapshot snapshot, int nkeys, ScanKey key, ParallelTableScanDesc parallel_scan, - uint32 flags); + uint32 flags, + int prefetch_maximum); extern void heap_setscanlimits(TableScanDesc sscan, BlockNumber startBlk, BlockNumber numBlks); extern void heap_prepare_pagescan(TableScanDesc sscan); @@ -425,11 +435,6 @@ extern bool HeapTupleHeaderIsOnlyLocked(HeapTupleHeader tuple); extern bool HeapTupleIsSurelyDead(HeapTuple htup, struct GlobalVisState *vistest); -/* in heapam_handler.c */ -extern void BitmapAdjustPrefetchIterator(BitmapHeapScanState *node); -extern void BitmapAdjustPrefetchTarget(BitmapHeapScanState *node); -extern void BitmapPrefetch(BitmapHeapScanState *node, TableScanDesc scan); - /* * To avoid leaking too much knowledge about reorderbuffer implementation * details this is implemented in reorderbuffer.c not heapam_visibility.c diff --git a/src/include/access/relscan.h b/src/include/access/relscan.h index e014abcdc7..f02176692d 100644 --- a/src/include/access/relscan.h +++ b/src/include/access/relscan.h @@ -43,8 +43,13 @@ typedef struct TableScanDescData */ union { - /* Iterator for Bitmap Table Scans */ - TBMIterator tbmiterator; + /* State for Bitmap Table Scans */ + struct + { + struct ParallelBitmapHeapState *pstate; + TBMIterator tbmiterator; + TBMIterator prefetch_iterator; + } bts; /* * Range of ItemPointers for table_scan_getnextslot_tidrange() to diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index b0f1a79623..0b4cacb631 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -324,12 +324,16 @@ typedef struct TableAmRoutine * the scan's behaviour (ScanOptions's SO_ALLOW_*, several may be * specified, an AM may ignore unsupported ones) and whether the snapshot * needs to be deallocated at scan_end (ScanOptions's SO_TEMP_SNAPSHOT). + * + * `prefetch_maximum` is the maximum prefetch distance for use by bitmap + * table scans as needed. */ TableScanDesc (*scan_begin) (Relation rel, Snapshot snapshot, int nkeys, struct ScanKeyData *key, ParallelTableScanDesc pscan, - uint32 flags); + uint32 flags, + int prefetch_maximum); /* * Release resources and deallocate scan. If TableScanDesc.temp_snap, @@ -780,9 +784,10 @@ typedef struct TableAmRoutine */ /* - * Prepare to fetch / check / return tuples from `blockno` as part of a - * bitmap table scan. `scan` was started via table_beginscan_bm(). Return - * false if the bitmap is exhausted and true otherwise. + * Prepare to fetch / check / return tuples from the next block yielded by + * the iterator as part of a bitmap table scan. `scan` was started via + * table_beginscan_bm(). Return false if the bitmap is exhausted and true + * otherwise. * * This will typically read and pin the target block, and do the necessary * work to allow scan_bitmap_next_tuple() to return tuples (e.g. it might @@ -797,28 +802,14 @@ typedef struct TableAmRoutine * always need to be rechecked, but some non-lossy pages' tuples may also * require recheck. * - * `blockno` is the current block and is set by the table AM. The table AM - * is responsible for advancing the main iterator, but the bitmap table - * scan code still advances the prefetch iterator. `blockno` is used by - * bitmap table scan code to validate that the prefetch block stays ahead - * of the current block. - * - * XXX: Currently this may only be implemented if the AM uses md.c as its - * storage manager, and uses ItemPointer->ip_blkid in a manner that maps - * blockids directly to the underlying storage. nodeBitmapHeapscan.c - * performs prefetching directly using that interface. This probably - * needs to be rectified at a later point. - * - * XXX: Currently this may only be implemented if the AM uses the - * visibilitymap, as nodeBitmapHeapscan.c unconditionally accesses it to - * perform prefetching. This probably needs to be rectified at a later - * point. + * Prefetching future blocks indicated in the bitmap is left to the table + * AM. * * Optional callback, but either both scan_bitmap_next_block and * scan_bitmap_next_tuple need to exist, or neither. */ bool (*scan_bitmap_next_block) (TableScanDesc scan, - BlockNumber *blockno, bool *recheck, + bool *recheck, uint64 *lossy_pages, uint64 *exact_pages); /* @@ -914,7 +905,7 @@ table_beginscan(Relation rel, Snapshot snapshot, uint32 flags = SO_TYPE_SEQSCAN | SO_ALLOW_STRAT | SO_ALLOW_SYNC | SO_ALLOW_PAGEMODE; - return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); + return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags, 0); } /* @@ -943,7 +934,7 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, if (allow_sync) flags |= SO_ALLOW_SYNC; - return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); + return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags, 0); } /* @@ -954,14 +945,20 @@ table_beginscan_strat(Relation rel, Snapshot snapshot, */ static inline TableScanDesc table_beginscan_bm(Relation rel, Snapshot snapshot, - int nkeys, struct ScanKeyData *key, bool need_tuple) + struct ParallelBitmapHeapState *pstate, + int nkeys, struct ScanKeyData *key, bool need_tuple, + int prefetch_maximum) { + TableScanDesc result; uint32 flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE; if (need_tuple) flags |= SO_NEED_TUPLES; - return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); + result = rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags, + prefetch_maximum); + result->st.bts.pstate = pstate; + return result; } /* @@ -986,7 +983,7 @@ table_beginscan_sampling(Relation rel, Snapshot snapshot, if (allow_pagemode) flags |= SO_ALLOW_PAGEMODE; - return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags); + return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags, 0); } /* @@ -999,7 +996,7 @@ table_beginscan_tid(Relation rel, Snapshot snapshot) { uint32 flags = SO_TYPE_TIDSCAN; - return rel->rd_tableam->scan_begin(rel, snapshot, 0, NULL, NULL, flags); + return rel->rd_tableam->scan_begin(rel, snapshot, 0, NULL, NULL, flags, 0); } /* @@ -1012,7 +1009,7 @@ table_beginscan_analyze(Relation rel) { uint32 flags = SO_TYPE_ANALYZE; - return rel->rd_tableam->scan_begin(rel, NULL, 0, NULL, NULL, flags); + return rel->rd_tableam->scan_begin(rel, NULL, 0, NULL, NULL, flags, 0); } /* @@ -1091,7 +1088,7 @@ table_beginscan_tidrange(Relation rel, Snapshot snapshot, TableScanDesc sscan; uint32 flags = SO_TYPE_TIDRANGESCAN | SO_ALLOW_PAGEMODE; - sscan = rel->rd_tableam->scan_begin(rel, snapshot, 0, NULL, NULL, flags); + sscan = rel->rd_tableam->scan_begin(rel, snapshot, 0, NULL, NULL, flags, 0); /* Set the range of TIDs to scan */ sscan->rs_rd->rd_tableam->scan_set_tidrange(sscan, mintid, maxtid); @@ -1963,16 +1960,12 @@ table_relation_estimate_size(Relation rel, int32 *attr_widths, * `recheck` is set by the table AM to indicate whether or not the tuples * from this block should be rechecked. * - * `blockno` is the current block and is set by the table AM and is used by - * bitmap table scan code to validate that the prefetch block stays ahead of - * the current block. - * * Note, this is an optionally implemented function, therefore should only be * used after verifying the presence (at plan time or such). */ static inline bool table_scan_bitmap_next_block(TableScanDesc scan, - BlockNumber *blockno, bool *recheck, + bool *recheck, uint64 *lossy_pages, uint64 *exact_pages) { @@ -1985,7 +1978,7 @@ table_scan_bitmap_next_block(TableScanDesc scan, elog(ERROR, "unexpected table_scan_bitmap_next_block call during logical decoding"); return scan->rs_rd->rd_tableam->scan_bitmap_next_block(scan, - blockno, recheck, + recheck, lossy_pages, exact_pages); } diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 2def5895fe..d5bca3ac73 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -1833,18 +1833,11 @@ typedef struct SharedBitmapHeapInstrumentation * * bitmapqualorig execution state for bitmapqualorig expressions * tbm bitmap obtained from child index scan(s) - * pvmbuffer buffer for visibility-map lookups of prefetched pages * stats execution statistics - * prefetch_iterator iterator for prefetching ahead of current page - * prefetch_pages # pages prefetch iterator is ahead of current - * prefetch_target current target prefetch distance - * prefetch_maximum maximum value for prefetch_target * initialized is node is ready to iterate * pstate shared state for parallel bitmap scan * sinstrument statistics for parallel workers * recheck do current page's tuples need recheck - * blockno used to validate pf and current block in sync - * pfblockno used to validate pf stays ahead of current block * ---------------- */ typedef struct BitmapHeapScanState @@ -1852,7 +1845,6 @@ typedef struct BitmapHeapScanState ScanState ss; /* its first field is NodeTag */ ExprState *bitmapqualorig; TIDBitmap *tbm; - Buffer pvmbuffer; BitmapHeapScanInstrumentation stats; TBMIterator prefetch_iterator; int prefetch_pages; @@ -1862,8 +1854,6 @@ typedef struct BitmapHeapScanState ParallelBitmapHeapState *pstate; SharedBitmapHeapInstrumentation *sinstrument; bool recheck; - BlockNumber blockno; - BlockNumber pfblockno; } BitmapHeapScanState; /* ---------------- -- 2.45.2