visibility-map-v1.patch

*** a/src/backend/access/heap/heapam.c
--- b/src/backend/access/heap/heapam.c
***************
*** 4044,4049 **** log_heap_freeze(Relation reln, Buffer buffer,
--- 4044,4081 ----
  }
  
  /*
+  * Perform XLogInsert for a heap-visible operation.	 'block' is the block
+  * being marked all-visible, and vm_buffer is the buffer containing the
+  * corresponding visibility map block.  Both should have already been modified
+  * and dirtied.
+  */
+ XLogRecPtr
+ log_heap_visible(RelFileNode rnode, BlockNumber block, Buffer vm_buffer)
+ {
+ 	xl_heap_visible xlrec;
+ 	XLogRecPtr	recptr;
+ 	XLogRecData rdata[2];
+ 
+ 	xlrec.node = rnode;
+ 	xlrec.block = block;
+ 
+ 	rdata[0].data = (char *) &xlrec;
+ 	rdata[0].len = SizeOfHeapVisible;
+ 	rdata[0].buffer = InvalidBuffer;
+ 	rdata[0].next = &(rdata[1]);
+ 
+ 	rdata[1].data = NULL;
+ 	rdata[1].len = 0;
+ 	rdata[1].buffer = vm_buffer;
+ 	rdata[1].buffer_std = false;
+ 	rdata[1].next = NULL;
+ 
+ 	recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_VISIBLE, rdata);
+ 
+ 	return recptr;
+ }
+ 
+ /*
   * Perform XLogInsert for a heap-update operation.	Caller must already
   * have modified the buffer(s) and marked them dirty.
   */
***************
*** 4331,4336 **** heap_xlog_freeze(XLogRecPtr lsn, XLogRecord *record)
--- 4363,4440 ----
  	UnlockReleaseBuffer(buffer);
  }
  
+ /*
+  * Replay XLOG_HEAP2_VISIBLE record.
+  *
+  * The critical integrity requirement here is that we must never end up with
+  * a situation where the visibility map bit is set, and the page-level
+  * PD_ALL_VISIBLE bit is clear.  If that were to occur, then a subsequent
+  * page modification would fail to clear the visibility map bit.
+  */
+ static void
+ heap_xlog_visible(XLogRecPtr lsn, XLogRecord *record)
+ {
+ 	xl_heap_visible *xlrec = (xl_heap_visible *) XLogRecGetData(record);
+ 	Buffer		buffer;
+ 	Page		page;
+ 
+ 	/*
+ 	 * Read the heap page, if it still exists.  If the heap file has been
+ 	 * dropped or truncated later in recovery, this might fail.  In that case,
+ 	 * there's no point in doing anything further, since the visibility map
+ 	 * will have to be cleared out at the same time.
+ 	 */
+ 	buffer = XLogReadBufferExtended(xlrec->node, MAIN_FORKNUM, xlrec->block,
+ 									RBM_NORMAL);
+ 	if (!BufferIsValid(buffer))
+ 		return;
+ 	page = (Page) BufferGetPage(buffer);
+ 
+ 	/*
+ 	 * We don't bump the LSN of the heap page when setting the visibility
+ 	 * map bit, because that would generate an unworkable volume of
+ 	 * full-page writes.  This exposes us to torn page hazards, but since
+ 	 * we're not inspecting the existing page contents in any way, we
+ 	 * don't care.
+ 	 *
+ 	 * However, all operations that clear the visibility map bit *do* bump
+ 	 * the LSN, and those operations will only be replayed if the XLOG LSN
+ 	 * precedes the page LSN.  Thus, if the page LSN has advanced past our
+ 	 * XLOG record's LSN, we mustn't mark the page all-visible, because
+ 	 * the subsequent update won't be replayed to clear the flag.
+ 	 */
+ 	if (XLByteLE(lsn, PageGetLSN(page)))
+ 	{
+ 		PageSetAllVisible(page);
+ 		MarkBufferDirty(buffer);
+ 	}
+ 
+ 	/* Done with heap page. */
+ 	UnlockReleaseBuffer(buffer);
+ 
+ 	/*
+ 	 * Even we skipped the heap page update due to the LSN interlock, it's
+ 	 * still safe to update the visibility map.  Any WAL record that clears
+ 	 * the visibility map bit does so before checking the page LSN, so any
+ 	 * bits that need to be cleared will still be cleared.
+ 	 */
+ 	if (record->xl_info & XLR_BKP_BLOCK_1)
+ 		RestoreBkpBlocks(lsn, record, false);
+ 	else
+ 	{
+ 		Relation	reln;
+ 		Buffer		vmbuffer = InvalidBuffer;
+ 
+ 		reln = CreateFakeRelcacheEntry(xlrec->node);
+ 		visibilitymap_pin(reln, xlrec->block, &vmbuffer);
+ 		/* Don't set the bit if replay has already passed this point. */
+ 		if (XLByteLE(lsn, PageGetLSN(BufferGetPage(vmbuffer))))
+ 			visibilitymap_set(reln, xlrec->block, lsn, &vmbuffer);
+ 		ReleaseBuffer(vmbuffer);
+ 		FreeFakeRelcacheEntry(reln);
+ 	}
+ }
+ 
  static void
  heap_xlog_newpage(XLogRecPtr lsn, XLogRecord *record)
  {
***************
*** 4923,4928 **** heap2_redo(XLogRecPtr lsn, XLogRecord *record)
--- 5027,5035 ----
  		case XLOG_HEAP2_CLEANUP_INFO:
  			heap_xlog_cleanup_info(lsn, record);
  			break;
+ 		case XLOG_HEAP2_VISIBLE:
+ 			heap_xlog_visible(lsn, record);
+ 			break;
  		default:
  			elog(PANIC, "heap2_redo: unknown op code %u", info);
  	}
***************
*** 5052,5057 **** heap2_desc(StringInfo buf, uint8 xl_info, char *rec)
--- 5159,5172 ----
  		appendStringInfo(buf, "cleanup info: remxid %u",
  						 xlrec->latestRemovedXid);
  	}
+ 	else if (info == XLOG_HEAP2_VISIBLE)
+ 	{
+ 		xl_heap_visible *xlrec = (xl_heap_visible *) rec;
+ 
+ 		appendStringInfo(buf, "visible: rel %u/%u/%u; blk %u",
+ 						 xlrec->node.spcNode, xlrec->node.dbNode,
+ 						 xlrec->node.relNode, xlrec->block);
+ 	}
  	else
  		appendStringInfo(buf, "UNKNOWN");
  }
*** a/src/backend/access/heap/visibilitymap.c
--- b/src/backend/access/heap/visibilitymap.c
***************
*** 76,94 ****
   * index would still be visible to all other backends, and deletions wouldn't
   * be visible to other backends yet.  (But HOT breaks that argument, no?)
   *
-  * There's another hole in the way the PD_ALL_VISIBLE flag is set. When
-  * vacuum observes that all tuples are visible to all, it sets the flag on
-  * the heap page, and also sets the bit in the visibility map. If we then
-  * crash, and only the visibility map page was flushed to disk, we'll have
-  * a bit set in the visibility map, but the corresponding flag on the heap
-  * page is not set. If the heap page is then updated, the updater won't
-  * know to clear the bit in the visibility map.  (Isn't that prevented by
-  * the LSN interlock?)
-  *
   *-------------------------------------------------------------------------
   */
  #include "postgres.h"
  
  #include "access/visibilitymap.h"
  #include "storage/bufmgr.h"
  #include "storage/bufpage.h"
--- 76,86 ----
   * index would still be visible to all other backends, and deletions wouldn't
   * be visible to other backends yet.  (But HOT breaks that argument, no?)
   *
   *-------------------------------------------------------------------------
   */
  #include "postgres.h"
  
+ #include "access/heapam.h"
  #include "access/visibilitymap.h"
  #include "storage/bufmgr.h"
  #include "storage/bufpage.h"
***************
*** 196,208 **** visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *buf)
  /*
   *	visibilitymap_set - set a bit on a previously pinned page
   *
!  * recptr is the LSN of the heap page. The LSN of the visibility map page is
!  * advanced to that, to make sure that the visibility map doesn't get flushed
!  * to disk before the update to the heap page that made all tuples visible.
   *
!  * This is an opportunistic function. It does nothing, unless *buf
!  * contains the bit for heapBlk. Call visibilitymap_pin first to pin
!  * the right map page. This function doesn't do any I/O.
   */
  void
  visibilitymap_set(Relation rel, BlockNumber heapBlk, XLogRecPtr recptr,
--- 188,201 ----
  /*
   *	visibilitymap_set - set a bit on a previously pinned page
   *
!  * recptr is the LSN of the XLOG record we're replaying, if we're in recovery,
!  * or InvalidXLogRecPtr in normal running.  The page LSN is advanced to the
!  * one provided; in normal running, we generate a new XLOG record and set the
!  * page LSN to that value.
   *
!  * You must pass a buffer containing the correct map page to this function.
!  * Call visibilitymap_pin first to pin the right one. This function doesn't do
!  * any I/O.
   */
  void
  visibilitymap_set(Relation rel, BlockNumber heapBlk, XLogRecPtr recptr,
***************
*** 220,226 **** visibilitymap_set(Relation rel, BlockNumber heapBlk, XLogRecPtr recptr,
  
  	/* Check that we have the right page pinned */
  	if (!BufferIsValid(*buf) || BufferGetBlockNumber(*buf) != mapBlock)
! 		return;
  
  	page = BufferGetPage(*buf);
  	map = PageGetContents(page);
--- 213,219 ----
  
  	/* Check that we have the right page pinned */
  	if (!BufferIsValid(*buf) || BufferGetBlockNumber(*buf) != mapBlock)
! 		elog(ERROR, "wrong buffer passed to visibilitymap_set");
  
  	page = BufferGetPage(*buf);
  	map = PageGetContents(page);
***************
*** 230,236 **** visibilitymap_set(Relation rel, BlockNumber heapBlk, XLogRecPtr recptr,
  	{
  		map[mapByte] |= (1 << mapBit);
  
! 		if (XLByteLT(PageGetLSN(page), recptr))
  			PageSetLSN(page, recptr);
  		PageSetTLI(page, ThisTimeLineID);
  		MarkBufferDirty(*buf);
--- 223,232 ----
  	{
  		map[mapByte] |= (1 << mapBit);
  
! 		if (XLogRecPtrIsInvalid(recptr) && RelationNeedsWAL(rel))
! 			recptr = log_heap_visible(rel->rd_node, heapBlk, *buf);
! 
! 		if (!XLogRecPtrIsInvalid(recptr))
  			PageSetLSN(page, recptr);
  		PageSetTLI(page, ThisTimeLineID);
  		MarkBufferDirty(*buf);
*** a/src/backend/access/transam/transam.c
--- b/src/backend/access/transam/transam.c
***************
*** 24,29 ****
--- 24,31 ----
  #include "access/transam.h"
  #include "utils/snapmgr.h"
  
+ /* Handy constant for an invalid xlog recptr */
+ const XLogRecPtr InvalidXLogRecPtr = {0, 0};
  
  /*
   * Single-item cache for results of TransactionLogFetch.  It's worth having
***************
*** 35,43 **** static TransactionId cachedFetchXid = InvalidTransactionId;
  static XidStatus cachedFetchXidStatus;
  static XLogRecPtr cachedCommitLSN;
  
- /* Handy constant for an invalid xlog recptr */
- static const XLogRecPtr InvalidXLogRecPtr = {0, 0};
- 
  /* Local functions */
  static XidStatus TransactionLogFetch(TransactionId transactionId);
  
--- 37,42 ----
*** a/src/backend/access/transam/xlog.c
--- b/src/backend/access/transam/xlog.c
***************
*** 5410,5416 **** exitArchiveRecovery(TimeLineID endTLI, uint32 endLogId, uint32 endLogSeg)
  {
  	char		recoveryPath[MAXPGPATH];
  	char		xlogpath[MAXPGPATH];
- 	XLogRecPtr	InvalidXLogRecPtr = {0, 0};
  
  	/*
  	 * We are no longer in archive recovery state.
--- 5410,5415 ----
***************
*** 7997,8004 **** CreateRestartPoint(int flags)
  	if (XLogRecPtrIsInvalid(lastCheckPointRecPtr) ||
  		XLByteLE(lastCheckPoint.redo, ControlFile->checkPointCopy.redo))
  	{
- 		XLogRecPtr	InvalidXLogRecPtr = {0, 0};
- 
  		ereport(DEBUG2,
  				(errmsg("skipping restartpoint, already performed at %X/%X",
  				  lastCheckPoint.redo.xlogid, lastCheckPoint.redo.xrecoff)));
--- 7996,8001 ----
*** a/src/backend/commands/vacuumlazy.c
--- b/src/backend/commands/vacuumlazy.c
***************
*** 479,485 **** lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats,
  				visibilitymap_pin(onerel, blkno, &vmbuffer);
  				LockBuffer(buf, BUFFER_LOCK_SHARE);
  				if (PageIsAllVisible(page))
! 					visibilitymap_set(onerel, blkno, PageGetLSN(page), &vmbuffer);
  				LockBuffer(buf, BUFFER_LOCK_UNLOCK);
  			}
  
--- 479,486 ----
  				visibilitymap_pin(onerel, blkno, &vmbuffer);
  				LockBuffer(buf, BUFFER_LOCK_SHARE);
  				if (PageIsAllVisible(page))
! 					visibilitymap_set(onerel, blkno, InvalidXLogRecPtr,
! 									  &vmbuffer);
  				LockBuffer(buf, BUFFER_LOCK_UNLOCK);
  			}
  
*** a/src/include/access/heapam.h
--- b/src/include/access/heapam.h
***************
*** 136,141 **** extern XLogRecPtr log_heap_clean(Relation reln, Buffer buffer,
--- 136,143 ----
  extern XLogRecPtr log_heap_freeze(Relation reln, Buffer buffer,
  				TransactionId cutoff_xid,
  				OffsetNumber *offsets, int offcnt);
+ extern XLogRecPtr log_heap_visible(RelFileNode rnode, BlockNumber block,
+ 				 Buffer vm_buffer);
  extern XLogRecPtr log_newpage(RelFileNode *rnode, ForkNumber forkNum,
  			BlockNumber blk, Page page);
  
*** a/src/include/access/htup.h
--- b/src/include/access/htup.h
***************
*** 606,611 **** typedef HeapTupleData *HeapTuple;
--- 606,612 ----
  #define XLOG_HEAP2_CLEAN		0x10
  /* 0x20 is free, was XLOG_HEAP2_CLEAN_MOVE */
  #define XLOG_HEAP2_CLEANUP_INFO 0x30
+ #define XLOG_HEAP2_VISIBLE		0x40
  
  /*
   * All what we need to find changed tuple
***************
*** 750,755 **** typedef struct xl_heap_freeze
--- 751,765 ----
  
  #define SizeOfHeapFreeze (offsetof(xl_heap_freeze, cutoff_xid) + sizeof(TransactionId))
  
+ /* This is what we need to know about setting a visibility map bit */
+ typedef struct xl_heap_visible
+ {
+ 	RelFileNode node;
+ 	BlockNumber block;
+ } xl_heap_visible;
+ 
+ #define SizeOfHeapVisible (offsetof(xl_heap_visible, block) + sizeof(BlockNumber))
+ 
  extern void HeapTupleHeaderAdvanceLatestRemovedXid(HeapTupleHeader tuple,
  									   TransactionId *latestRemovedXid);
  
*** a/src/include/access/transam.h
--- b/src/include/access/transam.h
***************
*** 135,140 **** extern bool TransactionStartedDuringRecovery(void);
--- 135,143 ----
  /* in transam/varsup.c */
  extern PGDLLIMPORT VariableCache ShmemVariableCache;
  
+ /* in transam/transam.c */
+ extern const XLogRecPtr InvalidXLogRecPtr;
+ 
  
  /*
   * prototypes for functions in transam/transam.c