cascading_standby_stuck_v2.patch

diff -rc a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
*** a/src/backend/access/transam/xlog.c	2016-10-25 05:17:41.000000000 +0900
--- b/src/backend/access/transam/xlog.c	2016-11-15 12:26:35.000000000 +0900
***************
*** 589,596 ****
  /* State information for XLOG reading */
  static XLogRecPtr ReadRecPtr;	/* start of last record read */
  static XLogRecPtr EndRecPtr;	/* end+1 of last record read */
! static TimeLineID lastPageTLI = 0;
! static TimeLineID lastSegmentTLI = 0;
  
  static XLogRecPtr minRecoveryPoint;		/* local copy of
  										 * ControlFile->minRecoveryPoint */
--- 589,596 ----
  /* State information for XLOG reading */
  static XLogRecPtr ReadRecPtr;	/* start of last record read */
  static XLogRecPtr EndRecPtr;	/* end+1 of last record read */
! static XLogRecPtr latestPagePtr;	/* start of last page read */
! static TimeLineID latestPageTLI = 0;
  
  static XLogRecPtr minRecoveryPoint;		/* local copy of
  										 * ControlFile->minRecoveryPoint */
***************
*** 678,684 ****
  static void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force);
  static XLogRecord *ReadRecord(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt);
  static void CheckRecoveryConsistency(void);
! static bool ValidXLOGHeader(XLogPageHeader hdr, int emode, bool segmentonly);
  static XLogRecord *ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt);
  static List *readTimeLineHistory(TimeLineID targetTLI);
  static bool existsTimeLineHistory(TimeLineID probeTLI);
--- 678,684 ----
  static void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force);
  static XLogRecord *ReadRecord(XLogRecPtr *RecPtr, int emode, bool fetching_ckpt);
  static void CheckRecoveryConsistency(void);
! static bool ValidXLOGHeader(XLogPageHeader hdr, int emode);
  static XLogRecord *ReadCheckpointRecord(XLogRecPtr RecPtr, int whichChkpt);
  static List *readTimeLineHistory(TimeLineID targetTLI);
  static bool existsTimeLineHistory(TimeLineID probeTLI);
***************
*** 3992,4005 ****
  					(errmsg("invalid record offset at %X/%X",
  							RecPtr->xlogid, RecPtr->xrecoff)));
  
- 		/*
- 		 * Since we are going to a random position in WAL, forget any prior
- 		 * state about what timeline we were in, and allow it to be any
- 		 * timeline in expectedTLIs.  We also set a flag to allow curFileTLI
- 		 * to go backwards (but we can't reset that variable right here, since
- 		 * we might not change files at all).
- 		 */
- 		lastPageTLI = lastSegmentTLI = 0;	/* see comment in ValidXLOGHeader */
  		randAccess = true;		/* allow curFileTLI to go backwards too */
  	}
  
--- 3992,3997 ----
***************
*** 4317,4323 ****
   * ReadRecord.  It's not intended for use from anywhere else.
   */
  static bool
! ValidXLOGHeader(XLogPageHeader hdr, int emode, bool segmentonly)
  {
  	XLogRecPtr	recaddr;
  
--- 4309,4315 ----
   * ReadRecord.  It's not intended for use from anywhere else.
   */
  static bool
! ValidXLOGHeader(XLogPageHeader hdr, int emode)
  {
  	XLogRecPtr	recaddr;
  
***************
*** 4411,4441 ****
  	 * immediate parent's TLI, we should never see TLI go backwards across
  	 * successive pages of a consistent WAL sequence.
  	 *
! 	 * Of course this check should only be applied when advancing sequentially
! 	 * across pages; therefore ReadRecord resets lastPageTLI and
! 	 * lastSegmentTLI to zero when going to a random page.
! 	 *
! 	 * Sometimes we re-open a segment that's already been partially replayed.
! 	 * In that case we cannot perform the normal TLI check: if there is a
! 	 * timeline switch within the segment, the first page has a smaller TLI
! 	 * than later pages following the timeline switch, and we might've read
! 	 * them already. As a weaker test, we still check that it's not smaller
! 	 * than the TLI we last saw at the beginning of a segment. Pass
! 	 * segmentonly = true when re-validating the first page like that, and the
! 	 * page you're actually interested in comes later.
  	 */
! 	if (hdr->xlp_tli < (segmentonly ? lastSegmentTLI : lastPageTLI))
  	{
! 		ereport(emode_for_corrupt_record(emode, recaddr),
! 				(errmsg("out-of-sequence timeline ID %u (after %u) in log file %u, segment %u, offset %u",
! 						hdr->xlp_tli,
! 						segmentonly ? lastSegmentTLI : lastPageTLI,
! 						readId, readSeg, readOff)));
! 		return false;
  	}
! 	lastPageTLI = hdr->xlp_tli;
! 	if (readOff == 0)
! 		lastSegmentTLI = hdr->xlp_tli;
  
  	return true;
  }
--- 4403,4447 ----
  	 * immediate parent's TLI, we should never see TLI go backwards across
  	 * successive pages of a consistent WAL sequence.
  	 *
! 	 * Sometimes we re-read a segment that's already been (partially) read. So
! 	 * we only verify TLIs for pages that are later than the last remembered
! 	 * LSN. An example of re-read which sees a smaller TLI is:
! 	 *
! 	 * In a streaming replication configuration where node1 is the primary,
! 	 * node2 is the standby, and node3 is the cascading standby. They use a
! 	 * shared WAL archive directory. node1's timeline is 1. It wrote a WAL
! 	 * record at the end of WAL segment 10. The WAL record didn't fit the last
! 	 * page, so it was split across segments 10 and 11. WAL segment 10 was
! 	 * archived. node1 got down, and node2 is promoted. As part of the
! 	 * recovery process, node2 retrieves WAL segment 10 from archive. It*
! 	 * found a WAL record fragment at the end of the segment but could not
! 	 * find the remaining fragment in segment 11, so node2 stops recovery
! 	 * there. node2 becomes the primary, and its timeline becomes 2. node3 is
! 	 * disconnected by node2 (but later reconnects to node2.) node3 retrieves
! 	 * and applies WAL segment 10 from archive. node3 found .history file for
! 	 * timeline 2 and renews its timeline to 2. Because node3 found a WAL
! 	 * record fragment at the end of segment 10, it expects to find the
! 	 * remaining fragment at the beginning of WAL segment 11 streamed from
! 	 * node2. But there was a fragment of a different WAL record, because
! 	 * node2 overwrote a different WAL record at the end of segment 10 across
! 	 * to 11. node3 then retrieves segment 10 from archive again to get the
! 	 * WAL record at the end of segment 10. While node3's timeline is already
! 	 * 2, it sees the timeline 1 at the beginning of segment 10.
  	 */
! 	if (XLByteLT(latestPagePtr, recaddr))
  	{
! 		if (hdr->xlp_tli < latestPageTLI)
! 		{
! 			ereport(emode_for_corrupt_record(emode, recaddr),
! 					(errmsg("out-of-sequence timeline ID %u (after %u) in log file %u, segment %u, offset %u",
! 							hdr->xlp_tli,
! 							latestPageTLI,
! 							readId, readSeg, readOff)));
! 			return false;
! 		}
  	}
! 	latestPagePtr = recaddr;
! 	latestPageTLI = hdr->xlp_tli;
  
  	return true;
  }
***************
*** 10817,10823 ****
  							readId, readSeg, readOff)));
  			goto next_record_is_invalid;
  		}
! 		if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode, true))
  			goto next_record_is_invalid;
  	}
  
--- 10823,10829 ----
  							readId, readSeg, readOff)));
  			goto next_record_is_invalid;
  		}
! 		if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode))
  			goto next_record_is_invalid;
  	}
  
***************
*** 10839,10845 ****
  				readId, readSeg, readOff)));
  		goto next_record_is_invalid;
  	}
! 	if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode, false))
  		goto next_record_is_invalid;
  
  	Assert(targetId == readId);
--- 10845,10851 ----
  				readId, readSeg, readOff)));
  		goto next_record_is_invalid;
  	}
! 	if (!ValidXLOGHeader((XLogPageHeader) readBuf, emode))
  		goto next_record_is_invalid;
  
  	Assert(targetId == readId);
b/src/backend/access/transamだけに発見: xlog.c.orig