0003-Refactor-some-basebackup-code-to-increase-reusabilit_v4.patch

From 21866fd73f852c4064c3e588f62964fe1bd52440 Mon Sep 17 00:00:00 2001
From: Asif Rehman <asif.rehman@highgo.ca>
Date: Wed, 9 Oct 2019 12:39:41 +0500
Subject: [PATCH 3/6] Refactor some basebackup code to increase reusability, in
 anticipation of adding parallel backup

---
 src/backend/access/transam/xlog.c    | 192 +++++-----
 src/backend/replication/basebackup.c | 512 ++++++++++++++-------------
 src/include/access/xlog.h            |   2 +
 3 files changed, 371 insertions(+), 335 deletions(-)

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 2e3cc51006..aa7d82a045 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -10286,10 +10286,6 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
 	PG_ENSURE_ERROR_CLEANUP(pg_start_backup_callback, (Datum) BoolGetDatum(exclusive));
 	{
 		bool		gotUniqueStartpoint = false;
-		DIR		   *tblspcdir;
-		struct dirent *de;
-		tablespaceinfo *ti;
-		int			datadirpathlen;
 
 		/*
 		 * Force an XLOG file switch before the checkpoint, to ensure that the
@@ -10415,93 +10411,7 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p,
 		if (exclusive)
 			tblspcmapfile = makeStringInfo();
 
-		datadirpathlen = strlen(DataDir);
-
-		/* Collect information about all tablespaces */
-		tblspcdir = AllocateDir("pg_tblspc");
-		while ((de = ReadDir(tblspcdir, "pg_tblspc")) != NULL)
-		{
-			char		fullpath[MAXPGPATH + 10];
-			char		linkpath[MAXPGPATH];
-			char	   *relpath = NULL;
-			int			rllen;
-			StringInfoData buflinkpath;
-			char	   *s = linkpath;
-
-			/* Skip special stuff */
-			if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
-				continue;
-
-			snprintf(fullpath, sizeof(fullpath), "pg_tblspc/%s", de->d_name);
-
-#if defined(HAVE_READLINK) || defined(WIN32)
-			rllen = readlink(fullpath, linkpath, sizeof(linkpath));
-			if (rllen < 0)
-			{
-				ereport(WARNING,
-						(errmsg("could not read symbolic link \"%s\": %m",
-								fullpath)));
-				continue;
-			}
-			else if (rllen >= sizeof(linkpath))
-			{
-				ereport(WARNING,
-						(errmsg("symbolic link \"%s\" target is too long",
-								fullpath)));
-				continue;
-			}
-			linkpath[rllen] = '\0';
-
-			/*
-			 * Add the escape character '\\' before newline in a string to
-			 * ensure that we can distinguish between the newline in the
-			 * tablespace path and end of line while reading tablespace_map
-			 * file during archive recovery.
-			 */
-			initStringInfo(&buflinkpath);
-
-			while (*s)
-			{
-				if ((*s == '\n' || *s == '\r') && needtblspcmapfile)
-					appendStringInfoChar(&buflinkpath, '\\');
-				appendStringInfoChar(&buflinkpath, *s++);
-			}
-
-			/*
-			 * Relpath holds the relative path of the tablespace directory
-			 * when it's located within PGDATA, or NULL if it's located
-			 * elsewhere.
-			 */
-			if (rllen > datadirpathlen &&
-				strncmp(linkpath, DataDir, datadirpathlen) == 0 &&
-				IS_DIR_SEP(linkpath[datadirpathlen]))
-				relpath = linkpath + datadirpathlen + 1;
-
-			ti = palloc(sizeof(tablespaceinfo));
-			ti->oid = pstrdup(de->d_name);
-			ti->path = pstrdup(buflinkpath.data);
-			ti->rpath = relpath ? pstrdup(relpath) : NULL;
-			ti->size = infotbssize ? sendTablespace(fullpath, true) : -1;
-
-			if (tablespaces)
-				*tablespaces = lappend(*tablespaces, ti);
-
-			appendStringInfo(tblspcmapfile, "%s %s\n", ti->oid, ti->path);
-
-			pfree(buflinkpath.data);
-#else
-
-			/*
-			 * If the platform does not have symbolic links, it should not be
-			 * possible to have tablespaces - clearly somebody else created
-			 * them. Warn about it and ignore.
-			 */
-			ereport(WARNING,
-					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-					 errmsg("tablespaces are not supported on this platform")));
-#endif
-		}
-		FreeDir(tblspcdir);
+		collectTablespaces(tablespaces, tblspcmapfile, infotbssize, needtblspcmapfile);
 
 		/*
 		 * Construct backup label file
@@ -12277,3 +12187,103 @@ XLogRequestWalReceiverReply(void)
 {
 	doRequestWalReceiverReply = true;
 }
+
+/*
+ * Collect information about all tablespaces.
+ */
+void
+collectTablespaces(List **tablespaces, StringInfo tblspcmapfile,
+				   bool infotbssize, bool needtblspcmapfile)
+{
+	DIR		   *tblspcdir;
+	struct dirent *de;
+	tablespaceinfo *ti;
+	int			datadirpathlen;
+
+	datadirpathlen = strlen(DataDir);
+
+	tblspcdir = AllocateDir("pg_tblspc");
+	while ((de = ReadDir(tblspcdir, "pg_tblspc")) != NULL)
+	{
+		char		fullpath[MAXPGPATH + 10];
+		char		linkpath[MAXPGPATH];
+		char	   *relpath = NULL;
+		int			rllen;
+		StringInfoData buflinkpath;
+		char	   *s = linkpath;
+
+		/* Skip special stuff */
+		if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
+			continue;
+
+		snprintf(fullpath, sizeof(fullpath), "pg_tblspc/%s", de->d_name);
+
+#if defined(HAVE_READLINK) || defined(WIN32)
+		rllen = readlink(fullpath, linkpath, sizeof(linkpath));
+		if (rllen < 0)
+		{
+			ereport(WARNING,
+					(errmsg("could not read symbolic link \"%s\": %m",
+							fullpath)));
+			continue;
+		}
+		else if (rllen >= sizeof(linkpath))
+		{
+			ereport(WARNING,
+					(errmsg("symbolic link \"%s\" target is too long",
+							fullpath)));
+			continue;
+		}
+		linkpath[rllen] = '\0';
+
+		/*
+		 * Add the escape character '\\' before newline in a string to
+		 * ensure that we can distinguish between the newline in the
+		 * tablespace path and end of line while reading tablespace_map
+		 * file during archive recovery.
+		 */
+		initStringInfo(&buflinkpath);
+
+		while (*s)
+		{
+			if ((*s == '\n' || *s == '\r') && needtblspcmapfile)
+				appendStringInfoChar(&buflinkpath, '\\');
+			appendStringInfoChar(&buflinkpath, *s++);
+		}
+
+		/*
+		 * Relpath holds the relative path of the tablespace directory
+		 * when it's located within PGDATA, or NULL if it's located
+		 * elsewhere.
+		 */
+		if (rllen > datadirpathlen &&
+			strncmp(linkpath, DataDir, datadirpathlen) == 0 &&
+			IS_DIR_SEP(linkpath[datadirpathlen]))
+			relpath = linkpath + datadirpathlen + 1;
+
+		ti = palloc(sizeof(tablespaceinfo));
+		ti->oid = pstrdup(de->d_name);
+		ti->path = pstrdup(buflinkpath.data);
+		ti->rpath = relpath ? pstrdup(relpath) : NULL;
+		ti->size = infotbssize ? sendTablespace(fullpath, true) : -1;
+
+		if (tablespaces)
+			*tablespaces = lappend(*tablespaces, ti);
+
+		appendStringInfo(tblspcmapfile, "%s %s\n", ti->oid, ti->path);
+
+		pfree(buflinkpath.data);
+#else
+
+		/*
+		 * If the platform does not have symbolic links, it should not be
+		 * possible to have tablespaces - clearly somebody else created
+		 * them. Warn about it and ignore.
+		 */
+		ereport(WARNING,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("tablespaces are not supported on this platform")));
+#endif
+	}
+	FreeDir(tblspcdir);
+}
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 9442486b66..b8e3daf711 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -68,10 +68,12 @@ static void send_int8_string(StringInfoData *buf, int64 intval);
 static void SendBackupHeader(List *tablespaces);
 static void base_backup_cleanup(int code, Datum arg);
 static void perform_base_backup(basebackup_options *opt);
+static void include_wal_files(XLogRecPtr endptr);
 static void parse_basebackup_options(List *options, basebackup_options *opt);
 static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
 static int	compareWalFileNames(const ListCell *a, const ListCell *b);
 static void throttle(size_t increment);
+static void setup_throttle(int maxrate);
 static bool is_checksummed_file(const char *fullpath, const char *filename);
 
 /* Was the backup currently in-progress initiated in recovery mode? */
@@ -294,29 +296,7 @@ perform_base_backup(basebackup_options *opt)
 	/* Send tablespace header */
 	SendBackupHeader(tablespaces);
 
-	/* Setup and activate network throttling, if client requested it */
-	if (opt->maxrate > 0)
-	{
-		throttling_sample =
-			(int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
-
-		/*
-		 * The minimum amount of time for throttling_sample bytes to be
-		 * transferred.
-		 */
-		elapsed_min_unit = USECS_PER_SEC / THROTTLING_FREQUENCY;
-
-		/* Enable throttling. */
-		throttling_counter = 0;
-
-		/* The 'real data' starts now (header was ignored). */
-		throttled_last = GetCurrentTimestamp();
-	}
-	else
-	{
-		/* Disable throttling. */
-		throttling_counter = -1;
-	}
+	setup_throttle(opt->maxrate);
 
 	/* Send off our tablespaces one by one */
 	foreach(lc, tablespaces)
@@ -382,227 +362,7 @@ perform_base_backup(basebackup_options *opt)
 		 * We've left the last tar file "open", so we can now append the
 		 * required WAL files to it.
 		 */
-		char		pathbuf[MAXPGPATH];
-		XLogSegNo	segno;
-		XLogSegNo	startsegno;
-		XLogSegNo	endsegno;
-		struct stat statbuf;
-		List	   *historyFileList = NIL;
-		List	   *walFileList = NIL;
-		char		firstoff[MAXFNAMELEN];
-		char		lastoff[MAXFNAMELEN];
-		DIR		   *dir;
-		struct dirent *de;
-		ListCell   *lc;
-		TimeLineID	tli;
-
-		/*
-		 * I'd rather not worry about timelines here, so scan pg_wal and
-		 * include all WAL files in the range between 'startptr' and 'endptr',
-		 * regardless of the timeline the file is stamped with. If there are
-		 * some spurious WAL files belonging to timelines that don't belong in
-		 * this server's history, they will be included too. Normally there
-		 * shouldn't be such files, but if there are, there's little harm in
-		 * including them.
-		 */
-		XLByteToSeg(startptr, startsegno, wal_segment_size);
-		XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
-		XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
-		XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
-
-		dir = AllocateDir("pg_wal");
-		while ((de = ReadDir(dir, "pg_wal")) != NULL)
-		{
-			/* Does it look like a WAL segment, and is it in the range? */
-			if (IsXLogFileName(de->d_name) &&
-				strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
-				strcmp(de->d_name + 8, lastoff + 8) <= 0)
-			{
-				walFileList = lappend(walFileList, pstrdup(de->d_name));
-			}
-			/* Does it look like a timeline history file? */
-			else if (IsTLHistoryFileName(de->d_name))
-			{
-				historyFileList = lappend(historyFileList, pstrdup(de->d_name));
-			}
-		}
-		FreeDir(dir);
-
-		/*
-		 * Before we go any further, check that none of the WAL segments we
-		 * need were removed.
-		 */
-		CheckXLogRemoved(startsegno, ThisTimeLineID);
-
-		/*
-		 * Sort the WAL filenames.  We want to send the files in order from
-		 * oldest to newest, to reduce the chance that a file is recycled
-		 * before we get a chance to send it over.
-		 */
-		list_sort(walFileList, compareWalFileNames);
-
-		/*
-		 * There must be at least one xlog file in the pg_wal directory, since
-		 * we are doing backup-including-xlog.
-		 */
-		if (walFileList == NIL)
-			ereport(ERROR,
-					(errmsg("could not find any WAL files")));
-
-		/*
-		 * Sanity check: the first and last segment should cover startptr and
-		 * endptr, with no gaps in between.
-		 */
-		XLogFromFileName((char *) linitial(walFileList),
-						 &tli, &segno, wal_segment_size);
-		if (segno != startsegno)
-		{
-			char		startfname[MAXFNAMELEN];
-
-			XLogFileName(startfname, ThisTimeLineID, startsegno,
-						 wal_segment_size);
-			ereport(ERROR,
-					(errmsg("could not find WAL file \"%s\"", startfname)));
-		}
-		foreach(lc, walFileList)
-		{
-			char	   *walFileName = (char *) lfirst(lc);
-			XLogSegNo	currsegno = segno;
-			XLogSegNo	nextsegno = segno + 1;
-
-			XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
-			if (!(nextsegno == segno || currsegno == segno))
-			{
-				char		nextfname[MAXFNAMELEN];
-
-				XLogFileName(nextfname, ThisTimeLineID, nextsegno,
-							 wal_segment_size);
-				ereport(ERROR,
-						(errmsg("could not find WAL file \"%s\"", nextfname)));
-			}
-		}
-		if (segno != endsegno)
-		{
-			char		endfname[MAXFNAMELEN];
-
-			XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
-			ereport(ERROR,
-					(errmsg("could not find WAL file \"%s\"", endfname)));
-		}
-
-		/* Ok, we have everything we need. Send the WAL files. */
-		foreach(lc, walFileList)
-		{
-			char	   *walFileName = (char *) lfirst(lc);
-			FILE	   *fp;
-			char		buf[TAR_SEND_SIZE];
-			size_t		cnt;
-			pgoff_t		len = 0;
-
-			snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
-			XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
-
-			fp = AllocateFile(pathbuf, "rb");
-			if (fp == NULL)
-			{
-				int			save_errno = errno;
-
-				/*
-				 * Most likely reason for this is that the file was already
-				 * removed by a checkpoint, so check for that to get a better
-				 * error message.
-				 */
-				CheckXLogRemoved(segno, tli);
-
-				errno = save_errno;
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("could not open file \"%s\": %m", pathbuf)));
-			}
-
-			if (fstat(fileno(fp), &statbuf) != 0)
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("could not stat file \"%s\": %m",
-								pathbuf)));
-			if (statbuf.st_size != wal_segment_size)
-			{
-				CheckXLogRemoved(segno, tli);
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("unexpected WAL file size \"%s\"", walFileName)));
-			}
-
-			/* send the WAL file itself */
-			_tarWriteHeader(pathbuf, NULL, &statbuf, false);
-
-			while ((cnt = fread(buf, 1,
-								Min(sizeof(buf), wal_segment_size - len),
-								fp)) > 0)
-			{
-				CheckXLogRemoved(segno, tli);
-				/* Send the chunk as a CopyData message */
-				if (pq_putmessage('d', buf, cnt))
-					ereport(ERROR,
-							(errmsg("base backup could not send data, aborting backup")));
-
-				len += cnt;
-				throttle(cnt);
-
-				if (len == wal_segment_size)
-					break;
-			}
-
-			CHECK_FREAD_ERROR(fp, pathbuf);
-
-			if (len != wal_segment_size)
-			{
-				CheckXLogRemoved(segno, tli);
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("unexpected WAL file size \"%s\"", walFileName)));
-			}
-
-			/* wal_segment_size is a multiple of 512, so no need for padding */
-
-			FreeFile(fp);
-
-			/*
-			 * Mark file as archived, otherwise files can get archived again
-			 * after promotion of a new node. This is in line with
-			 * walreceiver.c always doing an XLogArchiveForceDone() after a
-			 * complete segment.
-			 */
-			StatusFilePath(pathbuf, walFileName, ".done");
-			sendFileWithContent(pathbuf, "");
-		}
-
-		/*
-		 * Send timeline history files too. Only the latest timeline history
-		 * file is required for recovery, and even that only if there happens
-		 * to be a timeline switch in the first WAL segment that contains the
-		 * checkpoint record, or if we're taking a base backup from a standby
-		 * server and the target timeline changes while the backup is taken.
-		 * But they are small and highly useful for debugging purposes, so
-		 * better include them all, always.
-		 */
-		foreach(lc, historyFileList)
-		{
-			char	   *fname = lfirst(lc);
-
-			snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
-
-			if (lstat(pathbuf, &statbuf) != 0)
-				ereport(ERROR,
-						(errcode_for_file_access(),
-						 errmsg("could not stat file \"%s\": %m", pathbuf)));
-
-			sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid);
-
-			/* unconditionally mark file as archived */
-			StatusFilePath(pathbuf, fname, ".done");
-			sendFileWithContent(pathbuf, "");
-		}
+		include_wal_files(endptr);
 
 		/* Send CopyDone message for the last tar file */
 		pq_putemptymessage('c');
@@ -1741,3 +1501,267 @@ throttle(size_t increment)
 	 */
 	throttled_last = GetCurrentTimestamp();
 }
+
+/*
+ * Append the required WAL files to the backup tar file. It assumes that the
+ * last tar file is "open" and the WALs will be appended to it.
+ */
+static void
+include_wal_files(XLogRecPtr endptr)
+{
+	/*
+	 * We've left the last tar file "open", so we can now append the
+	 * required WAL files to it.
+	 */
+	char		pathbuf[MAXPGPATH];
+	XLogSegNo	segno;
+	XLogSegNo	startsegno;
+	XLogSegNo	endsegno;
+	struct stat statbuf;
+	List	   *historyFileList = NIL;
+	List	   *walFileList = NIL;
+	char		firstoff[MAXFNAMELEN];
+	char		lastoff[MAXFNAMELEN];
+	DIR		   *dir;
+	struct dirent *de;
+	ListCell   *lc;
+	TimeLineID	tli;
+
+	/*
+	 * I'd rather not worry about timelines here, so scan pg_wal and
+	 * include all WAL files in the range between 'startptr' and 'endptr',
+	 * regardless of the timeline the file is stamped with. If there are
+	 * some spurious WAL files belonging to timelines that don't belong in
+	 * this server's history, they will be included too. Normally there
+	 * shouldn't be such files, but if there are, there's little harm in
+	 * including them.
+	 */
+	XLByteToSeg(startptr, startsegno, wal_segment_size);
+	XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
+	XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
+	XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
+
+	dir = AllocateDir("pg_wal");
+	while ((de = ReadDir(dir, "pg_wal")) != NULL)
+	{
+		/* Does it look like a WAL segment, and is it in the range? */
+		if (IsXLogFileName(de->d_name) &&
+			strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
+			strcmp(de->d_name + 8, lastoff + 8) <= 0)
+		{
+			walFileList = lappend(walFileList, pstrdup(de->d_name));
+		}
+		/* Does it look like a timeline history file? */
+		else if (IsTLHistoryFileName(de->d_name))
+		{
+			historyFileList = lappend(historyFileList, pstrdup(de->d_name));
+		}
+	}
+	FreeDir(dir);
+
+	/*
+	 * Before we go any further, check that none of the WAL segments we
+	 * need were removed.
+	 */
+	CheckXLogRemoved(startsegno, ThisTimeLineID);
+
+	/*
+	 * Sort the WAL filenames.  We want to send the files in order from
+	 * oldest to newest, to reduce the chance that a file is recycled
+	 * before we get a chance to send it over.
+	 */
+	list_sort(walFileList, compareWalFileNames);
+
+	/*
+	 * There must be at least one xlog file in the pg_wal directory, since
+	 * we are doing backup-including-xlog.
+	 */
+	if (walFileList == NIL)
+		ereport(ERROR,
+				(errmsg("could not find any WAL files")));
+
+	/*
+	 * Sanity check: the first and last segment should cover startptr and
+	 * endptr, with no gaps in between.
+	 */
+	XLogFromFileName((char *) linitial(walFileList),
+					 &tli, &segno, wal_segment_size);
+	if (segno != startsegno)
+	{
+		char		startfname[MAXFNAMELEN];
+
+		XLogFileName(startfname, ThisTimeLineID, startsegno,
+					 wal_segment_size);
+		ereport(ERROR,
+				(errmsg("could not find WAL file \"%s\"", startfname)));
+	}
+	foreach(lc, walFileList)
+	{
+		char	   *walFileName = (char *) lfirst(lc);
+		XLogSegNo	currsegno = segno;
+		XLogSegNo	nextsegno = segno + 1;
+
+		XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
+		if (!(nextsegno == segno || currsegno == segno))
+		{
+			char		nextfname[MAXFNAMELEN];
+
+			XLogFileName(nextfname, ThisTimeLineID, nextsegno,
+						 wal_segment_size);
+			ereport(ERROR,
+					(errmsg("could not find WAL file \"%s\"", nextfname)));
+		}
+	}
+	if (segno != endsegno)
+	{
+		char		endfname[MAXFNAMELEN];
+
+		XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
+		ereport(ERROR,
+				(errmsg("could not find WAL file \"%s\"", endfname)));
+	}
+
+	/* Ok, we have everything we need. Send the WAL files. */
+	foreach(lc, walFileList)
+	{
+		char	   *walFileName = (char *) lfirst(lc);
+		FILE	   *fp;
+		char		buf[TAR_SEND_SIZE];
+		size_t		cnt;
+		pgoff_t		len = 0;
+
+		snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
+		XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
+
+		fp = AllocateFile(pathbuf, "rb");
+		if (fp == NULL)
+		{
+			int			save_errno = errno;
+
+			/*
+			 * Most likely reason for this is that the file was already
+			 * removed by a checkpoint, so check for that to get a better
+			 * error message.
+			 */
+			CheckXLogRemoved(segno, tli);
+
+			errno = save_errno;
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not open file \"%s\": %m", pathbuf)));
+		}
+
+		if (fstat(fileno(fp), &statbuf) != 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m",
+							pathbuf)));
+		if (statbuf.st_size != wal_segment_size)
+		{
+			CheckXLogRemoved(segno, tli);
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("unexpected WAL file size \"%s\"", walFileName)));
+		}
+
+		/* send the WAL file itself */
+		_tarWriteHeader(pathbuf, NULL, &statbuf, false);
+
+		while ((cnt = fread(buf, 1,
+							Min(sizeof(buf), wal_segment_size - len),
+							fp)) > 0)
+		{
+			CheckXLogRemoved(segno, tli);
+			/* Send the chunk as a CopyData message */
+			if (pq_putmessage('d', buf, cnt))
+				ereport(ERROR,
+						(errmsg("base backup could not send data, aborting backup")));
+
+			len += cnt;
+			throttle(cnt);
+
+			if (len == wal_segment_size)
+				break;
+		}
+
+		CHECK_FREAD_ERROR(fp, pathbuf);
+
+		if (len != wal_segment_size)
+		{
+			CheckXLogRemoved(segno, tli);
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("unexpected WAL file size \"%s\"", walFileName)));
+		}
+
+		/* wal_segment_size is a multiple of 512, so no need for padding */
+
+		FreeFile(fp);
+
+		/*
+		 * Mark file as archived, otherwise files can get archived again
+		 * after promotion of a new node. This is in line with
+		 * walreceiver.c always doing an XLogArchiveForceDone() after a
+		 * complete segment.
+		 */
+		StatusFilePath(pathbuf, walFileName, ".done");
+		sendFileWithContent(pathbuf, "");
+	}
+
+	/*
+	 * Send timeline history files too. Only the latest timeline history
+	 * file is required for recovery, and even that only if there happens
+	 * to be a timeline switch in the first WAL segment that contains the
+	 * checkpoint record, or if we're taking a base backup from a standby
+	 * server and the target timeline changes while the backup is taken.
+	 * But they are small and highly useful for debugging purposes, so
+	 * better include them all, always.
+	 */
+	foreach(lc, historyFileList)
+	{
+		char	   *fname = lfirst(lc);
+
+		snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
+
+		if (lstat(pathbuf, &statbuf) != 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m", pathbuf)));
+
+		sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid);
+
+		/* unconditionally mark file as archived */
+		StatusFilePath(pathbuf, fname, ".done");
+		sendFileWithContent(pathbuf, "");
+	}
+}
+
+/*
+ * Setup and activate network throttling, if client requested it
+ */
+static void
+setup_throttle(int maxrate)
+{
+	if (maxrate > 0)
+	{
+		throttling_sample =
+			(int64) maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
+
+		/*
+		 * The minimum amount of time for throttling_sample bytes to be
+		 * transferred.
+		 */
+		elapsed_min_unit = USECS_PER_SEC / THROTTLING_FREQUENCY;
+
+		/* Enable throttling. */
+		throttling_counter = 0;
+
+		/* The 'real data' starts now (header was ignored). */
+		throttled_last = GetCurrentTimestamp();
+	}
+	else
+	{
+		/* Disable throttling. */
+		throttling_counter = -1;
+	}
+}
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index d519252aad..5b0aa8ae85 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -350,6 +350,8 @@ extern XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast,
 									 bool needtblspcmapfile);
 extern XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive,
 									TimeLineID *stoptli_p);
+extern void collectTablespaces(List **tablespaces, StringInfo tblspcmapfile,
+							   bool infotbssize, bool needtblspcmapfile);
 extern void do_pg_abort_backup(void);
 extern SessionBackupState get_backup_status(void);
 
-- 
2.21.0 (Apple Git-122)