v1-0010-Create-and-use-bbarchiver-implementations-for-tar.patch

From 84ea8b5d51b7dcac17b250c0deaeb0d652471f46 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Thu, 7 May 2020 18:10:30 -0400
Subject: [PATCH v1 10/11] Create and use bbarchiver implementations for tar
 and tar sizing.

---
 src/backend/replication/Makefile         |   1 +
 src/backend/replication/basebackup.c     | 428 ++++++++++-------------
 src/backend/replication/basebackup_tar.c | 266 ++++++++++++++
 3 files changed, 454 insertions(+), 241 deletions(-)
 create mode 100644 src/backend/replication/basebackup_tar.c

diff --git a/src/backend/replication/Makefile b/src/backend/replication/Makefile
index aacccd350d..6b3c77f2c0 100644
--- a/src/backend/replication/Makefile
+++ b/src/backend/replication/Makefile
@@ -21,6 +21,7 @@ OBJS = \
 	basebackup_libpq.o \
 	basebackup_progress.o \
 	basebackup_sink.o \
+	basebackup_tar.o \
 	basebackup_throttle.o \
 	repl_gram.o \
 	slot.o \
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 1655806f1f..c606e7cf58 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -26,7 +26,7 @@
 #include "port.h"
 #include "postmaster/syslogger.h"
 #include "replication/basebackup.h"
-#include "replication/basebackup_sink.h"
+#include "replication/basebackup_archiver.h"
 #include "replication/backup_manifest.h"
 #include "replication/walsender.h"
 #include "replication/walsender_private.h"
@@ -55,20 +55,26 @@ typedef struct
 	pg_checksum_type manifest_checksum_type;
 } basebackup_options;
 
-static int64 sendTablespace(bbsink *sink, char *path, char *oid, bool sizeonly,
-							struct backup_manifest_info *manifest);
-static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
-					 List *tablespaces, bool sendtblspclinks,
-					 backup_manifest_info *manifest, const char *spcoid);
-static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
-					 struct stat *statbuf, bool missing_ok, Oid dboid,
-					 backup_manifest_info *manifest, const char *spcoid);
-static void sendFileWithContent(bbsink *sink, const char *filename,
-								const char *content,
-								backup_manifest_info *manifest);
-static int64 _tarWriteHeader(bbsink *sink, const char *filename,
-							 const char *linktarget, struct stat *statbuf,
-							 bool sizeonly);
+static void archive_database_cluster(List *tablespaces, bbarchiver *archiver,
+									 StringInfo labelfile,
+									 StringInfo tblspc_map_file,
+									 bool leave_main_tablespace_open,
+									 backup_manifest_info *manifest);
+static void archive_tablespace(bbarchiver *archiver, char *path, char *oid,
+							   struct backup_manifest_info *manifest);
+static void archive_directory(bbarchiver *archiver, const char *path,
+							  int basepathlen, List *tablespaces,
+							  bool sendtblspclinks,
+							  backup_manifest_info *manifest,
+							  const char *spcoid);
+static void archive_file(bbarchiver *archiver, const char *readfilename,
+						 const char *tarfilename, struct stat *statbuf,
+						 bool missing_ok, Oid dboid,
+						 backup_manifest_info *manifest, const char *spcoid);
+static void archive_file_with_content(bbarchiver *archiver,
+									  const char *filename,
+									  const char *content,
+									  backup_manifest_info *manifest);
 static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
 static void perform_base_backup(basebackup_options *opt);
 static void parse_basebackup_options(List *options, basebackup_options *opt);
@@ -242,6 +248,8 @@ perform_base_backup(basebackup_options *opt)
 	List	   *tablespaces = NIL;
 	bbsink	   *sink = bbsink_libpq_new();
 	bbsink	   *progress_sink;
+	bbarchiver *archiver;
+	bbarchiver *size_archiver;
 
 	/* Set up network throttling, if client requested it */
 	if (opt->maxrate > 0)
@@ -250,6 +258,10 @@ perform_base_backup(basebackup_options *opt)
 	/* Set up progress reporting. */
 	sink = progress_sink = bbsink_progress_new(sink, opt->progress);
 
+	/* Set up tar archiving. */
+	archiver = bbarchiver_tar_new(sink);
+	size_archiver = bbarchiver_tarsize_new();
+
 	/* we're going to use a BufFile, so we need a ResourceOwner */
 	Assert(CurrentResourceOwner == NULL);
 	CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
@@ -269,6 +281,8 @@ perform_base_backup(basebackup_options *opt)
 	startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
 								  labelfile, &tablespaces,
 								  tblspc_map_file, opt->sendtblspcmapfile);
+	if (!opt->sendtblspcmapfile)
+		tblspc_map_file = NULL;
 
 	/*
 	 * Once do_pg_start_backup has been called, ensure that any failure causes
@@ -279,7 +293,6 @@ perform_base_backup(basebackup_options *opt)
 
 	PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
 	{
-		ListCell   *lc;
 		tablespaceinfo *ti;
 
 		/*
@@ -299,89 +312,26 @@ perform_base_backup(basebackup_options *opt)
 		ti->size = -1;
 		tablespaces = lappend(tablespaces, ti);
 
-		/*
-		 * Calculate the total backup size by summing up the size of each
-		 * tablespace
-		 */
+		/* estimate sizes of all tablespaces, if PROGRESS option was given */
 		if (opt->progress)
 		{
 			basebackup_progress_estimate_backup_size();
 
-			foreach(lc, tablespaces)
-			{
-				tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
-
-				if (tmp->path == NULL)
-					tmp->size = sendDir(sink, ".", 1, true, tablespaces, true, NULL,
-										NULL);
-				else
-					tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
-											   NULL);
-			}
+			archive_database_cluster(tablespaces, size_archiver, labelfile,
+									 tblspc_map_file, false, NULL);
 		}
 
 		/* notify basebackup sink about start of backup */
 		bbsink_begin_backup(sink, startptr, starttli, tablespaces);
 
-		/* Send off our tablespaces one by one */
-		foreach(lc, tablespaces)
-		{
-			tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
-
-			if (ti->path == NULL)
-			{
-				struct stat statbuf;
-				bool	sendtblspclinks = true;
-
-				bbsink_begin_archive(sink, "base.tar");
-
-				/* In the main tar, include the backup_label first... */
-				sendFileWithContent(sink, BACKUP_LABEL_FILE, labelfile->data,
-									&manifest);
-
-				/* Then the tablespace_map file, if required... */
-				if (opt->sendtblspcmapfile)
-				{
-					sendFileWithContent(sink, TABLESPACE_MAP, tblspc_map_file->data,
-										&manifest);
-					sendtblspclinks = false;
-				}
-
-				/* Then the bulk of the files... */
-				sendDir(sink, ".", 1, false, tablespaces, sendtblspclinks,
-						&manifest, NULL);
-
-				/* ... and pg_control after everything else. */
-				if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
-					ereport(ERROR,
-							(errcode_for_file_access(),
-							 errmsg("could not stat file \"%s\": %m",
-									XLOG_CONTROL_FILE)));
-				sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
-						 false, InvalidOid, &manifest, NULL);
-			}
-			else
-			{
-				char   *archive_name = psprintf("%s.tar", ti->oid);
-
-				bbsink_begin_archive(sink, archive_name);
-
-				sendTablespace(sink, ti->path, ti->oid, false, &manifest);
-			}
-
-			/*
-			 * If we're including WAL, and this is the main data directory we
-			 * don't treat this as the end of the tablespace. Instead, we will
-			 * include the xlog files below and stop afterwards. This is safe
-			 * since the main data directory is always sent *last*.
-			 */
-			if (opt->includewal && ti->path == NULL)
-			{
-				Assert(lnext(tablespaces, lc) == NULL);
-			}
-			else
-				bbsink_end_archive(sink);
-		}
+		/*
+		 * Back up all of the tablespaces.
+		 *
+		 * If the backup is to include WAL, leave the main tablespace open,
+		 * so that we can archive the WAL files as well.
+		 */
+		archive_database_cluster(tablespaces, archiver, labelfile,
+								 tblspc_map_file, opt->includewal, &manifest);
 
 		basebackup_progress_wait_wal_archive(progress_sink);
 		endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
@@ -549,14 +499,13 @@ perform_base_backup(basebackup_options *opt)
 			}
 
 			/* send the WAL file itself */
-			_tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
-
+			bbarchiver_begin_file(archiver, pathbuf, &statbuf);
 			while ((cnt = fread(buf, 1,
 								Min(sizeof(buf), wal_segment_size - len),
 								fp)) > 0)
 			{
 				CheckXLogRemoved(segno, tli);
-				bbsink_archive_contents(sink, buf, cnt);
+				bbarchiver_file_contents(archiver, buf, cnt);
 
 				len += cnt;
 
@@ -574,11 +523,7 @@ perform_base_backup(basebackup_options *opt)
 						 errmsg("unexpected WAL file size \"%s\"", walFileName)));
 			}
 
-			/*
-			 * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
-			 * for padding.
-			 */
-			Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
+			bbarchiver_end_file(archiver);
 
 			FreeFile(fp);
 
@@ -589,7 +534,7 @@ perform_base_backup(basebackup_options *opt)
 			 * complete segment.
 			 */
 			StatusFilePath(pathbuf, walFileName, ".done");
-			sendFileWithContent(sink, pathbuf, "", &manifest);
+			archive_file_with_content(archiver, pathbuf, "", &manifest);
 		}
 
 		/*
@@ -612,12 +557,12 @@ perform_base_backup(basebackup_options *opt)
 						(errcode_for_file_access(),
 						 errmsg("could not stat file \"%s\": %m", pathbuf)));
 
-			sendFile(sink, pathbuf, pathbuf, &statbuf, false, InvalidOid,
-					 &manifest, NULL);
+			archive_file(archiver, pathbuf, pathbuf, &statbuf, false,
+						 InvalidOid, &manifest, NULL);
 
 			/* unconditionally mark file as archived */
 			StatusFilePath(pathbuf, fname, ".done");
-			sendFileWithContent(sink, pathbuf, "", &manifest);
+			archive_file_with_content(archiver, pathbuf, "", &manifest);
 		}
 
 		bbsink_end_archive(sink);
@@ -646,6 +591,74 @@ perform_base_backup(basebackup_options *opt)
 	basebackup_progress_done();
 }
 
+/*
+ * Iterate over the entire cluster and feed each tablespace to the archiver
+ * in turn.
+ */
+static void
+archive_database_cluster(List *tablespaces, bbarchiver *archiver,
+						 StringInfo labelfile, StringInfo tblspc_map_file,
+						 bool leave_main_tablespace_open,
+						 backup_manifest_info *manifest)
+{
+	ListCell   *lc;
+
+	/* Send off our tablespaces one by one */
+	foreach(lc, tablespaces)
+	{
+		tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
+
+		bbarchiver_begin_tablespace(archiver, ti);
+
+		if (ti->path == NULL)
+		{
+			struct stat statbuf;
+			bool	sendtblspclinks = true;
+
+			/* For the main tablespace, archive the backup_label first... */
+			archive_file_with_content(archiver, BACKUP_LABEL_FILE,
+									  labelfile->data, manifest);
+
+			/* Then the tablespace_map file, if present... */
+			if (tblspc_map_file != NULL)
+			{
+				archive_file_with_content(archiver, TABLESPACE_MAP,
+										  tblspc_map_file->data,
+										  manifest);
+				sendtblspclinks = false;
+			}
+
+			/* Then the bulk of the files... */
+			archive_directory(archiver, ".", 1, tablespaces,
+							  sendtblspclinks, manifest, NULL);
+
+			/* ... and pg_control after everything else. */
+			if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
+				ereport(ERROR,
+						(errcode_for_file_access(),
+						 errmsg("could not stat file \"%s\": %m",
+								XLOG_CONTROL_FILE)));
+			archive_file(archiver, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE,
+						 &statbuf, false, InvalidOid, manifest, NULL);
+		}
+		else
+			archive_tablespace(archiver, ti->path, ti->oid, manifest);
+
+		/*
+		 * If we were asked to leave the main tablespace open, then do so.
+		 * This is safe since the main data directory is always sent *last*,
+		 * so we'll not try to begin another tablespace without ending this
+		 * one.
+		 */
+		if (leave_main_tablespace_open && ti->path == NULL)
+		{
+			Assert(lnext(tablespaces, lc) == NULL);
+		}
+		else
+			bbarchiver_end_tablespace(archiver);
+	}
+}
+
 /*
  * list_sort comparison function, to compare log/seg portion of WAL segment
  * filenames, ignoring the timeline portion.
@@ -854,18 +867,20 @@ SendBaseBackup(BaseBackupCmd *cmd)
 }
 
 /*
- * Inject a file with given name and content in the output tar stream.
+ * Feed a file to the archiver that does not actually exist in the source
+ * directory. We use this to inject things like the backup_label file into
+ * the backup.
  */
 static void
-sendFileWithContent(bbsink *sink, const char *filename, const char *content,
-					backup_manifest_info *manifest)
+archive_file_with_content(bbarchiver *archiver, const char *filename,
+						  const char *content, backup_manifest_info *manifest)
 {
 	struct stat statbuf;
-	int			pad,
-				len;
+	int			len;
 	pg_checksum_context checksum_ctx;
 
-	pg_checksum_init(&checksum_ctx, manifest->checksum_type);
+	if (manifest != NULL)
+		pg_checksum_init(&checksum_ctx, manifest->checksum_type);
 
 	len = strlen(content);
 
@@ -885,22 +900,17 @@ sendFileWithContent(bbsink *sink, const char *filename, const char *content,
 	statbuf.st_mode = pg_file_create_mode;
 	statbuf.st_size = len;
 
-	_tarWriteHeader(sink, filename, NULL, &statbuf, false);
-	bbsink_archive_contents(sink, content, len);
+	bbarchiver_begin_file(archiver, filename, &statbuf);
+	if (bbarchiver_needs_file_contents(archiver))
+		bbarchiver_file_contents(archiver, content, len);
+	bbarchiver_end_file(archiver);
 
-	/* Pad to a multiple of the tar block size. */
-	pad = tarPaddingBytesRequired(len);
-	if (pad > 0)
+	if (manifest != NULL)
 	{
-		char		buf[TAR_BLOCK_SIZE];
-
-		MemSet(buf, 0, pad);
-		bbsink_archive_contents(sink, buf, pad);
+		pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
+		AddFileToBackupManifest(manifest, NULL, filename, len,
+								(pg_time_t) statbuf.st_mtime, &checksum_ctx);
 	}
-
-	pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
-	AddFileToBackupManifest(manifest, NULL, filename, len,
-							(pg_time_t) statbuf.st_mtime, &checksum_ctx);
 }
 
 /*
@@ -910,11 +920,10 @@ sendFileWithContent(bbsink *sink, const char *filename, const char *content,
  *
  * Only used to send auxiliary tablespaces, not PGDATA.
  */
-static int64
-sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
-			   backup_manifest_info *manifest)
+static void
+archive_tablespace(bbarchiver *archiver, char *path, char *spcoid,
+				   backup_manifest_info *manifest)
 {
-	int64		size;
 	char		pathbuf[MAXPGPATH];
 	struct stat statbuf;
 
@@ -938,17 +947,14 @@ sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
 							pathbuf)));
 
 		/* If the tablespace went away while scanning, it's no error. */
-		return 0;
+		return;
 	}
 
-	size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
-						   sizeonly);
+	bbarchiver_directory(archiver, TABLESPACE_VERSION_DIRECTORY, &statbuf);
 
 	/* Send all the files in the tablespace version directory */
-	size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
-					spcoid);
-
-	return size;
+	archive_directory(archiver, pathbuf, strlen(path), NIL, true, manifest,
+					  spcoid);
 }
 
 /*
@@ -963,16 +969,15 @@ sendTablespace(bbsink *sink, char *path, char *spcoid, bool sizeonly,
  * information in the tar file. If not, we can skip that
  * as it will be sent separately in the tablespace_map file.
  */
-static int64
-sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
-		List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
-		const char *spcoid)
+static void
+archive_directory(bbarchiver *archiver, const char *path, int basepathlen,
+				  List *tablespaces, bool sendtblspclinks,
+				  backup_manifest_info *manifest, const char *spcoid)
 {
 	DIR		   *dir;
 	struct dirent *de;
 	char		pathbuf[MAXPGPATH * 2];
 	struct stat statbuf;
-	int64		size = 0;
 	const char *lastDir;		/* Split last dir from parent path. */
 	bool		isDbDir = false;	/* Does this directory contain relations? */
 
@@ -1125,8 +1130,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 			{
 				elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
 				convert_link_to_directory(pathbuf, &statbuf);
-				size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
-										&statbuf, sizeonly);
+				bbarchiver_directory(archiver, pathbuf + basepathlen + 1,
+									 &statbuf);
 				excludeFound = true;
 				break;
 			}
@@ -1143,8 +1148,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 		{
 			elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
 			convert_link_to_directory(pathbuf, &statbuf);
-			size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
-									&statbuf, sizeonly);
+			bbarchiver_directory(archiver, pathbuf + basepathlen + 1,
+								 &statbuf);
 			continue;
 		}
 
@@ -1157,15 +1162,15 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 		{
 			/* If pg_wal is a symlink, write it as a directory anyway */
 			convert_link_to_directory(pathbuf, &statbuf);
-			size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
-									&statbuf, sizeonly);
+			bbarchiver_directory(archiver, pathbuf + basepathlen + 1,
+								 &statbuf);
 
 			/*
 			 * Also send archive_status directory (by hackishly reusing
 			 * statbuf from above ...).
 			 */
-			size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
-									&statbuf, sizeonly);
+			bbarchiver_directory(archiver, "./pg_wal/archive_status",
+								 &statbuf);
 
 			continue;			/* don't recurse into pg_wal */
 		}
@@ -1196,8 +1201,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 								pathbuf)));
 			linkpath[rllen] = '\0';
 
-			size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
-									&statbuf, sizeonly);
+			bbarchiver_symbolic_link(archiver, pathbuf + basepathlen + 1,
+									 linkpath, &statbuf);
 #else
 
 			/*
@@ -1220,8 +1225,8 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 			 * Store a directory entry in the tar file so we can get the
 			 * permissions right.
 			 */
-			size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
-									sizeonly);
+			bbarchiver_directory(archiver, pathbuf + basepathlen + 1,
+								 &statbuf);
 
 			/*
 			 * Call ourselves recursively for a directory, unless it happens
@@ -1252,36 +1257,21 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 				skip_this_dir = true;
 
 			if (!skip_this_dir)
-				size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
-								sendtblspclinks, manifest, spcoid);
+				archive_directory(archiver, pathbuf, basepathlen, tablespaces,
+								  sendtblspclinks, manifest, spcoid);
 		}
 		else if (S_ISREG(statbuf.st_mode))
 		{
-			bool		sent = false;
-
-			if (!sizeonly)
-				sent = sendFile(sink, pathbuf, pathbuf + basepathlen + 1, &statbuf,
-								true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
-								manifest, spcoid);
-
-			if (sent || sizeonly)
-			{
-				/* Add size. */
-				size += statbuf.st_size;
-
-				/* Pad to a multiple of the tar block size. */
-				size += tarPaddingBytesRequired(statbuf.st_size);
-
-				/* Size of the header for the file. */
-				size += TAR_BLOCK_SIZE;
-			}
+			archive_file(archiver, pathbuf, pathbuf + basepathlen + 1,
+						 &statbuf, true,
+						 isDbDir ? atooid(lastDir + 1) : InvalidOid,
+						 manifest, spcoid);
 		}
 		else
 			ereport(WARNING,
 					(errmsg("skipping special file \"%s\"", pathbuf)));
 	}
 	FreeDir(dir);
-	return size;
 }
 
 /*
@@ -1332,14 +1322,11 @@ is_checksummed_file(const char *fullpath, const char *filename)
  *
  * If dboid is anything other than InvalidOid then any checksum failures detected
  * will get reported to the stats collector.
- *
- * Returns true if the file was successfully sent, false if 'missing_ok',
- * and the file did not exist.
  */
-static bool
-sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
-		 struct stat *statbuf, bool missing_ok, Oid dboid,
-		 backup_manifest_info *manifest, const char *spcoid)
+static void
+archive_file(bbarchiver *archiver, const char *readfilename,
+			 const char *tarfilename, struct stat *statbuf, bool missing_ok,
+			 Oid dboid, backup_manifest_info *manifest, const char *spcoid)
 {
 	FILE	   *fp;
 	BlockNumber blkno = 0;
@@ -1351,27 +1338,33 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
 	int			i;
 	pgoff_t		len = 0;
 	char	   *page;
-	size_t		pad;
 	PageHeader	phdr;
 	int			segmentno = 0;
 	char	   *segmentpath;
 	bool		verify_checksum = false;
 	pg_checksum_context checksum_ctx;
 
-	pg_checksum_init(&checksum_ctx, manifest->checksum_type);
+	if (manifest != NULL)
+		pg_checksum_init(&checksum_ctx, manifest->checksum_type);
+
+	bbarchiver_begin_file(archiver, tarfilename, statbuf);
+
+	if (!bbarchiver_needs_file_contents(archiver))
+	{
+		bbarchiver_end_file(archiver);
+		return;
+	}
 
 	fp = AllocateFile(readfilename, "rb");
 	if (fp == NULL)
 	{
 		if (errno == ENOENT && missing_ok)
-			return false;
+			return;
 		ereport(ERROR,
 				(errcode_for_file_access(),
 				 errmsg("could not open file \"%s\": %m", readfilename)));
 	}
 
-	_tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
-
 	if (!noverify_checksums && DataChecksumsEnabled())
 	{
 		char	   *filename;
@@ -1523,10 +1516,11 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
 			}
 		}
 
-		bbsink_archive_contents(sink, buf, cnt);
+		bbarchiver_file_contents(archiver, buf, cnt);
 
 		/* Also feed it to the checksum machinery. */
-		pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
+		if (manifest != NULL)
+			pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
 
 		len += cnt;
 
@@ -1550,23 +1544,14 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
 		while (len < statbuf->st_size)
 		{
 			cnt = Min(sizeof(buf), statbuf->st_size - len);
-			bbsink_archive_contents(sink, buf, cnt);
-			pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
+			bbarchiver_file_contents(archiver, buf, cnt);
+			if (manifest != NULL)
+				pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
 			len += cnt;
 		}
 	}
 
-	/*
-	 * Pad to a block boundary, per tar format requirements. (This small
-	 * piece of data is probably not worth throttling, and is not checksummed
-	 * because it's not actually part of the file.)
-	 */
-	pad = tarPaddingBytesRequired(len);
-	if (pad > 0)
-	{
-		MemSet(buf, 0, pad);
-		bbsink_archive_contents(sink, buf, pad);
-	}
+	bbarchiver_end_file(archiver);
 
 	FreeFile(fp);
 
@@ -1583,54 +1568,15 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
 
 	total_checksum_failures += checksum_failures;
 
-	AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
-							(pg_time_t) statbuf->st_mtime, &checksum_ctx);
-
-	return true;
-}
-
-
-static int64
-_tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
-				struct stat *statbuf, bool sizeonly)
-{
-	char		h[TAR_BLOCK_SIZE];
-	enum tarError rc;
-
-	if (!sizeonly)
-	{
-		rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
-							 statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
-							 statbuf->st_mtime);
-
-		switch (rc)
-		{
-			case TAR_OK:
-				break;
-			case TAR_NAME_TOO_LONG:
-				ereport(ERROR,
-						(errmsg("file name too long for tar format: \"%s\"",
-								filename)));
-				break;
-			case TAR_SYMLINK_TOO_LONG:
-				ereport(ERROR,
-						(errmsg("symbolic link target too long for tar format: "
-								"file name \"%s\", target \"%s\"",
-								filename, linktarget)));
-				break;
-			default:
-				elog(ERROR, "unrecognized tar error: %d", rc);
-		}
-
-		bbsink_archive_contents(sink, h, sizeof(h));
-	}
-
-	return sizeof(h);
+	if (manifest != NULL)
+		AddFileToBackupManifest(manifest, spcoid, tarfilename,
+								statbuf->st_size,
+								(pg_time_t) statbuf->st_mtime, &checksum_ctx);
 }
 
 /*
  * If the entry in statbuf is a link, then adjust statbuf to make it look like a
- * directory, so that it will be written that way.
+ * directory.
  */
 static void
 convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
diff --git a/src/backend/replication/basebackup_tar.c b/src/backend/replication/basebackup_tar.c
new file mode 100644
index 0000000000..8618f2a0ec
--- /dev/null
+++ b/src/backend/replication/basebackup_tar.c
@@ -0,0 +1,266 @@
+#include "postgres.h"
+
+#include "pgtar.h"
+#include "replication/basebackup_archiver.h"
+
+typedef struct bbarchiver_tar
+{
+	bbarchiver	base;
+	bbsink	   *sink;
+	size_t		file_len;
+} bbarchiver_tar;
+
+typedef struct bbarchiver_tarsize
+{
+	bbarchiver	base;
+	tablespaceinfo *tsinfo;
+} bbarchiver_tarsize;
+
+static void bbarchiver_tar_begin_tablespace(bbarchiver *archiver,
+											tablespaceinfo *tsinfo);
+static void bbarchiver_tar_end_tablespace(bbarchiver *archiver);
+static void bbarchiver_tar_begin_file(bbarchiver *archiver,
+									  const char *relative_path,
+									  struct stat *statbuf);
+static void bbarchiver_tar_file_contents(bbarchiver *archiver,
+										 const char *data,
+										 size_t len);
+static void bbarchiver_tar_end_file(bbarchiver *archiver);
+static void bbarchiver_tar_directory(bbarchiver *archiver,
+									 const char *relative_path,
+									 struct stat *statbuf);
+static void bbarchiver_tar_symbolic_link(bbarchiver *archiver,
+										 const char *relative_path,
+										 const char *linktarget,
+										 struct stat *statbuf);
+static void report_tar_error(enum tarError rc, const char *filename,
+							 const char *linktarget);
+
+static void bbarchiver_tarsize_begin_tablespace(bbarchiver *archiver,
+												tablespaceinfo *tsinfo);
+static void bbarchiver_tarsize_begin_file(bbarchiver *archiver,
+										  const char *relative_path,
+										  struct stat *statbuf);
+static void bbarchiver_tarsize_directory(bbarchiver *archiver,
+										 const char *relative_path,
+										 struct stat *statbuf);
+static void bbarchiver_tarsize_symbolic_link(bbarchiver *archiver,
+											 const char *relative_path,
+											 const char *linktarget,
+											 struct stat *statbuf);
+static void add_tar_size(bbarchiver *archiver, uint64 file_size);
+
+const		bbarchiver_ops bbarchiver_tar_ops = {
+	.begin_tablespace = bbarchiver_tar_begin_tablespace,
+	.end_tablespace = bbarchiver_tar_end_tablespace,
+	.begin_file = bbarchiver_tar_begin_file,
+	.file_contents = bbarchiver_tar_file_contents,
+	.end_file = bbarchiver_tar_end_file,
+	.directory = bbarchiver_tar_directory,
+	.symbolic_link = bbarchiver_tar_symbolic_link,
+};
+
+const		bbarchiver_ops bbarchiver_tarsize_ops = {
+	.begin_tablespace = bbarchiver_tarsize_begin_tablespace,
+	.end_tablespace = bbarchiver_noop_end_tablespace,
+	.begin_file = bbarchiver_tarsize_begin_file,
+	.file_contents = NULL,
+	.end_file = bbarchiver_noop_end_file,
+	.directory = bbarchiver_tarsize_directory,
+	.symbolic_link = bbarchiver_tarsize_symbolic_link,
+};
+
+bbarchiver *
+bbarchiver_tar_new(bbsink *sink)
+{
+	bbarchiver_tar *archiver = palloc0(sizeof(bbarchiver_tar));
+
+	*((const bbarchiver_ops **) &archiver->base.bba_ops) = &bbarchiver_tar_ops;
+	archiver->base.bba_next = NULL;
+	archiver->sink = sink;
+
+	return &archiver->base;
+}
+
+static void
+bbarchiver_tar_begin_tablespace(bbarchiver *archiver, tablespaceinfo *tsinfo)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+	char	   *archive_name = "base.tar";
+
+	if (tsinfo->path != NULL)
+		archive_name = psprintf("%s.tar", tsinfo->oid);
+
+	bbsink_begin_archive(myarchiver->sink, archive_name);
+}
+
+static void
+bbarchiver_tar_end_tablespace(bbarchiver *archiver)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+
+	bbsink_end_archive(myarchiver->sink);
+}
+
+static void
+bbarchiver_tar_begin_file(bbarchiver *archiver, const char *relative_path,
+						  struct stat *statbuf)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+	char		h[TAR_BLOCK_SIZE];
+	enum tarError rc;
+
+	myarchiver->file_len = 0;
+
+	rc = tarCreateHeader(h, relative_path, NULL, statbuf->st_size,
+						 statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
+						 statbuf->st_mtime);
+	if (rc != TAR_OK)
+		report_tar_error(rc, relative_path, NULL);
+
+	bbsink_archive_contents(myarchiver->sink, h, sizeof(h));
+}
+
+static void
+bbarchiver_tar_file_contents(bbarchiver *archiver, const char *data,
+							 size_t len)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+
+	myarchiver->file_len += len;
+	bbsink_archive_contents(myarchiver->sink, data, len);
+}
+
+static void
+bbarchiver_tar_end_file(bbarchiver *archiver)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+	int			pad;
+
+	/* Pad to a block boundary, per tar format requirements. */
+	pad = tarPaddingBytesRequired(myarchiver->file_len);
+	if (pad > 0)
+	{
+		char		buf[TAR_BLOCK_SIZE];
+
+		MemSet(buf, 0, pad);
+		bbsink_archive_contents(myarchiver->sink, buf, pad);
+	}
+}
+
+static void
+bbarchiver_tar_directory(bbarchiver *archiver, const char *relative_path,
+						 struct stat *statbuf)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+	char		h[TAR_BLOCK_SIZE];
+	enum tarError rc;
+
+	rc = tarCreateHeader(h, relative_path, NULL, statbuf->st_size,
+						 statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
+						 statbuf->st_mtime);
+	if (rc != TAR_OK)
+		report_tar_error(rc, relative_path, NULL);
+
+	bbsink_archive_contents(myarchiver->sink, h, sizeof(h));
+}
+
+static void
+bbarchiver_tar_symbolic_link(bbarchiver *archiver, const char *relative_path,
+							 const char *linktarget, struct stat *statbuf)
+{
+	bbarchiver_tar *myarchiver = (bbarchiver_tar *) archiver;
+	char		h[TAR_BLOCK_SIZE];
+	enum tarError rc;
+
+	rc = tarCreateHeader(h, relative_path, linktarget, statbuf->st_size,
+						 statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
+						 statbuf->st_mtime);
+	if (rc != TAR_OK)
+		report_tar_error(rc, relative_path, NULL);
+
+	bbsink_archive_contents(myarchiver->sink, h, sizeof(h));
+}
+
+static void
+report_tar_error(enum tarError rc, const char *filename,
+				 const char *linktarget)
+{
+	switch (rc)
+	{
+		case TAR_OK:
+			break;
+		case TAR_NAME_TOO_LONG:
+			ereport(ERROR,
+					(errmsg("file name too long for tar format: \"%s\"",
+							filename)));
+			break;
+		case TAR_SYMLINK_TOO_LONG:
+			Assert(linktarget != NULL);
+			ereport(ERROR,
+					(errmsg("symbolic link target too long for tar format: "
+							"file name \"%s\", target \"%s\"",
+							filename, linktarget)));
+			break;
+		default:
+			elog(ERROR, "unrecognized tar error: %d", rc);
+	}
+}
+
+/*
+ * Create an archiver that calculates an estimated size for a tar file built
+ * from the files visited.
+ */
+bbarchiver *
+bbarchiver_tarsize_new(void)
+{
+	bbarchiver_tarsize *archiver = palloc0(sizeof(bbarchiver_tarsize));
+
+	*((const bbarchiver_ops **) &archiver->base.bba_ops) =
+		&bbarchiver_tarsize_ops;
+	archiver->base.bba_next = NULL;
+
+	return &archiver->base;
+}
+
+static void
+bbarchiver_tarsize_begin_tablespace(bbarchiver *archiver,
+									tablespaceinfo *tsinfo)
+{
+	bbarchiver_tarsize *myarchiver = (bbarchiver_tarsize *) archiver;
+
+	myarchiver->tsinfo = tsinfo;
+	tsinfo->size = 0;
+}
+
+static void
+bbarchiver_tarsize_begin_file(bbarchiver *archiver, const char *relative_path,
+							  struct stat *statbuf)
+{
+	add_tar_size(archiver, statbuf->st_size);
+}
+
+static void
+bbarchiver_tarsize_directory(bbarchiver *archiver, const char *relative_path,
+							 struct stat *statbuf)
+{
+	add_tar_size(archiver, 0);
+}
+
+static void
+bbarchiver_tarsize_symbolic_link(bbarchiver *archiver,
+								 const char *relative_path,
+								 const char *linktarget,
+								 struct stat *statbuf)
+{
+	add_tar_size(archiver, 0);
+}
+
+static void
+add_tar_size(bbarchiver *archiver, uint64 file_size)
+{
+	bbarchiver_tarsize *myarchiver = (bbarchiver_tarsize *) archiver;
+
+	myarchiver->tsinfo->size +=
+		TAR_BLOCK_SIZE + file_size + tarPaddingBytesRequired(file_size);
+}
-- 
2.24.2 (Apple Git-127)