v1-0008-pg_verifybackup-Add-backup-format-and-compression.patch

From 9da0442e9f3f440d9e7a09f2ba2a4df7c10946ac Mon Sep 17 00:00:00 2001
From: Amul Sul <amul.sul@enterprisedb.com>
Date: Tue, 2 Jul 2024 10:26:35 +0530
Subject: [PATCH v1 08/10] pg_verifybackup: Add backup format and compression
 option

----
NOTE: This patch is not meant to be committed separately. It should
be squashed with the next patch that implements tar format support.
----
---
 src/bin/pg_verifybackup/pg_verifybackup.c | 143 +++++++++++++++++++++-
 1 file changed, 141 insertions(+), 2 deletions(-)

diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 375f196b300..0d458298f34 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <time.h>
 
+#include "common/compression.h"
 #include "common/parse_manifest.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
@@ -56,6 +57,8 @@ static void report_manifest_error(JsonManifestParseContext *context,
 								  const char *fmt,...)
 			pg_attribute_printf(2, 3) pg_attribute_noreturn();
 
+static char find_backup_format(verifier_context *context);
+static pg_compress_algorithm find_backup_compression(verifier_context *context);
 static void verify_backup_directory(verifier_context *context,
 									char *relpath, char *fullpath);
 static void verify_backup_file(verifier_context *context,
@@ -82,6 +85,9 @@ bool skip_checksums = false;
 static uint64 total_size = 0;
 static uint64 done_size = 0;
 
+char		format = '\0';		/* p(lain)/t(ar) */
+pg_compress_algorithm compress_algorithm = PG_COMPRESSION_NONE;
+
 /*
  * Main entry point.
  */
@@ -92,11 +98,13 @@ main(int argc, char **argv)
 		{"exit-on-error", no_argument, NULL, 'e'},
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
+		{"format", required_argument, NULL, 'F'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
 		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
+		{"compress", required_argument, NULL, 'Z'},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -107,6 +115,7 @@ main(int argc, char **argv)
 	bool		quiet = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
+	bool		tar_compression_specified = false;
 
 	pg_logging_init(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
@@ -149,7 +158,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:Z:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -168,6 +177,15 @@ main(int argc, char **argv)
 				manifest_path = pstrdup(optarg);
 				canonicalize_path(manifest_path);
 				break;
+			case 'F':
+				if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
+					format = 'p';
+				else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
+					format = 't';
+				else
+					pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
+							 optarg);
+				break;
 			case 'n':
 				no_parse_wal = true;
 				break;
@@ -184,6 +202,12 @@ main(int argc, char **argv)
 				wal_directory = pstrdup(optarg);
 				canonicalize_path(wal_directory);
 				break;
+			case 'Z':
+				if (!parse_compress_algorithm(optarg, &compress_algorithm))
+					pg_fatal("unrecognized compression algorithm: \"%s\"",
+							 optarg);
+				tar_compression_specified = true;
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -215,11 +239,41 @@ main(int argc, char **argv)
 		pg_fatal("cannot specify both %s and %s",
 				 "-P/--progress", "-q/--quiet");
 
+	/* Complain if compression method specified but the format isn't tar. */
+	if (format != 't' && tar_compression_specified)
+	{
+		pg_log_error("only tar mode backups can be compressed");
+		pg_log_error_hint("Try \"%s --help\" for more information.", progname);
+		exit(1);
+	}
+
+	/* Determine the backup format if it hasn't been specified. */
+	if (format == '\0')
+		format = find_backup_format(&context);
+
+	/*
+	 * Determine the tar backup compression method if it hasn't been
+	 * specified.
+	 */
+	if (format == 't' && !tar_compression_specified)
+		compress_algorithm = find_backup_compression(&context);
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
 		int			ret;
 
+		/*
+		 * XXX: In the future, we should consider enhancing pg_waldump to read
+		 * WAL files from the tar archive.
+		 */
+		if (format != 'p')
+		{
+			pg_log_error("pg_waldump does not support parsing WAL files from a tar archive.");
+			pg_log_error_hint("Try \"%s --help\" to skip parse WAL files option.", progname);
+			exit(1);
+		}
+
 		pg_waldump_path = pg_malloc(MAXPGPATH);
 		ret = find_other_exec(argv[0], "pg_waldump",
 							  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
@@ -274,8 +328,13 @@ main(int argc, char **argv)
 	/*
 	 * Now do the expensive work of verifying file checksums, unless we were
 	 * told to skip it.
+	 *
+	 * We were only checking the plain backup here. For the tar backup, file
+	 * checksums verification (if requested) will be done immediately when the
+	 * file is accessed, as we don't have random access to the files like we
+	 * do with plain backups.
 	 */
-	if (!skip_checksums)
+	if (!skip_checksums && format == 'p')
 		verify_backup_checksums(&context);
 
 	/*
@@ -1041,6 +1100,84 @@ progress_report(bool finished)
 	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
 }
 
+/*
+ * To detect the backup format, it checks for the PG_VERSION file in the backup
+ * directory. If found, it will be considered a plain-format backup; otherwise,
+ * it will be assumed to be a tar-format backup.
+ */
+static char
+find_backup_format(verifier_context *context)
+{
+	char		result;
+	char	   *path;
+	struct stat sb;
+
+	/* Should be here only if the backup format is unknown */
+	Assert(format == '\0');
+
+	/* Check PG_VERSION file. */
+	path = psprintf("%s/%s", context->backup_directory, "PG_VERSION");
+	result = (stat(path, &sb) == 0) ? 'p' : 't';
+	pfree(path);
+
+	return result;
+}
+
+/*
+ * To determine the compression format, we will search for the main data
+ * directory archive and its extension, which starts with base.tar, as
+ * pg_basebackup writes the main data directory to an archive file named
+ * base.tar followed by a compression type extension like .gz, .lz4, or .zst.
+ */
+static pg_compress_algorithm
+find_backup_compression(verifier_context *context)
+{
+	char	   *path;
+	struct stat sb;
+	bool		found;
+
+	/* Should be here only for tar backup */
+	Assert(format == 't');
+
+	/*
+	 * Is this a tar archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_NONE;
+
+	/*
+	 * Is this a .tar.gz archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.gz");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_GZIP;
+
+	/*
+	 * Is this a .tar.lz4 archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.lz4");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_LZ4;
+
+	/*
+	 * Is this a .tar.zst archive?
+	 */
+	path = psprintf("%s/%s", context->backup_directory, "base.tar.zst");
+	found = (stat(path, &sb) == 0);
+	pfree(path);
+	if (found)
+		return PG_COMPRESSION_ZSTD;
+
+	return PG_COMPRESSION_NONE; /* placate compiler */
+}
+
 /*
  * Print out usage information and exit.
  */
@@ -1053,11 +1190,13 @@ usage(void)
 	printf(_("  -e, --exit-on-error         exit immediately on error\n"));
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
+	printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
 	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
+	printf(_("  -Z, --compress=METHOD       compress method (gzip, lz4, zstd, none) \n"));
 	printf(_("  -V, --version               output version information, then exit\n"));
 	printf(_("  -?, --help                  show this help, then exit\n"));
 	printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
-- 
2.18.0