v1-0001-Add-progress-reporting-to-pg_verifybackup.patch

From 146923a725a918416482b5a9391e10b5df0ab8e5 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 6 Jan 2023 13:26:09 +0900
Subject: [PATCH v1] Add progress reporting to pg_verifybackup

This adds a new option to pg_verifybackup called -P/--progress,
showing every second some information about the state of checksum
verification.

Similarly to what is done for pg_rewind and pg_basebackup, the
information printed in the progress report consists of the current
amount of data computed and the total amount of data to compute.  This
could be extended later on.
---
 doc/src/sgml/ref/pg_verifybackup.sgml     | 15 ++++
 src/bin/pg_verifybackup/pg_verifybackup.c | 85 +++++++++++++++++++++--
 2 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml
index 5f83c98706..36335e0a18 100644
--- a/doc/src/sgml/ref/pg_verifybackup.sgml
+++ b/doc/src/sgml/ref/pg_verifybackup.sgml
@@ -178,6 +178,21 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-P</option></term>
+      <term><option>--progress</option></term>
+      <listitem>
+       <para>
+        Enable progress reporting. Turning this on will deliver a progress
+        report while verifying checksums.
+       </para>
+       <para>
+        This option cannot be used together with the option
+        <option>--quiet</option>.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>-q</option></term>
       <term><option>--quiet</option></term>
diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c
index 7634dfc285..b2d54ce25f 100644
--- a/src/bin/pg_verifybackup/pg_verifybackup.c
+++ b/src/bin/pg_verifybackup/pg_verifybackup.c
@@ -16,12 +16,14 @@
 #include <dirent.h>
 #include <fcntl.h>
 #include <sys/stat.h>
+#include <time.h>
 
 #include "common/hashfn.h"
 #include "common/logging.h"
 #include "fe_utils/simple_list.h"
 #include "getopt_long.h"
 #include "parse_manifest.h"
+#include "pgtime.h"
 
 /*
  * For efficiency, we'd like our hash table containing information about the
@@ -57,6 +59,8 @@ typedef struct manifest_file
 	bool		matched;
 	bool		bad;
 } manifest_file;
+#define should_check_checksums(m) \
+	(((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
 
 /*
  * Define a hash table which we can use to store information about the files
@@ -147,10 +151,19 @@ static void report_fatal_error(const char *pg_restrict fmt,...)
 			pg_attribute_printf(1, 2) pg_attribute_noreturn();
 static bool should_ignore_relpath(verifier_context *context, char *relpath);
 
+static void progress_report(bool finished);
 static void usage(void);
 
 static const char *progname;
 
+/* options */
+static bool show_progress = false;
+static bool skip_checksums = false;
+
+/* Progress indicators */
+static uint64 total_size = 0;
+static uint64 done_size = 0;
+
 /*
  * Main entry point.
  */
@@ -162,6 +175,7 @@ main(int argc, char **argv)
 		{"ignore", required_argument, NULL, 'i'},
 		{"manifest-path", required_argument, NULL, 'm'},
 		{"no-parse-wal", no_argument, NULL, 'n'},
+		{"progress", no_argument, NULL, 'P'},
 		{"quiet", no_argument, NULL, 'q'},
 		{"skip-checksums", no_argument, NULL, 's'},
 		{"wal-directory", required_argument, NULL, 'w'},
@@ -174,7 +188,6 @@ main(int argc, char **argv)
 	char	   *manifest_path = NULL;
 	bool		no_parse_wal = false;
 	bool		quiet = false;
-	bool		skip_checksums = false;
 	char	   *wal_directory = NULL;
 	char	   *pg_waldump_path = NULL;
 
@@ -219,7 +232,7 @@ main(int argc, char **argv)
 	simple_string_list_append(&context.ignore_list, "recovery.signal");
 	simple_string_list_append(&context.ignore_list, "standby.signal");
 
-	while ((c = getopt_long(argc, argv, "ei:m:nqsw:", long_options, NULL)) != -1)
+	while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
 	{
 		switch (c)
 		{
@@ -241,6 +254,9 @@ main(int argc, char **argv)
 			case 'n':
 				no_parse_wal = true;
 				break;
+			case 'P':
+				show_progress = true;
+				break;
 			case 'q':
 				quiet = true;
 				break;
@@ -277,6 +293,10 @@ main(int argc, char **argv)
 		exit(1);
 	}
 
+	/* Complain if the specified arguments conflict */
+	if (show_progress && quiet)
+		pg_fatal("cannot specify both --progress and --quiet");
+
 	/* Unless --no-parse-wal was specified, we will need pg_waldump. */
 	if (!no_parse_wal)
 	{
@@ -638,6 +658,10 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
 		m->bad = true;
 	}
 
+	/* Update statistics for progress report, if necessary */
+	if (show_progress && !skip_checksums && should_check_checksums(m))
+		total_size += m->size;
+
 	/*
 	 * We don't verify checksums at this stage. We first finish verifying that
 	 * we have the expected set of files with the expected sizes, and only
@@ -675,11 +699,12 @@ verify_backup_checksums(verifier_context *context)
 	manifest_files_iterator it;
 	manifest_file *m;
 
+	progress_report(false);
+
 	manifest_files_start_iterate(context->ht, &it);
 	while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
 	{
-		if (m->matched && !m->bad && m->checksum_type != CHECKSUM_TYPE_NONE &&
-			!should_ignore_relpath(context, m->pathname))
+		if (should_check_checksums(m) && !should_ignore_relpath(context, m->pathname))
 		{
 			char	   *fullpath;
 
@@ -694,6 +719,8 @@ verify_backup_checksums(verifier_context *context)
 			pfree(fullpath);
 		}
 	}
+
+	progress_report(true);
 }
 
 /*
@@ -740,6 +767,10 @@ verify_file_checksum(verifier_context *context, manifest_file *m,
 			close(fd);
 			return;
 		}
+
+		/* Report progress */
+		done_size += rc;
+		progress_report(false);
 	}
 	if (rc < 0)
 		report_backup_error(context, "could not read file \"%s\": %m",
@@ -894,6 +925,51 @@ hash_string_pointer(char *s)
 	return hash_bytes(ss, strlen(s));
 }
 
+/*
+ * Print a progress report based on the global variables.
+ *
+ * Progress report is written at maximum once per second, unless the finished
+ * parameter is set to true.
+ *
+ * If finished is set to true, this is the last progress report. The cursor
+ * is moved to the next line.
+ */
+static void
+progress_report(bool finished)
+{
+	static pg_time_t last_progress_report = 0;
+	pg_time_t	now;
+	int 	percent_size = 0;
+	char	totalsize_str[32];
+	char	donesize_str[32];
+
+	if (!show_progress)
+		return;
+
+	now = time(NULL);
+	if (now == last_progress_report && !finished)
+		return;					/* Max once per second */
+
+	last_progress_report = now;
+	percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
+
+	snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
+			 total_size / 1024);
+	snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
+			 done_size / 1024);
+
+	fprintf(stderr,
+			_("%*s/%s kB (%d%%) verified"),
+			(int) strlen(totalsize_str),
+			donesize_str, totalsize_str, percent_size);
+
+	/*
+	 * Stay on the same line if reporting to a terminal and we're not done
+	 * yet.
+	 */
+	fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
+}
+
 /*
  * Print out usage information and exit.
  */
@@ -907,6 +983,7 @@ usage(void)
 	printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
 	printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
 	printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
+	printf(_("  -P, --progress              show progress information\n"));
 	printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
 	printf(_("  -s, --skip-checksums        skip checksum verification\n"));
 	printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
-- 
2.31.1