v3-0002-Implementation-of-backup-validator.patch

From bdaa60fed1d2b6f9001c74d21197f15078cc64f2 Mon Sep 17 00:00:00 2001
From: Suraj Kharage <suraj.kharage@enterprisedb.com>
Date: Tue, 17 Dec 2019 10:56:42 +0530
Subject: [PATCH v3 2/3] Implementation of backup validator

Patch by Suraj Kharage, inputs from Robert Haas, review from Jeevan Chalke,
and Robert Haas.
---
 doc/src/sgml/ref/pg_basebackup.sgml   |  12 +
 src/bin/pg_basebackup/pg_basebackup.c | 416 ++++++++++++++++++++++++++++++++++
 src/common/encode.c                   |  18 ++
 src/include/common/encode.h           |   1 +
 src/tools/pgindent/typedefs.list      |   1 +
 5 files changed, 448 insertions(+)

diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index af7c731..043ee39 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -548,6 +548,18 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>--verify-backup </option></term>
+      <listitem>
+       <para>
+        Validate the given backup directory and detect the modification if any
+        without restarting the server. For plain backup, provide the backup
+        directory path with <option>--pgdata</option> option. Tar format
+        backups can be verified after untarring.
+       </para>
+      </listitem>
+     </varlistentry>
+
     </variablelist>
    </para>
 
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 7d5ed0d..c3c3c85 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -27,9 +27,12 @@
 #endif
 
 #include "access/xlog_internal.h"
+#include "common/checksum_utils.h"
+#include "common/encode.h"
 #include "common/file_perm.h"
 #include "common/file_utils.h"
 #include "common/logging.h"
+#include "common/sha2.h"
 #include "common/string.h"
 #include "fe_utils/recovery_gen.h"
 #include "fe_utils/string_utils.h"
@@ -44,6 +47,8 @@
 
 #define ERRCODE_DATA_CORRUPTED	"XX001"
 
+#define CHECKSUM_LENGTH 256
+
 typedef struct TablespaceListCell
 {
 	struct TablespaceListCell *next;
@@ -97,6 +102,30 @@ typedef struct WriteManifestState
 typedef void (*WriteDataCallback) (size_t nbytes, char *buf,
 								   void *callback_data);
 
+typedef	struct DataDirectoryFileInfo
+{
+	char 		filetype[10];
+	char	   *filename;
+	int			filesize;
+	char		checksum[CHECKSUM_LENGTH];
+	bool		matched;
+	uint32		status;			/* hash status */
+} DataDirectoryFileInfo;
+
+typedef struct manifesthash_hash *hashtab;
+
+#define SH_PREFIX manifesthash
+#define SH_ELEMENT_TYPE DataDirectoryFileInfo
+#define SH_KEY_TYPE char*
+#define SH_KEY filename
+#define SH_HASH_KEY(tb, key) string_hash_sdbm(key)
+#define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
+#define SH_SCOPE static inline
+#define SH_RAW_ALLOCATOR pg_malloc
+#define SH_DECLARE
+#define SH_DEFINE
+#include "lib/simplehash.h"
+
 /*
  * pg_xlog has been renamed to pg_wal in version 10.  This version number
  * should be compared with PQserverVersion().
@@ -142,6 +171,7 @@ static bool create_slot = false;
 static bool no_slot = false;
 static bool verify_checksums = true;
 static char *manifest_checksums = NULL;
+static enum ChecksumAlgorithm checksum_type = MC_NONE;
 
 static bool success = false;
 static bool made_new_pgdata = false;
@@ -201,6 +231,13 @@ static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
 static const char *get_tablespace_mapping(const char *dir);
 static void tablespace_list_append(const char *arg);
 
+static void VerifyBackup(void);
+static  manifesthash_hash *create_manifest_hash(char manifest_path[MAXPGPATH]);
+static void scan_data_directory(char *basedir, const char *subdirpath,
+								manifesthash_hash *hashtab);
+static void verify_file(struct dirent *de, char fn[MAXPGPATH],
+						struct stat st, char relative_path[MAXPGPATH],
+						manifesthash_hash *hashtab);
 
 static void
 cleanup_directories_atexit(void)
@@ -401,6 +438,7 @@ usage(void)
 			 "                         do not verify checksums\n"));
 	printf(_("      --manifest-checksums=SHA256|CRC32C|NONE\n"
 			 "                         calculate checksums for manifest files using provided algorithm\n"));
+	printf(_("      --verify-backup    validate the backup\n"));
 	printf(_("  -?, --help             show this help, then exit\n"));
 	printf(_("\nConnection options:\n"));
 	printf(_("  -d, --dbname=CONNSTR   connection string\n"));
@@ -2167,11 +2205,13 @@ main(int argc, char **argv)
 		{"no-slot", no_argument, NULL, 2},
 		{"no-verify-checksums", no_argument, NULL, 3},
 		{"manifest-checksums", required_argument, NULL, 'm'},
+		{"verify-backup", no_argument, NULL, 4},
 		{NULL, 0, NULL, 0}
 	};
 	int			c;
 
 	int			option_index;
+	bool		verify_backup = false;
 
 	pg_logging_init(argv[0]);
 	progname = get_progname(argv[0]);
@@ -2338,6 +2378,9 @@ main(int argc, char **argv)
 			case 'm':
 				manifest_checksums = pg_strdup(optarg);
 				break;
+			case 4:
+				verify_backup = true;
+				break;
 			default:
 
 				/*
@@ -2460,6 +2503,12 @@ main(int argc, char **argv)
 	}
 #endif
 
+	if(verify_backup)
+	{
+		VerifyBackup();
+		return 0;
+	}
+
 	/* connection in replication mode to server */
 	conn = GetConnection();
 	if (!conn)
@@ -2524,3 +2573,370 @@ main(int argc, char **argv)
 	success = true;
 	return 0;
 }
+
+/*
+ * Read the backup_manifest file and generate the hash table, then scan data
+ * directroy and verify each file. Finally, iterate on hash table to find
+ * out missing files.
+ */
+static void
+VerifyBackup(void)
+{
+	char		manifest_path[MAXPGPATH];
+	manifesthash_hash *hashtab;
+	manifesthash_iterator i;
+	DataDirectoryFileInfo *entry;
+
+	snprintf(manifest_path, sizeof(manifest_path), "%s/%s", basedir,
+			 "backup_manifest");
+
+	/* create hash table */
+	hashtab = create_manifest_hash(manifest_path);
+
+	scan_data_directory(basedir, NULL, hashtab);
+
+	manifesthash_start_iterate(hashtab, &i);
+	while ((entry = manifesthash_iterate(hashtab, &i)) != NULL)
+	{
+		if (!entry->matched)
+		{
+			pg_log_info("missing file: %s", entry->filename);
+		}
+	}
+}
+
+/*
+ * Given a file path, read that file and generate the hash table for same.
+ * Also generate the checksum for the records that are read from file and
+ * compare that with checksum written in backup_manifest file. If both
+ * checksums are identical then proceed, otherwise throw an error and abort.
+ */
+static  manifesthash_hash *
+create_manifest_hash(char manifest_path[MAXPGPATH])
+{
+	manifesthash_hash *hashtab;
+	FILE	   *file;
+	DataDirectoryFileInfo *entry;
+	PQExpBuffer	manifest;
+	char		file_checksum[256];
+	char		header[1024];
+	long		pos = 0;
+
+	manifest = createPQExpBuffer();
+	if (!manifest)
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+
+	file = fopen(manifest_path, "r");
+
+	if (!file)
+	{
+		pg_log_error("could not open backup_manifest");
+		exit(1);
+	}
+
+	/* read the file header */
+	if (fscanf(file, "%1023[^\n]\n", header) != 1)
+	{
+		pg_log_error("error while reading the header from backup_manifest");
+		exit(1);
+	}
+
+	appendPQExpBufferStr(manifest, header);
+	appendPQExpBufferStr(manifest, "\n");
+
+	hashtab  = manifesthash_create(1024, NULL);
+
+	while (!feof(file))
+	{
+		char	   *filename;
+		char		filetype[10];
+		int			filesize = 0;
+		char		mtime[24];
+		char		checksum_with_type[CHECKSUM_LENGTH];
+		bool		found;
+		char		checksum[CHECKSUM_LENGTH];
+
+		filename = (char*) pg_malloc(MAXPGPATH);
+
+		if (fscanf(file, "%s\t%s\t%d\t%23[^\t] %s\n", filetype, filename,
+				   &filesize, mtime, checksum_with_type) != 5)
+		{
+			/*
+			 * On failure, re-read the last line of record and check if it is
+			 * a last line where manifest checksum is written. If yes, then
+			 * parse it.
+			 */
+			if (fseek(file, pos, SEEK_SET) == -1)
+			{
+				pg_log_error("error while reading the backup_manifest file");
+				exit(1);
+			}
+
+			if (fscanf(file, "Manifest-Checksum\t%s\n", file_checksum) != 1)
+			{
+				pg_log_error("error while reading the backup_manifest file");
+				exit(1);
+			}
+
+			if (feof(file))
+				break;
+		}
+
+		pos = ftell(file);
+
+		entry = manifesthash_insert(hashtab, filename, &found);
+
+		if (found)
+		{
+			pg_log_info("duplicate file present: %s\n", filename);
+			pg_free(filename);
+		}
+		else
+		{
+			memcpy(entry->filetype, filetype, strlen(filetype));
+			entry->filetype[strlen(filetype)+1] = '\0';
+			entry->filesize = filesize;
+		}
+
+		if (strcmp(checksum_with_type, "-") == 0)
+		{
+			checksum_type = MC_NONE;
+		}
+		else
+		{
+			if (strncmp(checksum_with_type, "SHA256", 6) == 0)
+			{
+				checksum_type = MC_SHA256;
+				snprintf(checksum, CHECKSUM_LENGTH, "%s",
+						 &checksum_with_type[strlen("SHA256:")]);
+			}
+			else if (strncmp(checksum_with_type, "CRC32C", 6) == 0)
+			{
+				checksum_type = MC_CRC32C;
+				snprintf(checksum, CHECKSUM_LENGTH, "%s",
+						 &checksum_with_type[strlen("CRC32C:")]);
+			}
+			else
+			{
+				pg_log_error("unknown checksum method");
+				exit(1);
+			}
+
+			snprintf(entry->checksum, CHECKSUM_LENGTH, "%s", checksum);
+		}
+
+		appendPQExpBuffer(manifest, "File\t%s\t%d\t%s\t%s\n", filename,
+						  filesize, mtime, checksum_with_type);
+
+	}
+
+	/*
+	 * Once read all the records from  backup_manifest file, generate the
+	 * backup manifest checksum and compare it with the backup menifest
+	 * checksum written in manifest file.
+	 */
+	if (checksum_type != MC_NONE)
+	{
+
+		char		checksumbuf[CHECKSUM_LENGTH];
+		int			checksumbuflen;
+		ChecksumCtx	cCtx;
+		char 		encoded_checksum[CHECKSUM_LENGTH];
+		char		manifest_checksum[CHECKSUM_LENGTH];
+
+		initialize_checksum(&cCtx, checksum_type);
+		update_checksum(&cCtx, checksum_type, manifest->data,
+						manifest->len);
+		checksumbuflen = finalize_checksum(&cCtx, checksum_type,
+										   (char *) checksumbuf);
+		appendPQExpBuffer(manifest, "Manifest-Checksum\t");
+
+		switch (checksum_type)
+		{
+			case MC_SHA256:
+				appendPQExpBuffer(manifest, "SHA256:");
+				snprintf(manifest_checksum, CHECKSUM_LENGTH, "%s",
+						 &file_checksum[strlen("SHA256:")]);
+				break;
+			case MC_CRC32C:
+				appendPQExpBuffer(manifest, "CRC32C:");
+				snprintf(manifest_checksum, CHECKSUM_LENGTH, "%s",
+						 &file_checksum[strlen("CRC32C:")]);
+				break;
+			case MC_NONE:
+				break;
+		}
+
+		checksumbuflen = hex_encode(checksumbuf, checksumbuflen,
+									encoded_checksum);
+		encoded_checksum[checksumbuflen] = '\0';
+
+		/*
+		 * Compare the both checksums, if they are not same that means
+		 * backup_manifest file is changed. Throw an error and abort.
+		 */
+		if (strcmp(encoded_checksum, manifest_checksum) != 0)
+		{
+			pg_log_error("backup manifest checksum difference. Aborting");
+			exit(1);
+		}
+	}
+
+	return hashtab;
+}
+
+/*
+ * Scan the data directory and check whether each file entry present in hash
+ * table with the correct details, i.e. filesize and checksum.
+ */
+static void
+scan_data_directory(char *basedir, const char *subdirpath,
+					manifesthash_hash *hashtab)
+{
+	char		path[MAXPGPATH];
+	char		relative_path[MAXPGPATH] = "";
+	DIR		   *dir;
+	struct dirent *de;
+
+	if (subdirpath)
+	{
+		snprintf(path, sizeof(path), "%s/%s", basedir,
+				 subdirpath);
+		snprintf(relative_path, sizeof(relative_path), "%s/", subdirpath);
+
+	}
+	else
+		snprintf(path, sizeof(path), "%s", basedir);
+
+	dir = opendir(path);
+	if (!dir)
+	{
+		pg_log_error("could not open directory \"%s\": %m", path);
+		exit(1);
+	}
+
+	while ((de = readdir(dir)) != NULL)
+	{
+		char		fn[MAXPGPATH];
+		struct stat st;
+
+		if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0 ||
+			strcmp(de->d_name, "pg_wal") == 0)
+			continue;
+
+		snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
+		if (stat(fn, &st) < 0)
+		{
+			pg_log_error("could not stat file \"%s\": %m", fn);
+			exit(1);
+		}
+		if (S_ISREG(st.st_mode))
+		{
+			verify_file(de, fn, st, relative_path, hashtab);
+		}
+		else if (S_ISDIR(st.st_mode))
+		{
+			char		newsubdirpath[MAXPGPATH];
+
+			if (subdirpath)
+				snprintf(newsubdirpath, MAXPGPATH, "%s/%s", subdirpath,
+						 de->d_name);
+			else
+				snprintf(newsubdirpath, MAXPGPATH, "%s", de->d_name);
+
+			scan_data_directory(basedir, newsubdirpath, hashtab);
+		}
+	}
+	closedir(dir);
+}
+
+/*
+ * Given the file and its details, check whether it is present in hash table
+ * and if yes, then compare its details with hash table entry.
+ */
+static void
+verify_file(struct dirent *de, char fn[MAXPGPATH], struct stat st,
+			char relative_path[MAXPGPATH], manifesthash_hash *hashtab)
+{
+	PQExpBuffer	filename = NULL;
+	DataDirectoryFileInfo *record;
+
+	/* Skip backup manifest file. */
+	if (strcmp(de->d_name, "backup_manifest") == 0)
+		return;
+
+	filename = createPQExpBuffer();
+	if (!filename)
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+
+	appendPQExpBuffer(filename, "%s%s", relative_path, de->d_name);
+
+	/*
+	 * Compare the hash and if record found then we match the file size and
+	 * checksum (if enabled). Modified time cannot be compared with the
+	 * file in the backup directory and its entry in the manifest as
+	 * manifest entry gives mtime from server file whereas the same file in
+	 * the backup will have different mtime.
+	 */
+	record = manifesthash_lookup(hashtab, filename->data);;
+	if (record)
+	{
+		record->matched = true;
+		if (record->filesize != st.st_size)
+			pg_log_info("size changed for file: %s, original size: %d, current size: %zu",
+						filename->data, record->filesize, st.st_size);
+
+		/*
+		 * Read the file and generate the checksum based on checksum method
+		 * and compare that with the checksum present in hash entry.
+		 */
+		if (checksum_type != MC_NONE)
+		{
+			FILE	   *fp;
+			char		buf[1048576];  // 1MB chunk
+			pgoff_t		len = 0;
+			off_t		cnt;
+			char 		checksumbuf[CHECKSUM_LENGTH];
+			char 		encode_checksumbuf[CHECKSUM_LENGTH];
+			int	   		checksumbuflen;
+			ChecksumCtx	cCtx;
+
+			initialize_checksum(&cCtx, checksum_type);
+
+			fp = fopen(fn, "r");
+			if (!fp)
+			{
+				pg_log_error("could not open file \"%s\": %m", de->d_name);
+				exit(1);
+			}
+
+			/* Read file in chunks [1 MB each chunk]*/
+			while ((cnt = fread(buf, 1, Min(sizeof(buf), st.st_size - len), fp)) > 0)
+			{
+				update_checksum(&cCtx, checksum_type, buf, cnt);
+				len += cnt;
+			}
+
+			checksumbuflen = finalize_checksum(&cCtx, checksum_type,
+											   checksumbuf);
+
+			/* Convert checksum to hexadecimal. */
+			checksumbuflen = hex_encode(checksumbuf, checksumbuflen,
+										encode_checksumbuf);
+			encode_checksumbuf[checksumbuflen] = '\0';
+
+			fclose(fp);
+
+			if (strcmp(record->checksum, encode_checksumbuf) != 0)
+				pg_log_info("checksum difference for file: %s", filename->data);
+		}
+	}
+	else
+		pg_log_info("extra file found: %s", filename->data);
+}
diff --git a/src/common/encode.c b/src/common/encode.c
index a450c53..14f2ec2 100644
--- a/src/common/encode.c
+++ b/src/common/encode.c
@@ -36,3 +36,21 @@ hex_encode(const char *src, unsigned len, char *dst)
 	}
 	return len * 2;
 }
+
+/*
+ * Simple string hash function from http://www.cse.yorku.ca/~oz/hash.html
+ *
+ * The backend uses a more sophisticated function for hashing strings,
+ * but we don't really need that complexity here.
+ */
+uint32
+string_hash_sdbm(const char *key)
+{
+	uint32		hash = 0;
+	int			c;
+
+	while ((c = *key++))
+		hash = c + (hash << 6) + (hash << 16) - hash;
+
+	return hash;
+}
diff --git a/src/include/common/encode.h b/src/include/common/encode.h
index 63328bc..44a062f 100644
--- a/src/include/common/encode.h
+++ b/src/include/common/encode.h
@@ -16,5 +16,6 @@
 static const char hextbl[] = "0123456789abcdef";
 
 extern unsigned hex_encode(const char *src, unsigned len, char *dst);
+extern uint32 string_hash_sdbm(const char *key);
 
 #endif							/* COMMON_ENCODE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 8b2dae6..982ba29 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -486,6 +486,7 @@ DR_sqlfunction
 DR_transientrel
 DSA
 DWORD
+DataDirectoryFileInfo
 DataDumperPtr
 DataPageDeleteStack
 DateADT
-- 
1.8.3.1