0006-checksum-algo-option.patch

text/x-patch

Filename: 0006-checksum-algo-option.patch
Type: text/x-patch
Part: 0
Message: Re: backup manifests

Patch

Same data as JSON: GET /api/v1/attachments/:id/patch the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes. API reference →
Format: unified
Series: patch 0006
File+
src/backend/replication/basebackup.c 153 50
src/backend/replication/repl_gram.y 2 2
src/bin/pg_basebackup/pg_basebackup.c 8 8
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 66aa0fc..c6bb74d 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -54,6 +54,20 @@ typedef struct
 	bool		sendtblspcmapfile;
 } basebackup_options;
 
+/* Checksum algorithm option for manifest */
+enum manifestCheckSum
+{
+	MC_NONE = 0,
+	MC_SHA256,
+	MC_CRC
+};
+
+/* checksum algorithm context */
+typedef union checksumCtx
+{
+	pg_sha256_ctx	sha256_ctx;
+	pg_crc32c		crc_ctx;
+}	ChecksumCtx;
 
 static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
 					 List *tablespaces, bool sendtblspclinks,
@@ -72,7 +86,7 @@ static void SendBackupHeader(List *tablespaces);
 static void InitializeManifest(StringInfo manifest);
 static void AddFileToManifest(StringInfo manifest, const char *tsoid,
 							  const char *filename, size_t size, time_t mtime,
-							  uint8 *shabuf);
+							  ChecksumCtx *cCtx);
 static void SendBackupManifest(StringInfo manifest);
 static char *escape_field_for_manifest(const char *s);
 static void base_backup_cleanup(int code, Datum arg);
@@ -82,6 +96,9 @@ static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
 static int	compareWalFileNames(const ListCell *a, const ListCell *b);
 static void throttle(size_t increment);
 static bool is_checksummed_file(const char *fullpath, const char *filename);
+static void initilize_manifest_checksum(ChecksumCtx *cCtx);
+static void update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt);
+static int final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf);
 
 /* Was the backup currently in-progress initiated in recovery mode? */
 static bool backup_started_in_recovery = false;
@@ -132,8 +149,8 @@ static long long int total_checksum_failures;
 /* Do not verify checksums. */
 static bool noverify_checksums = false;
 
-/* Add file entry in to manifest with checksums. */
-static bool manifest_with_checksums = false;
+
+static enum manifestCheckSum manifest_with_checksums = MC_NONE;
 
 /*
  * The contents of these directories are removed or recreated during server
@@ -677,6 +694,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 	bool		o_maxrate = false;
 	bool		o_tablespace_map = false;
 	bool		o_noverify_checksums = false;
+	bool		o_manifest_with_checksums = false;
 
 	MemSet(opt, 0, sizeof(*opt));
 	foreach(lopt, options)
@@ -767,11 +785,23 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 		}
 		else if (strcmp(defel->defname, "manifest_with_checksums") == 0)
 		{
-			if (manifest_with_checksums)
+			char  *manifest_with_checksum_algo = NULL;
+			if (o_manifest_with_checksums)
 				ereport(ERROR,
 						(errcode(ERRCODE_SYNTAX_ERROR),
 						 errmsg("duplicate option \"%s\"", defel->defname)));
-			manifest_with_checksums = true;
+			manifest_with_checksum_algo = strVal(defel->arg);
+
+			if (pg_strcasecmp(manifest_with_checksum_algo, "SHA256") == 0)
+				manifest_with_checksums = MC_SHA256;
+			else if (pg_strcasecmp(manifest_with_checksum_algo, "CRC") == 0)
+				manifest_with_checksums = MC_CRC;
+			else if (pg_strcasecmp(manifest_with_checksum_algo, "NONE") == 0)
+				manifest_with_checksums = MC_NONE;
+			else
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("invalid manifest_with_checksums option \"%s\"", manifest_with_checksum_algo)));
 		}
 
 		else
@@ -907,14 +937,16 @@ InitializeManifest(StringInfo manifest)
 static void
 AddFileToManifest(StringInfo manifest, const char *tsoid,
 				  const char *filename, size_t size, time_t mtime,
-				  uint8 *shabuf)
+				  ChecksumCtx *cCtx)
 {
 	char	pathbuf[MAXPGPATH];
 	char   *escaped_filename;
 	static char timebuf[128];
-	static char shatextbuf[PG_SHA256_DIGEST_LENGTH * 2 + 1];
-	int		shatextlen;
+	static char checksumbuf[256];
+	char encode_checksumbuf[256];
 	struct pg_tm *tm;
+	char *checksumlabel = NULL;
+	int	   checksumbuflen;
 
 	/*
 	 * If this file is part of a tablespace, the filename passed to this
@@ -941,19 +973,32 @@ AddFileToManifest(StringInfo manifest, const char *tsoid,
 		elog(ERROR, "could not convert epoch to timestamp: %m");
 	pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z", tm);
 
-	/* Convert checksum to hexadecimal. */
-	if (manifest_with_checksums)
+	/* Generate final checksum and Convert it to hexadecimal. */
+	if (manifest_with_checksums != MC_NONE)
 	{
-		shatextlen =
-			hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, shatextbuf);
-		Assert(shatextlen + 1 == sizeof(shatextbuf));
-		shatextbuf[shatextlen] = '\0';
+		checksumbuflen = final_manifest_checksum(cCtx, checksumbuf);
+		switch (manifest_with_checksums)
+		{
+			case MC_SHA256:
+				checksumlabel = "SHA256:";
+				break;
+			case MC_CRC:
+				checksumlabel = "CRC:";
+				break;
+			case MC_NONE:
+				break;
+		}
+		checksumbuflen = hex_encode(checksumbuf,
+									checksumbuflen,
+									encode_checksumbuf);
+		encode_checksumbuf[checksumbuflen] = '\0';
 	}
 
 	/* Add to manifest. */
-	appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s\n",
+	appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s%s\n",
 					 escaped_filename == NULL ? filename : escaped_filename,
-					 size, timebuf, manifest_with_checksums ? shatextbuf : "-");
+					 size, timebuf, checksumlabel ? checksumlabel : "",
+					 manifest_with_checksums != MC_NONE ? encode_checksumbuf : "-");
 
 	/* Avoid leaking memory. */
 	if (escaped_filename != NULL)
@@ -966,24 +1011,34 @@ AddFileToManifest(StringInfo manifest, const char *tsoid,
 static void
 SendBackupManifest(StringInfo manifest)
 {
-	pg_sha256_ctx	sha256_ctx;
-	uint8			shabuf[PG_SHA256_DIGEST_LENGTH];
+	char			checksumbuf[256];
 	StringInfoData	protobuf;
-	int				shastringlen;
+	int				checksumbuflen;
+	ChecksumCtx		cCtx;
+
 
 	/* Checksum the manifest. */
-	if (manifest_with_checksums)
+	if (manifest_with_checksums != MC_NONE)
 	{
-		pg_sha256_init(&sha256_ctx);
-		pg_sha256_update(&sha256_ctx, (uint8 *) manifest->data, manifest->len);
-		pg_sha256_final(&sha256_ctx, shabuf);
+		initilize_manifest_checksum(&cCtx);
+		update_manifest_checksum(&cCtx, manifest->data, manifest->len);
+		checksumbuflen = final_manifest_checksum(&cCtx, (char *) checksumbuf);
 		appendStringInfoString(manifest, "Manifest-Checksum\t");
-		shastringlen = PG_SHA256_DIGEST_LENGTH * 2;
-		enlargeStringInfo(manifest, shastringlen);
-		shastringlen = hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH,
+		switch (manifest_with_checksums)
+		{
+			case MC_SHA256:
+				appendStringInfoString(manifest, "SHA256:");
+				break;
+			case MC_CRC:
+				appendStringInfoString(manifest, "CRC:");
+				break;
+			case MC_NONE:
+				break;
+		}
+		enlargeStringInfo(manifest, checksumbuflen * 2);
+		checksumbuflen = hex_encode(checksumbuf, checksumbuflen,
 				manifest->data + manifest->len);
-		Assert(shastringlen == PG_SHA256_DIGEST_LENGTH * 2);
-		manifest->len += shastringlen;
+		manifest->len += checksumbuflen;
 		appendStringInfoChar(manifest, '\n');
 	}
 
@@ -1115,11 +1170,7 @@ sendFileWithContent(const char *filename, const char *content,
 	struct stat statbuf;
 	int			pad,
 				len;
-	pg_sha256_ctx	sha256_ctx;
-	uint8		shabuf[PG_SHA256_DIGEST_LENGTH];
-
-	if (manifest_with_checksums)
-		pg_sha256_init(&sha256_ctx);
+	ChecksumCtx cCtx;
 
 	len = strlen(content);
 
@@ -1153,14 +1204,14 @@ sendFileWithContent(const char *filename, const char *content,
 		pq_putmessage('d', buf, pad);
 	}
 
-	if (manifest_with_checksums)
+	if (manifest_with_checksums != MC_NONE)
 	{
-		pg_sha256_update(&sha256_ctx, (uint8 *) content, len);
-		pg_sha256_final(&sha256_ctx, shabuf);
+		initilize_manifest_checksum(&cCtx);
+		update_manifest_checksum(&cCtx, content, len);
 	}
 
 	AddFileToManifest(manifest, NULL, filename, len, statbuf.st_mtime,
-					  shabuf);
+					  &cCtx);
 }
 
 /*
@@ -1559,7 +1610,6 @@ is_checksummed_file(const char *fullpath, const char *filename)
  * Copied from pg_dump, but modified to work with libpq for sending
  */
 
-
 /*
  * Given the member, write the TAR header & send the file.
  *
@@ -1591,11 +1641,9 @@ sendFile(const char *readfilename, const char *tarfilename,
 	int			segmentno = 0;
 	char	   *segmentpath;
 	bool		verify_checksum = false;
-	pg_sha256_ctx	sha256_ctx;
-	uint8		shabuf[PG_SHA256_DIGEST_LENGTH];
+	ChecksumCtx cCtx;
 
-	if (manifest_with_checksums)
-		pg_sha256_init(&sha256_ctx);
+	initilize_manifest_checksum(&cCtx);
 
 	fp = AllocateFile(readfilename, "rb");
 	if (fp == NULL)
@@ -1766,8 +1814,7 @@ sendFile(const char *readfilename, const char *tarfilename,
 					(errmsg("base backup could not send data, aborting backup")));
 
 		/* Also feed it to the checksum machinery. */
-		if (manifest_with_checksums)
-			pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
+		update_manifest_checksum(&cCtx, buf, cnt);
 
 		len += cnt;
 		throttle(cnt);
@@ -1793,8 +1840,7 @@ sendFile(const char *readfilename, const char *tarfilename,
 		{
 			cnt = Min(sizeof(buf), statbuf->st_size - len);
 			pq_putmessage('d', buf, cnt);
-			if (manifest_with_checksums)
-				pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
+			update_manifest_checksum(&cCtx, buf, cnt);
 			len += cnt;
 			throttle(cnt);
 		}
@@ -1826,11 +1872,8 @@ sendFile(const char *readfilename, const char *tarfilename,
 	}
 
 	total_checksum_failures += checksum_failures;
-
-	if (manifest_with_checksums)
-		pg_sha256_final(&sha256_ctx, shabuf);
 	AddFileToManifest(manifest, tsoid, tarfilename, statbuf->st_size,
-					  statbuf->st_mtime, shabuf);
+					  statbuf->st_mtime, &cCtx);
 
 	return true;
 }
@@ -1966,3 +2009,63 @@ throttle(size_t increment)
 	 */
 	throttled_last = GetCurrentTimestamp();
 }
+
+/*
+ * Initilize the manifest checksum context according to the provided algorithm.
+ */
+static void
+initilize_manifest_checksum(ChecksumCtx *cCtx)
+{
+	switch (manifest_with_checksums)
+	{
+		case MC_SHA256:
+			pg_sha256_init(&cCtx->sha256_ctx);
+			break;
+		case MC_CRC:
+			INIT_CRC32C(cCtx->crc_ctx);
+			break;
+		case MC_NONE:
+			break;
+	}
+}
+
+static void
+update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt)
+{
+	switch (manifest_with_checksums)
+	{
+		case MC_SHA256:
+			pg_sha256_update(&cCtx->sha256_ctx, (uint8 *) buf, cnt);
+			break;
+		case MC_CRC:
+			COMP_CRC32C(cCtx->crc_ctx, buf, cnt);
+			break;
+		case MC_NONE:
+			break;
+	}
+}
+
+/*
+ * Function calculate the final checksum for the provided context and returns
+ * the length of checksum.
+ */
+static int
+final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf)
+{
+	int checksumlen = 0;
+	switch (manifest_with_checksums)
+	{
+		case MC_SHA256:
+			pg_sha256_final(&cCtx->sha256_ctx, (uint8 *)checksumbuf);
+			checksumlen = PG_SHA256_DIGEST_LENGTH;
+			break;
+		case MC_CRC:
+			FIN_CRC32C(cCtx->crc_ctx);
+			pg_ltoa(cCtx->crc_ctx, checksumbuf);
+			checksumlen = strlen(checksumbuf);
+			break;
+		case MC_NONE:
+			break;
+	}
+	return checksumlen;
+}
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 542a3f7..409bd23 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -215,10 +215,10 @@ base_backup_opt:
 				  $$ = makeDefElem("noverify_checksums",
 								   (Node *)makeInteger(true), -1);
 				}
-			| K_MANIFEST_WITH_CHECKSUMS
+			| K_MANIFEST_WITH_CHECKSUMS SCONST
 				{
 				  $$ = makeDefElem("manifest_with_checksums",
-								   (Node *)makeInteger(true), -1);
+								   (Node *)makeString($2), -1);
 				}
 			;
 
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index f4f8ffe..f7b16ab 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -141,7 +141,7 @@ static bool temp_replication_slot = true;
 static bool create_slot = false;
 static bool no_slot = false;
 static bool verify_checksums = true;
-static bool manifest_with_checksums = false;
+static char *manifest_with_checksums = NULL;
 
 static bool success = false;
 static bool made_new_pgdata = false;
@@ -400,7 +400,7 @@ usage(void)
 	printf(_("      --no-verify-checksums\n"
 			 "                         do not verify checksums\n"));
 	printf(_("      --manifest-with-checksums\n"
-			 "                         do calculate checksums for manifest files\n"));
+			 "                         calculate checksums for manifest files using provided algorithm\n"));
 	printf(_("  -?, --help             show this help, then exit\n"));
 	printf(_("\nConnection options:\n"));
 	printf(_("  -d, --dbname=CONNSTR   connection string\n"));
@@ -1824,7 +1824,7 @@ BaseBackup(void)
 	}
 
 	basebkp =
-		psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s",
+		psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s MANIFEST_WITH_CHECKSUMS '%s'",
 				 escaped_label,
 				 showprogress ? "PROGRESS" : "",
 				 includewal == FETCH_WAL ? "WAL" : "",
@@ -1833,7 +1833,7 @@ BaseBackup(void)
 				 maxrate_clause ? maxrate_clause : "",
 				 format == 't' ? "TABLESPACE_MAP" : "",
 				 verify_checksums ? "" : "NOVERIFY_CHECKSUMS",
-				 manifest_with_checksums ? "MANIFEST_WITH_CHECKSUMS" : "");
+				 manifest_with_checksums ? manifest_with_checksums : "NONE");
 
 	if (PQsendQuery(conn, basebkp) == 0)
 	{
@@ -2166,7 +2166,7 @@ main(int argc, char **argv)
 		{"waldir", required_argument, NULL, 1},
 		{"no-slot", no_argument, NULL, 2},
 		{"no-verify-checksums", no_argument, NULL, 3},
-		{"manifest-with-checksums", no_argument, NULL, 4},
+		{"manifest-with-checksums", required_argument, NULL, 'm'},
 		{NULL, 0, NULL, 0}
 	};
 	int			c;
@@ -2194,7 +2194,7 @@ main(int argc, char **argv)
 
 	atexit(cleanup_directories_atexit);
 
-	while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
+	while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvPm:",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -2335,8 +2335,8 @@ main(int argc, char **argv)
 			case 3:
 				verify_checksums = false;
 				break;
-			case 4:
-				manifest_with_checksums = true;
+			case 'm':
+				manifest_with_checksums = pg_strdup(optarg);
 				break;
 			default: