0006-checksum-algo-option.patch
text/x-patch
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: unified
Series: patch 0006
| File | + | − |
|---|---|---|
| src/backend/replication/basebackup.c | 153 | 50 |
| src/backend/replication/repl_gram.y | 2 | 2 |
| src/bin/pg_basebackup/pg_basebackup.c | 8 | 8 |
diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c
index 66aa0fc..c6bb74d 100644
--- a/src/backend/replication/basebackup.c
+++ b/src/backend/replication/basebackup.c
@@ -54,6 +54,20 @@ typedef struct
bool sendtblspcmapfile;
} basebackup_options;
+/* Checksum algorithm option for manifest */
+enum manifestCheckSum
+{
+ MC_NONE = 0,
+ MC_SHA256,
+ MC_CRC
+};
+
+/* checksum algorithm context */
+typedef union checksumCtx
+{
+ pg_sha256_ctx sha256_ctx;
+ pg_crc32c crc_ctx;
+} ChecksumCtx;
static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
List *tablespaces, bool sendtblspclinks,
@@ -72,7 +86,7 @@ static void SendBackupHeader(List *tablespaces);
static void InitializeManifest(StringInfo manifest);
static void AddFileToManifest(StringInfo manifest, const char *tsoid,
const char *filename, size_t size, time_t mtime,
- uint8 *shabuf);
+ ChecksumCtx *cCtx);
static void SendBackupManifest(StringInfo manifest);
static char *escape_field_for_manifest(const char *s);
static void base_backup_cleanup(int code, Datum arg);
@@ -82,6 +96,9 @@ static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
static int compareWalFileNames(const ListCell *a, const ListCell *b);
static void throttle(size_t increment);
static bool is_checksummed_file(const char *fullpath, const char *filename);
+static void initilize_manifest_checksum(ChecksumCtx *cCtx);
+static void update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt);
+static int final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf);
/* Was the backup currently in-progress initiated in recovery mode? */
static bool backup_started_in_recovery = false;
@@ -132,8 +149,8 @@ static long long int total_checksum_failures;
/* Do not verify checksums. */
static bool noverify_checksums = false;
-/* Add file entry in to manifest with checksums. */
-static bool manifest_with_checksums = false;
+
+static enum manifestCheckSum manifest_with_checksums = MC_NONE;
/*
* The contents of these directories are removed or recreated during server
@@ -677,6 +694,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
bool o_maxrate = false;
bool o_tablespace_map = false;
bool o_noverify_checksums = false;
+ bool o_manifest_with_checksums = false;
MemSet(opt, 0, sizeof(*opt));
foreach(lopt, options)
@@ -767,11 +785,23 @@ parse_basebackup_options(List *options, basebackup_options *opt)
}
else if (strcmp(defel->defname, "manifest_with_checksums") == 0)
{
- if (manifest_with_checksums)
+ char *manifest_with_checksum_algo = NULL;
+ if (o_manifest_with_checksums)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("duplicate option \"%s\"", defel->defname)));
- manifest_with_checksums = true;
+ manifest_with_checksum_algo = strVal(defel->arg);
+
+ if (pg_strcasecmp(manifest_with_checksum_algo, "SHA256") == 0)
+ manifest_with_checksums = MC_SHA256;
+ else if (pg_strcasecmp(manifest_with_checksum_algo, "CRC") == 0)
+ manifest_with_checksums = MC_CRC;
+ else if (pg_strcasecmp(manifest_with_checksum_algo, "NONE") == 0)
+ manifest_with_checksums = MC_NONE;
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid manifest_with_checksums option \"%s\"", manifest_with_checksum_algo)));
}
else
@@ -907,14 +937,16 @@ InitializeManifest(StringInfo manifest)
static void
AddFileToManifest(StringInfo manifest, const char *tsoid,
const char *filename, size_t size, time_t mtime,
- uint8 *shabuf)
+ ChecksumCtx *cCtx)
{
char pathbuf[MAXPGPATH];
char *escaped_filename;
static char timebuf[128];
- static char shatextbuf[PG_SHA256_DIGEST_LENGTH * 2 + 1];
- int shatextlen;
+ static char checksumbuf[256];
+ char encode_checksumbuf[256];
struct pg_tm *tm;
+ char *checksumlabel = NULL;
+ int checksumbuflen;
/*
* If this file is part of a tablespace, the filename passed to this
@@ -941,19 +973,32 @@ AddFileToManifest(StringInfo manifest, const char *tsoid,
elog(ERROR, "could not convert epoch to timestamp: %m");
pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z", tm);
- /* Convert checksum to hexadecimal. */
- if (manifest_with_checksums)
+ /* Generate final checksum and Convert it to hexadecimal. */
+ if (manifest_with_checksums != MC_NONE)
{
- shatextlen =
- hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, shatextbuf);
- Assert(shatextlen + 1 == sizeof(shatextbuf));
- shatextbuf[shatextlen] = '\0';
+ checksumbuflen = final_manifest_checksum(cCtx, checksumbuf);
+ switch (manifest_with_checksums)
+ {
+ case MC_SHA256:
+ checksumlabel = "SHA256:";
+ break;
+ case MC_CRC:
+ checksumlabel = "CRC:";
+ break;
+ case MC_NONE:
+ break;
+ }
+ checksumbuflen = hex_encode(checksumbuf,
+ checksumbuflen,
+ encode_checksumbuf);
+ encode_checksumbuf[checksumbuflen] = '\0';
}
/* Add to manifest. */
- appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s\n",
+ appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s%s\n",
escaped_filename == NULL ? filename : escaped_filename,
- size, timebuf, manifest_with_checksums ? shatextbuf : "-");
+ size, timebuf, checksumlabel ? checksumlabel : "",
+ manifest_with_checksums != MC_NONE ? encode_checksumbuf : "-");
/* Avoid leaking memory. */
if (escaped_filename != NULL)
@@ -966,24 +1011,34 @@ AddFileToManifest(StringInfo manifest, const char *tsoid,
static void
SendBackupManifest(StringInfo manifest)
{
- pg_sha256_ctx sha256_ctx;
- uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
+ char checksumbuf[256];
StringInfoData protobuf;
- int shastringlen;
+ int checksumbuflen;
+ ChecksumCtx cCtx;
+
/* Checksum the manifest. */
- if (manifest_with_checksums)
+ if (manifest_with_checksums != MC_NONE)
{
- pg_sha256_init(&sha256_ctx);
- pg_sha256_update(&sha256_ctx, (uint8 *) manifest->data, manifest->len);
- pg_sha256_final(&sha256_ctx, shabuf);
+ initilize_manifest_checksum(&cCtx);
+ update_manifest_checksum(&cCtx, manifest->data, manifest->len);
+ checksumbuflen = final_manifest_checksum(&cCtx, (char *) checksumbuf);
appendStringInfoString(manifest, "Manifest-Checksum\t");
- shastringlen = PG_SHA256_DIGEST_LENGTH * 2;
- enlargeStringInfo(manifest, shastringlen);
- shastringlen = hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH,
+ switch (manifest_with_checksums)
+ {
+ case MC_SHA256:
+ appendStringInfoString(manifest, "SHA256:");
+ break;
+ case MC_CRC:
+ appendStringInfoString(manifest, "CRC:");
+ break;
+ case MC_NONE:
+ break;
+ }
+ enlargeStringInfo(manifest, checksumbuflen * 2);
+ checksumbuflen = hex_encode(checksumbuf, checksumbuflen,
manifest->data + manifest->len);
- Assert(shastringlen == PG_SHA256_DIGEST_LENGTH * 2);
- manifest->len += shastringlen;
+ manifest->len += checksumbuflen;
appendStringInfoChar(manifest, '\n');
}
@@ -1115,11 +1170,7 @@ sendFileWithContent(const char *filename, const char *content,
struct stat statbuf;
int pad,
len;
- pg_sha256_ctx sha256_ctx;
- uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
-
- if (manifest_with_checksums)
- pg_sha256_init(&sha256_ctx);
+ ChecksumCtx cCtx;
len = strlen(content);
@@ -1153,14 +1204,14 @@ sendFileWithContent(const char *filename, const char *content,
pq_putmessage('d', buf, pad);
}
- if (manifest_with_checksums)
+ if (manifest_with_checksums != MC_NONE)
{
- pg_sha256_update(&sha256_ctx, (uint8 *) content, len);
- pg_sha256_final(&sha256_ctx, shabuf);
+ initilize_manifest_checksum(&cCtx);
+ update_manifest_checksum(&cCtx, content, len);
}
AddFileToManifest(manifest, NULL, filename, len, statbuf.st_mtime,
- shabuf);
+ &cCtx);
}
/*
@@ -1559,7 +1610,6 @@ is_checksummed_file(const char *fullpath, const char *filename)
* Copied from pg_dump, but modified to work with libpq for sending
*/
-
/*
* Given the member, write the TAR header & send the file.
*
@@ -1591,11 +1641,9 @@ sendFile(const char *readfilename, const char *tarfilename,
int segmentno = 0;
char *segmentpath;
bool verify_checksum = false;
- pg_sha256_ctx sha256_ctx;
- uint8 shabuf[PG_SHA256_DIGEST_LENGTH];
+ ChecksumCtx cCtx;
- if (manifest_with_checksums)
- pg_sha256_init(&sha256_ctx);
+ initilize_manifest_checksum(&cCtx);
fp = AllocateFile(readfilename, "rb");
if (fp == NULL)
@@ -1766,8 +1814,7 @@ sendFile(const char *readfilename, const char *tarfilename,
(errmsg("base backup could not send data, aborting backup")));
/* Also feed it to the checksum machinery. */
- if (manifest_with_checksums)
- pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
+ update_manifest_checksum(&cCtx, buf, cnt);
len += cnt;
throttle(cnt);
@@ -1793,8 +1840,7 @@ sendFile(const char *readfilename, const char *tarfilename,
{
cnt = Min(sizeof(buf), statbuf->st_size - len);
pq_putmessage('d', buf, cnt);
- if (manifest_with_checksums)
- pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt);
+ update_manifest_checksum(&cCtx, buf, cnt);
len += cnt;
throttle(cnt);
}
@@ -1826,11 +1872,8 @@ sendFile(const char *readfilename, const char *tarfilename,
}
total_checksum_failures += checksum_failures;
-
- if (manifest_with_checksums)
- pg_sha256_final(&sha256_ctx, shabuf);
AddFileToManifest(manifest, tsoid, tarfilename, statbuf->st_size,
- statbuf->st_mtime, shabuf);
+ statbuf->st_mtime, &cCtx);
return true;
}
@@ -1966,3 +2009,63 @@ throttle(size_t increment)
*/
throttled_last = GetCurrentTimestamp();
}
+
+/*
+ * Initilize the manifest checksum context according to the provided algorithm.
+ */
+static void
+initilize_manifest_checksum(ChecksumCtx *cCtx)
+{
+ switch (manifest_with_checksums)
+ {
+ case MC_SHA256:
+ pg_sha256_init(&cCtx->sha256_ctx);
+ break;
+ case MC_CRC:
+ INIT_CRC32C(cCtx->crc_ctx);
+ break;
+ case MC_NONE:
+ break;
+ }
+}
+
+static void
+update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt)
+{
+ switch (manifest_with_checksums)
+ {
+ case MC_SHA256:
+ pg_sha256_update(&cCtx->sha256_ctx, (uint8 *) buf, cnt);
+ break;
+ case MC_CRC:
+ COMP_CRC32C(cCtx->crc_ctx, buf, cnt);
+ break;
+ case MC_NONE:
+ break;
+ }
+}
+
+/*
+ * Function calculate the final checksum for the provided context and returns
+ * the length of checksum.
+ */
+static int
+final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf)
+{
+ int checksumlen = 0;
+ switch (manifest_with_checksums)
+ {
+ case MC_SHA256:
+ pg_sha256_final(&cCtx->sha256_ctx, (uint8 *)checksumbuf);
+ checksumlen = PG_SHA256_DIGEST_LENGTH;
+ break;
+ case MC_CRC:
+ FIN_CRC32C(cCtx->crc_ctx);
+ pg_ltoa(cCtx->crc_ctx, checksumbuf);
+ checksumlen = strlen(checksumbuf);
+ break;
+ case MC_NONE:
+ break;
+ }
+ return checksumlen;
+}
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 542a3f7..409bd23 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -215,10 +215,10 @@ base_backup_opt:
$$ = makeDefElem("noverify_checksums",
(Node *)makeInteger(true), -1);
}
- | K_MANIFEST_WITH_CHECKSUMS
+ | K_MANIFEST_WITH_CHECKSUMS SCONST
{
$$ = makeDefElem("manifest_with_checksums",
- (Node *)makeInteger(true), -1);
+ (Node *)makeString($2), -1);
}
;
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index f4f8ffe..f7b16ab 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -141,7 +141,7 @@ static bool temp_replication_slot = true;
static bool create_slot = false;
static bool no_slot = false;
static bool verify_checksums = true;
-static bool manifest_with_checksums = false;
+static char *manifest_with_checksums = NULL;
static bool success = false;
static bool made_new_pgdata = false;
@@ -400,7 +400,7 @@ usage(void)
printf(_(" --no-verify-checksums\n"
" do not verify checksums\n"));
printf(_(" --manifest-with-checksums\n"
- " do calculate checksums for manifest files\n"));
+ " calculate checksums for manifest files using provided algorithm\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nConnection options:\n"));
printf(_(" -d, --dbname=CONNSTR connection string\n"));
@@ -1824,7 +1824,7 @@ BaseBackup(void)
}
basebkp =
- psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s",
+ psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s MANIFEST_WITH_CHECKSUMS '%s'",
escaped_label,
showprogress ? "PROGRESS" : "",
includewal == FETCH_WAL ? "WAL" : "",
@@ -1833,7 +1833,7 @@ BaseBackup(void)
maxrate_clause ? maxrate_clause : "",
format == 't' ? "TABLESPACE_MAP" : "",
verify_checksums ? "" : "NOVERIFY_CHECKSUMS",
- manifest_with_checksums ? "MANIFEST_WITH_CHECKSUMS" : "");
+ manifest_with_checksums ? manifest_with_checksums : "NONE");
if (PQsendQuery(conn, basebkp) == 0)
{
@@ -2166,7 +2166,7 @@ main(int argc, char **argv)
{"waldir", required_argument, NULL, 1},
{"no-slot", no_argument, NULL, 2},
{"no-verify-checksums", no_argument, NULL, 3},
- {"manifest-with-checksums", no_argument, NULL, 4},
+ {"manifest-with-checksums", required_argument, NULL, 'm'},
{NULL, 0, NULL, 0}
};
int c;
@@ -2194,7 +2194,7 @@ main(int argc, char **argv)
atexit(cleanup_directories_atexit);
- while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
+ while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvPm:",
long_options, &option_index)) != -1)
{
switch (c)
@@ -2335,8 +2335,8 @@ main(int argc, char **argv)
case 3:
verify_checksums = false;
break;
- case 4:
- manifest_with_checksums = true;
+ case 'm':
+ manifest_with_checksums = pg_strdup(optarg);
break;
default: