From 5c3baefd9c9caca252f949528667d2427b037579 Mon Sep 17 00:00:00 2001 From: Rushabh Lathia Date: Wed, 13 Nov 2019 15:25:54 +0530 Subject: [PATCH] POC of backup manifest with file names, sizes, timestamps, checksums. --- src/backend/access/transam/xlog.c | 3 +- src/backend/replication/basebackup.c | 238 ++++++++++++++++++++++++++++++---- src/bin/pg_basebackup/pg_basebackup.c | 116 ++++++++++++++++- src/include/replication/basebackup.h | 4 +- 4 files changed, 334 insertions(+), 27 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 3b766e6..b7f1fe5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -10495,7 +10495,8 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, ti->oid = pstrdup(de->d_name); ti->path = pstrdup(buflinkpath.data); ti->rpath = relpath ? pstrdup(relpath) : NULL; - ti->size = infotbssize ? sendTablespace(fullpath, true) : -1; + ti->size = infotbssize ? + sendTablespace(fullpath, ti->oid, true, NULL) : -1; if (tablespaces) *tablespaces = lappend(*tablespaces, ti); diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 1fa4551..9812f2a 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -19,6 +19,7 @@ #include "access/xlog_internal.h" /* for pg_start/stop_backup */ #include "catalog/pg_type.h" #include "common/file_perm.h" +#include "common/sha2.h" #include "lib/stringinfo.h" #include "libpq/libpq.h" #include "libpq/pqformat.h" @@ -55,16 +56,25 @@ typedef struct static int64 sendDir(const char *path, int basepathlen, bool sizeonly, - List *tablespaces, bool sendtblspclinks); + List *tablespaces, bool sendtblspclinks, + StringInfo manifest, const char *tsoid); static bool sendFile(const char *readfilename, const char *tarfilename, - struct stat *statbuf, bool missing_ok, Oid dboid); -static void sendFileWithContent(const char *filename, const char *content); + struct stat *statbuf, bool missing_ok, Oid dboid, + StringInfo manifest, const char *tsoid); +static void sendFileWithContent(const char *filename, const char *content, + StringInfo manifest); static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly); static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); +static void InitializeManifest(StringInfo manifest); +static void AddFileToManifest(StringInfo manifest, const char *tsoid, + const char *filename, size_t size, time_t mtime, + uint8 *shabuf); +static void SendBackupManifest(StringInfo manifest); +static char *escape_field_for_manifest(const char *s); static void base_backup_cleanup(int code, Datum arg); static void perform_base_backup(basebackup_options *opt); static void parse_basebackup_options(List *options, basebackup_options *opt); @@ -241,6 +251,7 @@ perform_base_backup(basebackup_options *opt) TimeLineID endtli; StringInfo labelfile; StringInfo tblspc_map_file = NULL; + StringInfo manifest; int datadirpathlen; List *tablespaces = NIL; @@ -250,6 +261,8 @@ perform_base_backup(basebackup_options *opt) labelfile = makeStringInfo(); tblspc_map_file = makeStringInfo(); + manifest = makeStringInfo(); + InitializeManifest(manifest); total_checksum_failures = 0; @@ -286,7 +299,10 @@ perform_base_backup(basebackup_options *opt) /* Add a node for the base directory at the end */ ti = palloc0(sizeof(tablespaceinfo)); - ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1; + if (opt->progress) + ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL); + else + ti->size = -1; tablespaces = lappend(tablespaces, ti); /* Send tablespace header */ @@ -333,7 +349,8 @@ perform_base_backup(basebackup_options *opt) struct stat statbuf; /* In the main tar, include the backup_label first... */ - sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data); + sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data, + manifest); /* * Send tablespace_map file if required and then the bulk of @@ -341,11 +358,12 @@ perform_base_backup(basebackup_options *opt) */ if (tblspc_map_file && opt->sendtblspcmapfile) { - sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data); - sendDir(".", 1, false, tablespaces, false); + sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data, + manifest); + sendDir(".", 1, false, tablespaces, false, manifest, NULL); } else - sendDir(".", 1, false, tablespaces, true); + sendDir(".", 1, false, tablespaces, true, manifest, NULL); /* ... and pg_control after everything else. */ if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) @@ -353,10 +371,11 @@ perform_base_backup(basebackup_options *opt) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", XLOG_CONTROL_FILE))); - sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false, InvalidOid); + sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, + false, InvalidOid, manifest, NULL); } else - sendTablespace(ti->path, false); + sendTablespace(ti->path, ti->oid, false, manifest); /* * If we're including WAL, and this is the main data directory we @@ -575,7 +594,7 @@ perform_base_backup(basebackup_options *opt) * complete segment. */ StatusFilePath(pathbuf, walFileName, ".done"); - sendFileWithContent(pathbuf, ""); + sendFileWithContent(pathbuf, "", manifest); } /* @@ -598,16 +617,20 @@ perform_base_backup(basebackup_options *opt) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathbuf))); - sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid); + sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid, manifest, + NULL); /* unconditionally mark file as archived */ StatusFilePath(pathbuf, fname, ".done"); - sendFileWithContent(pathbuf, ""); + sendFileWithContent(pathbuf, "", manifest); } /* Send CopyDone message for the last tar file */ pq_putemptymessage('c'); } + + SendBackupManifest(manifest); + SendXlogRecPtrResult(endptr, endtli); if (total_checksum_failures) @@ -860,6 +883,151 @@ SendBackupHeader(List *tablespaces) pq_puttextmessage('C', "SELECT"); } +static void +InitializeManifest(StringInfo manifest) +{ + appendStringInfoString(manifest, "PostgreSQL-Backup-Manifest-Version 1\n"); +} + +/* + * Add an entry to the backup manifest for a file. + */ +static void +AddFileToManifest(StringInfo manifest, const char *tsoid, + const char *filename, size_t size, time_t mtime, + uint8 *shabuf) +{ + char pathbuf[MAXPGPATH]; + char *escaped_filename; + static char timebuf[128]; + static char shatextbuf[PG_SHA256_DIGEST_LENGTH * 2 + 1]; + int shatextlen; + + /* + * If this file is part of a tablespace, the filename passed to this + * function will be relative to the tar file that contains it. We want + * the pathname relative to the data directory (ignoring the intermediate + * symlink traversal). + */ + if (tsoid != NULL) + { + snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", tsoid, filename); + filename = pathbuf; + } + + /* Escape filename, if necessary. */ + escaped_filename = escape_field_for_manifest(filename); + + /* + * Convert time to a string. Since it's not clear what time zone to use + * and since time zone definitions can change, possibly causing confusion, + * use GMT always. + */ + pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z", + pg_gmtime(&mtime)); + + /* Convert checksum to hexadecimal. */ + shatextlen = + hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, shatextbuf); + Assert(shatextlen + 1 == sizeof(shatextbuf)); + shatextbuf[shatextlen] = '\0'; + + /* Add to manifest. */ + appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s\n", + escaped_filename == NULL ? filename : escaped_filename, + size, timebuf, shatextbuf); + + /* Avoid leaking memory. */ + if (escaped_filename != NULL) + pfree(escaped_filename); +} + +/* + * Finalize the backup manifest, and send it to the client. + */ +static void +SendBackupManifest(StringInfo manifest) +{ + pg_sha256_ctx sha256_ctx; + uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; + StringInfoData protobuf; + int shastringlen; + + /* Checksum the manifest. */ + pg_sha256_init(&sha256_ctx); + pg_sha256_update(&sha256_ctx, (uint8 *) manifest->data, manifest->len); + pg_sha256_final(&sha256_ctx, shabuf); + appendStringInfoString(manifest, "Manifest-Checksum\t"); + shastringlen = PG_SHA256_DIGEST_LENGTH * 2; + enlargeStringInfo(manifest, shastringlen); + shastringlen = hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, + manifest->data + manifest->len); + Assert(shastringlen == PG_SHA256_DIGEST_LENGTH * 2); + manifest->len += shastringlen; + appendStringInfoChar(manifest, '\n'); + + /* Send CopyOutResponse message */ + pq_beginmessage(&protobuf, 'H'); + pq_sendbyte(&protobuf, 0); /* overall format */ + pq_sendint16(&protobuf, 0); /* natts */ + pq_endmessage(&protobuf); + + /* Send CopyData message */ + pq_putmessage('d', manifest->data, manifest->len); + + /* And finally CopyDone message */ + pq_putemptymessage('c'); +} + +/* + * Escape a field for inclusion in a manifest. + * + * We use the following escaping rule: If a field contains \t, \r, or \n, + * the field must be surrounded by double-quotes, and any internal double + * quotes must be doubled. Otherwise, no escaping is required. + * + * The return value is a new palloc'd string with escaping added, or NULL + * if no escaping is required. + */ +static char * +escape_field_for_manifest(const char *s) +{ + bool escaping_required = false; + int escaped_length = 2; + const char *t; + char *result; + char *r; + + for (t = s; *t != '\0'; ++t) + { + if (*t == '\t' || *t == '\r' || *t == '\n') + escaping_required = true; + if (*t == '"') + ++escaped_length; + ++escaped_length; + } + + if (!escaping_required) + return NULL; + + result = palloc(escaped_length + 1); + result[0] = '"'; + result[escaped_length - 1] = '"'; + result[escaped_length] = '\0'; + r = result + 1; + + for (t = s; *t != '\0'; ++t) + { + *(r++) = *t; + if (*t == '"') + *(r++) = *t; + } + + Assert(r == &result[escaped_length - 1]); + + return result; +} + /* * Send a single resultset containing just a single * XLogRecPtr record (in text format) @@ -920,11 +1088,16 @@ SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli) * Inject a file with given name and content in the output tar stream. */ static void -sendFileWithContent(const char *filename, const char *content) +sendFileWithContent(const char *filename, const char *content, + StringInfo manifest) { struct stat statbuf; int pad, len; + pg_sha256_ctx sha256_ctx; + uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; + + pg_sha256_init(&sha256_ctx); len = strlen(content); @@ -957,6 +1130,11 @@ sendFileWithContent(const char *filename, const char *content) MemSet(buf, 0, pad); pq_putmessage('d', buf, pad); } + + pg_sha256_update(&sha256_ctx, (uint8 *) content, len); + pg_sha256_final(&sha256_ctx, shabuf); + AddFileToManifest(manifest, NULL, filename, len, statbuf.st_mtime, + shabuf); } /* @@ -967,7 +1145,7 @@ sendFileWithContent(const char *filename, const char *content) * Only used to send auxiliary tablespaces, not PGDATA. */ int64 -sendTablespace(char *path, bool sizeonly) +sendTablespace(char *path, char *oid, bool sizeonly, StringInfo manifest) { int64 size; char pathbuf[MAXPGPATH]; @@ -1000,7 +1178,7 @@ sendTablespace(char *path, bool sizeonly) sizeonly); /* Send all the files in the tablespace version directory */ - size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true); + size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest, oid); return size; } @@ -1019,7 +1197,7 @@ sendTablespace(char *path, bool sizeonly) */ static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, - bool sendtblspclinks) + bool sendtblspclinks, StringInfo manifest, const char *tsoid) { DIR *dir; struct dirent *de; @@ -1295,7 +1473,8 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, skip_this_dir = true; if (!skip_this_dir) - size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks); + size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, + sendtblspclinks, manifest, tsoid); } else if (S_ISREG(statbuf.st_mode)) { @@ -1303,7 +1482,8 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, if (!sizeonly) sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf, - true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid); + true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid, + manifest, tsoid); if (sent || sizeonly) { @@ -1366,8 +1546,9 @@ is_checksummed_file(const char *fullpath, const char *filename) * and the file did not exist. */ static bool -sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, - bool missing_ok, Oid dboid) +sendFile(const char *readfilename, const char *tarfilename, + struct stat *statbuf, bool missing_ok, Oid dboid, + StringInfo manifest, const char *tsoid) { FILE *fp; BlockNumber blkno = 0; @@ -1384,6 +1565,10 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf int segmentno = 0; char *segmentpath; bool verify_checksum = false; + pg_sha256_ctx sha256_ctx; + uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; + + pg_sha256_init(&sha256_ctx); fp = AllocateFile(readfilename, "rb"); if (fp == NULL) @@ -1553,6 +1738,9 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf ereport(ERROR, (errmsg("base backup could not send data, aborting backup"))); + /* Also feed it to the checksum machinery. */ + pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt); + len += cnt; throttle(cnt); @@ -1577,6 +1765,7 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf { cnt = Min(sizeof(buf), statbuf->st_size - len); pq_putmessage('d', buf, cnt); + pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt); len += cnt; throttle(cnt); } @@ -1584,7 +1773,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf /* * Pad to 512 byte boundary, per tar format requirements. (This small - * piece of data is probably not worth throttling.) + * piece of data is probably not worth throttling, and is not checksummed + * because it's not actually part of the file.) */ pad = ((len + 511) & ~511) - len; if (pad > 0) @@ -1608,6 +1798,10 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf total_checksum_failures += checksum_failures; + pg_sha256_final(&sha256_ctx, shabuf); + AddFileToManifest(manifest, tsoid, tarfilename, statbuf->st_size, + statbuf->st_mtime, shabuf); + return true; } diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 0565212..c56246f 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -88,6 +88,12 @@ typedef struct UnpackTarState FILE *file; } UnpackTarState; +typedef struct WriteManifestState +{ + char filename[MAXPGPATH]; + FILE *file; +} WriteManifestState; + typedef void (*WriteDataCallback) (size_t nbytes, char *buf, void *callback_data); @@ -180,6 +186,12 @@ static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data); static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum); static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data); +static void ReceiveBackupManifest(PGconn *conn); +static void ReceiveBackupManifestChunk(size_t r, char *copybuf, + void *callback_data); +static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf); +static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, + void *callback_data); static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, @@ -924,8 +936,8 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback, res = PQgetResult(conn); if (PQresultStatus(res) != PGRES_COPY_OUT) { - pg_log_error("could not get COPY data stream: %s", - PQerrorMessage(conn)); + pg_log_error("could not get COPY data stream: %s [%s]", + PQerrorMessage(conn), PQresStatus(PQresultStatus(res))); exit(1); } PQclear(res); @@ -1170,6 +1182,31 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) } } + /* + * Normally, we emit the backup manifest as a separate file, but when + * we're writing a tarfile to stdout, we don't have that option, so + * include it in the one tarfile we've got. + */ + if (strcmp(basedir, "-") == 0) + { + char header[512]; + PQExpBufferData buf; + + initPQExpBuffer(&buf); + ReceiveBackupManifestInMemory(conn, &buf); + if (PQExpBufferBroken(&buf)) + { + pg_log_error("out of memory"); + exit(1); + } + tarCreateHeader(header, "backup_manifest", NULL, buf.len, + pg_file_create_mode, 04000, 02000, + time(NULL)); + writeTarData(&state, header, sizeof(header)); + writeTarData(&state, buf.data, buf.len); + termPQExpBuffer(&buf); + } + /* 2 * 512 bytes empty data at end of file */ writeTarData(&state, zerobuf, sizeof(zerobuf)); @@ -1417,6 +1454,64 @@ get_tablespace_mapping(const char *dir) /* + * Receive the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifest(PGconn *conn) +{ + WriteManifestState state; + + snprintf(state.filename, sizeof(state.filename), + "%s/backup_manifest", basedir); + state.file = fopen(state.filename, "wb"); + if (state.file == NULL) + { + pg_log_error("could not create file \"%s\": %m", state.filename); + exit(1); + } + + ReceiveCopyData(conn, ReceiveBackupManifestChunk, &state); + + fclose(state.file); +} + +/* + * Receive one chunk of the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data) +{ + WriteManifestState *state = callback_data; + + if (fwrite(copybuf, r, 1, state->file) != 1) + { + pg_log_error("could not write to file \"%s\": %m", state->filename); + exit(1); + } +} + +/* + * Receive the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf) +{ + ReceiveCopyData(conn, ReceiveBackupManifestInMemoryChunk, buf); +} + +/* + * Receive one chunk of the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, + void *callback_data) +{ + PQExpBuffer buf = callback_data; + + appendPQExpBuffer(buf, copybuf, r); +} + +/* * Receive a tar format stream from the connection to the server, and unpack * the contents of it into a directory. Only files, directories and * symlinks are supported, no other kinds of special files. @@ -1658,6 +1753,7 @@ BaseBackup(void) maxServerMajor; int serverVersion, serverMajor; + int writing_to_stdout; Assert(conn != NULL); @@ -1821,7 +1917,8 @@ BaseBackup(void) /* * When writing to stdout, require a single tablespace */ - if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1) + writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0; + if (writing_to_stdout && PQntuples(res) > 1) { pg_log_error("can only write single tablespace to stdout, database has %d", PQntuples(res)); @@ -1850,6 +1947,19 @@ BaseBackup(void) ReceiveAndUnpackTarFile(conn, res, i); } /* Loop over all tablespaces */ + /* + * Now receive backup manifest, if appropriate. + * + * If we're writing a tarfile to stdout, ReceiveTarFile will have already + * processed the backup manifest and included it in the output tarfile. + * Such a configuration doesn't allow for writing multiple files. + * + * If we're talking to an older server, it won't send a backup manifest, + * so don't try to receive one. + */ + if (!writing_to_stdout && serverMajor >= 1300) + ReceiveBackupManifest(conn); + if (showprogress) { progress_report(PQntuples(res), NULL, true); diff --git a/src/include/replication/basebackup.h b/src/include/replication/basebackup.h index 503a5b9..8fe0136 100644 --- a/src/include/replication/basebackup.h +++ b/src/include/replication/basebackup.h @@ -12,6 +12,7 @@ #ifndef _BASEBACKUP_H #define _BASEBACKUP_H +#include "lib/stringinfo.h" #include "nodes/replnodes.h" /* @@ -31,6 +32,7 @@ typedef struct extern void SendBaseBackup(BaseBackupCmd *cmd); -extern int64 sendTablespace(char *path, bool sizeonly); +extern int64 sendTablespace(char *path, char *oid, bool sizeonly, + StringInfo manifest); #endif /* _BASEBACKUP_H */ -- 1.8.3.1