v3-0004-Add-clone-and-copy-file-range-copy-strategies-to-.patch
application/octet-stream
Filename: v3-0004-Add-clone-and-copy-file-range-copy-strategies-to-.patch
Type: application/octet-stream
Part: 3
Message:
Re: pg_upgrade --copy-file-range
Patch
Format: format-patch
Series: patch v3-0004
Subject: Add --clone and --copy-file-range copy strategies to pg_combinebackup using pg_copyfile_offload().
| File | + | − |
|---|---|---|
| src/bin/pg_combinebackup/copy_file.c | 12 | 52 |
| src/bin/pg_combinebackup/copy_file.h | 4 | 1 |
| src/bin/pg_combinebackup/pg_combinebackup.c | 113 | 117 |
| src/bin/pg_combinebackup/reconstruct.c | 46 | 65 |
| src/bin/pg_combinebackup/reconstruct.h | 9 | 13 |
From 7925d17874d2f173899caff7894d348560674069 Mon Sep 17 00:00:00 2001
From: Jakub Wartak <jakub.wartak@enterprisedb.com>
Date: Fri, 5 Jan 2024 11:15:24 +0100
Subject: [PATCH v3 4/4] Add --clone and --copy-file-range copy strategies to
pg_combinebackup using pg_copyfile_offload().
Discussion: https://www.postgresql.org/message-id/flat/CA%2BhUKGJvLLNQtzb%3DZWcTsYF8kv8cR_%3DH17CX-eL8qNixeC4DAw%40mail.gmail.com#ce606227e39df74c6b2abf80b8eab04a
---
src/bin/pg_combinebackup/copy_file.c | 64 +-----
src/bin/pg_combinebackup/copy_file.h | 5 +-
src/bin/pg_combinebackup/pg_combinebackup.c | 230 ++++++++++----------
src/bin/pg_combinebackup/reconstruct.c | 111 ++++------
src/bin/pg_combinebackup/reconstruct.h | 22 +-
5 files changed, 184 insertions(+), 248 deletions(-)
diff --git a/src/bin/pg_combinebackup/copy_file.c b/src/bin/pg_combinebackup/copy_file.c
index 40a55e3087..448bfcd642 100644
--- a/src/bin/pg_combinebackup/copy_file.c
+++ b/src/bin/pg_combinebackup/copy_file.c
@@ -10,14 +10,12 @@
*/
#include "postgres_fe.h"
-#ifdef HAVE_COPYFILE_H
-#include <copyfile.h>
-#endif
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common/file_perm.h"
+#include "common/file_utils.h"
#include "common/logging.h"
#include "copy_file.h"
@@ -35,7 +33,8 @@ static void copy_file_copyfile(const char *src, const char *dst);
*/
void
copy_file(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx, bool dry_run)
+ pg_checksum_context *checksum_ctx, bool dry_run,
+ CopyFileMethod copy_strategy)
{
/*
* In dry-run mode, we don't actually copy anything, nor do we read any
@@ -49,6 +48,8 @@ copy_file(const char *src, const char *dst,
pg_fatal("could not open \"%s\": %m", src);
if (close(fd) < 0)
pg_fatal("could not close \"%s\": %m", src);
+
+ return;
}
/*
@@ -56,56 +57,13 @@ copy_file(const char *src, const char *dst,
* operating system primitives that we know about to copy the file; this
* may be quicker than a naive block copy.
*/
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- {
- char *strategy_name = NULL;
- void (*strategy_implementation) (const char *, const char *) = NULL;
-
-#ifdef WIN32
- strategy_name = "CopyFile";
- strategy_implementation = copy_file_copyfile;
-#endif
-
- if (strategy_name != NULL)
- {
- if (dry_run)
- pg_log_debug("would copy \"%s\" to \"%s\" using strategy %s",
- src, dst, strategy_name);
- else
- {
- pg_log_debug("copying \"%s\" to \"%s\" using strategy %s",
- src, dst, strategy_name);
- (*strategy_implementation) (src, dst);
- }
- return;
- }
- }
-
- /*
- * Fall back to the simple approach of reading and writing all the blocks,
- * feeding them into the checksum context as we go.
- */
- if (dry_run)
- {
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- pg_log_debug("would copy \"%s\" to \"%s\"",
- src, dst);
- else
- pg_log_debug("would copy \"%s\" to \"%s\" and checksum with %s",
- src, dst, pg_checksum_type_name(checksum_ctx->type));
- }
+ if (checksum_ctx->type == CHECKSUM_TYPE_NONE && copy_strategy != 0)
+ pg_copyfile_offload(src, dst, NULL, copy_strategy);
else
- {
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- pg_log_debug("copying \"%s\" to \"%s\"",
- src, dst);
- else
- pg_log_debug("copying \"%s\" to \"%s\" and checksumming with %s",
- src, dst, pg_checksum_type_name(checksum_ctx->type));
- copy_file_blocks(src, dst, checksum_ctx);
- }
+ pg_copyfile(src, dst, NULL, checksum_ctx);
}
+#if 0
/*
* Copy a file block by block, and optionally compute a checksum as we go.
*/
@@ -138,7 +96,8 @@ copy_file_blocks(const char *src, const char *dst,
if (wb < 0)
pg_fatal("could not write file \"%s\": %m", dst);
else
- pg_fatal("could not write file \"%s\": wrote only %d of %d bytes at offset %u",
+ pg_fatal("could not write file \"%s\": wrote only %d of %d bytes at "
+ "offset %u",
dst, (int) wb, (int) rb, offset);
}
@@ -167,3 +126,4 @@ copy_file_copyfile(const char *src, const char *dst)
}
}
#endif /* WIN32 */
+#endif
diff --git a/src/bin/pg_combinebackup/copy_file.h b/src/bin/pg_combinebackup/copy_file.h
index 031030bacb..cafaa0bc9c 100644
--- a/src/bin/pg_combinebackup/copy_file.h
+++ b/src/bin/pg_combinebackup/copy_file.h
@@ -11,9 +11,12 @@
#ifndef COPY_FILE_H
#define COPY_FILE_H
+#include "c.h"
#include "common/checksum_helper.h"
+#include "common/file_utils.h"
extern void copy_file(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx, bool dry_run);
+ pg_checksum_context *checksum_ctx, bool dry_run,
+ CopyFileMethod copy_strategy);
#endif /* COPY_FILE_H */
diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c
index cef4941d84..64dbc6c5fe 100644
--- a/src/bin/pg_combinebackup/pg_combinebackup.c
+++ b/src/bin/pg_combinebackup/pg_combinebackup.c
@@ -25,15 +25,15 @@
#include "common/logging.h"
#include "copy_file.h"
#include "fe_utils/option_utils.h"
+#include "getopt_long.h"
#include "lib/stringinfo.h"
#include "load_manifest.h"
-#include "getopt_long.h"
#include "reconstruct.h"
#include "write_manifest.h"
/* Incremental file naming convention. */
-#define INCREMENTAL_PREFIX "INCREMENTAL."
-#define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
+#define INCREMENTAL_PREFIX "INCREMENTAL."
+#define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
/*
* Tracking for directories that need to be removed, or have their contents
@@ -69,6 +69,7 @@ typedef struct cb_options
pg_checksum_type manifest_checksums;
bool no_manifest;
DataDirSyncMethod sync_method;
+ CopyFileMethod copy_method;
} cb_options;
/*
@@ -98,15 +99,10 @@ static void cleanup_directories_atexit(void);
static void create_output_directory(char *dirname, cb_options *opt);
static void help(const char *progname);
static bool parse_oid(char *s, Oid *result);
-static void process_directory_recursively(Oid tsoid,
- char *input_directory,
- char *output_directory,
- char *relative_path,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- manifest_writer *mwriter,
- cb_options *opt);
+static void process_directory_recursively(
+ Oid tsoid, char *input_directory, char *output_directory,
+ char *relative_path, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, manifest_writer *mwriter, cb_options *opt);
static int read_pg_version_file(char *directory);
static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
static void reset_directory_cleanup_list(void);
@@ -129,8 +125,9 @@ main(int argc, char *argv[])
{"manifest-checksums", required_argument, NULL, 1},
{"no-manifest", no_argument, NULL, 2},
{"sync-method", required_argument, NULL, 3},
- {NULL, 0, NULL, 0}
- };
+ {"clone", no_argument, NULL, 4},
+ {"copy-file-range", no_argument, NULL, 5},
+ {NULL, 0, NULL, 0}};
const char *progname;
char *last_input_dir;
@@ -154,10 +151,11 @@ main(int argc, char *argv[])
memset(&opt, 0, sizeof(opt));
opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
+ opt.copy_method = 0;
/* process command-line options */
- while ((c = getopt_long(argc, argv, "dnNPo:T:",
- long_options, &optindex)) != -1)
+ while ((c = getopt_long(argc, argv, "dnNPo:T:", long_options, &optindex)) !=
+ -1)
{
switch (c)
{
@@ -178,10 +176,8 @@ main(int argc, char *argv[])
add_tablespace_mapping(&opt, optarg);
break;
case 1:
- if (!pg_checksum_parse_type(optarg,
- &opt.manifest_checksums))
- pg_fatal("unrecognized checksum algorithm: \"%s\"",
- optarg);
+ if (!pg_checksum_parse_type(optarg, &opt.manifest_checksums))
+ pg_fatal("unrecognized checksum algorithm: \"%s\"", optarg);
break;
case 2:
opt.no_manifest = true;
@@ -190,6 +186,12 @@ main(int argc, char *argv[])
if (!parse_sync_method(optarg, &opt.sync_method))
exit(1);
break;
+ case 4:
+ opt.copy_method = PG_COPYFILE_IOCTL_FICLONE;
+ break;
+ case 5:
+ opt.copy_method = PG_COPYFILE_COPY_FILE_RANGE;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -211,6 +213,14 @@ main(int argc, char *argv[])
if (opt.no_manifest)
opt.manifest_checksums = CHECKSUM_TYPE_NONE;
+ /*
+ * We cannot provide file copy/clone offload in case when we need to
+ * calculate checksums
+ */
+ if (opt.copy_method != 0 && opt.manifest_checksums != CHECKSUM_TYPE_NONE)
+ pg_fatal("unable to use accelerated copy when manifest checksums "
+ "are to be calculated. Use --no-manifest");
+
/* Read the server version from the final backup. */
version = read_pg_version_file(argv[argc - 1]);
@@ -263,7 +273,8 @@ main(int argc, char *argv[])
* won't have the WAL ranges for the resulting manifest.
*/
if (manifests[n_prior_backups] == NULL)
- pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
+ pg_fatal("can't generate a manifest because no manifest is available for "
+ "the final input backup");
}
else
mwriter = NULL;
@@ -275,15 +286,15 @@ main(int argc, char *argv[])
{
pg_log_debug("generating \"%s/backup_label\"", opt.output);
last_backup_label->cursor = 0;
- write_backup_label(opt.output, last_backup_label,
- opt.manifest_checksums, mwriter);
+ write_backup_label(opt.output, last_backup_label, opt.manifest_checksums,
+ mwriter);
}
/* Process everything that's not part of a user-defined tablespace. */
pg_log_debug("processing backup directory \"%s\"", last_input_dir);
- process_directory_recursively(InvalidOid, last_input_dir, opt.output,
- NULL, n_prior_backups, prior_backup_dirs,
- manifests, mwriter, &opt);
+ process_directory_recursively(InvalidOid, last_input_dir, opt.output, NULL,
+ n_prior_backups, prior_backup_dirs, manifests,
+ mwriter, &opt);
/* Process user-defined tablespaces. */
for (ts = tablespaces; ts != NULL; ts = ts->next)
@@ -299,16 +310,15 @@ main(int argc, char *argv[])
{
char linkpath[MAXPGPATH];
- snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
- ts->oid);
+ snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output, ts->oid);
if (opt.dry_run)
pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
linkpath, ts->new_dir);
else
{
- pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
- linkpath, ts->new_dir);
+ pg_log_debug("creating symbolic link from \"%s\" to \"%s\"", linkpath,
+ ts->new_dir);
if (symlink(ts->new_dir, linkpath) != 0)
pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
linkpath, ts->new_dir);
@@ -322,21 +332,19 @@ main(int argc, char *argv[])
{
pg_log_debug("creating directory \"%s\"", ts->new_dir);
if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
- pg_fatal("could not create directory \"%s\": %m",
- ts->new_dir);
+ pg_fatal("could not create directory \"%s\": %m", ts->new_dir);
}
}
/* OK, now handle the directory contents. */
- process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
- NULL, n_prior_backups, prior_backup_dirs,
- manifests, mwriter, &opt);
+ process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir, NULL,
+ n_prior_backups, prior_backup_dirs, manifests,
+ mwriter, &opt);
}
/* Finalize the backup_manifest, if we're generating one. */
if (mwriter != NULL)
- finalize_manifest(mwriter,
- manifests[n_prior_backups]->first_wal_range);
+ finalize_manifest(mwriter, manifests[n_prior_backups]->first_wal_range);
/* fsync that output directory unless we've been told not to do so */
if (!opt.no_sync)
@@ -392,7 +400,9 @@ add_tablespace_mapping(cb_options *opt, char *arg)
*dst_ptr++ = *arg_ptr;
}
if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
- pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
+ pg_fatal(
+ "invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"",
+ arg);
/*
* All tablespaces are created with absolute directories, so specifying a
@@ -464,8 +474,8 @@ check_backup_label_files(int n_backups, char **backup_dirs)
pg_fatal("could not close \"%s\": %m", pathbuf);
/* Parse the file contents. */
- parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
- &previous_tli, &previous_lsn);
+ parse_backup_label(pathbuf, buf, &start_tli, &start_lsn, &previous_tli,
+ &previous_lsn);
/*
* Sanity checks.
@@ -476,18 +486,19 @@ check_backup_label_files(int n_backups, char **backup_dirs)
* we don't have that information.
*/
if (i > 0 && previous_tli == 0)
- pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
+ pg_fatal("backup at \"%s\" is a full backup, but only the first backup "
+ "should be a full backup",
backup_dirs[i]);
if (i == 0 && previous_tli != 0)
- pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
+ pg_fatal("backup at \"%s\" is an incremental backup, but the first "
+ "backup should be a full backup",
backup_dirs[i]);
if (i < n_backups - 1 && start_tli != check_tli)
pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
backup_dirs[i], start_tli, check_tli);
if (i < n_backups - 1 && start_lsn != check_lsn)
pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
- backup_dirs[i],
- LSN_FORMAT_ARGS(start_lsn),
+ backup_dirs[i], LSN_FORMAT_ARGS(start_lsn),
LSN_FORMAT_ARGS(check_lsn));
check_tli = previous_tli;
check_lsn = previous_lsn;
@@ -542,8 +553,7 @@ check_control_files(int n_backups, char **backup_dirs)
/* Can't interpret control file if not current version. */
if (control_file->pg_control_version != PG_CONTROL_VERSION)
- pg_fatal("%s: unexpected control file version",
- controlpath);
+ pg_fatal("%s: unexpected control file version", controlpath);
/* System identifiers should all match. */
if (i == n_backups - 1)
@@ -667,14 +677,23 @@ help(const char *progname)
printf(_("\nOptions:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" -n, --dry-run don't actually do anything\n"));
- printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
+ printf(_(" -N, --no-sync do not wait for changes to be written "
+ "safely to disk\n"));
printf(_(" -o, --output output directory\n"));
- printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
+ printf(_(
+ " -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
" relocate tablespace in OLDDIR to NEWDIR\n"));
- printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
+ printf(
+ _(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
" use algorithm for manifest checksums\n"));
- printf(_(" --no-manifest suppress generation of backup manifest\n"));
- printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(
+ " --no-manifest suppress generation of backup manifest\n"));
+ printf(
+ _(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --clone clone (reflink) instead of copying "
+ "files\n"));
+ printf(
+ _(" --copy-file-range copy using copy_file_range() syscall\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
@@ -721,15 +740,10 @@ parse_oid(char *s, Oid *result)
* the locations of those previous backups.
*/
static void
-process_directory_recursively(Oid tsoid,
- char *input_directory,
- char *output_directory,
- char *relative_path,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- manifest_writer *mwriter,
- cb_options *opt)
+process_directory_recursively(
+ Oid tsoid, char *input_directory, char *output_directory,
+ char *relative_path, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, manifest_writer *mwriter, cb_options *opt)
{
char ifulldir[MAXPGPATH];
char ofulldir[MAXPGPATH];
@@ -782,13 +796,11 @@ process_directory_recursively(Oid tsoid,
}
else
{
- snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
- relative_path);
- snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
- relative_path);
+ snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory, relative_path);
+ snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory, relative_path);
if (OidIsValid(tsoid))
- snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
- tsoid, relative_path);
+ snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/", tsoid,
+ relative_path);
else
snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
}
@@ -824,8 +836,7 @@ process_directory_recursively(Oid tsoid,
pg_checksum_context checksum_ctx;
/* Ignore "." and ".." entries. */
- if (strcmp(de->d_name, ".") == 0 ||
- strcmp(de->d_name, "..") == 0)
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
continue;
/* Construct input path. */
@@ -861,11 +872,9 @@ process_directory_recursively(Oid tsoid,
de->d_name);
/* And recurse. */
- process_directory_recursively(tsoid,
- input_directory, output_directory,
- new_relative_path,
- n_prior_backups, prior_backup_dirs,
- manifests, mwriter, opt);
+ process_directory_recursively(tsoid, input_directory, output_directory,
+ new_relative_path, n_prior_backups,
+ prior_backup_dirs, manifests, mwriter, opt);
continue;
}
@@ -883,46 +892,37 @@ process_directory_recursively(Oid tsoid,
* Skip the backup_label and backup_manifest files; they require
* special handling and are handled elsewhere.
*/
- if (relative_path == NULL &&
- (strcmp(de->d_name, "backup_label") == 0 ||
- strcmp(de->d_name, "backup_manifest") == 0))
+ if (relative_path == NULL && (strcmp(de->d_name, "backup_label") == 0 ||
+ strcmp(de->d_name, "backup_manifest") == 0))
continue;
/*
* If it's an incremental file, hand it off to the reconstruction
* code, which will figure out what to do.
*/
- if (strncmp(de->d_name, INCREMENTAL_PREFIX,
- INCREMENTAL_PREFIX_LENGTH) == 0)
+ if (strncmp(de->d_name, INCREMENTAL_PREFIX, INCREMENTAL_PREFIX_LENGTH) ==
+ 0)
{
/* Output path should not include "INCREMENTAL." prefix. */
snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
-
/* Manifest path likewise omits incremental prefix. */
snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
/* Reconstruction logic will do the rest. */
- reconstruct_from_incremental_file(ifullpath, ofullpath,
- relative_path,
- de->d_name + INCREMENTAL_PREFIX_LENGTH,
- n_prior_backups,
- prior_backup_dirs,
- manifests,
- manifest_path,
- checksum_type,
- &checksum_length,
- &checksum_payload,
- opt->debug,
- opt->dry_run);
+ reconstruct_from_incremental_file(
+ ifullpath, ofullpath, relative_path,
+ de->d_name + INCREMENTAL_PREFIX_LENGTH, n_prior_backups,
+ prior_backup_dirs, manifests, manifest_path, checksum_type,
+ &checksum_length, &checksum_payload, opt->debug, opt->dry_run,
+ opt->copy_method);
}
else
{
/* Construct the path that the backup_manifest will use. */
- snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
- de->d_name);
+ snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, de->d_name);
/*
* It's not an incremental file, so we need to copy the entire
@@ -932,13 +932,11 @@ process_directory_recursively(Oid tsoid,
* backup_manifest for the final input directory, we can save some
* work by reusing that checksum instead of computing a new one.
*/
- if (checksum_type != CHECKSUM_TYPE_NONE &&
- latest_manifest != NULL)
+ if (checksum_type != CHECKSUM_TYPE_NONE && latest_manifest != NULL)
{
manifest_file *mfile;
- mfile = manifest_files_lookup(latest_manifest->files,
- manifest_path);
+ mfile = manifest_files_lookup(latest_manifest->files, manifest_path);
if (mfile == NULL)
{
char *bmpath;
@@ -947,10 +945,9 @@ process_directory_recursively(Oid tsoid,
* The directory is out of sync with the backup_manifest,
* so emit a warning.
*/
- bmpath = psprintf("%s/%s", input_directory,
- "backup_manifest");
- pg_log_warning("\"%s\" contains no entry for \"%s\"",
- bmpath, manifest_path);
+ bmpath = psprintf("%s/%s", input_directory, "backup_manifest");
+ pg_log_warning("\"%s\" contains no entry for \"%s\"", bmpath,
+ manifest_path);
pfree(bmpath);
}
else if (mfile->checksum_type == checksum_type)
@@ -972,7 +969,8 @@ process_directory_recursively(Oid tsoid,
/* Actually copy the file. */
snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
- copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run);
+ copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run,
+ opt->copy_method);
/*
* If copy_file() performed a checksum calculation for us, then
@@ -982,8 +980,7 @@ process_directory_recursively(Oid tsoid,
if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
{
checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- checksum_length = pg_checksum_final(&checksum_ctx,
- checksum_payload);
+ checksum_length = pg_checksum_final(&checksum_ctx, checksum_payload);
}
}
@@ -1009,10 +1006,8 @@ process_directory_recursively(Oid tsoid,
pg_fatal("could not stat file \"%s\": %m", ofullpath);
/* OK, now do the work. */
- add_file_to_manifest(mwriter, manifest_path,
- sb.st_size, sb.st_mtime,
- checksum_type, checksum_length,
- checksum_payload);
+ add_file_to_manifest(mwriter, manifest_path, sb.st_size, sb.st_mtime,
+ checksum_type, checksum_length, checksum_payload);
}
/* Avoid leaking memory. */
@@ -1120,7 +1115,8 @@ reset_directory_cleanup_list(void)
* final backup in the backup chain.
*/
static cb_tablespace *
-scan_for_existing_tablespaces(char *pathname, cb_options *opt)
+scan_for_existing_tablespaces(char *pathname,
+ cb_options *opt)
{
char pg_tblspc[MAXPGPATH];
DIR *dir;
@@ -1153,7 +1149,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Ignore any file name that doesn't look like a proper OID. */
if (!parse_oid(de->d_name, &oid))
{
- pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
+ pg_log_debug(
+ "skipping \"%s\" because the filename is not a legal tablespace OID",
tblspcdir);
continue;
}
@@ -1164,7 +1161,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
exit(1);
if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
{
- pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
+ pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor "
+ "a directory",
tblspcdir);
continue;
}
@@ -1184,8 +1182,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Read the link target. */
link_length = readlink(tblspcdir, link_target, sizeof(link_target));
if (link_length < 0)
- pg_fatal("could not read symbolic link \"%s\": %m",
- tblspcdir);
+ pg_fatal("could not read symbolic link \"%s\": %m", tblspcdir);
if (link_length >= sizeof(link_target))
pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
link_target[link_length] = '\0';
@@ -1212,8 +1209,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Every non-in-place tablespace must be mapped. */
if (tsmap == NULL)
- pg_fatal("tablespace at \"%s\" has no tablespace mapping",
- link_target);
+ pg_fatal("tablespace at \"%s\" has no tablespace mapping", link_target);
}
else
{
@@ -1274,8 +1270,8 @@ slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
if (rb < 0)
pg_fatal("could not read file \"%s\": %m", filename);
else
- pg_fatal("could not read file \"%s\": read only %d of %d bytes",
- filename, (int) rb, (int) st.st_size);
+ pg_fatal("could not read file \"%s\": read only %d of %d bytes", filename,
+ (int) rb, (int) st.st_size);
}
/* Adjust buffer length for new data and restore trailing-\0 invariant */
diff --git a/src/bin/pg_combinebackup/reconstruct.c b/src/bin/pg_combinebackup/reconstruct.c
index 874e6cd150..3f66eb9ad6 100644
--- a/src/bin/pg_combinebackup/reconstruct.c
+++ b/src/bin/pg_combinebackup/reconstruct.c
@@ -15,8 +15,8 @@
#include <unistd.h>
#include "backup/basebackup_incremental.h"
-#include "common/logging.h"
#include "common/file_perm.h"
+#include "common/logging.h"
#include "copy_file.h"
#include "lib/stringinfo.h"
#include "reconstruct.h"
@@ -46,20 +46,16 @@ typedef struct rfile
off_t highest_offset_read;
} rfile;
-static void debug_reconstruction(int n_source,
- rfile **sources,
- bool dry_run);
+static void debug_reconstruction(int n_source, rfile **sources, bool dry_run);
static unsigned find_reconstructed_block_length(rfile *s);
static rfile *make_incremental_rfile(char *filename);
static rfile *make_rfile(char *filename, bool missing_ok);
static void write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length,
- rfile **sourcemap,
+ unsigned block_length, rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug,
- bool dry_run);
+ bool debug, bool dry_run);
static void read_bytes(rfile *rf, void *buffer, unsigned length);
/*
@@ -78,19 +74,13 @@ static void read_bytes(rfile *rf, void *buffer, unsigned length);
* an array of pathnames where those backups can be found.
*/
void
-reconstruct_from_incremental_file(char *input_filename,
- char *output_filename,
- char *relative_path,
- char *bare_file_name,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- char *manifest_path,
- pg_checksum_type checksum_type,
- int *checksum_length,
- uint8 **checksum_payload,
- bool debug,
- bool dry_run)
+reconstruct_from_incremental_file(
+ char *input_filename, char *output_filename, char *relative_path,
+ char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, char *manifest_path,
+ pg_checksum_type checksum_type, int *checksum_length,
+ uint8 **checksum_payload, bool debug, bool dry_run,
+ CopyFileMethod copy_method)
{
rfile **source;
rfile *latest_source = NULL;
@@ -167,8 +157,8 @@ reconstruct_from_incremental_file(char *input_filename,
* Look for the full file in the previous backup. If not found, then
* look for an incremental file instead.
*/
- snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
- prior_backup_dirs[sidx], relative_path, bare_file_name);
+ snprintf(source_filename, MAXPGPATH, "%s/%s/%s", prior_backup_dirs[sidx],
+ relative_path, bare_file_name);
if ((s = make_rfile(source_filename, true)) == NULL)
{
snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
@@ -231,8 +221,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
uint64 expected_length;
- expected_length =
- (uint64) latest_source->truncation_block_length;
+ expected_length = (uint64) latest_source->truncation_block_length;
expected_length *= BLCKSZ;
if (expected_length == sb.st_size)
{
@@ -253,8 +242,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
BlockNumber b = s->relative_block_numbers[i];
- if (b < latest_source->truncation_block_length &&
- sourcemap[b] == NULL)
+ if (b < latest_source->truncation_block_length && sourcemap[b] == NULL)
{
sourcemap[b] = s;
offsetmap[b] = s->header_length + (i * BLCKSZ);
@@ -283,16 +271,16 @@ reconstruct_from_incremental_file(char *input_filename,
manifest_path);
if (mfile == NULL)
{
- char *path = psprintf("%s/backup_manifest",
- prior_backup_dirs[copy_source_index]);
+ char *path =
+ psprintf("%s/backup_manifest", prior_backup_dirs[copy_source_index]);
/*
* The directory is out of sync with the backup_manifest, so emit
* a warning.
*/
- /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
- pg_log_warning("\"%s\" contains no entry for \"%s\"",
- path,
+ /*- translator: the first %s is a backup manifest file, the second is a
+ * file absent therein */
+ pg_log_warning("\"%s\" contains no entry for \"%s\"", path,
manifest_path);
pfree(path);
}
@@ -300,8 +288,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
*checksum_length = mfile->checksum_length;
*checksum_payload = pg_malloc(*checksum_length);
- memcpy(*checksum_payload, mfile->checksum_payload,
- *checksum_length);
+ memcpy(*checksum_payload, mfile->checksum_payload, *checksum_length);
checksum_type = CHECKSUM_TYPE_NONE;
}
}
@@ -318,13 +305,13 @@ reconstruct_from_incremental_file(char *input_filename,
* Otherwise, reconstruct.
*/
if (copy_source != NULL)
- copy_file(copy_source->filename, output_filename,
- &checksum_ctx, dry_run);
+ copy_file(copy_source->filename, output_filename, &checksum_ctx, dry_run,
+ copy_method);
else
{
- write_reconstructed_file(input_filename, output_filename,
- block_length, sourcemap, offsetmap,
- &checksum_ctx, debug, dry_run);
+ write_reconstructed_file(input_filename, output_filename, block_length,
+ sourcemap, offsetmap, &checksum_ctx, debug,
+ dry_run);
debug_reconstruction(n_prior_backups + 1, source, dry_run);
}
@@ -332,8 +319,7 @@ reconstruct_from_incremental_file(char *input_filename,
if (checksum_type != CHECKSUM_TYPE_NONE)
{
*checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- *checksum_length = pg_checksum_final(&checksum_ctx,
- *checksum_payload);
+ *checksum_length = pg_checksum_final(&checksum_ctx, *checksum_payload);
}
/*
@@ -378,11 +364,11 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
/* Debug logging. */
if (dry_run)
- pg_log_debug("would have read %u blocks from \"%s\"",
- s->num_blocks_read, s->filename);
+ pg_log_debug("would have read %u blocks from \"%s\"", s->num_blocks_read,
+ s->filename);
else
- pg_log_debug("read %u blocks from \"%s\"",
- s->num_blocks_read, s->filename);
+ pg_log_debug("read %u blocks from \"%s\"", s->num_blocks_read,
+ s->filename);
/*
* In dry-run mode, we don't actually try to read data from the file,
@@ -401,8 +387,7 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
pg_fatal("could not stat \"%s\": %m", s->filename);
if (sb.st_size < s->highest_offset_read)
pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
- s->filename,
- (unsigned long long) s->highest_offset_read,
+ s->filename, (unsigned long long) s->highest_offset_read,
(unsigned long long) sb.st_size);
}
}
@@ -455,7 +440,8 @@ make_incremental_rfile(char *filename)
read_bytes(rf, &rf->truncation_block_length,
sizeof(rf->truncation_block_length));
if (rf->truncation_block_length > RELSEG_SIZE)
- pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
+ pg_fatal("file \"%s\" has truncation block length %u in excess of segment "
+ "size %u",
filename, rf->truncation_block_length, RELSEG_SIZE);
/* Read block numbers if there are any. */
@@ -522,12 +508,10 @@ read_bytes(rfile *rf, void *buffer, unsigned length)
static void
write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length,
- rfile **sourcemap,
+ unsigned block_length, rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug,
- bool dry_run)
+ bool debug, bool dry_run)
{
int wfd = -1;
unsigned i;
@@ -570,14 +554,13 @@ write_reconstructed_file(char *input_filename,
if (current_block == start_of_range)
appendStringInfo(&debug_buf, " %u:zero", current_block);
else
- appendStringInfo(&debug_buf, " %u-%u:zero",
- start_of_range, current_block);
+ appendStringInfo(&debug_buf, " %u-%u:zero", start_of_range,
+ current_block);
}
else
{
if (current_block == start_of_range)
- appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
- current_block,
+ appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT, current_block,
s == NULL ? "ZERO" : s->filename,
(uint64) offsetmap[current_block]);
else
@@ -604,8 +587,7 @@ write_reconstructed_file(char *input_filename,
/* Open the output file, except in dry_run mode. */
if (!dry_run &&
- (wfd = open(output_filename,
- O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
+ (wfd = open(output_filename, O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
pg_file_create_mode)) < 0)
pg_fatal("could not open file \"%s\": %m", output_filename);
@@ -622,8 +604,8 @@ write_reconstructed_file(char *input_filename,
else
{
s->num_blocks_read++;
- s->highest_offset_read = Max(s->highest_offset_read,
- offsetmap[i] + BLCKSZ);
+ s->highest_offset_read =
+ Max(s->highest_offset_read, offsetmap[i] + BLCKSZ);
}
/* Skip the rest of this in dry-run mode. */
@@ -650,9 +632,9 @@ write_reconstructed_file(char *input_filename,
if (rb < 0)
pg_fatal("could not read file \"%s\": %m", s->filename);
else
- pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
- s->filename, rb, BLCKSZ,
- (unsigned long long) offsetmap[i]);
+ pg_fatal("could not read file \"%s\": read only %d of %d bytes at "
+ "offset %llu",
+ s->filename, rb, BLCKSZ, (unsigned long long) offsetmap[i]);
}
}
@@ -668,8 +650,7 @@ write_reconstructed_file(char *input_filename,
/* Update the checksum computation. */
if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
- pg_fatal("could not update checksum of file \"%s\"",
- output_filename);
+ pg_fatal("could not update checksum of file \"%s\"", output_filename);
}
/* Debugging output. */
diff --git a/src/bin/pg_combinebackup/reconstruct.h b/src/bin/pg_combinebackup/reconstruct.h
index d689aeb5c2..6623ca932e 100644
--- a/src/bin/pg_combinebackup/reconstruct.h
+++ b/src/bin/pg_combinebackup/reconstruct.h
@@ -13,21 +13,17 @@
#ifndef RECONSTRUCT_H
#define RECONSTRUCT_H
+#include "c.h"
#include "common/checksum_helper.h"
+#include "common/file_utils.h"
#include "load_manifest.h"
-extern void reconstruct_from_incremental_file(char *input_filename,
- char *output_filename,
- char *relative_path,
- char *bare_file_name,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- char *manifest_path,
- pg_checksum_type checksum_type,
- int *checksum_length,
- uint8 **checksum_payload,
- bool debug,
- bool dry_run);
+extern void reconstruct_from_incremental_file(
+ char *input_filename, char *output_filename, char *relative_path,
+ char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, char *manifest_path,
+ pg_checksum_type checksum_type, int *checksum_length,
+ uint8 **checksum_payload, bool debug, bool dry_run,
+ CopyFileMethod copy_method);
#endif
--
2.30.2