From bf494ca68028e22fdd8984170bc7c41801e4b82a Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Sun, 27 Oct 2019 23:27:03 +0500 Subject: [PATCH 3/5] add 'exclusive' backup option in parallel backup --- src/backend/replication/basebackup.c | 99 ++++++++++++++++++++++++-- src/backend/replication/repl_gram.y | 6 ++ src/backend/replication/repl_scanner.l | 1 + 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 4a382c4558..1c657e247a 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -55,6 +55,7 @@ typedef struct bool sendtblspcmapfile; const char *tablespace_path; XLogRecPtr wal_location; + bool exclusive; } basebackup_options; typedef struct @@ -104,6 +105,7 @@ static void StartBackup(basebackup_options *opt); static void StopBackup(basebackup_options *opt); static void SendBackupManifest(basebackup_options *opt); static void SendBackupFiles(basebackup_options *opt, List *filenames, bool missing_ok); +static char *readfile(const char *readfilename, bool missing_ok); /* Was the backup currently in-progress initiated in recovery mode? */ static bool backup_started_in_recovery = false; @@ -256,7 +258,14 @@ static const char *const noChecksumFiles[] = { static void base_backup_cleanup(int code, Datum arg) { - do_pg_abort_backup(); + bool exclusive = DatumGetBool(arg); + + /* + * do_pg_abort_backup is only for non-exclusive backups, exclusive backup + * is terminated by calling pg_stop_backup(). + */ + if (!exclusive) + do_pg_abort_backup(); } /* @@ -443,6 +452,7 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_noverify_checksums = false; bool o_tablespace_path = false; bool o_wal_location = false; + bool o_exclusive = false; MemSet(opt, 0, sizeof(*opt)); foreach(lopt, options) @@ -554,6 +564,16 @@ parse_basebackup_options(List *options, basebackup_options *opt) opt->wal_location = pg_lsn_in_internal(wal_location, &have_error); o_wal_location = true; } + else if (strcmp(defel->defname, "exclusive") == 0) + { + if (o_exclusive) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("duplicate option \"%s\"", defel->defname))); + + opt->exclusive = intVal(defel->arg); + o_exclusive = true; + } else elog(ERROR, "option \"%s\" not recognized", defel->defname); @@ -1944,7 +1964,7 @@ StartBackup(basebackup_options *opt) total_checksum_failures = 0; startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli, - labelfile, &tablespaces, + opt->exclusive? NULL : labelfile, &tablespaces, tblspc_map_file, opt->progress, opt->sendtblspcmapfile); @@ -1955,7 +1975,7 @@ StartBackup(basebackup_options *opt) * do_pg_stop_backup() should be inside the error cleanup block! */ - PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0); + PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) BoolGetDatum(opt->exclusive)); { tablespaceinfo *ti; @@ -1984,6 +2004,25 @@ StartBackup(basebackup_options *opt) /* Setup and activate network throttling, if client requested it */ setup_throttle(opt->maxrate); + /* + * In exclusive mode, pg_start_backup creates backup_label and + * tablespace_map files and does not return their contents in + * *labelfile and *tblspcmapfile. So we read them from these files to + * return to frontend. + * + * In non-exclusive mode, contents of these files are available in + * *labelfile and *tblspcmapfile and are returned directly. + */ + if (opt->exclusive) + { + resetStringInfo(labelfile); + resetStringInfo(tblspc_map_file); + + appendStringInfoString(labelfile, readfile(BACKUP_LABEL_FILE, false)); + if (opt->sendtblspcmapfile) + appendStringInfoString(tblspc_map_file, readfile(TABLESPACE_MAP, false)); + } + if ((tblspc_map_file && tblspc_map_file->len <= 0) || !opt->sendtblspcmapfile) tblspc_map_file = NULL; @@ -1991,14 +2030,14 @@ StartBackup(basebackup_options *opt) /* send backup_label and tablespace_map to frontend */ SendStartBackupResult(labelfile, tblspc_map_file); } - PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0); + PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) BoolGetDatum(opt->exclusive)); } /* * StopBackup() - ends an online backup * * The function is called at the end of an online backup. It sends out pg_control - * file, optionaly WAL segments and ending WAL location. + * file, optionally WAL segments and ending WAL location. */ static void StopBackup(basebackup_options *opt) @@ -2009,7 +2048,7 @@ StopBackup(basebackup_options *opt) StringInfoData buf; char *labelfile = NULL; - PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0); + PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) BoolGetDatum(opt->exclusive)); { /* Setup and activate network throttling, if client requested it */ setup_throttle(opt->maxrate); @@ -2028,6 +2067,8 @@ StopBackup(basebackup_options *opt) sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false, InvalidOid); /* stop backup */ + if (!opt->exclusive) + labelfile = (char *) opt->label; endptr = do_pg_stop_backup(labelfile, !opt->nowait, &endtli); if (opt->includewal) @@ -2036,7 +2077,7 @@ StopBackup(basebackup_options *opt) pq_putemptymessage('c'); /* CopyDone */ SendXlogRecPtrResult(endptr, endtli); } - PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0); + PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) BoolGetDatum(opt->exclusive)); } /* @@ -2271,3 +2312,47 @@ SendBackupFiles(basebackup_options *opt, List *filenames, bool missing_ok) errmsg("checksum verification failure during base backup"))); } } + +static char * +readfile(const char *readfilename, bool missing_ok) +{ + struct stat statbuf; + FILE *fp; + char *data; + int r; + + if (stat(readfilename, &statbuf)) + { + if (errno == ENOENT && missing_ok) + return NULL; + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat file \"%s\": %m", + readfilename))); + } + + fp = AllocateFile(readfilename, "r"); + if (!fp) + { + if (errno == ENOENT && missing_ok) + return NULL; + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", readfilename))); + } + + data = palloc(statbuf.st_size + 1); + r = fread(data, statbuf.st_size, 1, fp); + data[statbuf.st_size] = '\0'; + + /* Close the file */ + if (r != 1 || ferror(fp) || FreeFile(fp)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", + readfilename))); + + return data; +} diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 9e2499814b..94c6aafbed 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -93,6 +93,7 @@ static SQLCmd *make_sqlcmd(void); %token K_STOP_BACKUP %token K_START_WAL_LOCATION %token K_TABLESPACE_PATH +%token K_EXCLUSIVE %type command %type base_backup start_replication start_logical_replication @@ -262,6 +263,11 @@ base_backup_opt: $$ = makeDefElem("tablespace_path", (Node *)makeString($2), -1); } + | K_EXCLUSIVE + { + $$ = makeDefElem("exclusive", + (Node *)makeInteger(true), -1); + } ; backup_files: diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 7a1bb54da8..ad0dd04cb1 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -113,6 +113,7 @@ SEND_BACKUP_FILES { return K_SEND_BACKUP_FILES; } STOP_BACKUP { return K_STOP_BACKUP; } START_WAL_LOCATION { return K_START_WAL_LOCATION; } TABLESPACE_PATH { return K_TABLESPACE_PATH; } +EXCLUSIVE { return K_EXCLUSIVE; } "," { return ','; } -- 2.21.0 (Apple Git-122)