diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c index e4d057e..7ed3522 100644 --- a/contrib/pg_standby/pg_standby.c +++ b/contrib/pg_standby/pg_standby.c @@ -33,9 +33,12 @@ #include "pg_getopt.h" #include "access/xlog_internal.h" +#include "access/xlogreader.h" const char *progname; +uint32 XLogSegSize; + /* Options and defaults */ int sleeptime = 5; /* amount of time to sleep between file checks */ int waittime = -1; /* how long we have been waiting, -1 no wait @@ -100,6 +103,54 @@ int nextWALFileType; struct stat stat_buf; +static bool SetWALFileNameForCleanup(void); + +/* Set XLogSegSize from the WAL file header */ +static bool +RetrieveXLogSegSize() +{ + int fd; + bool ret = false; + + if (stat(WALFilePath, &stat_buf) == 0) + { + if ((fd = open(WALFilePath, O_RDWR, 0)) >= 0) + { + char *buf = (char *) malloc(XLOG_BLCKSZ); + + if (read(fd, buf, XLOG_BLCKSZ) != XLOG_BLCKSZ) + { + fprintf(stderr, "could not read from log segment %s", + nextWALFileName); + fflush(stderr); + } + else + { + XLogPageHeader hdr = (XLogPageHeader) buf; + XLogLongPageHeader NewLongPage = (XLogLongPageHeader) hdr; + + XLogSegSize = NewLongPage->xlp_seg_size; + need_cleanup = SetWALFileNameForCleanup(); + + if (debug) + { + fprintf(stderr, + _("WAL segment size: %d \n"), NewLongPage->xlp_seg_size); + fprintf(stderr, "Keep archive history: "); + if (need_cleanup) + fprintf(stderr, "%s and later\n", exclusiveCleanupFileName); + else + fprintf(stderr, "no cleanup required\n"); + fflush(stderr); + } + ret = true; + } + } + close(fd); + } + return ret; +} + /* ===================================================================== * * Customizable section @@ -184,42 +235,47 @@ CustomizableNextWALFileReady(void) nextWALFileType = XLOG_BACKUP_LABEL; return true; } - else if (stat_buf.st_size == XLOG_SEG_SIZE) + else { + /* Check if we can read the WAL header and set the XLogSegSize */ + if (!RetrieveXLogSegSize()) + return false; + + if (stat_buf.st_size == XLOG_SEG_SIZE) + { #ifdef WIN32 - /* - * Windows 'cp' sets the final file size before the copy is - * complete, and not yet ready to be opened by pg_standby. So we - * wait for sleeptime secs before attempting to restore. If that - * is not enough, we will rely on the retry/holdoff mechanism. - * GNUWin32's cp does not have this problem. - */ - pg_usleep(sleeptime * 1000000L); + /* + * Windows 'cp' sets the final file size before the copy is + * complete, and not yet ready to be opened by pg_standby. So + * we wait for sleeptime secs before attempting to restore. If + * that is not enough, we will rely on the retry/holdoff + * mechanism. GNUWin32's cp does not have this problem. + */ + pg_usleep(sleeptime * 1000000L); #endif - nextWALFileType = XLOG_DATA; - return true; - } + nextWALFileType = XLOG_DATA; + return true; + } - /* - * If still too small, wait until it is the correct size - */ - if (stat_buf.st_size > XLOG_SEG_SIZE) - { - if (debug) + /* + * If still too small, wait until it is the correct size + */ + if (stat_buf.st_size > XLOG_SEG_SIZE) { - fprintf(stderr, "file size greater than expected\n"); - fflush(stderr); + if (debug) + { + fprintf(stderr, "file size greater than expected\n"); + fflush(stderr); + } + exit(3); } - exit(3); } } return false; } -#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE ) - static void CustomizableCleanupPriorWALFiles(void) { @@ -315,6 +371,7 @@ SetWALFileNameForCleanup(void) uint32 log_diff = 0, seg_diff = 0; bool cleanup = false; + int MaxSegmentsPerLogFile = (0xFFFFFFFF / XLogSegSize); if (restartWALFileName) { @@ -708,8 +765,6 @@ main(int argc, char **argv) CustomizableInitialize(); - need_cleanup = SetWALFileNameForCleanup(); - if (debug) { fprintf(stderr, "Trigger file: %s\n", triggerPath ? triggerPath : ""); @@ -721,11 +776,6 @@ main(int argc, char **argv) fprintf(stderr, "Max wait interval: %d %s\n", maxwaittime, (maxwaittime > 0 ? "seconds" : "forever")); fprintf(stderr, "Command for restore: %s\n", restoreCommand); - fprintf(stderr, "Keep archive history: "); - if (need_cleanup) - fprintf(stderr, "%s and later\n", exclusiveCleanupFileName); - else - fprintf(stderr, "no cleanup required\n"); fflush(stderr); } diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index aa56a6b..461b397 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -110,6 +110,7 @@ int wal_retrieve_retry_interval = 5000; bool XLOG_DEBUG = false; #endif +uint32 XLogSegSize = XLOG_SEG_SIZE; /* * Number of WAL insertion locks to use. A higher value allows more insertions * to happen concurrently, but adds some CPU overhead to flushing the WAL, diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 0a4944d..c9db0ce 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -2388,6 +2388,9 @@ main(int argc, char **argv) exit(1); } + if (!RetrieveXLogSegSize(conn)) + disconnect_and_exit(1); + /* Create transaction log symlink, if required */ if (strcmp(xlog_dir, "") != 0) { diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c index 15348ad..986c4e6 100644 --- a/src/bin/pg_basebackup/pg_receivewal.c +++ b/src/bin/pg_basebackup/pg_receivewal.c @@ -663,6 +663,9 @@ main(int argc, char **argv) if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name)) disconnect_and_exit(1); + if (!RetrieveXLogSegSize(conn)) + disconnect_and_exit(1); + /* * Check that there is a database associated with connection, none should * be defined in this context. diff --git a/src/bin/pg_basebackup/receivelog.h b/src/bin/pg_basebackup/receivelog.h index 42e93ac..9265dc0 100644 --- a/src/bin/pg_basebackup/receivelog.h +++ b/src/bin/pg_basebackup/receivelog.h @@ -16,6 +16,7 @@ #include "walmethods.h" #include "access/xlogdefs.h" +#include "access/xlog_internal.h" /* * Called before trying to read more data or when a segment is @@ -54,4 +55,5 @@ extern bool CheckServerVersionForStreaming(PGconn *conn); extern bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream); +uint32 XLogSegSize; #endif /* RECEIVELOG_H */ diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index 1f2b406..777e74f 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -30,6 +30,9 @@ #define ERRCODE_DUPLICATE_OBJECT "42710" +/* The SHOW command for replication connection was introduced in version 10 */ +#define MINIMUM_VERSION_FOR_SHOW_CMD 100000 + const char *progname; char *connection_string = NULL; char *dbhost = NULL; @@ -231,6 +234,71 @@ GetConnection(void) } /* + * From version 10, explicitly set XLogSegSize using SHOW wal_segment_size + * since ControlFile is not accessible here + */ +bool +RetrieveXLogSegSize(PGconn *conn) +{ + PGresult *res; + char *tmp_result; + char xlog_unit[3]; + int xlog_val, + multiplier = 1; + + /* Check connection existence */ + Assert(conn != NULL); + + /* Previous versions do not support SHOW command */ + if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD) + { + const char *serverver = PQparameterStatus(conn, "server_version"); + fprintf(stderr, _("%s: incompatible server version %s; client does not support SHOW command from server versions older than %s\n"), + progname, + serverver ? serverver : "'unknown'", + "10"); + return false; + } + + res = PQexec(conn, "SHOW wal_segment_size"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + fprintf(stderr, _("%s: could not send replication command \"%s\": %s"), + progname, "SHOW wal_segment_size", PQerrorMessage(conn)); + + PQclear(res); + return false; + } + if (PQntuples(res) != 1 || PQnfields(res) < 1) + { + fprintf(stderr, + _("%s: could not fetch XLogSegSize: got %d rows and %d fields, expected %d rows and %d or more fields\n"), + progname, PQntuples(res), PQnfields(res), 1, 1); + + PQclear(res); + return false; + } + + /* wal_segment_size ranges from 1MB to 1GB */ + tmp_result = pg_strdup(PQgetvalue(res, 0, 0)); + + /* Fetch the xlog value and unit from the result */ + sscanf(tmp_result, "%d%s", &xlog_val, xlog_unit); + + /* XLogSegSize should be in bytes, set the multiplier based on unit */ + if (strcmp(xlog_unit, "MB") == 0) + multiplier = 1024 * 1024; + else if (strcmp(xlog_unit, "GB") == 0) + multiplier = 1024 * 1024 * 1024; + + /* Set the XLogSegSize */ + XLogSegSize = xlog_val * multiplier; + + PQclear(res); + return true; +} + +/* * Run IDENTIFY_SYSTEM through a given connection and give back to caller * some result information if requested: * - System identifier diff --git a/src/bin/pg_basebackup/streamutil.h b/src/bin/pg_basebackup/streamutil.h index 460dcb5..8b24d74 100644 --- a/src/bin/pg_basebackup/streamutil.h +++ b/src/bin/pg_basebackup/streamutil.h @@ -39,6 +39,7 @@ extern bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name); +extern bool RetrieveXLogSegSize(PGconn *conn); extern TimestampTz feGetCurrentTimestamp(void); extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs); diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 2ea8931..bbd2c1f 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -27,6 +27,8 @@ #include "pg_getopt.h" +uint32 XLogSegSize; + static void usage(const char *progname) { @@ -164,6 +166,9 @@ main(int argc, char *argv[]) "Either the file is corrupt, or it has a different layout than this program\n" "is expecting. The results below are untrustworthy.\n\n")); + /* Set the XLogSegSize */ + XLogSegSize = ControlFile->xlog_seg_size; + /* * This slightly-chintzy coding will work as long as the control file * timestamps are within the range of time_t; that should be the case in diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 27bd9b0..8e61782 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -56,6 +56,7 @@ #include "storage/large_object.h" #include "pg_getopt.h" +uint32 XLogSegSize; static ControlFileData ControlFile; /* pg_control values */ static XLogSegNo newXlogSegNo; /* new XLOG segment # */ @@ -93,6 +94,7 @@ main(int argc, char *argv[]) char *endptr; char *endptr2; char *DataDir = NULL; + char *log_fname = NULL; int fd; set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_resetwal")); @@ -264,7 +266,12 @@ main(int argc, char *argv[]) fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - XLogFromFileName(optarg, &minXlogTli, &minXlogSegNo); + + /* + * XLogFromFileName requires XLogSegSize which is not yet + * defined. Hence XLog details are set later on. + */ + log_fname = pg_strdup(optarg); break; default: @@ -346,6 +353,9 @@ main(int argc, char *argv[]) if (!ReadControlFile()) GuessControlValues(); + if (log_fname != NULL) + XLogFromFileName(log_fname, &minXlogTli, &minXlogSegNo); + /* * Also look at existing segment files to set up newXlogSegNo */ @@ -509,6 +519,7 @@ ReadControlFile(void) { /* Valid data... */ memcpy(&ControlFile, buffer, sizeof(ControlFile)); + XLogSegSize = ControlFile.xlog_seg_size; return true; } @@ -516,6 +527,7 @@ ReadControlFile(void) progname); /* We will use the data anyway, but treat it as guessed. */ memcpy(&ControlFile, buffer, sizeof(ControlFile)); + XLogSegSize = ControlFile.xlog_seg_size; guessed = true; return true; } diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2014fee..47b205a 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -44,6 +44,7 @@ static ControlFileData ControlFile_target; static ControlFileData ControlFile_source; const char *progname; +uint32 XLogSegSize; /* Configuration options */ char *datadir_target = NULL; @@ -631,6 +632,9 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) memcpy(ControlFile, src, sizeof(ControlFileData)); + /* Set XLogSegSize */ + XLogSegSize = ControlFile->xlog_seg_size; + /* Additional checks on control file */ checkControlFile(ControlFile); } diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index bfe44b8..8b14f20 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -22,9 +22,11 @@ #include "common/fe_memutils.h" #include "getopt_long.h" #include "rmgrdesc.h" +#include "sys/stat.h" static const char *progname; +uint32 XLogSegSize; typedef struct XLogDumpPrivate { @@ -217,6 +219,87 @@ fuzzy_open_file(const char *directory, const char *fname) return -1; } +static bool +ReadXLogFromDir(char *archive_loc) +{ + bool ret_val = false; + DIR *xldir; + + if ((xldir = opendir(archive_loc)) != NULL) + { + struct dirent *xlde; + + while ((xlde = readdir(xldir)) != NULL) + { + if (IsXLogFileName(xlde->d_name)) + { + int fd = fuzzy_open_file(archive_loc, xlde->d_name); + char *buf = (char *) malloc(XLOG_BLCKSZ); + + if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ) + { + XLogPageHeader hdr = (XLogPageHeader) buf; + XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr; + + XLogSegSize = longhdr->xlp_seg_size; + ret_val = true; + break; + } + close(fd); + } + } + } + closedir(xldir); + return ret_val; +} + +/* + * Scan through the archive location to set XLogSegsize from the first WAL file + */ +static bool +RetrieveXLogSegSize(char *archive_path) +{ + char fpath[MAXPGPATH]; + + if (archive_path == NULL) + { + /* Check in the current directory */ + if (getcwd(fpath, MAXPGPATH)) + if (ReadXLogFromDir(fpath)) + return true; + + /* Check if there is XLOGDIR in current location */ + snprintf(fpath, MAXPGPATH, "%s", XLOGDIR); + if (ReadXLogFromDir(fpath)) + return true; + else + { + /* Check in $PGDATA / XLOGDIR / */ + const char *datadir = getenv("PGDATA"); + + if (datadir != NULL) + { + snprintf(fpath, MAXPGPATH, "%s/%s", datadir, XLOGDIR); + if (ReadXLogFromDir(fpath)) + return true; + } + } + } + else + { + /* Check in the path provided */ + snprintf(fpath, MAXPGPATH, "%s", archive_path); + if (ReadXLogFromDir(fpath)) + return true; + + /* Check in the path / XLOGDIR */ + snprintf(fpath, MAXPGPATH, "%s/%s", archive_path, XLOGDIR); + if (ReadXLogFromDir(fpath)) + return true; + } + return false; +} + /* * Read count bytes from a segment file in the specified directory, for the * given timeline, containing the specified record pointer; store the data in @@ -897,6 +980,12 @@ main(int argc, char **argv) } } + if (!RetrieveXLogSegSize(private.inpath)) + { + fprintf(stderr, _("%s: coulnot access the archive location\n"), progname); + return EXIT_FAILURE; + } + /* parse files as start/end boundaries, extract path if not specified */ if (optind < argc) { diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index c09c0f8..00682fc 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -89,7 +89,7 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader; * The XLOG is split into WAL segments (physical files) of the size indicated * by XLOG_SEG_SIZE. */ -#define XLogSegSize ((uint32) XLOG_SEG_SIZE) +extern uint32 XLogSegSize; #define XLogSegmentsPerXLogId (UINT64CONST(0x100000000) / XLOG_SEG_SIZE) #define XLogSegNoOffsetToRecPtr(segno, offset, dest) \