02-initdb-configurable-walsegsize.patch
application/octet-stream
Filename: 02-initdb-configurable-walsegsize.patch
Type: application/octet-stream
Part: 0
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: unified
| File | + | − |
|---|---|---|
| configure | 0 | 55 |
| configure.in | 0 | 31 |
| contrib/pg_standby/pg_standby.c | 74 | 11 |
| doc/src/sgml/backup.sgml | 1 | 1 |
| doc/src/sgml/installation.sgml | 0 | 14 |
| doc/src/sgml/ref/initdb.sgml | 15 | 0 |
| doc/src/sgml/wal.sgml | 6 | 7 |
| src/backend/access/transam/xlog.c | 70 | 20 |
| src/backend/access/transam/xlogreader.c | 1 | 1 |
| src/backend/bootstrap/bootstrap.c | 9 | 1 |
| src/backend/postmaster/postmaster.c | 4 | 0 |
| src/backend/utils/misc/guc.c | 7 | 9 |
| src/backend/utils/misc/postgresql.conf.sample | 1 | 1 |
| src/bin/initdb/initdb.c | 63 | 1 |
| src/bin/pg_basebackup/pg_basebackup.c | 4 | 0 |
| src/bin/pg_basebackup/pg_receivewal.c | 6 | 2 |
| src/bin/pg_basebackup/receivelog.c | 5 | 5 |
| src/bin/pg_basebackup/streamutil.c | 76 | 0 |
| src/bin/pg_basebackup/streamutil.h | 1 | 0 |
| src/bin/pg_controldata/pg_controldata.c | 9 | 0 |
| src/bin/pg_resetwal/pg_resetwal.c | 28 | 10 |
| src/bin/pg_rewind/pg_rewind.c | 7 | 0 |
| src/bin/pg_test_fsync/pg_test_fsync.c | 3 | 3 |
| src/bin/pg_waldump/pg_waldump.c | 153 | 56 |
| src/include/access/xlog.h | 2 | 0 |
| src/include/access/xlog_internal.h | 19 | 3 |
| src/include/pg_config.h.in | 0 | 7 |
| src/include/pg_config_manual.h | 6 | 0 |
| src/tools/msvc/Solution.pm | 0 | 2 |
diff --git a/configure b/configure
index a2f9a25..13f1da2 100755
--- a/configure
+++ b/configure
@@ -821,7 +821,6 @@ enable_tap_tests
with_blocksize
with_segsize
with_wal_blocksize
-with_wal_segsize
with_CC
enable_depend
enable_cassert
@@ -1518,8 +1517,6 @@ Optional Packages:
--with-segsize=SEGSIZE set table segment size in GB [1]
--with-wal-blocksize=BLOCKSIZE
set WAL block size in kB [8]
- --with-wal-segsize=SEGSIZE
- set WAL segment size in MB [16]
--with-CC=CMD set compiler (deprecated)
--with-icu build with ICU support
--with-tcl build Tcl modules (PL/Tcl)
@@ -3732,58 +3729,6 @@ cat >>confdefs.h <<_ACEOF
#define XLOG_BLCKSZ ${XLOG_BLCKSZ}
_ACEOF
-
-#
-# WAL segment size
-#
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WAL segment size" >&5
-$as_echo_n "checking for WAL segment size... " >&6; }
-
-
-
-# Check whether --with-wal-segsize was given.
-if test "${with_wal_segsize+set}" = set; then :
- withval=$with_wal_segsize;
- case $withval in
- yes)
- as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5
- ;;
- no)
- as_fn_error $? "argument required for --with-wal-segsize option" "$LINENO" 5
- ;;
- *)
- wal_segsize=$withval
- ;;
- esac
-
-else
- wal_segsize=16
-fi
-
-
-case ${wal_segsize} in
- 1) ;;
- 2) ;;
- 4) ;;
- 8) ;;
- 16) ;;
- 32) ;;
- 64) ;;
- 128) ;;
- 256) ;;
- 512) ;;
- 1024) ;;
- *) as_fn_error $? "Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024." "$LINENO" 5
-esac
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${wal_segsize}MB" >&5
-$as_echo "${wal_segsize}MB" >&6; }
-
-
-cat >>confdefs.h <<_ACEOF
-#define XLOG_SEG_SIZE (${wal_segsize} * 1024 * 1024)
-_ACEOF
-
-
#
# C compiler
#
diff --git a/configure.in b/configure.in
index e94fba5..241e89c 100644
--- a/configure.in
+++ b/configure.in
@@ -344,37 +344,6 @@ AC_DEFINE_UNQUOTED([XLOG_BLCKSZ], ${XLOG_BLCKSZ}, [
])
#
-# WAL segment size
-#
-AC_MSG_CHECKING([for WAL segment size])
-PGAC_ARG_REQ(with, wal-segsize, [SEGSIZE], [set WAL segment size in MB [16]],
- [wal_segsize=$withval],
- [wal_segsize=16])
-case ${wal_segsize} in
- 1) ;;
- 2) ;;
- 4) ;;
- 8) ;;
- 16) ;;
- 32) ;;
- 64) ;;
- 128) ;;
- 256) ;;
- 512) ;;
- 1024) ;;
- *) AC_MSG_ERROR([Invalid WAL segment size. Allowed values are 1,2,4,8,16,32,64,128,256,512,1024.])
-esac
-AC_MSG_RESULT([${wal_segsize}MB])
-
-AC_DEFINE_UNQUOTED([XLOG_SEG_SIZE], [(${wal_segsize} * 1024 * 1024)], [
- XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
- and larger than XLOG_BLCKSZ (preferably, a great deal larger than
- XLOG_BLCKSZ).
-
- Changing XLOG_SEG_SIZE requires an initdb.
-])
-
-#
# C compiler
#
diff --git a/contrib/pg_standby/pg_standby.c b/contrib/pg_standby/pg_standby.c
index d7fa2a8..641c933 100644
--- a/contrib/pg_standby/pg_standby.c
+++ b/contrib/pg_standby/pg_standby.c
@@ -33,9 +33,12 @@
#include "pg_getopt.h"
#include "access/xlog_internal.h"
+#include "access/xlogreader.h"
const char *progname;
+uint32 XLogSegSize;
+
/* Options and defaults */
int sleeptime = 5; /* amount of time to sleep between file checks */
int waittime = -1; /* how long we have been waiting, -1 no wait
@@ -100,6 +103,72 @@ int nextWALFileType;
struct stat stat_buf;
+static bool SetWALFileNameForCleanup(void);
+
+/* Set wal segment size from the WAL file specified by WALFilePath */
+static bool
+RetrieveXLogSegSize(void)
+{
+ bool ret_val = false;
+ int fd;
+ char *buf = (char *) malloc(XLOG_BLCKSZ);
+
+ /* Already set a valid XLogSegSize? */
+ if (IsValidXLogSegSize(XLogSegSize))
+ return true;
+
+ if ((fd = open(WALFilePath, O_RDWR, 0)) < 0)
+ {
+ fprintf(stderr, "%s: couldn't open WAL file \"%s\"\n",
+ progname, WALFilePath);
+ return false;
+ }
+ if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
+ {
+ XLogPageHeader hdr = (XLogPageHeader) buf;
+ XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr;
+
+ XLogSegSize = longhdr->xlp_seg_size;
+
+ if (IsValidXLogSegSize(XLogSegSize))
+ {
+ /* successfully retrieved WAL segment size */
+ ret_val = true;
+
+ /* check if clean up is necessary */
+ need_cleanup = SetWALFileNameForCleanup();
+ if (debug)
+ {
+ fprintf(stderr,
+ _("WAL segment size: %d \n"), XLogSegSize);
+ fprintf(stderr, "Keep archive history: ");
+
+ if (need_cleanup)
+ fprintf(stderr, "%s and later\n",
+ exclusiveCleanupFileName);
+ else
+ fprintf(stderr, "no cleanup required\n");
+ }
+ }
+ else
+ fprintf(stderr,
+ "%s: WAL segment size must be a power of two between 1MB and 1GB, but the WAL file header specifies %d bytes\n",
+ progname, XLogSegSize);
+ close(fd);
+ }
+ else
+ {
+ if (errno != 0)
+ fprintf(stderr, "could not read file \"%s\": %s",
+ WALFilePath, strerror(errno));
+ else
+ fprintf(stderr, "not enough data in file \"%s\"", WALFilePath);
+ }
+
+ fflush(stderr);
+ return ret_val;
+}
+
/* =====================================================================
*
* Customizable section
@@ -184,7 +253,9 @@ CustomizableNextWALFileReady(void)
nextWALFileType = XLOG_BACKUP_LABEL;
return true;
}
- else if (stat_buf.st_size == XLOG_SEG_SIZE)
+ else if (!RetrieveXLogSegSize())
+ return false;
+ else if (stat_buf.st_size == XLogSegSize)
{
#ifdef WIN32
@@ -204,7 +275,7 @@ CustomizableNextWALFileReady(void)
/*
* If still too small, wait until it is the correct size
*/
- if (stat_buf.st_size > XLOG_SEG_SIZE)
+ if (stat_buf.st_size > XLogSegSize)
{
if (debug)
{
@@ -218,8 +289,6 @@ CustomizableNextWALFileReady(void)
return false;
}
-#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
-
static void
CustomizableCleanupPriorWALFiles(void)
{
@@ -315,6 +384,7 @@ SetWALFileNameForCleanup(void)
uint32 log_diff = 0,
seg_diff = 0;
bool cleanup = false;
+ int MaxSegmentsPerLogFile = (0xFFFFFFFF / XLogSegSize);
if (restartWALFileName)
{
@@ -708,8 +778,6 @@ main(int argc, char **argv)
CustomizableInitialize();
- need_cleanup = SetWALFileNameForCleanup();
-
if (debug)
{
fprintf(stderr, "Trigger file: %s\n", triggerPath ? triggerPath : "<not set>");
@@ -721,11 +789,6 @@ main(int argc, char **argv)
fprintf(stderr, "Max wait interval: %d %s\n",
maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
fprintf(stderr, "Command for restore: %s\n", restoreCommand);
- fprintf(stderr, "Keep archive history: ");
- if (need_cleanup)
- fprintf(stderr, "%s and later\n", exclusiveCleanupFileName);
- else
- fprintf(stderr, "no cleanup required\n");
fflush(stderr);
}
diff --git a/doc/src/sgml/backup.sgml b/doc/src/sgml/backup.sgml
index 0e7c6e2..93712af 100644
--- a/doc/src/sgml/backup.sgml
+++ b/doc/src/sgml/backup.sgml
@@ -562,7 +562,7 @@ tar -cf backup.tar /usr/local/pgsql/data
produces an indefinitely long sequence of WAL records. The system
physically divides this sequence into WAL <firstterm>segment
files</>, which are normally 16MB apiece (although the segment size
- can be altered when building <productname>PostgreSQL</>). The segment
+ can be altered during <application>initdb</>). The segment
files are given numeric names that reflect their position in the
abstract WAL sequence. When not using WAL archiving, the system
normally creates just a few segment files and then
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 12866b4..0f512ba 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -1059,20 +1059,6 @@ su - postgres
</varlistentry>
<varlistentry>
- <term><option>--with-wal-segsize=<replaceable>SEGSIZE</replaceable></option></term>
- <listitem>
- <para>
- Set the <firstterm>WAL segment size</>, in megabytes. This is
- the size of each individual file in the WAL log. It may be useful
- to adjust this size to control the granularity of WAL log shipping.
- The default size is 16 megabytes.
- The value must be a power of 2 between 1 and 1024 (megabytes).
- Note that changing this value requires an initdb.
- </para>
- </listitem>
- </varlistentry>
-
- <varlistentry>
<term><option>--with-wal-blocksize=<replaceable>BLOCKSIZE</replaceable></option></term>
<listitem>
<para>
diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml
index 6efb2e4..732feca 100644
--- a/doc/src/sgml/ref/initdb.sgml
+++ b/doc/src/sgml/ref/initdb.sgml
@@ -316,6 +316,21 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
+ <term><option>--wal-segsize=<replaceable>SEGSIZE</replaceable></option></term>
+ <listitem>
+ <para>
+ Set the <firstterm>WAL segment size</>, in megabytes. This is
+ the size of each individual file in the WAL log. It may be useful
+ to adjust this size to control the granularity of WAL log shipping.
+ This option can only be set during initialization, and cannot be
+ changed later.
+ The default size is 16 megabytes.
+ The value must be a power of 2 between 1 and 1024 (megabytes).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-X <replaceable class="parameter">directory</replaceable></option></term>
<term><option>--waldir=<replaceable class="parameter">directory</replaceable></option></term>
<listitem>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 940c37b..58c13d3 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -752,13 +752,12 @@
<acronym>WAL</acronym> logs are stored in the directory
<filename>pg_wal</filename> under the data directory, as a set of
segment files, normally each 16 MB in size (but the size can be changed
- by altering the <option>--with-wal-segsize</> configure option when
- building the server). Each segment is divided into pages, normally
- 8 kB each (this size can be changed via the <option>--with-wal-blocksize</>
- configure option). The log record headers are described in
- <filename>access/xlogrecord.h</filename>; the record content is dependent
- on the type of event that is being logged. Segment files are given
- ever-increasing numbers as names, starting at
+ by altering the <option>--wal-segsize</> initdb option). Each segment is
+ divided into pages, normally 8 kB each (this size can be changed via the
+ <option>--with-wal-blocksize</> configure option). The log record headers
+ are described in <filename>access/xlogrecord.h</filename>; the record content
+ is dependent on the type of event that is being logged. Segment files are
+ given ever-increasing numbers as names, starting at
<filename>000000010000000000000000</filename>. The numbers do not wrap,
but it will take a very, very long time to exhaust the
available stock of numbers.
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3d5fb3a..06cbf4f 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -86,6 +86,7 @@ extern uint32 bootstrap_data_checksum_version;
/* User-settable parameters */
+int wal_segment_size = DEFAULT_XLOG_SEG_SIZE / XLOG_BLCKSZ;
int max_wal_size_mb = 1024; /* 1 GB */
int min_wal_size_mb = 80; /* 80 MB */
int wal_keep_segments = 0;
@@ -110,6 +111,8 @@ int wal_retrieve_retry_interval = 5000;
bool XLOG_DEBUG = false;
#endif
+uint32 XLogSegSize;
+
/*
* Number of WAL insertion locks to use. A higher value allows more insertions
* to happen concurrently, but adds some CPU overhead to flushing the WAL,
@@ -731,14 +734,16 @@ static ControlFileData *ControlFile = NULL;
(((recptr) / XLOG_BLCKSZ) % (XLogCtl->XLogCacheBlck + 1))
/*
- * These are the number of bytes in a WAL page and segment usable for WAL data.
+ * These are the number of bytes in a WAL page usable for WAL data.
*/
#define UsableBytesInPage (XLOG_BLCKSZ - SizeOfXLogShortPHD)
-#define UsableBytesInSegment ((XLOG_SEG_SIZE / XLOG_BLCKSZ) * UsableBytesInPage - (SizeOfXLogLongPHD - SizeOfXLogShortPHD))
/* Convert min_wal_size_mb and max wal_size_mb to equivalent segment count */
#define ConvertToXSegs(x) \
- (x / (XLOG_SEG_SIZE / (1024 * 1024)))
+ (x / (wal_segment_size * XLOG_BLCKSZ/ (1024 * 1024)))
+
+/* The number of bytes in a WAL segment usable for WAL data. */
+static int UsableBytesInSegment;
/*
* Private, possibly out-of-date copy of shared LogwrtResult.
@@ -1309,8 +1314,8 @@ ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr)
*StartPos = XLogBytePosToRecPtr(startbytepos);
*EndPos = XLogBytePosToEndRecPtr(endbytepos);
- segleft = XLOG_SEG_SIZE - (XLogSegmentOffset(*EndPos));
- if (segleft != XLOG_SEG_SIZE)
+ segleft = XLogSegSize - (XLogSegmentOffset(*EndPos));
+ if (segleft != XLogSegSize)
{
/* consume the rest of the segment */
*EndPos += segleft;
@@ -1539,7 +1544,7 @@ CopyXLogRecordToWAL(int write_len, bool isLogSwitch, XLogRecData *rdata,
/*
* We do this one page at a time, to make sure we don't deadlock
- * against ourselves if wal_buffers < XLOG_SEG_SIZE.
+ * against ourselves if wal_buffers < XLogSegSize.
*/
Assert(XLogSegmentOffset(EndPos) == 0);
@@ -2260,8 +2265,8 @@ XLOGfileslop(XLogRecPtr PriorRedoPtr)
* correspond to. Always recycle enough segments to meet the minimum, and
* remove enough segments to stay below the maximum.
*/
- minSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(min_wal_size_mb) - 1;
- maxSegNo = PriorRedoPtr / XLOG_SEG_SIZE + ConvertToXSegs(max_wal_size_mb) - 1;
+ minSegNo = PriorRedoPtr / XLogSegSize + ConvertToXSegs(min_wal_size_mb) - 1;
+ maxSegNo = PriorRedoPtr / XLogSegSize + ConvertToXSegs(max_wal_size_mb) - 1;
/*
* Between those limits, recycle enough segments to get us through to the
@@ -2290,7 +2295,7 @@ XLOGfileslop(XLogRecPtr PriorRedoPtr)
/* add 10% for good measure. */
distance *= 1.10;
- recycleSegNo = (XLogSegNo) ceil(((double) PriorRedoPtr + distance) / XLOG_SEG_SIZE);
+ recycleSegNo = (XLogSegNo) ceil(((double) PriorRedoPtr + distance) / XLogSegSize);
if (recycleSegNo < minSegNo)
recycleSegNo = minSegNo;
@@ -4399,7 +4404,7 @@ WriteControlFile(void)
ControlFile->blcksz = BLCKSZ;
ControlFile->relseg_size = RELSEG_SIZE;
ControlFile->xlog_blcksz = XLOG_BLCKSZ;
- ControlFile->xlog_seg_size = XLOG_SEG_SIZE;
+ ControlFile->xlog_seg_size = XLogSegSize;
ControlFile->nameDataLen = NAMEDATALEN;
ControlFile->indexMaxKeys = INDEX_MAX_KEYS;
@@ -4467,6 +4472,8 @@ ReadControlFile(void)
{
pg_crc32c crc;
int fd;
+ uint32 wal_segsz;
+ static char wal_segsz_str[20];
/*
* Read data...
@@ -4569,13 +4576,6 @@ ReadControlFile(void)
" but the server was compiled with XLOG_BLCKSZ %d.",
ControlFile->xlog_blcksz, XLOG_BLCKSZ),
errhint("It looks like you need to recompile or initdb.")));
- if (ControlFile->xlog_seg_size != XLOG_SEG_SIZE)
- ereport(FATAL,
- (errmsg("database files are incompatible with server"),
- errdetail("The database cluster was initialized with XLOG_SEG_SIZE %d,"
- " but the server was compiled with XLOG_SEG_SIZE %d.",
- ControlFile->xlog_seg_size, XLOG_SEG_SIZE),
- errhint("It looks like you need to recompile or initdb.")));
if (ControlFile->nameDataLen != NAMEDATALEN)
ereport(FATAL,
(errmsg("database files are incompatible with server"),
@@ -4637,6 +4637,31 @@ ReadControlFile(void)
errhint("It looks like you need to recompile or initdb.")));
#endif
+ XLogSegSize = ControlFile->xlog_seg_size;
+
+ if (!IsValidXLogSegSize(XLogSegSize))
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("WAL segment size must be a power of two between 1MB and 1GB, but the control file specifies %d bytes",
+ XLogSegSize)));
+
+ /* set wal_segment_size */
+ wal_segsz = XLogSegSize / XLOG_BLCKSZ;
+ snprintf(wal_segsz_str, sizeof(wal_segsz_str), "%d", wal_segsz);
+ SetConfigOption("wal_segment_size", wal_segsz_str, PGC_INTERNAL,
+ PGC_S_OVERRIDE);
+
+ /* check and update variables dependent on XLogSegSize */
+ if (ConvertToXSegs(min_wal_size_mb) < 2)
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"min_wal_size\" must be at least twice \"wal_segment_size\".")));
+
+ if (ConvertToXSegs(max_wal_size_mb) < 2)
+ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"max_wal_size\" must be at least twice \"wal_segment_size\".")));
+
+ CalculateUsableBytesInSegment();
+ CalculateCheckpointSegments();
+
/* Make the initdb settings visible as GUC variables, too */
SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no",
PGC_INTERNAL, PGC_S_OVERRIDE);
@@ -4757,8 +4782,8 @@ XLOGChooseNumBuffers(void)
int xbuffers;
xbuffers = NBuffers / 32;
- if (xbuffers > XLOG_SEG_SIZE / XLOG_BLCKSZ)
- xbuffers = XLOG_SEG_SIZE / XLOG_BLCKSZ;
+ if (xbuffers > wal_segment_size)
+ xbuffers = wal_segment_size;
if (xbuffers < 8)
xbuffers = 8;
return xbuffers;
@@ -4817,6 +4842,18 @@ XLOGShmemSize(void)
{
char buf[32];
+ /*
+ * The calculation of XLOGbuffers requires the run-time parameter
+ * XLogSegSize which is set from the control file. This value is
+ * required to create the shared memory segment. Hence, temporarily
+ * allocate space for reading the control file.
+ */
+ if (!IsBootstrapProcessingMode())
+ {
+ ControlFile = palloc(sizeof(ControlFileData));
+ ReadControlFile();
+ ControlFile = NULL;
+ }
snprintf(buf, sizeof(buf), "%d", XLOGChooseNumBuffers());
SetConfigOption("wal_buffers", buf, PGC_POSTMASTER, PGC_S_OVERRIDE);
}
@@ -6188,6 +6225,16 @@ CheckRequiredParameterValues(void)
}
/*
+ * Calculate UsableBytesInSegment based on XLogSegSize
+ */
+void
+CalculateUsableBytesInSegment(void)
+{
+ UsableBytesInSegment = (XLogSegSize / XLOG_BLCKSZ * UsableBytesInPage) -
+ (SizeOfXLogLongPHD - SizeOfXLogShortPHD);
+}
+
+/*
* This must be called ONCE during postmaster or standalone-backend startup
*/
void
@@ -8150,6 +8197,9 @@ InitXLOGAccess(void)
ThisTimeLineID = XLogCtl->ThisTimeLineID;
Assert(ThisTimeLineID != 0 || IsBootstrapProcessingMode());
+ /* set XLogSegSize */
+ XLogSegSize = ControlFile->xlog_seg_size;
+
/* Use GetRedoRecPtr to copy the RedoRecPtr safely */
(void) GetRedoRecPtr();
/* Also update our copy of doPageWrites. */
@@ -8480,7 +8530,7 @@ UpdateCheckPointDistanceEstimate(uint64 nbytes)
* more.
*
* When checkpoints are triggered by max_wal_size, this should converge to
- * CheckpointSegments * XLOG_SEG_SIZE,
+ * CheckpointSegments * XLogSegSize,
*
* Note: This doesn't pay any attention to what caused the checkpoint.
* Checkpoints triggered manually with CHECKPOINT command, or by e.g.
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 60523d1..02e38f6 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -778,7 +778,7 @@ ValidXLogPageHeader(XLogReaderState *state, XLogRecPtr recptr,
else if (longhdr->xlp_seg_size != XLogSegSize)
{
report_invalid_record(state,
- "WAL file is from different database system: incorrect XLOG_SEG_SIZE in page header");
+ "WAL file is from different database system: incorrect XLogSegSize in page header");
return false;
}
else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ)
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index b3f0b3c..9f7f4fc 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -19,6 +19,7 @@
#include "access/htup_details.h"
#include "access/xact.h"
+#include "access/xlog_internal.h"
#include "bootstrap/bootstrap.h"
#include "catalog/index.h"
#include "catalog/pg_collation.h"
@@ -222,7 +223,7 @@ AuxiliaryProcessMain(int argc, char *argv[])
/* If no -x argument, we are a CheckerProcess */
MyAuxProcType = CheckerProcess;
- while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:-:")) != -1)
+ while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:x:X:-:")) != -1)
{
switch (flag)
{
@@ -257,6 +258,13 @@ AuxiliaryProcessMain(int argc, char *argv[])
case 'x':
MyAuxProcType = atoi(optarg);
break;
+ case 'X':
+ XLogSegSize = strtoul(optarg, NULL, 0);
+ if (!IsValidXLogSegSize(XLogSegSize))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("-X requires a power of 2 value between 1MB and 1GB")));
+ break;
case 'c':
case '-':
{
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 95180b2..9879e70 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -521,6 +521,7 @@ typedef struct
HANDLE PostmasterHandle;
HANDLE initial_signal_pipe;
HANDLE syslogPipe[2];
+ uint32 XLogSegSize;
#else
int postmaster_alive_fds[2];
int syslogPipe[2];
@@ -6008,6 +6009,7 @@ save_backend_variables(BackendParameters *param, Port *port,
pgwin32_create_signal_listener(childPid),
childProcess))
return false;
+ param->XLogSegSize = XLogSegSize;
#else
memcpy(¶m->postmaster_alive_fds, &postmaster_alive_fds,
sizeof(postmaster_alive_fds));
@@ -6237,6 +6239,8 @@ restore_backend_variables(BackendParameters *param, Port *port)
#ifdef WIN32
PostmasterHandle = param->PostmasterHandle;
pgwin32_initial_signal_pipe = param->initial_signal_pipe;
+ XLogSegSize = param->XLogSegSize;
+ CalculateUsableBytesInSegment();
#else
memcpy(&postmaster_alive_fds, ¶m->postmaster_alive_fds,
sizeof(postmaster_alive_fds));
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 246fea8..b03187d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -514,7 +514,6 @@ static int block_size;
static int segment_size;
static int wal_block_size;
static bool data_checksums;
-static int wal_segment_size;
static bool integer_datetimes;
static bool assert_enabled;
@@ -714,9 +713,6 @@ typedef struct
#if XLOG_BLCKSZ < 1024 || XLOG_BLCKSZ > (1024*1024)
#error XLOG_BLCKSZ must be between 1KB and 1MB
#endif
-#if XLOG_SEG_SIZE < (1024*1024) || XLOG_SEG_SIZE > (1024*1024*1024)
-#error XLOG_SEG_SIZE must be between 1MB and 1GB
-#endif
static const char *memory_units_hint = gettext_noop("Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\".");
@@ -2259,7 +2255,8 @@ static struct config_int ConfigureNamesInt[] =
GUC_UNIT_MB
},
&min_wal_size_mb,
- 5 * (XLOG_SEG_SIZE / (1024 * 1024)), 2, MAX_KILOBYTES,
+ DEFAULT_MIN_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
+ 2, MAX_KILOBYTES,
NULL, NULL, NULL
},
@@ -2270,7 +2267,8 @@ static struct config_int ConfigureNamesInt[] =
GUC_UNIT_MB
},
&max_wal_size_mb,
- 64 * (XLOG_SEG_SIZE / (1024 * 1024)), 2, MAX_KILOBYTES,
+ DEFAULT_MAX_WAL_SEGS * (DEFAULT_XLOG_SEG_SIZE / (1024 * 1024)),
+ 2, MAX_KILOBYTES,
NULL, assign_max_wal_size, NULL
},
@@ -2637,9 +2635,9 @@ static struct config_int ConfigureNamesInt[] =
GUC_UNIT_XBLOCKS | GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE
},
&wal_segment_size,
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
- (XLOG_SEG_SIZE / XLOG_BLCKSZ),
+ DEFAULT_XLOG_SEG_SIZE / XLOG_BLCKSZ,
+ XLogSegMinSize / XLOG_BLCKSZ,
+ XLogSegMaxSize / XLOG_BLCKSZ,
NULL, NULL, NULL
},
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index df5d2f3..1a79258 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -234,7 +234,7 @@
#max_wal_senders = 10 # max number of walsender processes
# (change requires restart)
-#wal_keep_segments = 0 # in logfile segments, 16MB each; 0 disables
+#wal_keep_segments = 0 # in logfile segments; 0 disables
#wal_sender_timeout = 60s # in milliseconds; 0 disables
#max_replication_slots = 10 # max number of replication slots
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7303bbe..7fc2fc8 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -52,6 +52,7 @@
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
+#include <math.h>
#include <signal.h>
#include <time.h>
@@ -59,6 +60,7 @@
#include "sys/mman.h"
#endif
+#include "access/xlog_internal.h"
#include "catalog/catalog.h"
#include "catalog/pg_authid.h"
#include "catalog/pg_class.h"
@@ -141,6 +143,8 @@ static bool sync_only = false;
static bool show_setting = false;
static bool data_checksums = false;
static char *xlog_dir = "";
+static char *str_wal_segment_size = "";
+static int wal_segment_size;
/* internal vars */
@@ -1000,6 +1004,25 @@ test_config_settings(void)
}
/*
+ * Calculate the default wal_size in proper unit.
+ */
+static char *
+pretty_wal_size(int segment_count)
+{
+ double val = wal_segment_size / (1024 * 1024) * segment_count;
+ double temp_val;
+ char *result = malloc(10);
+
+ temp_val = val / 1024;
+ if (ceilf(temp_val) == temp_val)
+ snprintf(result, 10, "%dGB", (int) temp_val);
+ else
+ snprintf(result, 10, "%dMB", (int) val);
+
+ return result;
+}
+
+/*
* set up all the config files
*/
static void
@@ -1043,6 +1066,15 @@ setup_config(void)
conflines = replace_token(conflines, "#port = 5432", repltok);
#endif
+ /* set default max_wal_size and min_wal_size */
+ snprintf(repltok, sizeof(repltok), "min_wal_size = %s",
+ pretty_wal_size(DEFAULT_MIN_WAL_SEGS));
+ conflines = replace_token(conflines, "#min_wal_size = 80MB", repltok);
+
+ snprintf(repltok, sizeof(repltok), "max_wal_size = %s",
+ pretty_wal_size(DEFAULT_MAX_WAL_SEGS));
+ conflines = replace_token(conflines, "#max_wal_size = 1GB", repltok);
+
snprintf(repltok, sizeof(repltok), "lc_messages = '%s'",
escape_quotes(lc_messages));
conflines = replace_token(conflines, "#lc_messages = 'C'", repltok);
@@ -1356,8 +1388,9 @@ bootstrap_template1(void)
unsetenv("PGCLIENTENCODING");
snprintf(cmd, sizeof(cmd),
- "\"%s\" --boot -x1 %s %s %s",
+ "\"%s\" --boot -x1 -X %u %s %s %s",
backend_exec,
+ wal_segment_size,
data_checksums ? "-k" : "",
boot_options, talkargs);
@@ -2291,6 +2324,7 @@ usage(const char *progname)
printf(_(" -U, --username=NAME database superuser name\n"));
printf(_(" -W, --pwprompt prompt for a password for the new superuser\n"));
printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n"));
+ printf(_(" --wal-segsize=SIZE size of wal segment size\n"));
printf(_("\nLess commonly used options:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" -k, --data-checksums use data page checksums\n"));
@@ -2985,6 +3019,7 @@ main(int argc, char *argv[])
{"no-sync", no_argument, NULL, 'N'},
{"sync-only", no_argument, NULL, 'S'},
{"waldir", required_argument, NULL, 'X'},
+ {"wal-segsize", required_argument, NULL, 12},
{"data-checksums", no_argument, NULL, 'k'},
{NULL, 0, NULL, 0}
};
@@ -3118,6 +3153,9 @@ main(int argc, char *argv[])
case 'X':
xlog_dir = pg_strdup(optarg);
break;
+ case 12:
+ str_wal_segment_size = pg_strdup(optarg);
+ break;
default:
/* getopt_long already emitted a complaint */
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
@@ -3180,6 +3218,30 @@ main(int argc, char *argv[])
check_need_password(authmethodlocal, authmethodhost);
+ /* set wal segment size */
+ if (str_wal_segment_size == NULL || !strlen(str_wal_segment_size))
+ wal_segment_size = DEFAULT_XLOG_SEG_SIZE;
+ else
+ {
+ char *endptr;
+
+ /* check that the argument is a number */
+ wal_segment_size = strtol(str_wal_segment_size, &endptr, 10);
+
+ /* verify that wal segment size is valid */
+ if (*endptr != '\0' ||
+ !IsValidXLogSegSize(wal_segment_size * 1024 * 1024))
+ {
+ fprintf(stderr,
+ _("%s: --wal-segsize must be a power of two between 1 and 1024\n"),
+ progname);
+ exit(1);
+ }
+
+ /* convert to bytes */
+ wal_segment_size = wal_segment_size * 1024 * 1024;
+ }
+
get_restricted_token(progname);
setup_pgdata();
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 79c95fd..6886e35 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -2398,6 +2398,10 @@ main(int argc, char **argv)
exit(1);
}
+ /* determine remote server's xlog segment size */
+ if (!RetrieveXLogSegSize(conn))
+ disconnect_and_exit(1);
+
/* Create pg_wal symlink, if required */
if (strcmp(xlog_dir, "") != 0)
{
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 318e626..442d5bc 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -255,7 +255,7 @@ FindStreamingStart(uint32 *tli)
disconnect_and_exit(1);
}
- if (statbuf.st_size != XLOG_SEG_SIZE)
+ if (statbuf.st_size != XLogSegSize)
{
fprintf(stderr,
_("%s: segment file \"%s\" has incorrect size %d, skipping\n"),
@@ -296,7 +296,7 @@ FindStreamingStart(uint32 *tli)
bytes_out = (buf[3] << 24) | (buf[2] << 16) |
(buf[1] << 8) | buf[0];
- if (bytes_out != XLOG_SEG_SIZE)
+ if (bytes_out != XLogSegSize)
{
fprintf(stderr,
_("%s: compressed segment file \"%s\" has incorrect uncompressed size %d, skipping\n"),
@@ -665,6 +665,10 @@ main(int argc, char **argv)
if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name))
disconnect_and_exit(1);
+ /* determine remote server's xlog segment size */
+ if (!RetrieveXLogSegSize(conn))
+ disconnect_and_exit(1);
+
/*
* Check that there is a database associated with connection, none should
* be defined in this context.
diff --git a/src/bin/pg_basebackup/receivelog.c b/src/bin/pg_basebackup/receivelog.c
index 4885f31..fd3ce0a 100644
--- a/src/bin/pg_basebackup/receivelog.c
+++ b/src/bin/pg_basebackup/receivelog.c
@@ -203,7 +203,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
if (stream->partial_suffix)
{
- if (currpos == XLOG_SEG_SIZE)
+ if (currpos == XLogSegSize)
r = stream->walmethod->close(walfile, CLOSE_NORMAL);
else
{
@@ -231,7 +231,7 @@ close_walfile(StreamCtl *stream, XLogRecPtr pos)
* new node. This is in line with walreceiver.c always doing a
* XLogArchiveForceDone() after a complete segment.
*/
- if (currpos == XLOG_SEG_SIZE && stream->mark_done)
+ if (currpos == XLogSegSize && stream->mark_done)
{
/* writes error message if failed */
if (!mark_file_as_archived(stream, current_walfile_name))
@@ -1149,10 +1149,10 @@ ProcessXLogDataMsg(PGconn *conn, StreamCtl *stream, char *copybuf, int len,
/*
* If crossing a WAL boundary, only write up until we reach
- * XLOG_SEG_SIZE.
+ * XLogSegSize.
*/
- if (xlogoff + bytes_left > XLOG_SEG_SIZE)
- bytes_to_write = XLOG_SEG_SIZE - xlogoff;
+ if (xlogoff + bytes_left > XLogSegSize)
+ bytes_to_write = XLogSegSize - xlogoff;
else
bytes_to_write = bytes_left;
diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c
index 9d40744..83fb9d0 100644
--- a/src/bin/pg_basebackup/streamutil.c
+++ b/src/bin/pg_basebackup/streamutil.c
@@ -25,12 +25,18 @@
#include "receivelog.h"
#include "streamutil.h"
+#include "access/xlog_internal.h"
#include "pqexpbuffer.h"
#include "common/fe_memutils.h"
#include "datatype/timestamp.h"
#define ERRCODE_DUPLICATE_OBJECT "42710"
+uint32 XLogSegSize;
+
+/* SHOW command for replication connection was introduced in version 10 */
+#define MINIMUM_VERSION_FOR_SHOW_CMD 100000
+
const char *progname;
char *connection_string = NULL;
char *dbhost = NULL;
@@ -232,6 +238,76 @@ GetConnection(void)
}
/*
+ * From version 10, explicitly set XLogSegSize using SHOW wal_segment_size
+ * since ControlFile is not accessible here.
+ */
+bool
+RetrieveXLogSegSize(PGconn *conn)
+{
+ PGresult *res;
+ char xlog_unit[3];
+ int xlog_val,
+ multiplier = 1;
+
+ /* check connection existence */
+ Assert(conn != NULL);
+
+ /* for previous versions set the default xlog seg size */
+ if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD)
+ {
+ XLogSegSize = DEFAULT_XLOG_SEG_SIZE;
+ return true;
+ }
+
+ res = PQexec(conn, "SHOW wal_segment_size");
+ if (PQresultStatus(res) != PGRES_TUPLES_OK)
+ {
+ fprintf(stderr, _("%s: could not send replication command \"%s\": %s\n"),
+ progname, "SHOW wal_segment_size", PQerrorMessage(conn));
+
+ PQclear(res);
+ return false;
+ }
+ if (PQntuples(res) != 1 || PQnfields(res) < 1)
+ {
+ fprintf(stderr,
+ _("%s: could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields\n"),
+ progname, PQntuples(res), PQnfields(res), 1, 1);
+
+ PQclear(res);
+ return false;
+ }
+
+ /* fetch xlog value and unit from the result */
+ if (sscanf(PQgetvalue(res, 0, 0), "%d%s", &xlog_val, xlog_unit) != 2)
+ {
+ fprintf(stderr, _("%s: WAL segment size could not be parsed\n"),
+ progname);
+ return false;
+ }
+
+ /* set the multiplier based on unit to convert XLogSegSize to bytes */
+ if (strcmp(xlog_unit, "MB") == 0)
+ multiplier = 1024 * 1024;
+ else if (strcmp(xlog_unit, "GB") == 0)
+ multiplier = 1024 * 1024 * 1024;
+
+ /* convert and set XLogSegSize */
+ XLogSegSize = xlog_val * multiplier;
+
+ if (!IsValidXLogSegSize(XLogSegSize))
+ {
+ fprintf(stderr,
+ _("%s: WAL segment size must be a power of two between 1MB and 1GB, but the remote server reported a value of %d bytes\n"),
+ progname, XLogSegSize);
+ return false;
+ }
+
+ PQclear(res);
+ return true;
+}
+
+/*
* Run IDENTIFY_SYSTEM through a given connection and give back to caller
* some result information if requested:
* - System identifier
diff --git a/src/bin/pg_basebackup/streamutil.h b/src/bin/pg_basebackup/streamutil.h
index 6f68786..6fb85ba 100644
--- a/src/bin/pg_basebackup/streamutil.h
+++ b/src/bin/pg_basebackup/streamutil.h
@@ -39,6 +39,7 @@ extern bool RunIdentifySystem(PGconn *conn, char **sysid,
TimeLineID *starttli,
XLogRecPtr *startpos,
char **db_name);
+extern bool RetrieveXLogSegSize(PGconn *conn);
extern TimestampTz feGetCurrentTimestamp(void);
extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time,
long *secs, int *microsecs);
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 2ea8931..b796039 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -26,6 +26,7 @@
#include "common/controldata_utils.h"
#include "pg_getopt.h"
+uint32 XLogSegSize;
static void
usage(const char *progname)
@@ -164,6 +165,14 @@ main(int argc, char *argv[])
"Either the file is corrupt, or it has a different layout than this program\n"
"is expecting. The results below are untrustworthy.\n\n"));
+ /* set XLogSegSize */
+ XLogSegSize = ControlFile->xlog_seg_size;
+
+ if (!IsValidXLogSegSize(XLogSegSize))
+ fprintf(stderr,
+ _("WARNING: WAL segment size specified, %d bytes, is not a power of two between 1MB and 1GB. The file is corrupt and the results below are untrustworthy."),
+ XLogSegSize);
+
/*
* This slightly-chintzy coding will work as long as the control file
* timestamps are within the range of time_t; that should be the case in
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index ac67831..7e8d934 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -56,6 +56,7 @@
#include "storage/large_object.h"
#include "pg_getopt.h"
+uint32 XLogSegSize;
static ControlFileData ControlFile; /* pg_control values */
static XLogSegNo newXlogSegNo; /* new XLOG segment # */
@@ -94,6 +95,7 @@ main(int argc, char *argv[])
char *endptr;
char *endptr2;
char *DataDir = NULL;
+ char *log_fname = NULL;
int fd;
set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_resetwal"));
@@ -265,7 +267,12 @@ main(int argc, char *argv[])
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
- XLogFromFileName(optarg, &minXlogTli, &minXlogSegNo);
+
+ /*
+ * XLogFromFileName requires XLogSegSize which is not yet
+ * set. Hence XLog details are set later on.
+ */
+ log_fname = pg_strdup(optarg);
break;
default:
@@ -350,6 +357,9 @@ main(int argc, char *argv[])
if (!ReadControlFile())
GuessControlValues();
+ if (log_fname != NULL)
+ XLogFromFileName(log_fname, &minXlogTli, &minXlogSegNo);
+
/*
* Also look at existing segment files to set up newXlogSegNo
*/
@@ -573,18 +583,26 @@ ReadControlFile(void)
offsetof(ControlFileData, crc));
FIN_CRC32C(crc);
- if (EQ_CRC32C(crc, ((ControlFileData *) buffer)->crc))
+ if (!EQ_CRC32C(crc, ((ControlFileData *) buffer)->crc))
{
- /* Valid data... */
- memcpy(&ControlFile, buffer, sizeof(ControlFile));
- return true;
+ /* We will use the data but treat it as guessed. */
+ fprintf(stderr, _("%s: pg_control exists but has invalid CRC; proceed with caution\n"),
+ progname);
+ guessed = true;
}
- fprintf(stderr, _("%s: pg_control exists but has invalid CRC; proceed with caution\n"),
- progname);
- /* We will use the data anyway, but treat it as guessed. */
memcpy(&ControlFile, buffer, sizeof(ControlFile));
- guessed = true;
+ XLogSegSize = ControlFile.xlog_seg_size;
+
+ /* return false if XLogSegSize is not valid */
+ if (!IsValidXLogSegSize(XLogSegSize))
+ {
+ fprintf(stderr,
+ _("%s: WAL segment size must be a power of two between 1MB and 1GB; ignoring control file specifying %d bytes\n"),
+ progname, XLogSegSize);
+ return false;
+ }
+
return true;
}
@@ -660,7 +678,7 @@ GuessControlValues(void)
ControlFile.blcksz = BLCKSZ;
ControlFile.relseg_size = RELSEG_SIZE;
ControlFile.xlog_blcksz = XLOG_BLCKSZ;
- ControlFile.xlog_seg_size = XLOG_SEG_SIZE;
+ ControlFile.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE;
ControlFile.nameDataLen = NAMEDATALEN;
ControlFile.indexMaxKeys = INDEX_MAX_KEYS;
ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 4bd1a75..e169af35 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -44,6 +44,7 @@ static ControlFileData ControlFile_target;
static ControlFileData ControlFile_source;
const char *progname;
+uint32 XLogSegSize;
/* Configuration options */
char *datadir_target = NULL;
@@ -631,6 +632,12 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
memcpy(ControlFile, src, sizeof(ControlFileData));
+ /* set and validate XLogSegSize */
+ XLogSegSize = ControlFile->xlog_seg_size;
+
+ if (!IsValidXLogSegSize(XLogSegSize))
+ pg_fatal("WAL segment size must be a power of two between 1MB and 1GB, but the control file specifies %d bytes\n", XLogSegSize);
+
/* Additional checks on control file */
checkControlFile(ControlFile);
}
diff --git a/src/bin/pg_test_fsync/pg_test_fsync.c b/src/bin/pg_test_fsync/pg_test_fsync.c
index 266b2b6..dccb702 100644
--- a/src/bin/pg_test_fsync/pg_test_fsync.c
+++ b/src/bin/pg_test_fsync/pg_test_fsync.c
@@ -63,7 +63,7 @@ static const char *progname;
static int secs_per_test = 5;
static int needs_unlink = 0;
-static char full_buf[XLOG_SEG_SIZE],
+static char full_buf[DEFAULT_XLOG_SEG_SIZE],
*buf,
*filename = FSYNC_FILENAME;
static struct timeval start_t,
@@ -205,7 +205,7 @@ prepare_buf(void)
int ops;
/* write random data into buffer */
- for (ops = 0; ops < XLOG_SEG_SIZE; ops++)
+ for (ops = 0; ops < DEFAULT_XLOG_SEG_SIZE; ops++)
full_buf[ops] = random();
buf = (char *) TYPEALIGN(XLOG_BLCKSZ, full_buf);
@@ -222,7 +222,7 @@ test_open(void)
if ((tmpfile = open(filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR)) == -1)
die("could not open output file");
needs_unlink = 1;
- if (write(tmpfile, full_buf, XLOG_SEG_SIZE) != XLOG_SEG_SIZE)
+ if (write(tmpfile, full_buf, DEFAULT_XLOG_SEG_SIZE) != DEFAULT_XLOG_SEG_SIZE)
die("write failed");
/* fsync now so that dirty buffers don't skew later tests */
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index ed73ffe..b2315a0 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -22,10 +22,13 @@
#include "common/fe_memutils.h"
#include "getopt_long.h"
#include "rmgrdesc.h"
+#include "sys/stat.h"
static const char *progname;
+uint32 XLogSegSize;
+
typedef struct XLogDumpPrivate
{
TimeLineID timeline;
@@ -144,77 +147,168 @@ split_path(const char *path, char **dir, char **fname)
}
/*
- * Try to find the file in several places:
- * if directory == NULL:
- * fname
- * XLOGDIR / fname
- * $PGDATA / XLOGDIR / fname
- * else
- * directory / fname
- * directory / XLOGDIR / fname
+ * Open the file in the valid target directory.
*
* return a read only fd
*/
static int
-fuzzy_open_file(const char *directory, const char *fname)
+open_file_in_directory(const char *directory, const char *fname)
{
int fd = -1;
char fpath[MAXPGPATH];
- if (directory == NULL)
+ Assert(directory != NULL);
+
+ snprintf(fpath, MAXPGPATH, "%s/%s", directory, fname);
+ fd = open(fpath, O_RDONLY | PG_BINARY, 0);
+
+ if (fd < 0 && errno != ENOENT)
+ fatal_error("could not open file \"%s\": %s",
+ fname, strerror(errno));
+ return fd;
+}
+
+/*
+ * Try to find fname in the given directory. Returns true if it is found,
+ * false otherwise. If fname is NULL, search the complete directory for any
+ * file with a valid WAL file name. If file is successfully opened, set the
+ * XLogSegSize.
+ */
+static bool
+search_directory(char *directory, char *fname)
+{
+ int fd = -1;
+ DIR *xldir;
+
+ /* open file if valid filename is provided */
+ if (fname != NULL)
+ fd = open_file_in_directory(directory, fname);
+
+ /*
+ * A valid file name is not passed so search the complete directory. If
+ * we find any file whose name is like a valid WAL file name then try to
+ * open it. If we can not open it then bail out.
+ */
+ else if ((xldir = opendir(directory)) != NULL)
+ {
+ struct dirent *xlde;
+
+ while ((xlde = readdir(xldir)) != NULL)
+ {
+ if (IsXLogFileName(xlde->d_name))
+ {
+ fd = open_file_in_directory(directory, xlde->d_name);
+ fname = xlde->d_name;
+ break;
+ }
+ }
+
+ closedir(xldir);
+ }
+
+ /* set XLogSegSize if file is successfully opened */
+ if (fd >= 0)
+ {
+ char *buf = (char *) malloc(XLOG_BLCKSZ);
+
+ if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
+ {
+ XLogPageHeader hdr = (XLogPageHeader) buf;
+ XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr;
+
+ XLogSegSize = longhdr->xlp_seg_size;
+
+ if (!IsValidXLogSegSize(XLogSegSize))
+ fatal_error("WAL segment size must be a power of two between 1MB and 1GB, but the WAL file \"%s\" header specifies %d bytes",
+ fname, XLogSegSize);
+ }
+ else
+ {
+ if (errno != 0)
+ fatal_error("could not read file \"%s\": %s",
+ fname, strerror(errno));
+ else
+ fatal_error("not enough data in file \"%s\"", fname);
+ }
+ free(buf);
+ close(fd);
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * Identify the target directory and set XLogSegSize.
+ *
+ * Try to find the file in several places:
+ * if directory != NULL:
+ * directory /
+ * directory / XLOGDIR /
+ * else
+ * .
+ * XLOGDIR /
+ * $PGDATA / XLOGDIR /
+ *
+ * Set the valid target directory in private->inpath.
+ */
+static void
+identify_target_directory(XLogDumpPrivate *private, char *directory,
+ char *fname)
+{
+ char fpath[MAXPGPATH];
+
+ if (directory != NULL)
+ {
+ if (search_directory(directory, fname))
+ {
+ private->inpath = strdup(directory);
+ return;
+ }
+
+ /* directory / XLOGDIR */
+ snprintf(fpath, MAXPGPATH, "%s/%s", directory, XLOGDIR);
+ if (search_directory(fpath, fname))
+ {
+ private->inpath = strdup(fpath);
+ return;
+ }
+ }
+ else
{
const char *datadir;
- /* fname */
- fd = open(fname, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
-
- /* XLOGDIR / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s",
- XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
+ /* current directory */
+ if (search_directory(".", fname))
+ {
+ private->inpath = strdup(".");
+ return;
+ }
+ /* XLOGDIR */
+ if (search_directory(XLOGDIR, fname))
+ {
+ private->inpath = strdup(XLOGDIR);
+ return;
+ }
datadir = getenv("PGDATA");
- /* $PGDATA / XLOGDIR / fname */
+ /* $PGDATA / XLOGDIR */
if (datadir != NULL)
{
- snprintf(fpath, MAXPGPATH, "%s/%s/%s",
- datadir, XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
+ snprintf(fpath, MAXPGPATH, "%s/%s", datadir, XLOGDIR);
+ if (search_directory(fpath, fname))
+ {
+ private->inpath = strdup(fpath);
+ return;
+ }
}
}
+
+ /* could not locate WAL file */
+ if (fname)
+ fatal_error("could not locate WAL file \"%s\"", fname);
else
- {
- /* directory / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s",
- directory, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
-
- /* directory / XLOGDIR / fname */
- snprintf(fpath, MAXPGPATH, "%s/%s/%s",
- directory, XLOGDIR, fname);
- fd = open(fpath, O_RDONLY | PG_BINARY, 0);
- if (fd < 0 && errno != ENOENT)
- return -1;
- else if (fd >= 0)
- return fd;
- }
- return -1;
+ fatal_error("could not find any WAL file");
}
/*
@@ -267,7 +361,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
*/
for (tries = 0; tries < 10; tries++)
{
- sendFile = fuzzy_open_file(directory, fname);
+ sendFile = open_file_in_directory(directory, fname);
if (sendFile >= 0)
break;
if (errno == ENOENT)
@@ -935,7 +1029,8 @@ main(int argc, char **argv)
private.inpath, strerror(errno));
}
- fd = fuzzy_open_file(private.inpath, fname);
+ identify_target_directory(&private, private.inpath, fname);
+ fd = open_file_in_directory(private.inpath, fname);
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);
@@ -968,7 +1063,7 @@ main(int argc, char **argv)
/* ignore directory, already have that */
split_path(argv[optind + 1], &directory, &fname);
- fd = fuzzy_open_file(private.inpath, fname);
+ fd = open_file_in_directory(private.inpath, fname);
if (fd < 0)
fatal_error("could not open file \"%s\"", fname);
close(fd);
@@ -1000,6 +1095,8 @@ main(int argc, char **argv)
goto bad_argument;
}
}
+ else
+ identify_target_directory(&private, private.inpath, NULL);
/* we don't know what to print */
if (XLogRecPtrIsInvalid(private.startptr))
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 66bfb77..705f88c 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -94,6 +94,7 @@ extern PGDLLIMPORT XLogRecPtr XactLastCommitEnd;
extern bool reachedConsistency;
/* these variables are GUC parameters related to XLOG */
+extern int wal_segment_size;
extern int min_wal_size_mb;
extern int max_wal_size_mb;
extern int wal_keep_segments;
@@ -262,6 +263,7 @@ extern Size XLOGShmemSize(void);
extern void XLOGShmemInit(void);
extern void BootStrapXLOG(void);
extern void StartupXLOG(void);
+extern void CalculateUsableBytesInSegment(void);
extern void ShutdownXLOG(int code, Datum arg);
extern void InitXLOGAccess(void);
extern void CreateCheckPoint(int flags);
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 2a01b03..b760c95 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -89,11 +89,27 @@ typedef XLogLongPageHeaderData *XLogLongPageHeader;
* The XLOG is split into WAL segments (physical files) of the size indicated
* by XLOG_SEG_SIZE.
*/
-#define XLogSegSize ((uint32) XLOG_SEG_SIZE)
-#define XLogSegmentsPerXLogId (UINT64CONST(0x100000000) / XLOG_SEG_SIZE)
+
+extern uint32 XLogSegSize;
+
+/* default number of min and max wal segments */
+#define DEFAULT_MIN_WAL_SEGS 5
+#define DEFAULT_MAX_WAL_SEGS 64
+
+/* XLogSegSize can range from 1MB to 1GB */
+#define XLogSegMinSize 1024 * 1024
+#define XLogSegMaxSize 1024 * 1024 * 1024
+
+/* check that the given size is a valid XLogSegSize */
+#define IsPowerOf2(x) (((x) & ((x)-1)) == 0)
+#define IsValidXLogSegSize(size) \
+ (IsPowerOf2(size) && \
+ ((size) >= XLogSegMinSize && (size) <= XLogSegMaxSize))
+
+#define XLogSegmentsPerXLogId (UINT64CONST(0x100000000) / XLogSegSize)
#define XLogSegNoOffsetToRecPtr(segno, offset, dest) \
- (dest) = (segno) * XLOG_SEG_SIZE + (offset)
+ (dest) = (segno) * (XLogSegSize) + (offset)
#define XLogSegmentOffset(xlogptr) \
((xlogptr) & (XLogSegSize - 1))
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index dcb7a1a..54dd462 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,13 +899,6 @@
*/
#undef XLOG_BLCKSZ
-/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
- and larger than XLOG_BLCKSZ (preferably, a great deal larger than
- XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
-#undef XLOG_SEG_SIZE
-
-
-
/* Number of bits in a file offset, on hosts where this is settable. */
#undef _FILE_OFFSET_BITS
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index f3b3529..477e7e1 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -14,6 +14,12 @@
*/
/*
+ * This is default value for wal_segment_size to be used at intidb when run
+ * without --walsegsize option.
+ */
+#define DEFAULT_XLOG_SEG_SIZE (16*1024*1024)
+
+/*
* Maximum length for identifiers (e.g. table names, column names,
* function names). Names actually are limited to one less byte than this,
* because the length must include a trailing zero byte.
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index 01e5846..d5e3649 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -179,8 +179,6 @@ s{PG_VERSION_STR "[^"]+"}{PG_VERSION_STR "PostgreSQL $self->{strver}$extraver, c
1024, "\n";
print $o "#define XLOG_BLCKSZ ",
1024 * $self->{options}->{wal_blocksize}, "\n";
- print $o "#define XLOG_SEG_SIZE (", $self->{options}->{wal_segsize},
- " * 1024 * 1024)\n";
if ($self->{options}->{float4byval})
{