v12-0006-pg_waldump-Remove-the-restriction-on-the-order-o.patch

From 77e29d3162afec46d9ed9f3d592bdbeee6347b37 Mon Sep 17 00:00:00 2001
From: Amul Sul <sulamul@gmail.com>
Date: Tue, 27 Jan 2026 15:38:34 +0530
Subject: [PATCH v12 6/9] pg_waldump: Remove the restriction on the order of
 archived WAL files.

With previous patch, pg_waldump would stop decoding if WAL files were
not in the required sequence. With this patch, decoding will now
continue.  Any WAL file that is out of order will be written to a
temporary location, from which it will be read later. Once a temporary
file has been read, it will be removed.
---
 doc/src/sgml/ref/pg_waldump.sgml     |   8 +-
 src/bin/pg_waldump/archive_waldump.c | 171 +++++++++++++++++++++++++--
 src/bin/pg_waldump/pg_waldump.c      |  31 ++++-
 src/bin/pg_waldump/pg_waldump.h      |   3 +
 src/bin/pg_waldump/t/001_basic.pl    |   3 +-
 5 files changed, 196 insertions(+), 20 deletions(-)

diff --git a/doc/src/sgml/ref/pg_waldump.sgml b/doc/src/sgml/ref/pg_waldump.sgml
index d004bb0f67e..27adf77755c 100644
--- a/doc/src/sgml/ref/pg_waldump.sgml
+++ b/doc/src/sgml/ref/pg_waldump.sgml
@@ -149,8 +149,12 @@ PostgreSQL documentation
         of <envar>PGDATA</envar>.
        </para>
        <para>
-        If a tar archive is provided, its WAL segment files must be in
-        sequential order; otherwise, an error will be reported.
+        If a tar archive is provided and its WAL segment files are not in
+        sequential order, those files will be written to a temporary directory
+        named starting with <filename>waldump_tmp</filename>. This directory will be
+        created inside the directory specified by the <envar>TMPDIR</envar>
+        environment variable if it is set; otherwise, it will be created within
+        the same directory as the tar archive.
        </para>
       </listitem>
      </varlistentry>
diff --git a/src/bin/pg_waldump/archive_waldump.c b/src/bin/pg_waldump/archive_waldump.c
index 27a5a5c6d5d..b1353088c4a 100644
--- a/src/bin/pg_waldump/archive_waldump.c
+++ b/src/bin/pg_waldump/archive_waldump.c
@@ -17,6 +17,7 @@
 #include <unistd.h>
 
 #include "access/xlog_internal.h"
+#include "common/file_perm.h"
 #include "common/hashfn.h"
 #include "common/logging.h"
 #include "fe_utils/simple_list.h"
@@ -27,6 +28,9 @@
  */
 #define READ_CHUNK_SIZE				(128 * 1024)
 
+/* Temporary exported WAL file directory */
+char	   *TmpWalSegDir = NULL;
+
 /*
  * Check if the start segment number is zero; this indicates a request to read
  * any WAL file.
@@ -57,6 +61,8 @@ typedef struct ArchivedWALFile
 	const char *fname;			/* hash key: WAL segment name */
 
 	StringInfo	buf;			/* holds WAL bytes read from archive */
+	bool		spilled;		/* true if the WAL data was spilled to a
+								 * temporary file */
 
 	int			read_len;		/* total bytes of a WAL read from archive */
 } ArchivedWALFile;
@@ -84,6 +90,11 @@ static ArchivedWALFile *get_archive_wal_entry(const char *fname,
 											  XLogDumpPrivate *privateInfo,
 											  int WalSegSz);
 static int	read_archive_file(XLogDumpPrivate *privateInfo, Size count);
+static void setup_tmpwal_dir(const char *waldir);
+static void cleanup_tmpwal_dir_atexit(void);
+
+static FILE *prepare_tmp_write(const char *fname);
+static void perform_tmp_write(const char *fname, StringInfo buf, FILE *file);
 
 static astreamer *astreamer_waldump_new(XLogDumpPrivate *privateInfo);
 static void astreamer_waldump_content(astreamer *streamer,
@@ -137,7 +148,9 @@ is_archive_file(const char *fname, pg_compress_algorithm *compression)
 /*
  * Initializes the tar archive reader, creates a hash table for WAL entries,
  * checks for existing valid WAL segments in the archive file and retrieves the
- * segment size, and sets up filters for relevant entries.
+ * segment size, and sets up filters for relevant entries. It also configures a
+ * temporary directory for out-of-order WAL data and registers an exit callback
+ * to clean up temporary files.
  */
 void
 init_archive_reader(XLogDumpPrivate *privateInfo, const char *waldir,
@@ -230,6 +243,13 @@ init_archive_reader(XLogDumpPrivate *privateInfo, const char *waldir,
 		privateInfo->start_segno > segno ||
 		privateInfo->end_segno < segno)
 		free_archive_wal_entry(entry->fname, privateInfo);
+
+	/*
+	 * Setup temporary directory to store WAL segments and set up an exit
+	 * callback to remove it upon completion.
+	 */
+	setup_tmpwal_dir(waldir);
+	atexit(cleanup_tmpwal_dir_atexit);
 }
 
 /*
@@ -396,6 +416,17 @@ free_archive_wal_entry(const char *fname, XLogDumpPrivate *privateInfo)
 	destroyStringInfo(entry->buf);
 	entry->buf = NULL;
 
+	/* Remove temporary file if any */
+	if (entry->spilled)
+	{
+		char		fpath[MAXPGPATH];
+
+		snprintf(fpath, MAXPGPATH, "%s/%s", TmpWalSegDir, fname);
+
+		if (unlink(fpath) == 0)
+			pg_log_debug("removed file \"%s\"", fpath);
+	}
+
 	/* Set cur_file to NULL if it matches the entry being ignored */
 	if (privateInfo->cur_file == entry)
 		privateInfo->cur_file = NULL;
@@ -407,12 +438,16 @@ free_archive_wal_entry(const char *fname, XLogDumpPrivate *privateInfo)
  * Returns the archived WAL entry from the hash table if it exists.  Otherwise,
  * it invokes the routine to read the archived file, which then populates the
  * entry in the hash table if that WAL exists in the archive.
+ * If the archive streamer happens to be reading a
+ * WAL from archive file that is not currently needed, that WAL data is written
+ * to a temporary file.
  */
 static ArchivedWALFile *
 get_archive_wal_entry(const char *fname, XLogDumpPrivate *privateInfo,
 					  int WalSegSz)
 {
 	ArchivedWALFile *entry = NULL;
+	FILE	   *write_fp = NULL;
 
 	/* Search hash table */
 	entry = ArchivedWAL_lookup(privateInfo->archive_wal_htab, fname);
@@ -426,28 +461,59 @@ get_archive_wal_entry(const char *fname, XLogDumpPrivate *privateInfo,
 	 */
 	while (1)
 	{
+		/*
+		 * The WAL file entry currently being processed may change during
+		 * archive streamer execution. Therefore, maintain a local variable to
+		 * reference the previous entry, ensuring that any remaining data in
+		 * its buffer is successfully flushed to the temporary file before
+		 * switching to the next WAL entry.
+		 */
+		entry = privateInfo->cur_file;
+
 		/* Fetch more data */
-		if (read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0)
-			break;				/* archive file ended */
+		if (entry == NULL || entry->buf->len == 0)
+		{
+			if (read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0)
+				break;			/* archive file ended */
+		}
 
 		/*
 		 * Archived streamer is reading a non-WAL file or an irrelevant WAL
 		 * file.
 		 */
-		if (privateInfo->cur_file == NULL)
+		if (entry == NULL)
 			continue;
 
-		entry = privateInfo->cur_file;
-
 		/* Found the required entry */
 		if (strcmp(fname, entry->fname) == 0)
 			return entry;
 
-		/* WAL segments must be archived in order */
-		pg_log_error("WAL files are not archived in sequential order");
-		pg_log_error_detail("Expecting segment \"%s\" but found \"%s\".",
-							fname, entry->fname);
-		exit(1);
+		/*
+		 * Archive streamer is currently reading a file that isn't the one
+		 * asked for, but it's required in the future. It should be written to
+		 * a temporary location for retrieval when needed.
+		 */
+
+		/* Create a temporary file if one does not already exist */
+		if (!entry->spilled)
+		{
+			write_fp = prepare_tmp_write(entry->fname);
+			entry->spilled = true;
+		}
+
+		/* Flush data from the buffer to the file */
+		perform_tmp_write(entry->fname, entry->buf, write_fp);
+		resetStringInfo(entry->buf);
+
+		/*
+		 * The change in the current segment entry indicates that the reading
+		 * of this file has ended.
+		 */
+		if (entry != privateInfo->cur_file && write_fp != NULL)
+		{
+			fclose(write_fp);
+			write_fp = NULL;
+		}
 	}
 
 	/* Requested WAL segment not found */
@@ -485,7 +551,88 @@ read_archive_file(XLogDumpPrivate *privateInfo, Size count)
 }
 
 /*
- * Create an astreamer that can read WAL from a tar file.
+ * Set up a temporary directory to temporarily store WAL segments.
+ */
+static void
+setup_tmpwal_dir(const char *waldir)
+{
+	char	   *template;
+
+	/*
+	 * Use the directory specified by the TMPDIR environment variable. If it's
+	 * not set, use the provided WAL directory to extract WAL file
+	 * temporarily.
+	 */
+	template = psprintf("%s/waldump_tmp-XXXXXX",
+						getenv("TMPDIR") ? getenv("TMPDIR") : waldir);
+	TmpWalSegDir = mkdtemp(template);
+
+	if (TmpWalSegDir == NULL)
+		pg_fatal("could not create directory \"%s\": %m", template);
+
+	canonicalize_path(TmpWalSegDir);
+
+	pg_log_debug("created directory \"%s\"", TmpWalSegDir);
+}
+
+/*
+ * Remove temporary directory at exit, if any.
+ */
+static void
+cleanup_tmpwal_dir_atexit(void)
+{
+	rmtree(TmpWalSegDir, true);
+}
+
+/*
+ * Create an empty placeholder file and return its handle.
+ */
+static FILE *
+prepare_tmp_write(const char *fname)
+{
+	char		fpath[MAXPGPATH];
+	FILE	   *file;
+
+	snprintf(fpath, MAXPGPATH, "%s/%s", TmpWalSegDir, fname);
+
+	/* Create an empty placeholder */
+	file = fopen(fpath, PG_BINARY_W);
+	if (file == NULL)
+		pg_fatal("could not create file \"%s\": %m", fpath);
+
+#ifndef WIN32
+	if (chmod(fpath, pg_file_create_mode))
+		pg_fatal("could not set permissions on file \"%s\": %m",
+				 fpath);
+#endif
+
+	pg_log_debug("spilling to temporary file \"%s\"", fpath);
+
+	return file;
+}
+
+/*
+ * Write buffer data to the given file handle.
+ */
+static void
+perform_tmp_write(const char *fname, StringInfo buf, FILE *file)
+{
+	Assert(file);
+
+	errno = 0;
+	if (buf->len > 0 && fwrite(buf->data, buf->len, 1, file) != 1)
+	{
+		/*
+		 * If write didn't set errno, assume problem is no disk space
+		 */
+		if (errno == 0)
+			errno = ENOSPC;
+		pg_fatal("could not write to file \"%s\": %m", fname);
+	}
+}
+
+/*
+ * Create an astreamer that can read WAL from tar file.
  */
 static astreamer *
 astreamer_waldump_new(XLogDumpPrivate *privateInfo)
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 6d04462d039..faf300af2be 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -478,25 +478,46 @@ TarWALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
 		return -1;
 
 	/*
-	 * If the target page is in a different segment, free the buffer space
-	 * occupied by the previous segment data. Since pg_waldump never requests
-	 * the same WAL bytes twice, moving to a new segment implies the previous
-	 * buffer's data and that segment will not be needed again.
+	 * If the target page is in a different segment, free the buffer and/or
+	 * temporary file disk space occupied by the previous segment's data.
+	 * Since pg_waldump never requests the same WAL bytes twice, moving to a
+	 * new segment implies the previous buffer's data and that segment will
+	 * not be needed again.
+	 *
+	 * Afterward, check for the next required WAL segment's physical existence
+	 * in the temporary directory first before invoking the archive streamer.
 	 */
 	nextSegNo = state->seg.ws_segno;
 	if (!XLByteInSeg(targetPagePtr, nextSegNo, WalSegSz))
 	{
 		char		fname[MAXFNAMELEN];
 
+		if (state->seg.ws_file >= 0)
+		{
+			close(state->seg.ws_file);
+			state->seg.ws_file = -1;
+		}
+
 		XLogFileName(fname, state->seg.ws_tli, nextSegNo, WalSegSz);
 		free_archive_wal_entry(fname, private);
 
 		XLByteToSeg(targetPagePtr, nextSegNo, WalSegSz);
 		state->seg.ws_tli = private->timeline;
 		state->seg.ws_segno = nextSegNo;
+
+		/*
+		 * If the next segment exists, open it and continue reading from there
+		 */
+		XLogFileName(fname, state->seg.ws_tli, nextSegNo, WalSegSz);
+		state->seg.ws_file = open_file_in_directory(TmpWalSegDir, fname);
 	}
 
-	/* Read the WAL page from the archive streamer */
+	/* Continue reading from the open WAL segment, if any */
+	if (state->seg.ws_file >= 0)
+		return WALDumpReadPage(state, targetPagePtr, count, targetPtr,
+							   readBuff);
+
+	/* Otherwise, read the WAL page from the archive streamer */
 	return read_archive_wal_page(private, targetPagePtr, count, readBuff,
 								 WalSegSz);
 }
diff --git a/src/bin/pg_waldump/pg_waldump.h b/src/bin/pg_waldump/pg_waldump.h
index 02da2c43b08..476f74e2846 100644
--- a/src/bin/pg_waldump/pg_waldump.h
+++ b/src/bin/pg_waldump/pg_waldump.h
@@ -18,6 +18,9 @@
 struct ArchivedWALFile;
 struct ArchivedWAL_hash;
 
+/* Temporary directory */
+extern char *TmpWalSegDir;
+
 /* Contains the necessary information to drive WAL decoding */
 typedef struct XLogDumpPrivate
 {
diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl
index cae543c8990..55a21c71208 100644
--- a/src/bin/pg_waldump/t/001_basic.pl
+++ b/src/bin/pg_waldump/t/001_basic.pl
@@ -7,6 +7,7 @@ use Cwd;
 use PostgreSQL::Test::Cluster;
 use PostgreSQL::Test::Utils;
 use Test::More;
+use List::Util qw(shuffle);
 
 my $tar = $ENV{TAR};
 
@@ -272,7 +273,7 @@ sub generate_archive
 	}
 	closedir $dh;
 
-	@files = sort @files;
+	@files = shuffle @files;
 
 	# move into the WAL directory before archiving files
 	my $cwd = getcwd;
-- 
2.47.1