v17-changes.nocfbot

text/plain

Filename: v17-changes.nocfbot
Type: text/plain
Part: 3
Message: Re: pg_waldump: support decoding of WAL inside tarfile
diff --git a/src/bin/pg_waldump/archive_waldump.c b/src/bin/pg_waldump/archive_waldump.c
index 547a5154cb6..c93e02ece8b 100644
--- a/src/bin/pg_waldump/archive_waldump.c
+++ b/src/bin/pg_waldump/archive_waldump.c
@@ -53,7 +53,7 @@ char	   *TmpWalSegDir = NULL;
  *
  * To minimize the memory footprint, entries and their associated buffers are
  * freed immediately once consumed. Since pg_waldump does not request the same
- * bytes twice, a segment is discarded as soon as it moves past it.
+ * bytes twice, a segment is discarded as soon as pg_waldump moves past it.
  */
 typedef struct ArchivedWALFile
 {
@@ -142,10 +142,10 @@ init_archive_reader(XLogDumpPrivate *privateInfo, const char *waldir,
 
 	streamer = astreamer_waldump_new(privateInfo);
 
-	/* Before that we must parse the tar archive. */
+	/* We must first parse the tar archive. */
 	streamer = astreamer_tar_parser_new(streamer);
 
-	/* Before that we must decompress, if archive is compressed. */
+	/* If the archive is compressed, decompress before parsing. */
 	if (compression == PG_COMPRESSION_GZIP)
 		streamer = astreamer_gzip_decompressor_new(streamer);
 	else if (compression == PG_COMPRESSION_LZ4)
@@ -243,6 +243,23 @@ free_archive_reader(XLogDumpPrivate *privateInfo)
 	 */
 	astreamer_free(privateInfo->archive_streamer);
 
+	/* Free any remaining hash table entries and their buffers. */
+	if (privateInfo->archive_wal_htab != NULL)
+	{
+		ArchivedWAL_iterator iter;
+		ArchivedWALFile *entry;
+
+		ArchivedWAL_start_iterate(privateInfo->archive_wal_htab, &iter);
+		while ((entry = ArchivedWAL_iterate(privateInfo->archive_wal_htab,
+											&iter)) != NULL)
+		{
+			if (entry->buf != NULL)
+				destroyStringInfo(entry->buf);
+		}
+		ArchivedWAL_destroy(privateInfo->archive_wal_htab);
+		privateInfo->archive_wal_htab = NULL;
+	}
+
 	/* Free the reusable read buffer. */
 	if (privateInfo->archive_read_buf != NULL)
 	{
@@ -350,16 +367,21 @@ read_archive_wal_page(XLogDumpPrivate *privateInfo, XLogRecPtr targetPagePtr,
 			}
 
 			/*
-			 * Now, fetch more data; raise an error if it's not the current
-			 * segment being read by the archive streamer or if reading of the
-			 * archived file has finished.
+			 * Now, fetch more data.  Raise an error if the archive
+			 * streamer has moved past our segment (meaning the WAL file
+			 * in the archive is shorter than expected) or if reading the
+			 * archive reached EOF.
 			 */
-			if (privateInfo->cur_file != entry ||
-				read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0)
-				pg_fatal("could not read file \"%s\" from archive \"%s\": read %lld of %lld",
+			if (privateInfo->cur_file != entry)
+				pg_fatal("WAL segment \"%s\" in archive \"%s\" is too short: read %lld of %lld bytes",
 						 fname, privateInfo->archive_name,
 						 (long long int) (count - nbytes),
 						 (long long int) count);
+			if (read_archive_file(privateInfo, READ_CHUNK_SIZE) == 0)
+				pg_fatal("unexpected end of archive \"%s\" while reading \"%s\": read %lld of %lld bytes",
+						 privateInfo->archive_name, fname,
+						 (long long int) (count - nbytes),
+						 (long long int) count);
 		}
 	}
 
@@ -736,7 +758,7 @@ astreamer_waldump_finalize(astreamer *streamer)
 }
 
 /*
- * Free memory associated with a astreamer_waldump stream.
+ * Free memory associated with an astreamer_waldump stream.
  */
 static void
 astreamer_waldump_free(astreamer *streamer)
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index e970b007883..b13cedaa3e7 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -874,7 +874,7 @@ usage(void)
 	printf(_("  -F, --fork=FORK        only show records that modify blocks in fork FORK;\n"
 			 "                         valid names are main, fsm, vm, init\n"));
 	printf(_("  -n, --limit=N          number of records to display\n"));
-	printf(_("  -p, --path=PATH        tar archive or a directory in which to find WAL segment files or\n"
+	printf(_("  -p, --path=PATH        a tar archive or a directory in which to find WAL segment files or\n"
 			 "                         a directory with a ./pg_wal that contains such files\n"
 			 "                         (default: current directory, ./pg_wal, $PGDATA/pg_wal)\n"));
 	printf(_("  -q, --quiet            do not print any output, except for errors\n"));
@@ -1338,6 +1338,13 @@ main(int argc, char **argv)
 		goto bad_argument;
 	}
 
+	/* --follow is not supported with tar archives */
+	if (config.follow && private.archive_name)
+	{
+		pg_log_error("--follow is not supported when reading from a tar archive");
+		goto bad_argument;
+	}
+
 	/* done with argument parsing, do the actual work */
 
 	/* we have everything we need, start reading */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d849293e6fa..b7b78e2c958 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -147,6 +147,7 @@ ArchiveStartupCB
 ArchiveStreamState
 ArchivedWALFile
 ArchivedWAL_hash
+ArchivedWAL_iterator
 ArchiverOutput
 ArchiverStage
 ArrayAnalyzeExtraData