v1-0002-Add-function-to-make-pg_waldump-to-support-TDE.patch

application/octet-stream

Filename: v1-0002-Add-function-to-make-pg_waldump-to-support-TDE.patch
Type: application/octet-stream
Part: 3
Message: Re: [Proposal] Table-level Transparent Data Encryption (TDE) and Key Management Service (KMS)

Patch

Same data as JSON: GET /api/v1/attachments/:id/patch the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes. API reference →
Format: unified
Series: patch v1-0002
File+
src/bin/pg_waldump/compat.c 34 0
src/bin/pg_waldump/Makefile 7 2
src/bin/pg_waldump/pg_waldump.c 64 13
diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile
index 135979c..bae7abe 100644
--- a/src/bin/pg_waldump/Makefile
+++ b/src/bin/pg_waldump/Makefile
@@ -8,13 +8,15 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 OBJS = pg_waldump.o compat.o xlogreader.o rmgrdesc.o \
-	$(RMGRDESCOBJS) $(WIN32RES)
+	$(RMGRDESCOBJS) $(ENCRYPTIONOBJS) $(WIN32RES)
 
 override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
 
 RMGRDESCSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc.c)))
 RMGRDESCOBJS = $(patsubst %.c,%.o,$(RMGRDESCSOURCES))
 
+ENCRYPTIONSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/storage/encryption/*.c)))
+ENCRYPTIONOBJS = $(patsubst %.c,%.o,$(ENCRYPTIONSOURCES))
 
 all: pg_waldump
 
@@ -27,6 +29,9 @@ xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
 $(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/%
 	rm -f $@ && $(LN_S) $< .
 
+$(ENCRYPTIONSOURCES): % : $(top_srcdir)/src/backend/storage/encryption/%
+	rm -f $@ && $(LN_S) $< .
+
 install: all installdirs
 	$(INSTALL_PROGRAM) pg_waldump$(X) '$(DESTDIR)$(bindir)/pg_waldump$(X)'
 
@@ -37,7 +42,7 @@ uninstall:
 	rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)'
 
 clean distclean maintainer-clean:
-	rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c
+	rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) $(ENCRYPTIONSOURCES) xlogreader.c
 	rm -rf tmp_check
 
 check:
diff --git a/src/bin/pg_waldump/compat.c b/src/bin/pg_waldump/compat.c
index 7b389a2..d1fa140 100644
--- a/src/bin/pg_waldump/compat.c
+++ b/src/bin/pg_waldump/compat.c
@@ -22,6 +22,7 @@
 
 #include "lib/stringinfo.h"
 #include "utils/datetime.h"
+#include "common/logging.h"
 
 /* copied from timestamp.c */
 pg_time_t
@@ -69,11 +70,44 @@ timestamptz_to_str(TimestampTz dt)
  * be linked/called.
  */
 void
+initStringInfo(StringInfo str)
+{
+	int size = 1024;    /* initial default buffer size */
+
+	str->data = (char *) palloc(size);
+	str->maxlen = size;
+	str->data[0] = '\0';
+	str->len = 0;
+	str->cursor = 0;
+}
+
+void
 appendStringInfo(StringInfo str, const char *fmt,...)
 {
 	va_list		args;
 
 	va_start(args, fmt);
+	if (str)
+	{
+		int		avail;
+		size_t		nprinted;
+
+		avail = str->maxlen - str->len;
+		nprinted = pvsnprintf(str->data + str->len, (size_t) avail, fmt, args);
+
+		if (nprinted < (size_t) avail)
+		{
+			/* Success.  Note nprinted does not include trailing null. */
+			str->len += (int) nprinted;
+		}
+		else
+		{
+			pg_log_error(ERROR, "the string is too long");
+			exit(1);
+		}
+
+	}
+	else
 	vprintf(fmt, args);
 	va_end(args);
 }
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 8e4bff6..6b993cf 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -24,6 +24,9 @@
 #include "common/logging.h"
 #include "getopt_long.h"
 #include "rmgrdesc.h"
+#include "storage/kmgr.h"
+#include "catalog/pg_control.h"
+#include "storage/encryption.h"
 
 static const char *progname;
 
@@ -192,27 +195,42 @@ search_directory(const char *directory, const char *fname)
 		PGAlignedXLogBlock buf;
 		int			r;
 
-		r = read(fd, buf.data, XLOG_BLCKSZ);
-		if (r == XLOG_BLCKSZ)
+		if (DataEncryptionEnabled())
 		{
-			XLogLongPageHeader longhdr = (XLogLongPageHeader) buf.data;
-
-			WalSegSz = longhdr->xlp_seg_size;
+			WalSegSz = (int) lseek(fd, 0, SEEK_END);
+			if (WalSegSz <= 0)
+				fatal_error("Could not determine size of WAL segment \"%s\"", fname);
 
 			if (!IsValidWalSegSize(WalSegSz))
 				fatal_error(ngettext("WAL segment size must be a power of two between 1 MB and 1 GB, but the WAL file \"%s\" header specifies %d byte",
-									 "WAL segment size must be a power of two between 1 MB and 1 GB, but the WAL file \"%s\" header specifies %d bytes",
-									 WalSegSz),
-							fname, WalSegSz);
+					"WAL segment size must be a power of two between 1 MB and 1 GB, but the WAL file \"%s\" header specifies %d bytes",
+						WalSegSz),
+						fname, WalSegSz);
 		}
 		else
 		{
-			if (errno != 0)
-				fatal_error("could not read file \"%s\": %s",
-							fname, strerror(errno));
+			r = read(fd, buf.data, XLOG_BLCKSZ);
+			if (r == XLOG_BLCKSZ)
+			{
+				XLogLongPageHeader longhdr = (XLogLongPageHeader) buf.data;
+
+				WalSegSz = longhdr->xlp_seg_size;
+
+				if (!IsValidWalSegSize(WalSegSz))
+					fatal_error(ngettext("WAL segment size must be a power of two between 1 MB and 1 GB, but the WAL file \"%s\" header specifies %d byte",
+									 "WAL segment size must be a power of two between 1 MB and 1 GB, but the WAL file \"%s\" header specifies %d bytes",
+									 WalSegSz),
+							fname, WalSegSz);
+			}
 			else
-				fatal_error("could not read file \"%s\": read %d of %zu",
+			{
+				if (errno != 0)
+					fatal_error("could not read file \"%s\": %s",
+								fname, strerror(errno));
+				else
+					fatal_error("could not read file \"%s\": read %d of %zu",
 							fname, r, (Size) XLOG_BLCKSZ);
+			}
 		}
 		close(fd);
 		return true;
@@ -428,6 +446,17 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
 	XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
 					 readBuff, count);
 
+	if (DataEncryptionEnabled())
+	{
+		XLogSegNo       readSegNo;
+		uint32          readSegOff;
+
+		XLByteToSeg(targetPagePtr, readSegNo, WalSegSz);
+		readSegOff = targetPagePtr % WalSegSz;
+
+		DecryptXLog(readBuff, XLOG_BLCKSZ, readSegNo, readSegOff);
+	}
+
 	return count;
 }
 
@@ -775,6 +804,10 @@ usage(void)
 	printf(_("  %s [OPTION]... [STARTSEG [ENDSEG]]\n"), progname);
 	printf(_("\nOptions:\n"));
 	printf(_("  -b, --bkp-details      output detailed information about backup blocks\n"));
+	printf(_("  -c  --cluster-passphrase-command=COMMAND\n"
+		"                      set command to obtain passphrase for data encryption key\n"));
+
+	printf(_(" [-D, --pgdata=]DATADIR  data directory to get the Data encryption cipher\n"));
 	printf(_("  -e, --end=RECPTR       stop reading at WAL location RECPTR\n"));
 	printf(_("  -f, --follow           keep retrying after reaching end of WAL\n"));
 	printf(_("  -n, --limit=N          number of records to display\n"));
@@ -807,9 +840,13 @@ main(int argc, char **argv)
 	XLogRecPtr	first_record;
 	char	   *waldir = NULL;
 	char	   *errormsg;
+	char       *pgdataDir = NULL;
+	char	   *cluster_passphrase = NULL;
 
 	static struct option long_options[] = {
 		{"bkp-details", no_argument, NULL, 'b'},
+		{"pgdata", required_argument, NULL, 'D'},
+		{"cluster-passphrase-command", required_argument, NULL, 'c'},
 		{"end", required_argument, NULL, 'e'},
 		{"follow", no_argument, NULL, 'f'},
 		{"help", no_argument, NULL, '?'},
@@ -870,7 +907,7 @@ main(int argc, char **argv)
 		goto bad_argument;
 	}
 
-	while ((option = getopt_long(argc, argv, "be:fn:p:r:s:t:x:z",
+	while ((option = getopt_long(argc, argv, "bc:D:e:fn:p:r:s:t:x:z",
 								 long_options, &optindex)) != -1)
 	{
 		switch (option)
@@ -878,6 +915,12 @@ main(int argc, char **argv)
 			case 'b':
 				config.bkp_details = true;
 				break;
+			case 'c':
+				cluster_passphrase = pg_strdup(optarg);
+				break;
+			case 'D':
+				pgdataDir = pg_strdup(optarg);
+				break;
 			case 'e':
 				if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2)
 				{
@@ -973,6 +1016,14 @@ main(int argc, char **argv)
 		}
 	}
 
+	if (pgdataDir == NULL)
+	{
+		pgdataDir = getenv("PGDATA");
+	}
+
+	if (pgdataDir)
+		getKmgr(pgdataDir, cluster_passphrase, progname);
+
 	if ((optind + 2) < argc)
 	{
 		pg_log_error("too many command-line arguments (first is \"%s\")",