v5-0001-Extact-common-functions-from-pg_basebackup-into-s.patch

From fabcc0a53c88e4ac595780088c05af686070a445 Mon Sep 17 00:00:00 2001
From: Paul Guo <paulguo@gmail.com>
Date: Thu, 18 Apr 2019 18:55:09 +0800
Subject: [PATCH v5 1/3] Extact common functions from pg_basebackup into
 separate files for pg_rewind use.

---
 src/bin/pg_basebackup/Makefile        |   2 +-
 src/bin/pg_basebackup/backup_common.c | 176 ++++++++++++++++++++++++++
 src/bin/pg_basebackup/backup_common.h |  33 +++++
 src/bin/pg_basebackup/pg_basebackup.c | 174 +------------------------
 src/bin/pg_basebackup/streamutil.c    |   1 -
 src/bin/pg_rewind/Makefile            |  13 +-
 src/bin/pg_rewind/libpq_fetch.c       |   2 +-
 src/bin/pg_rewind/pg_rewind.c         |   4 +
 src/tools/msvc/Mkvcbuild.pm           |   2 +
 9 files changed, 228 insertions(+), 179 deletions(-)
 create mode 100644 src/bin/pg_basebackup/backup_common.c
 create mode 100644 src/bin/pg_basebackup/backup_common.h

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index d7a081b9bb..2ed77bd0d6 100644
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -21,7 +21,7 @@ include $(top_builddir)/src/Makefile.global
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
-OBJS=receivelog.o streamutil.o walmethods.o $(WIN32RES)
+OBJS=receivelog.o streamutil.o walmethods.o backup_common.o $(WIN32RES)
 
 all: pg_basebackup pg_receivewal pg_recvlogical
 
diff --git a/src/bin/pg_basebackup/backup_common.c b/src/bin/pg_basebackup/backup_common.c
new file mode 100644
index 0000000000..89a377630b
--- /dev/null
+++ b/src/bin/pg_basebackup/backup_common.c
@@ -0,0 +1,176 @@
+/*-------------------------------------------------------------------------
+ *
+ * backup_common.c - common code that is used in both pg_basebackup and pg_rewind.
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+#include "common/logging.h"
+#include "fe_utils/string_utils.h"
+#include "backup_common.h"
+
+/* Contents of configuration file to be generated */
+PQExpBuffer recoveryconfcontents = NULL;
+
+bool writerecoveryconf = false;
+char *replication_slot = NULL;
+PGconn	   *conn = NULL;
+
+void
+disconnect_atexit(void)
+{
+	if (conn != NULL)
+		PQfinish(conn);
+}
+
+/*
+ * Escape a string so that it can be used as a value in a key-value pair
+ * a configuration file.
+ */
+static char *
+escape_quotes(const char *src)
+{
+	char	   *result = escape_single_quotes_ascii(src);
+
+	if (!result)
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+	return result;
+}
+
+/*
+ * Create a configuration file in memory using a PQExpBuffer
+ */
+void
+GenerateRecoveryConf(void)
+{
+	PQconninfoOption *connOptions;
+	PQconninfoOption *option;
+	PQExpBufferData conninfo_buf;
+	char	   *escaped;
+
+	recoveryconfcontents = createPQExpBuffer();
+	if (!recoveryconfcontents)
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+
+	/*
+	 * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by
+	 * standby.signal to trigger a standby state at recovery.
+	 */
+	if (conn && PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
+		appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
+
+	connOptions = PQconninfo(conn);
+	if (connOptions == NULL)
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+
+	initPQExpBuffer(&conninfo_buf);
+	for (option = connOptions; option && option->keyword; option++)
+	{
+		/* Omit empty settings and those libpqwalreceiver overrides. */
+		if (strcmp(option->keyword, "replication") == 0 ||
+			strcmp(option->keyword, "dbname") == 0 ||
+			strcmp(option->keyword, "fallback_application_name") == 0 ||
+			(option->val == NULL) ||
+			(option->val != NULL && option->val[0] == '\0'))
+			continue;
+
+		/* Separate key-value pairs with spaces */
+		if (conninfo_buf.len != 0)
+			appendPQExpBufferChar(&conninfo_buf, ' ');
+
+		/*
+		 * Write "keyword=value" pieces, the value string is escaped and/or
+		 * quoted if necessary.
+		 */
+		appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
+		appendConnStrVal(&conninfo_buf, option->val);
+	}
+
+	/*
+	 * Escape the connection string, so that it can be put in the config file.
+	 * Note that this is different from the escaping of individual connection
+	 * options above!
+	 */
+	escaped = escape_quotes(conninfo_buf.data);
+	appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
+	free(escaped);
+
+	if (replication_slot)
+	{
+		/* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */
+		appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n",
+						  replication_slot);
+	}
+
+	if (PQExpBufferBroken(recoveryconfcontents) ||
+		PQExpBufferDataBroken(conninfo_buf))
+	{
+		pg_log_error("out of memory");
+		exit(1);
+	}
+
+	termPQExpBuffer(&conninfo_buf);
+
+	PQconninfoFree(connOptions);
+}
+
+/*
+ * Write the configuration file into the directory specified in basedir,
+ * with the contents already collected in memory appended.  Then write
+ * the signal file into the basedir.  If the server does not support
+ * recovery parameters as GUCs, the signal file is not necessary, and
+ * configuration is written to recovery.conf.
+ */
+void
+WriteRecoveryConf(char *target_dir)
+{
+	char		filename[MAXPGPATH];
+	FILE	   *cf;
+	bool		is_recovery_guc_supported = true;
+
+	if (conn && PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
+		is_recovery_guc_supported = false;
+
+	snprintf(filename, MAXPGPATH, "%s/%s", target_dir,
+			 is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf");
+
+	cf = fopen(filename, is_recovery_guc_supported ? "a" : "w");
+	if (cf == NULL)
+	{
+		pg_log_error("could not open file \"%s\": %m", filename);
+		exit(1);
+	}
+
+	if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
+	{
+		pg_log_error("could not write to file \"%s\": %m", filename);
+		exit(1);
+	}
+
+	fclose(cf);
+
+	if (is_recovery_guc_supported)
+	{
+		snprintf(filename, MAXPGPATH, "%s/%s", target_dir, "standby.signal");
+		cf = fopen(filename, "w");
+		if (cf == NULL)
+		{
+			pg_log_error("could not create file \"%s\": %m", filename);
+			exit(1);
+		}
+
+		fclose(cf);
+	}
+}
diff --git a/src/bin/pg_basebackup/backup_common.h b/src/bin/pg_basebackup/backup_common.h
new file mode 100644
index 0000000000..8f9f860485
--- /dev/null
+++ b/src/bin/pg_basebackup/backup_common.h
@@ -0,0 +1,33 @@
+/*-------------------------------------------------------------------------
+ *
+ * backup_common.h - common code that is used in both pg_basebackup and pg_rewind.
+ *
+ * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef BACKUP_COMMON_H
+#define BACKUP_COMMON_H
+
+#include "libpq-fe.h"
+#include "pqexpbuffer.h"
+
+/* Contents of configuration file to be generated */
+extern PQExpBuffer recoveryconfcontents;
+
+extern bool writerecoveryconf;
+extern char *replication_slot;
+PGconn	   *conn;
+
+extern void disconnect_atexit(void);
+extern void GenerateRecoveryConf(void);
+extern void WriteRecoveryConf(char *target_dir);
+
+/*
+ * recovery.conf is integrated into postgresql.conf from version 12.
+ */
+#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000
+
+
+#endif							/* BACKUP_COMMON_H */
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 7986872f10..169fc2429c 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -37,6 +37,7 @@
 #include "pqexpbuffer.h"
 #include "pgtar.h"
 #include "pgtime.h"
+#include "backup_common.h"
 #include "receivelog.h"
 #include "replication/basebackup.h"
 #include "streamutil.h"
@@ -67,11 +68,6 @@ typedef struct TablespaceList
  */
 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
 
-/*
- * recovery.conf is integrated into postgresql.conf from version 12.
- */
-#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000
-
 /*
  * Different ways to include WAL
  */
@@ -95,12 +91,10 @@ static int	verbose = 0;
 static int	compresslevel = 0;
 static IncludeWal includewal = STREAM_WAL;
 static bool fastcheckpoint = false;
-static bool writerecoveryconf = false;
 static bool do_sync = true;
 static int	standby_message_timeout = 10 * 1000;	/* 10 sec = default */
 static pg_time_t last_progress_report = 0;
 static int32 maxrate = 0;		/* no limit by default */
-static char *replication_slot = NULL;
 static bool temp_replication_slot = true;
 static bool create_slot = false;
 static bool no_slot = false;
@@ -137,9 +131,6 @@ static int	has_xlogendptr = 0;
 static volatile LONG has_xlogendptr = 0;
 #endif
 
-/* Contents of configuration file to be generated */
-static PQExpBuffer recoveryconfcontents = NULL;
-
 /* Function headers */
 static void usage(void);
 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
@@ -147,8 +138,6 @@ static void progress_report(int tablespacenum, const char *filename, bool force)
 
 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
-static void GenerateRecoveryConf(PGconn *conn);
-static void WriteRecoveryConf(void);
 static void BaseBackup(void);
 
 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
@@ -205,13 +194,6 @@ cleanup_directories_atexit(void)
 		pg_log_info("changes to tablespace directories will not be undone");
 }
 
-static void
-disconnect_atexit(void)
-{
-	if (conn != NULL)
-		PQfinish(conn);
-}
-
 #ifndef WIN32
 /*
  * On windows, our background thread dies along with the process. But on
@@ -1629,7 +1611,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
 		PQfreemem(copybuf);
 
 	if (basetablespace && writerecoveryconf)
-		WriteRecoveryConf();
+		WriteRecoveryConf(basedir);
 
 	/*
 	 * No data is synced here, everything is done for all tablespaces at the
@@ -1637,156 +1619,6 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
 	 */
 }
 
-/*
- * Escape a string so that it can be used as a value in a key-value pair
- * a configuration file.
- */
-static char *
-escape_quotes(const char *src)
-{
-	char	   *result = escape_single_quotes_ascii(src);
-
-	if (!result)
-	{
-		pg_log_error("out of memory");
-		exit(1);
-	}
-	return result;
-}
-
-/*
- * Create a configuration file in memory using a PQExpBuffer
- */
-static void
-GenerateRecoveryConf(PGconn *conn)
-{
-	PQconninfoOption *connOptions;
-	PQconninfoOption *option;
-	PQExpBufferData conninfo_buf;
-	char	   *escaped;
-
-	recoveryconfcontents = createPQExpBuffer();
-	if (!recoveryconfcontents)
-	{
-		pg_log_error("out of memory");
-		exit(1);
-	}
-
-	/*
-	 * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by
-	 * standby.signal to trigger a standby state at recovery.
-	 */
-	if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
-		appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
-
-	connOptions = PQconninfo(conn);
-	if (connOptions == NULL)
-	{
-		pg_log_error("out of memory");
-		exit(1);
-	}
-
-	initPQExpBuffer(&conninfo_buf);
-	for (option = connOptions; option && option->keyword; option++)
-	{
-		/* Omit empty settings and those libpqwalreceiver overrides. */
-		if (strcmp(option->keyword, "replication") == 0 ||
-			strcmp(option->keyword, "dbname") == 0 ||
-			strcmp(option->keyword, "fallback_application_name") == 0 ||
-			(option->val == NULL) ||
-			(option->val != NULL && option->val[0] == '\0'))
-			continue;
-
-		/* Separate key-value pairs with spaces */
-		if (conninfo_buf.len != 0)
-			appendPQExpBufferChar(&conninfo_buf, ' ');
-
-		/*
-		 * Write "keyword=value" pieces, the value string is escaped and/or
-		 * quoted if necessary.
-		 */
-		appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
-		appendConnStrVal(&conninfo_buf, option->val);
-	}
-
-	/*
-	 * Escape the connection string, so that it can be put in the config file.
-	 * Note that this is different from the escaping of individual connection
-	 * options above!
-	 */
-	escaped = escape_quotes(conninfo_buf.data);
-	appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
-	free(escaped);
-
-	if (replication_slot)
-	{
-		/* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */
-		appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n",
-						  replication_slot);
-	}
-
-	if (PQExpBufferBroken(recoveryconfcontents) ||
-		PQExpBufferDataBroken(conninfo_buf))
-	{
-		pg_log_error("out of memory");
-		exit(1);
-	}
-
-	termPQExpBuffer(&conninfo_buf);
-
-	PQconninfoFree(connOptions);
-}
-
-
-/*
- * Write the configuration file into the directory specified in basedir,
- * with the contents already collected in memory appended.  Then write
- * the signal file into the basedir.  If the server does not support
- * recovery parameters as GUCs, the signal file is not necessary, and
- * configuration is written to recovery.conf.
- */
-static void
-WriteRecoveryConf(void)
-{
-	char		filename[MAXPGPATH];
-	FILE	   *cf;
-	bool		is_recovery_guc_supported = true;
-
-	if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
-		is_recovery_guc_supported = false;
-
-	snprintf(filename, MAXPGPATH, "%s/%s", basedir,
-			 is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf");
-
-	cf = fopen(filename, is_recovery_guc_supported ? "a" : "w");
-	if (cf == NULL)
-	{
-		pg_log_error("could not open file \"%s\": %m", filename);
-		exit(1);
-	}
-
-	if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
-	{
-		pg_log_error("could not write to file \"%s\": %m", filename);
-		exit(1);
-	}
-
-	fclose(cf);
-
-	if (is_recovery_guc_supported)
-	{
-		snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal");
-		cf = fopen(filename, "w");
-		if (cf == NULL)
-		{
-			pg_log_error("could not create file \"%s\": %m", filename);
-			exit(1);
-		}
-
-		fclose(cf);
-	}
-}
-
 
 static void
 BaseBackup(void)
@@ -1843,7 +1675,7 @@ BaseBackup(void)
 	 * Build contents of configuration file if requested
 	 */
 	if (writerecoveryconf)
-		GenerateRecoveryConf(conn);
+		GenerateRecoveryConf();
 
 	/*
 	 * Run IDENTIFY_SYSTEM so we can get the timeline
diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c
index 8d8ac11b66..7c78835268 100644
--- a/src/bin/pg_basebackup/streamutil.c
+++ b/src/bin/pg_basebackup/streamutil.c
@@ -53,7 +53,6 @@ char	   *dbname = NULL;
 int			dbgetpassword = 0;	/* 0=auto, -1=never, 1=always */
 static bool have_password = false;
 static char password[100];
-PGconn	   *conn = NULL;
 
 /*
  * Connect to the server. Returns a valid PGconn pointer if connected,
diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile
index 859d3abc41..555f5cb31f 100644
--- a/src/bin/pg_rewind/Makefile
+++ b/src/bin/pg_rewind/Makefile
@@ -15,20 +15,23 @@ subdir = src/bin/pg_rewind
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS)
-LDFLAGS_INTERNAL += $(libpq_pgport)
+override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND -I$(top_srcdir)/src/bin/pg_basebackup $(CPPFLAGS)
+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 
 OBJS	= pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \
-	fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o \
+	fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o backup_common.o \
 	$(WIN32RES)
 
-EXTRA_CLEAN = xlogreader.c
+EXTRA_CLEAN = xlogreader.c backup_common.c
 
 all: pg_rewind
 
 pg_rewind: $(OBJS) | submake-libpq submake-libpgport
 	$(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
 
+backup_common.c: % : $(top_srcdir)/src/bin/pg_basebackup/%
+	rm -f $@ && $(LN_S) $< .
+
 xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
 	rm -f $@ && $(LN_S) $< .
 
@@ -42,7 +45,7 @@ uninstall:
 	rm -f '$(DESTDIR)$(bindir)/pg_rewind$(X)'
 
 clean distclean maintainer-clean:
-	rm -f pg_rewind$(X) $(OBJS) xlogreader.c
+	rm -f pg_rewind$(X) $(OBJS) xlogreader.c backup_common.c
 	rm -rf tmp_check
 
 check:
diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c
index 002776f696..2f41d56b3d 100644
--- a/src/bin/pg_rewind/libpq_fetch.c
+++ b/src/bin/pg_rewind/libpq_fetch.c
@@ -25,7 +25,7 @@
 #include "fe_utils/connect.h"
 #include "port/pg_bswap.h"
 
-static PGconn *conn = NULL;
+extern PGconn *conn;
 
 /*
  * Files are fetched max CHUNKSIZE bytes at a time.
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 15e3eab550..e8674c24bb 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -29,6 +29,7 @@
 #include "common/restricted_token.h"
 #include "getopt_long.h"
 #include "storage/bufpage.h"
+#include "backup_common.h"
 
 static void usage(const char *progname);
 
@@ -231,7 +232,10 @@ main(int argc, char **argv)
 
 	/* Connect to remote server */
 	if (connstr_source)
+	{
 		libpqConnect(connstr_source);
+		atexit(disconnect_atexit);
+	}
 
 	/*
 	 * Ok, we have all the options and we're ready to start. Read in all the
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 239f13cc12..0d4ae86c0f 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -375,7 +375,9 @@ sub mkvcbuild
 
 	my $pgrewind = AddSimpleFrontend('pg_rewind', 1);
 	$pgrewind->{name} = 'pg_rewind';
+	$pgrewind->AddIncludeDir('src/bin/pg_basebackup');
 	$pgrewind->AddFile('src/backend/access/transam/xlogreader.c');
+	$pgrewind->AddFile('src/bin/pg_basebackup/backup_common.c');
 	$pgrewind->AddLibrary('ws2_32.lib');
 	$pgrewind->AddDefine('FRONTEND');
 
-- 
2.17.2