v3-0001-Preserve-relfilenode-and-tablespace-OID-in-pg_upg.patch

From 2c00a37c4a3a00bb765232fea64e2bc02bfc8129 Mon Sep 17 00:00:00 2001
From: shruthikc-gowda <shruthi.kc@enterprisedb.com>
Date: Wed, 22 Sep 2021 23:33:41 +0530
Subject: [PATCH v3 1/2] Preserve relfilenode and tablespace OID in pg_upgrade

The patch aims to preserve the OIDs of relfilenode and tablespace during binary
upgrade so that the OIDs are same across old and new cluster.

Author: Shruthi KC, based on an earlier patch from Antonin Houska
Discussion: https://www.postgresql.org/message-id/7082.1562337694@localhost
---
 src/backend/bootstrap/bootparse.y                  |   3 +-
 src/backend/catalog/heap.c                         |  72 ++++++++++++--
 src/backend/catalog/index.c                        |  22 ++++-
 src/backend/commands/tablespace.c                  |  17 +++-
 src/backend/utils/adt/pg_upgrade_support.c         |  44 +++++++++
 src/bin/pg_dump/pg_dump.c                          | 104 +++++++++++++--------
 src/bin/pg_dump/pg_dumpall.c                       |   3 +
 src/bin/pg_upgrade/pg_upgrade.c                    |  13 +--
 src/include/catalog/binary_upgrade.h               |   5 +
 src/include/catalog/heap.h                         |   3 +-
 src/include/catalog/pg_proc.dat                    |  16 ++++
 .../spgist_name_ops/expected/spgist_name_ops.out   |  12 ++-
 12 files changed, 252 insertions(+), 62 deletions(-)

diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y
index 5fcd004..6560b19 100644
--- a/src/backend/bootstrap/bootparse.y
+++ b/src/backend/bootstrap/bootparse.y
@@ -212,7 +212,8 @@ Boot_CreateStmt:
 												   mapped_relation,
 												   true,
 												   &relfrozenxid,
-												   &relminmxid);
+												   &relminmxid,
+												   true);
 						elog(DEBUG4, "bootstrap relation created");
 					}
 					else
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 83746d3..851a4e2 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -91,7 +91,9 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_heap_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
 Oid			binary_upgrade_next_toast_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
 
 static void AddNewRelationTuple(Relation pg_class_desc,
 								Relation new_rel_desc,
@@ -286,7 +288,10 @@ SystemAttributeByName(const char *attname)
  *
  *		Note API change: the caller must now always provide the OID
  *		to use for the relation.  The relfilenode may (and, normally,
- *		should) be left unspecified.
+ *		should) be left unspecified unless explicitly set in binary-upgrade mode.
+ *
+ *		suppress_storage indicates whether or not to create the storage if the
+ *		caller passed a valid relfilenode.
  *
  *		rel->rd_rel is initialized by RelationBuildLocalRelation,
  *		and is mostly zeroes at return.
@@ -306,7 +311,8 @@ heap_create(const char *relname,
 			bool mapped_relation,
 			bool allow_system_table_mods,
 			TransactionId *relfrozenxid,
-			MultiXactId *relminmxid)
+			MultiXactId *relminmxid,
+			bool suppress_storage)
 {
 	bool		create_storage;
 	Relation	rel;
@@ -367,16 +373,22 @@ heap_create(const char *relname,
 	}
 
 	/*
-	 * Decide whether to create storage. If caller passed a valid relfilenode,
-	 * storage is already created, so don't do it here.  Also don't create it
-	 * for relkinds without physical storage.
+	 * Decide whether to create storage. If suppress_storage is true and a
+	 * valid relfilenode is passed, storage is already created, so don't do it
+	 * here. If suppress_storage is false and relfilenode is valid, create the
+	 * storage with the specified relfilenode. If relfilenode is unspecified
+	 * by the caller then create the storage with oid same as relid. Also,
+	 * don't create it for relkinds without physical storage.
 	 */
-	if (!RELKIND_HAS_STORAGE(relkind) || OidIsValid(relfilenode))
+	if (!RELKIND_HAS_STORAGE(relkind) || (OidIsValid(relfilenode) && suppress_storage))
 		create_storage = false;
 	else
 	{
 		create_storage = true;
-		relfilenode = relid;
+
+		/* Initialize relfilenode to relid */
+		if (!OidIsValid(relfilenode))
+			relfilenode = relid;
 	}
 
 	/*
@@ -1168,8 +1180,12 @@ heap_create_with_catalog(const char *relname,
 	Oid			existing_relid;
 	Oid			old_type_oid;
 	Oid			new_type_oid;
+
+	/* By default set to InvalidOid unless overridden by binary-upgrade */
+	Oid			relfilenode = InvalidOid;
 	TransactionId relfrozenxid;
 	MultiXactId relminmxid;
+	bool		suppress_storage = true;
 
 	pg_class_desc = table_open(RelationRelationId, RowExclusiveLock);
 
@@ -1230,7 +1246,7 @@ heap_create_with_catalog(const char *relname,
 	 */
 	if (!OidIsValid(relid))
 	{
-		/* Use binary-upgrade override for pg_class.oid/relfilenode? */
+		/* Use binary-upgrade override for pg_class.oid and relfilenode */
 		if (IsBinaryUpgrade &&
 			(relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE ||
 			 relkind == RELKIND_VIEW || relkind == RELKIND_MATVIEW ||
@@ -1244,7 +1260,27 @@ heap_create_with_catalog(const char *relname,
 
 			relid = binary_upgrade_next_heap_pg_class_oid;
 			binary_upgrade_next_heap_pg_class_oid = InvalidOid;
+
+			/*
+			 * Override the relfilenode and set the suppress_storage flag to
+			 * false so that the storage gets created with the specified
+			 * relfilenode during heap_create().
+			 */
+			if (relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE ||
+				relkind == RELKIND_MATVIEW)
+			{
+				if (!OidIsValid(binary_upgrade_next_heap_pg_class_relfilenode))
+					ereport(ERROR,
+							(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+							 errmsg("relfilenode value not set when in binary upgrade mode")));
+
+				relfilenode = binary_upgrade_next_heap_pg_class_relfilenode;
+				binary_upgrade_next_heap_pg_class_relfilenode = InvalidOid;
+
+				suppress_storage = false;
+			}
 		}
+
 		/* There might be no TOAST table, so we have to test for it. */
 		else if (IsBinaryUpgrade &&
 				 OidIsValid(binary_upgrade_next_toast_pg_class_oid) &&
@@ -1252,6 +1288,21 @@ heap_create_with_catalog(const char *relname,
 		{
 			relid = binary_upgrade_next_toast_pg_class_oid;
 			binary_upgrade_next_toast_pg_class_oid = InvalidOid;
+
+			/*
+			 * Override the toast relfilenode and set the suppress_storage
+			 * flag to false so that the storage gets created with the
+			 * specified relfilenode during heap_create().
+			 */
+			if (!OidIsValid(binary_upgrade_next_toast_pg_class_relfilenode))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("toast relfilenode value not set when in binary upgrade mode")));
+
+			relfilenode = binary_upgrade_next_toast_pg_class_relfilenode;
+			binary_upgrade_next_toast_pg_class_relfilenode = InvalidOid;
+
+			suppress_storage = false;
 		}
 		else
 			relid = GetNewRelFileNode(reltablespace, pg_class_desc,
@@ -1294,7 +1345,7 @@ heap_create_with_catalog(const char *relname,
 							   relnamespace,
 							   reltablespace,
 							   relid,
-							   InvalidOid,
+							   relfilenode,
 							   accessmtd,
 							   tupdesc,
 							   relkind,
@@ -1303,7 +1354,8 @@ heap_create_with_catalog(const char *relname,
 							   mapped_relation,
 							   allow_system_table_mods,
 							   &relfrozenxid,
-							   &relminmxid);
+							   &relminmxid,
+							   suppress_storage);
 
 	Assert(relid == RelationGetRelid(new_rel_desc));
 
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 26bfa74..56a4ef9 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -86,6 +86,7 @@
 
 /* Potentially set by pg_upgrade_support functions */
 Oid			binary_upgrade_next_index_pg_class_oid = InvalidOid;
+Oid			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
 
 /*
  * Pointer-free representation of variables used when reindexing system
@@ -732,6 +733,7 @@ index_create(Relation heapRelation,
 	char		relkind;
 	TransactionId relfrozenxid;
 	MultiXactId relminmxid;
+	bool		suppress_storage = true;
 
 	/* constraint flags can only be set when a constraint is requested */
 	Assert((constr_flags == 0) ||
@@ -903,7 +905,7 @@ index_create(Relation heapRelation,
 	 */
 	if (!OidIsValid(indexRelationId))
 	{
-		/* Use binary-upgrade override for pg_class.oid/relfilenode? */
+		/* Use binary-upgrade override for pg_class.oid and relfilenode */
 		if (IsBinaryUpgrade)
 		{
 			if (!OidIsValid(binary_upgrade_next_index_pg_class_oid))
@@ -913,6 +915,21 @@ index_create(Relation heapRelation,
 
 			indexRelationId = binary_upgrade_next_index_pg_class_oid;
 			binary_upgrade_next_index_pg_class_oid = InvalidOid;
+
+			/*
+			 * Overide the relfilenode and set the suppress_storage to false
+			 * so that the storage gets created with the specified relfilenode
+			 * during heap_create().
+			 */
+			if ((relkind == RELKIND_INDEX) &&
+				(!OidIsValid(binary_upgrade_next_index_pg_class_relfilenode)))
+				ereport(ERROR,
+						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+						 errmsg("index relfilenode value not set when in binary upgrade mode")));
+			relFileNode = binary_upgrade_next_index_pg_class_relfilenode;
+			binary_upgrade_next_index_pg_class_relfilenode = InvalidOid;
+
+			suppress_storage = false;
 		}
 		else
 		{
@@ -939,7 +956,8 @@ index_create(Relation heapRelation,
 								mapped_relation,
 								allow_system_table_mods,
 								&relfrozenxid,
-								&relminmxid);
+								&relminmxid,
+								suppress_storage);
 
 	Assert(relfrozenxid == InvalidTransactionId);
 	Assert(relminmxid == InvalidMultiXactId);
diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c
index 4b96eec..51ffa97 100644
--- a/src/backend/commands/tablespace.c
+++ b/src/backend/commands/tablespace.c
@@ -88,6 +88,7 @@
 char	   *default_tablespace = NULL;
 char	   *temp_tablespaces = NULL;
 
+Oid			binary_upgrade_next_pg_tablespace_oid = InvalidOid;
 
 static void create_tablespace_directories(const char *location,
 										  const Oid tablespaceoid);
@@ -335,8 +336,20 @@ CreateTableSpace(CreateTableSpaceStmt *stmt)
 
 	MemSet(nulls, false, sizeof(nulls));
 
-	tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
-									   Anum_pg_tablespace_oid);
+	if (IsBinaryUpgrade)
+	{
+		/* Use binary-upgrade override for tablespace oid */
+		if (!OidIsValid(binary_upgrade_next_pg_tablespace_oid))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+					 errmsg("pg_tablespace OID value not set when in binary upgrade mode")));
+
+		tablespaceoid = binary_upgrade_next_pg_tablespace_oid;
+		binary_upgrade_next_pg_tablespace_oid = InvalidOid;
+	}
+	else
+		tablespaceoid = GetNewOidWithIndex(rel, TablespaceOidIndexId,
+										   Anum_pg_tablespace_oid);
 	values[Anum_pg_tablespace_oid - 1] = ObjectIdGetDatum(tablespaceoid);
 	values[Anum_pg_tablespace_spcname - 1] =
 		DirectFunctionCall1(namein, CStringGetDatum(stmt->tablespacename));
diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c
index b5b46d7..135c382 100644
--- a/src/backend/utils/adt/pg_upgrade_support.c
+++ b/src/backend/utils/adt/pg_upgrade_support.c
@@ -30,6 +30,17 @@ do {															\
 } while (0)
 
 Datum
+binary_upgrade_set_next_pg_tablespace_oid(PG_FUNCTION_ARGS)
+{
+	Oid			tbspoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_pg_tablespace_oid = tbspoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_type_oid(PG_FUNCTION_ARGS)
 {
 	Oid			typoid = PG_GETARG_OID(0);
@@ -85,6 +96,17 @@ binary_upgrade_set_next_heap_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_heap_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_heap_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -96,6 +118,17 @@ binary_upgrade_set_next_index_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_index_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_index_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 {
 	Oid			reloid = PG_GETARG_OID(0);
@@ -107,6 +140,17 @@ binary_upgrade_set_next_toast_pg_class_oid(PG_FUNCTION_ARGS)
 }
 
 Datum
+binary_upgrade_set_next_toast_relfilenode(PG_FUNCTION_ARGS)
+{
+	Oid			nodeoid = PG_GETARG_OID(0);
+
+	CHECK_IS_BINARY_UPGRADE;
+	binary_upgrade_next_toast_pg_class_relfilenode = nodeoid;
+
+	PG_RETURN_VOID();
+}
+
+Datum
 binary_upgrade_set_next_pg_enum_oid(PG_FUNCTION_ARGS)
 {
 	Oid			enumoid = PG_GETARG_OID(0);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index a485fb2..b148919 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -4693,73 +4693,101 @@ binary_upgrade_set_pg_class_oids(Archive *fout,
 								 PQExpBuffer upgrade_buffer, Oid pg_class_oid,
 								 bool is_index)
 {
+	PQExpBuffer upgrade_query = createPQExpBuffer();
+	PGresult   *upgrade_res;
+	Oid			relfilenode;
+	Oid			toast_oid;
+	Oid			toast_relfilenode;
+	char		relkind;
+	Oid			toast_index_oid;
+	Oid			toast_index_relfilenode;
+
+	/*
+	 * Preserve the OID and relfilenode of the table, table's index, table's
+	 * toast table and toast table's index if any.
+	 *
+	 * One complexity is that the current table definition might not require
+	 * the creation of a TOAST table, but the old database might have a TOAST
+	 * table that was created earlier, before some wide columns were dropped.
+	 * By setting the TOAST oid we force creation of the TOAST heap and index
+	 * by the new backend, so we can copy the files during binary upgrade
+	 * without worrying about this case.
+	 */
+	appendPQExpBuffer(upgrade_query,
+					  "SELECT c.relkind, c.relfilenode, c.reltoastrelid, ct.relfilenode AS toast_relfilenode, i.indexrelid, cti.relfilenode AS toast_index_relfilenode "
+					  "FROM pg_catalog.pg_class c LEFT JOIN "
+					  "pg_catalog.pg_index i ON (c.reltoastrelid = i.indrelid AND i.indisvalid) "
+					  "LEFT JOIN pg_catalog.pg_class ct ON (c.reltoastrelid = ct.oid) "
+					  "LEFT JOIN pg_catalog.pg_class AS cti ON (i.indexrelid = cti.oid) "
+					  "WHERE c.oid = '%u'::pg_catalog.oid;",
+					  pg_class_oid);
+
+	upgrade_res = ExecuteSqlQueryForSingleRow(fout, upgrade_query->data);
+
+	relkind = *PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relkind"));
+
+	relfilenode = atooid(PQgetvalue(upgrade_res, 0,
+									PQfnumber(upgrade_res, "relfilenode")));
+	toast_oid = atooid(PQgetvalue(upgrade_res, 0,
+								  PQfnumber(upgrade_res, "reltoastrelid")));
+	toast_relfilenode = atooid(PQgetvalue(upgrade_res, 0,
+										  PQfnumber(upgrade_res, "toast_relfilenode")));
+	toast_index_oid = atooid(PQgetvalue(upgrade_res, 0,
+										PQfnumber(upgrade_res, "indexrelid")));
+	toast_index_relfilenode = atooid(PQgetvalue(upgrade_res, 0,
+												PQfnumber(upgrade_res, "toast_index_relfilenode")));
+
 	appendPQExpBufferStr(upgrade_buffer,
-						 "\n-- For binary upgrade, must preserve pg_class oids\n");
+						 "\n-- For binary upgrade, must preserve pg_class oids and relfilenodes\n");
 
 	if (!is_index)
 	{
-		PQExpBuffer upgrade_query = createPQExpBuffer();
-		PGresult   *upgrade_res;
-		Oid			pg_class_reltoastrelid;
-		char		pg_class_relkind;
-		Oid			pg_index_indexrelid;
-
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_heap_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
 
-		/*
-		 * Preserve the OIDs of the table's toast table and index, if any.
-		 * Indexes cannot have toast tables, so we need not make this probe in
-		 * the index code path.
-		 *
-		 * One complexity is that the current table definition might not
-		 * require the creation of a TOAST table, but the old database might
-		 * have a TOAST table that was created earlier, before some wide
-		 * columns were dropped.  By setting the TOAST oid we force creation
-		 * of the TOAST heap and index by the new backend, so we can copy the
-		 * files during binary upgrade without worrying about this case.
-		 */
-		appendPQExpBuffer(upgrade_query,
-						  "SELECT c.reltoastrelid, c.relkind, i.indexrelid "
-						  "FROM pg_catalog.pg_class c LEFT JOIN "
-						  "pg_catalog.pg_index i ON (c.reltoastrelid = i.indrelid AND i.indisvalid) "
-						  "WHERE c.oid = '%u'::pg_catalog.oid;",
-						  pg_class_oid);
-
-		upgrade_res = ExecuteSqlQueryForSingleRow(fout, upgrade_query->data);
-
-		pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0,
-												   PQfnumber(upgrade_res, "reltoastrelid")));
-		pg_class_relkind = *PQgetvalue(upgrade_res, 0,
-									   PQfnumber(upgrade_res, "relkind"));
-		pg_index_indexrelid = atooid(PQgetvalue(upgrade_res, 0,
-												PQfnumber(upgrade_res, "indexrelid")));
+		/* Not every relation has storage. */
+		if (OidIsValid(relfilenode))
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_heap_relfilenode('%u'::pg_catalog.oid);\n",
+							  relfilenode);
 
 		/*
 		 * In a pre-v12 database, partitioned tables might be marked as having
 		 * toast tables, but we should ignore them if so.
 		 */
-		if (OidIsValid(pg_class_reltoastrelid) &&
-			pg_class_relkind != RELKIND_PARTITIONED_TABLE)
+		if (OidIsValid(toast_oid) &&
+			relkind != RELKIND_PARTITIONED_TABLE)
 		{
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_toast_pg_class_oid('%u'::pg_catalog.oid);\n",
-							  pg_class_reltoastrelid);
+							  toast_oid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_toast_relfilenode('%u'::pg_catalog.oid);\n",
+							  toast_relfilenode);
 
 			/* every toast table has an index */
 			appendPQExpBuffer(upgrade_buffer,
 							  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
-							  pg_index_indexrelid);
+							  toast_index_oid);
+			appendPQExpBuffer(upgrade_buffer,
+							  "SELECT pg_catalog.binary_upgrade_set_next_index_relfilenode('%u'::pg_catalog.oid);\n",
+							  toast_index_relfilenode);
 		}
 
 		PQclear(upgrade_res);
 		destroyPQExpBuffer(upgrade_query);
 	}
 	else
+	{
+		/* Preserve the OID and relfilenode of the index */
 		appendPQExpBuffer(upgrade_buffer,
 						  "SELECT pg_catalog.binary_upgrade_set_next_index_pg_class_oid('%u'::pg_catalog.oid);\n",
 						  pg_class_oid);
+		appendPQExpBuffer(upgrade_buffer,
+						  "SELECT pg_catalog.binary_upgrade_set_next_index_relfilenode('%u'::pg_catalog.oid);\n",
+						  relfilenode);
+	}
 
 	appendPQExpBufferChar(upgrade_buffer, '\n');
 }
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index c291017..618187c 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -1265,6 +1265,9 @@ dumpTablespaces(PGconn *conn)
 		/* needed for buildACLCommands() */
 		fspcname = pg_strdup(fmtId(spcname));
 
+		appendPQExpBufferStr(buf, "\n-- For binary upgrade, must preserve pg_tablespace oid\n");
+		appendPQExpBuffer(buf, "SELECT pg_catalog.binary_upgrade_set_next_pg_tablespace_oid('%u'::pg_catalog.oid);\n", spcoid);
+
 		appendPQExpBuffer(buf, "CREATE TABLESPACE %s", fspcname);
 		appendPQExpBuffer(buf, " OWNER %s", fmtId(spcowner));
 
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index 3628bd7..acaf343 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -15,12 +15,13 @@
  *	oids are the same between old and new clusters.  This is important
  *	because toast oids are stored as toast pointers in user tables.
  *
- *	While pg_class.oid and pg_class.relfilenode are initially the same
- *	in a cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM
- *	FULL.  In the new cluster, pg_class.oid and pg_class.relfilenode will
- *	be the same and will match the old pg_class.oid value.  Because of
- *	this, old/new pg_class.relfilenode values will not match if CLUSTER,
- *	REINDEX, or VACUUM FULL have been performed in the old cluster.
+ *	While pg_class.oid and pg_class.relfilenode are initially the same in a
+ *	cluster, they can diverge due to CLUSTER, REINDEX, or VACUUM FULL. We
+ *	control assignments of pg_class.relfilenode because we want the filenames
+ *	to match between the old and new cluster.
+ *
+ *	We control assignment of pg_tablespace.oid because we want the oid to match
+ *	between the old and new cluster.
  *
  *	We control all assignments of pg_type.oid because these oids are stored
  *	in user composite type values.
diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h
index f6e82e7..4ba5748 100644
--- a/src/include/catalog/binary_upgrade.h
+++ b/src/include/catalog/binary_upgrade.h
@@ -14,14 +14,19 @@
 #ifndef BINARY_UPGRADE_H
 #define BINARY_UPGRADE_H
 
+extern PGDLLIMPORT Oid binary_upgrade_next_pg_tablespace_oid;
+
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_array_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_mrng_pg_type_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_mrng_array_pg_type_oid;
 
 extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_heap_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_index_pg_class_relfilenode;
 extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_oid;
+extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_relfilenode;
 
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_enum_oid;
 extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid;
diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h
index 6ce480b..2008758 100644
--- a/src/include/catalog/heap.h
+++ b/src/include/catalog/heap.h
@@ -59,7 +59,8 @@ extern Relation heap_create(const char *relname,
 							bool mapped_relation,
 							bool allow_system_table_mods,
 							TransactionId *relfrozenxid,
-							MultiXactId *relminmxid);
+							MultiXactId *relminmxid,
+							bool suppress_storage);
 
 extern Oid	heap_create_with_catalog(const char *relname,
 									 Oid relnamespace,
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index d068d65..cd0a20d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -11036,6 +11036,22 @@
   proname => 'binary_upgrade_set_missing_value', provolatile => 'v',
   proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text',
   prosrc => 'binary_upgrade_set_missing_value' },
+{ oid => '4544', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_heap_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_heap_relfilenode' },
+{ oid => '4545', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_index_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_index_relfilenode' },
+{ oid => '4546', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_toast_relfilenode', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_toast_relfilenode' },
+{ oid => '4547', descr => 'for use by pg_upgrade',
+  proname => 'binary_upgrade_set_next_pg_tablespace_oid', provolatile => 'v',
+  proparallel => 'u', prorettype => 'void', proargtypes => 'oid',
+  prosrc => 'binary_upgrade_set_next_pg_tablespace_oid' },
 
 # conversion functions
 { oid => '4302',
diff --git a/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out b/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
index ac0ddce..1ee65ed 100644
--- a/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
+++ b/src/test/modules/spgist_name_ops/expected/spgist_name_ops.out
@@ -52,14 +52,18 @@ select * from t
 ------------------------------------------------------+----+------------------------------------------------------
  binary_upgrade_set_next_array_pg_type_oid            |    | binary_upgrade_set_next_array_pg_type_oid
  binary_upgrade_set_next_heap_pg_class_oid            |    | binary_upgrade_set_next_heap_pg_class_oid
+ binary_upgrade_set_next_heap_relfilenode             |  1 | binary_upgrade_set_next_heap_relfilenode
  binary_upgrade_set_next_index_pg_class_oid           |  1 | binary_upgrade_set_next_index_pg_class_oid
+ binary_upgrade_set_next_index_relfilenode            |    | binary_upgrade_set_next_index_relfilenode
  binary_upgrade_set_next_multirange_array_pg_type_oid |  1 | binary_upgrade_set_next_multirange_array_pg_type_oid
  binary_upgrade_set_next_multirange_pg_type_oid       |  1 | binary_upgrade_set_next_multirange_pg_type_oid
  binary_upgrade_set_next_pg_authid_oid                |    | binary_upgrade_set_next_pg_authid_oid
  binary_upgrade_set_next_pg_enum_oid                  |    | binary_upgrade_set_next_pg_enum_oid
+ binary_upgrade_set_next_pg_tablespace_oid            |    | binary_upgrade_set_next_pg_tablespace_oid
  binary_upgrade_set_next_pg_type_oid                  |    | binary_upgrade_set_next_pg_type_oid
  binary_upgrade_set_next_toast_pg_class_oid           |  1 | binary_upgrade_set_next_toast_pg_class_oid
-(9 rows)
+ binary_upgrade_set_next_toast_relfilenode            |    | binary_upgrade_set_next_toast_relfilenode
+(13 rows)
 
 -- Verify clean failure when INCLUDE'd columns result in overlength tuple
 -- The error message details are platform-dependent, so show only SQLSTATE
@@ -97,14 +101,18 @@ select * from t
 ------------------------------------------------------+----+------------------------------------------------------
  binary_upgrade_set_next_array_pg_type_oid            |    | binary_upgrade_set_next_array_pg_type_oid
  binary_upgrade_set_next_heap_pg_class_oid            |    | binary_upgrade_set_next_heap_pg_class_oid
+ binary_upgrade_set_next_heap_relfilenode             |  1 | binary_upgrade_set_next_heap_relfilenode
  binary_upgrade_set_next_index_pg_class_oid           |  1 | binary_upgrade_set_next_index_pg_class_oid
+ binary_upgrade_set_next_index_relfilenode            |    | binary_upgrade_set_next_index_relfilenode
  binary_upgrade_set_next_multirange_array_pg_type_oid |  1 | binary_upgrade_set_next_multirange_array_pg_type_oid
  binary_upgrade_set_next_multirange_pg_type_oid       |  1 | binary_upgrade_set_next_multirange_pg_type_oid
  binary_upgrade_set_next_pg_authid_oid                |    | binary_upgrade_set_next_pg_authid_oid
  binary_upgrade_set_next_pg_enum_oid                  |    | binary_upgrade_set_next_pg_enum_oid
+ binary_upgrade_set_next_pg_tablespace_oid            |    | binary_upgrade_set_next_pg_tablespace_oid
  binary_upgrade_set_next_pg_type_oid                  |    | binary_upgrade_set_next_pg_type_oid
  binary_upgrade_set_next_toast_pg_class_oid           |  1 | binary_upgrade_set_next_toast_pg_class_oid
-(9 rows)
+ binary_upgrade_set_next_toast_relfilenode            |    | binary_upgrade_set_next_toast_relfilenode
+(13 rows)
 
 \set VERBOSITY sqlstate
 insert into t values(repeat('xyzzy', 12), 42, repeat('xyzzy', 4000));
-- 
1.8.3.1