v13-0003-Make-pg_upgrade-convert-multixact-offsets.patch

From e15f89143dd8aef70957e87d59c177fab66f9ce2 Mon Sep 17 00:00:00 2001
From: Maxim Orlov <m.orlov@postgrespro.ru>
Date: Tue, 13 Aug 2024 14:44:50 +0300
Subject: [PATCH v13 3/7] Make pg_upgrade convert multixact offsets.

Author: Heikki Linnakangas <heikki.linnakangas@iki.fi>
Author: Maxim Orlov <orlovmg@gmail.com>
Author: Yura Sokolov <y.sokolov@postgrespro.ru>
---
 src/backend/access/transam/multixact.c |  35 +--
 src/bin/pg_upgrade/Makefile            |   3 +
 src/bin/pg_upgrade/meson.build         |   3 +
 src/bin/pg_upgrade/multixact_old.c     | 338 +++++++++++++++++++++++++
 src/bin/pg_upgrade/multixact_old.h     |  12 +
 src/bin/pg_upgrade/multixact_rewrite.c | 238 +++++++++++++++++
 src/bin/pg_upgrade/pg_upgrade.c        |  29 ++-
 src/bin/pg_upgrade/pg_upgrade.h        |  13 +-
 src/bin/pg_upgrade/slru_io.c           | 211 +++++++++++++++
 src/bin/pg_upgrade/slru_io.h           |  23 ++
 10 files changed, 873 insertions(+), 32 deletions(-)
 create mode 100644 src/bin/pg_upgrade/multixact_old.c
 create mode 100644 src/bin/pg_upgrade/multixact_old.h
 create mode 100644 src/bin/pg_upgrade/multixact_rewrite.c
 create mode 100644 src/bin/pg_upgrade/slru_io.c
 create mode 100644 src/bin/pg_upgrade/slru_io.h

diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c
index cd9db52e95..d63ae17330 100644
--- a/src/backend/access/transam/multixact.c
+++ b/src/backend/access/transam/multixact.c
@@ -1103,7 +1103,6 @@ GetMultiXactIdMembers(MultiXactId multi, MultiXactMember **members,
 	MultiXactOffset *offptr;
 	MultiXactOffset offset;
 	int			length;
-	int			truelength;
 	MultiXactId oldestMXact;
 	MultiXactId nextMXact;
 	MultiXactId tmpMXact;
@@ -1202,15 +1201,6 @@ GetMultiXactIdMembers(MultiXactId multi, MultiXactMember **members,
 	 * we have just for this; the process in charge will signal the CV as soon
 	 * as it has finished writing the multixact offset.
 	 *
-	 * 3. Because GetNewMultiXactId increments offset zero to offset one to
-	 * handle case #2, there is an ambiguity near the point of offset
-	 * wraparound.  If we see next multixact's offset is one, is that our
-	 * multixact's actual endpoint, or did it end at zero with a subsequent
-	 * increment?  We handle this using the knowledge that if the zero'th
-	 * member slot wasn't filled, it'll contain zero, and zero isn't a valid
-	 * transaction ID so it can't be a multixact member.  Therefore, if we
-	 * read a zero from the members array, just ignore it.
-	 *
 	 * This is all pretty messy, but the mess occurs only in infrequent corner
 	 * cases, so it seems better than holding the MultiXactGenLock for a long
 	 * time on every multixact creation.
@@ -1297,6 +1287,9 @@ retry:
 	LWLockRelease(lock);
 	lock = NULL;
 
+	/* A multixid with zero members should not happen */
+	Assert(length > 0);
+
 	/*
 	 * If we slept above, clean up state; it's no longer needed.
 	 */
@@ -1305,7 +1298,6 @@ retry:
 
 	ptr = (MultiXactMember *) palloc(length * sizeof(MultiXactMember));
 
-	truelength = 0;
 	prev_pageno = -1;
 	for (int i = 0; i < length; i++, offset++)
 	{
@@ -1343,36 +1335,27 @@ retry:
 		xactptr = (TransactionId *)
 			(MultiXactMemberCtl->shared->page_buffer[slotno] + memberoff);
 
-		if (!TransactionIdIsValid(*xactptr))
-		{
-			/* Corner case 3: we must be looking at unused slot zero */
-			Assert(offset == 0);
-			continue;
-		}
+		Assert(TransactionIdIsValid(*xactptr));
 
 		flagsoff = MXOffsetToFlagsOffset(offset);
 		bshift = MXOffsetToFlagsBitShift(offset);
 		flagsptr = (uint32 *) (MultiXactMemberCtl->shared->page_buffer[slotno] + flagsoff);
 
-		ptr[truelength].xid = *xactptr;
-		ptr[truelength].status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK;
-		truelength++;
+		ptr[i].xid = *xactptr;
+		ptr[i].status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK;
 	}
 
 	LWLockRelease(lock);
 
-	/* A multixid with zero members should not happen */
-	Assert(truelength > 0);
-
 	/*
 	 * Copy the result into the local cache.
 	 */
-	mXactCachePut(multi, truelength, ptr);
+	mXactCachePut(multi, length, ptr);
 
 	debug_elog3(DEBUG2, "GetMembers: no cache for %s",
-				mxid_to_string(multi, truelength, ptr));
+				mxid_to_string(multi, length, ptr));
 	*members = ptr;
-	return truelength;
+	return length;
 }
 
 /*
diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile
index f83d2b5d30..b4ad01c00b 100644
--- a/src/bin/pg_upgrade/Makefile
+++ b/src/bin/pg_upgrade/Makefile
@@ -19,11 +19,14 @@ OBJS = \
 	file.o \
 	function.o \
 	info.o \
+	multixact_old.o \
+	multixact_rewrite.o \
 	option.o \
 	parallel.o \
 	pg_upgrade.o \
 	relfilenumber.o \
 	server.o \
+	slru_io.o \
 	tablespace.o \
 	task.o \
 	util.o \
diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build
index cc2ba97d9a..76c8f2005d 100644
--- a/src/bin/pg_upgrade/meson.build
+++ b/src/bin/pg_upgrade/meson.build
@@ -8,11 +8,14 @@ pg_upgrade_sources = files(
   'file.c',
   'function.c',
   'info.c',
+  'multixact_old.c',
+  'multixact_rewrite.c',
   'option.c',
   'parallel.c',
   'pg_upgrade.c',
   'relfilenumber.c',
   'server.c',
+  'slru_io.c',
   'tablespace.c',
   'task.c',
   'util.c',
diff --git a/src/bin/pg_upgrade/multixact_old.c b/src/bin/pg_upgrade/multixact_old.c
new file mode 100644
index 0000000000..0442928e89
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_old.c
@@ -0,0 +1,338 @@
+/*
+ *	multixact_old.c
+ *
+ *	Support for reading pre-v18 format pg_multixact files
+ *
+ *	Copyright (c) 2024, PostgreSQL Global Development Group
+ *	src/bin/pg_upgrade/multixact_old.c
+ */
+
+#include "postgres_fe.h"
+
+#include "access/transam.h"
+#include "pg_upgrade.h"
+#include "multixact_old.h"
+#include "slru_io.h"
+
+/*
+ * Below are a bunch of definitions that are copy-pasted from multixact.c from
+ * version 17. They shadow the new definitions in access/multixact.h, so it's
+ * important that we *don't* include that here. That's is a big reason this
+ * code has to be in a separate source file.
+ *
+ * All references to MultiXactOffset have been replaced with OldMultiXactOffset;
+ */
+typedef uint32 OldMultiXactOffset;
+
+#define FirstMultiXactId	((MultiXactId) 1)
+
+/*
+ * Possible multixact lock modes ("status").  The first four modes are for
+ * tuple locks (FOR KEY SHARE, FOR SHARE, FOR NO KEY UPDATE, FOR UPDATE); the
+ * next two are used for update and delete modes.
+ */
+typedef enum
+{
+	MultiXactStatusForKeyShare = 0x00,
+	MultiXactStatusForShare = 0x01,
+	MultiXactStatusForNoKeyUpdate = 0x02,
+	MultiXactStatusForUpdate = 0x03,
+	/* an update that doesn't touch "key" columns */
+	MultiXactStatusNoKeyUpdate = 0x04,
+	/* other updates, and delete */
+	MultiXactStatusUpdate = 0x05,
+} MultiXactStatus;
+
+/* does a status value correspond to a tuple update? */
+#define ISUPDATE_from_mxstatus(status) \
+			((status) > MultiXactStatusForUpdate)
+
+/*
+ * Defines for OldMultiXactOffset page sizes.  A page is the same BLCKSZ as is
+ * used everywhere else in Postgres.
+ *
+ * Note: because OldMultiXactOffsets are 32 bits and wrap around at 0xFFFFFFFF,
+ * MultiXact page numbering also wraps around at
+ * 0xFFFFFFFF/MULTIXACT_OFFSETS_PER_PAGE, and segment numbering at
+ * 0xFFFFFFFF/MULTIXACT_OFFSETS_PER_PAGE/SLRU_PAGES_PER_SEGMENT.  We need
+ * take no explicit notice of that fact in this module, except when comparing
+ * segment and page numbers in TruncateMultiXact (see
+ * OldMultiXactOffsetPagePrecedes).
+ */
+
+/* We need four bytes per offset */
+#define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(uint32))
+
+static inline int64
+MultiXactIdToOffsetPage(MultiXactId multi)
+{
+	return multi / MULTIXACT_OFFSETS_PER_PAGE;
+}
+
+static inline int
+MultiXactIdToOffsetEntry(MultiXactId multi)
+{
+	return multi % MULTIXACT_OFFSETS_PER_PAGE;
+}
+
+static inline int64
+MultiXactIdToOffsetSegment(MultiXactId multi)
+{
+	return MultiXactIdToOffsetPage(multi) / SLRU_PAGES_PER_SEGMENT;
+}
+
+/*
+ * The situation for members is a bit more complex: we store one byte of
+ * additional flag bits for each TransactionId.  To do this without getting
+ * into alignment issues, we store four bytes of flags, and then the
+ * corresponding 4 Xids.  Each such 5-word (20-byte) set we call a "group", and
+ * are stored as a whole in pages.  Thus, with 8kB BLCKSZ, we keep 409 groups
+ * per page.  This wastes 12 bytes per page, but that's OK -- simplicity (and
+ * performance) trumps space efficiency here.
+ *
+ * Note that the "offset" macros work with byte offset, not array indexes, so
+ * arithmetic must be done using "char *" pointers.
+ */
+/* We need eight bits per xact, so one xact fits in a byte */
+#define MXACT_MEMBER_BITS_PER_XACT			8
+#define MXACT_MEMBER_FLAGS_PER_BYTE			1
+#define MXACT_MEMBER_XACT_BITMASK	((1 << MXACT_MEMBER_BITS_PER_XACT) - 1)
+
+/* how many full bytes of flags are there in a group? */
+#define MULTIXACT_FLAGBYTES_PER_GROUP		4
+#define MULTIXACT_MEMBERS_PER_MEMBERGROUP	\
+	(MULTIXACT_FLAGBYTES_PER_GROUP * MXACT_MEMBER_FLAGS_PER_BYTE)
+/* size in bytes of a complete group */
+#define MULTIXACT_MEMBERGROUP_SIZE \
+	(sizeof(TransactionId) * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP)
+#define MULTIXACT_MEMBERGROUPS_PER_PAGE (BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE)
+#define MULTIXACT_MEMBERS_PER_PAGE	\
+	(MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP)
+
+/* page in which a member is to be found */
+static inline int64
+MXOffsetToMemberPage(OldMultiXactOffset offset)
+{
+	return offset / MULTIXACT_MEMBERS_PER_PAGE;
+}
+
+/* Location (byte offset within page) of flag word for a given member */
+static inline int
+MXOffsetToFlagsOffset(OldMultiXactOffset offset)
+{
+	OldMultiXactOffset group = offset / MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+	int			grouponpg = group % MULTIXACT_MEMBERGROUPS_PER_PAGE;
+	int			byteoff = grouponpg * MULTIXACT_MEMBERGROUP_SIZE;
+
+	return byteoff;
+}
+
+static inline int
+MXOffsetToFlagsBitShift(OldMultiXactOffset offset)
+{
+	int			member_in_group = offset % MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+	int			bshift = member_in_group * MXACT_MEMBER_BITS_PER_XACT;
+
+	return bshift;
+}
+
+/* Location (byte offset within page) of TransactionId of given member */
+static inline int
+MXOffsetToMemberOffset(OldMultiXactOffset offset)
+{
+	int			member_in_group = offset % MULTIXACT_MEMBERS_PER_MEMBERGROUP;
+
+	return MXOffsetToFlagsOffset(offset) +
+		MULTIXACT_FLAGBYTES_PER_GROUP +
+		member_in_group * sizeof(TransactionId);
+}
+
+typedef struct OldMultiXactReader
+{
+	MultiXactId nextMXact;
+	uint32	nextOffset;
+
+	SlruSegState *offset;
+	SlruSegState *members;
+} OldMultiXactReader;
+
+OldMultiXactReader *
+StartOldMultiXactRead(void)
+{
+	OldMultiXactReader *state;
+	char	   *dir;
+
+	state = pg_malloc(sizeof(OldMultiXactReader));
+	state->nextMXact = old_cluster.controldata.chkpnt_nxtmulti;
+	state->nextOffset = old_cluster.controldata.chkpnt_nxtmxoff;
+
+	dir = psprintf("%s/pg_multixact/offsets", old_cluster.pgdata);
+	state->offset = OpenSlruRead(dir);
+	pg_free(dir);
+
+	dir = psprintf("%s/pg_multixact/members", old_cluster.pgdata);
+	state->members = OpenSlruRead(dir);
+	pg_free(dir);
+
+	return state;
+}
+
+/*
+ * This is a simplified version of the GetMultiXactIdMembers() server function.
+ *
+ * - Only return the updating member, if any. Upgrade only cares about the updaters.
+ *   If there is no updating member, return the first locking-only member. We don't
+ *   have any way to represent "no members", but we also don't need to preserve all
+ *   the locking members.
+ *
+ * - We don't need to worry about locking and some corner cases because there's
+ *   no concurrent activity.
+ */
+void
+GetOldMultiXactIdSingleMember(OldMultiXactReader *state, MultiXactId multi,
+							  TransactionId *result, bool *isupdate)
+{
+	TransactionId result_xid;
+	bool		result_isupdate;
+	int64		pageno;
+	int64		prev_pageno;
+	int			entryno;
+	OldMultiXactOffset *offptr;
+	OldMultiXactOffset offset;
+	int			length;
+	MultiXactId nextMXact;
+	MultiXactId tmpMXact;
+	OldMultiXactOffset nextOffset;
+	char	   *buf;
+
+	nextMXact = state->nextMXact;
+	nextOffset = state->nextOffset;
+
+	/*
+	 * Find out the offset at which we need to start reading MultiXactMembers
+	 * and the number of members in the multixact.  We determine the latter as
+	 * the difference between this multixact's starting offset and the next
+	 * one's.  However, there are some corner cases to worry about:
+	 *
+	 * 1. This multixact may be the latest one created, in which case there is
+	 * no next one to look at.  In this case the nextOffset value we just
+	 * saved is the correct endpoint.
+	 *
+	 * 2. (this cannot happen during upgrade)
+	 *
+	 * 3. Because GetNewMultiXactId increments offset zero to offset one to
+	 * handle case #2, there is an ambiguity near the point of offset
+	 * wraparound.  If we see next multixact's offset is one, is that our
+	 * multixact's actual endpoint, or did it end at zero with a subsequent
+	 * increment?  We handle this using the knowledge that if the zero'th
+	 * member slot wasn't filled, it'll contain zero, and zero isn't a valid
+	 * transaction ID so it can't be a multixact member.  Therefore, if we
+	 * read a zero from the members array, just ignore it.
+	 */
+	pageno = MultiXactIdToOffsetPage(multi);
+	entryno = MultiXactIdToOffsetEntry(multi);
+
+	buf = SlruReadSwitchPage(state->offset, pageno);
+	offptr = (OldMultiXactOffset *) buf;
+	offptr += entryno;
+	offset = *offptr;
+
+	Assert(offset != 0);
+
+	/*
+	 * Use the same increment rule as GetNewMultiXactId(), that is, don't
+	 * handle wraparound explicitly until needed.
+	 */
+	tmpMXact = multi + 1;
+
+	if (nextMXact == tmpMXact)
+	{
+		/* Corner case 1: there is no next multixact */
+		length = nextOffset - offset;
+	}
+	else
+	{
+		OldMultiXactOffset nextMXOffset;
+
+		/* handle wraparound if needed */
+		if (tmpMXact < FirstMultiXactId)
+			tmpMXact = FirstMultiXactId;
+
+		prev_pageno = pageno;
+
+		pageno = MultiXactIdToOffsetPage(tmpMXact);
+		entryno = MultiXactIdToOffsetEntry(tmpMXact);
+
+		if (pageno != prev_pageno)
+		{
+			buf = SlruReadSwitchPage(state->offset, pageno);
+		}
+
+		offptr = (OldMultiXactOffset *) buf;
+		offptr += entryno;
+		nextMXOffset = *offptr;
+
+		if (nextMXOffset == 0)
+		{
+			/* Corner case 2: next multixact is still being filled in */
+			Assert(false); /* shouldn't happen during upgrade */
+		}
+
+		length = nextMXOffset - offset;
+	}
+
+	result_xid = InvalidTransactionId;
+	result_isupdate = false;
+	prev_pageno = -1;
+	for (int i = 0; i < length; i++, offset++)
+	{
+		TransactionId *xactptr;
+		uint32	   *flagsptr;
+		int			flagsoff;
+		int			bshift;
+		int			memberoff;
+		MultiXactStatus status;
+
+		pageno = MXOffsetToMemberPage(offset);
+		memberoff = MXOffsetToMemberOffset(offset);
+
+		if (pageno != prev_pageno)
+		{
+			buf = SlruReadSwitchPage(state->members, pageno);
+			prev_pageno = pageno;
+		}
+
+		xactptr = (TransactionId *) (buf + memberoff);
+
+		if (!TransactionIdIsValid(*xactptr))
+		{
+			/* Corner case 3: we must be looking at unused slot zero */
+			Assert(offset == 0);
+			continue;
+		}
+
+		flagsoff = MXOffsetToFlagsOffset(offset);
+		bshift = MXOffsetToFlagsBitShift(offset);
+		flagsptr = (uint32 *) (buf + flagsoff);
+
+		status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK;
+
+		/* Verify that there is a single update Xid among the given members. */
+		if (ISUPDATE_from_mxstatus(status))
+		{
+			if (result_isupdate)
+				pg_fatal("multixact %u has more than one updating member",
+						 multi);
+			result_xid = *xactptr;
+			result_isupdate = true;
+		}
+		else if (!TransactionIdIsValid(result_xid))
+			result_xid = *xactptr;
+	}
+
+	/* A multixid with zero members should not happen */
+	Assert(TransactionIdIsValid(result_xid));
+
+	*result = result_xid;
+	*isupdate = result_isupdate;
+}
diff --git a/src/bin/pg_upgrade/multixact_old.h b/src/bin/pg_upgrade/multixact_old.h
new file mode 100644
index 0000000000..70800c1cda
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_old.h
@@ -0,0 +1,12 @@
+/*
+ *	multixact_old.h
+ *
+ *	Copyright (c) 2010-2024, PostgreSQL Global Development Group
+ *	src/bin/pg_upgrade/multixact_old.h
+ */
+
+typedef struct OldMultiXactReader OldMultiXactReader;
+
+extern OldMultiXactReader *StartOldMultiXactRead(void);
+extern void GetOldMultiXactIdSingleMember(OldMultiXactReader *state, MultiXactId multi,
+										  TransactionId *result, bool *isupdate);
diff --git a/src/bin/pg_upgrade/multixact_rewrite.c b/src/bin/pg_upgrade/multixact_rewrite.c
new file mode 100644
index 0000000000..8c3f538cc9
--- /dev/null
+++ b/src/bin/pg_upgrade/multixact_rewrite.c
@@ -0,0 +1,238 @@
+/*
+ *	multixact_rewrite.c
+ *
+ *	Rewrite pre-v18 multixacts to new format with 64-bit MultiXactOffsets
+ *
+ *	Copyright (c) 2024, PostgreSQL Global Development Group
+ *	src/bin/pg_upgrade/multixact_rewrite.c
+ */
+
+#include "postgres_fe.h"
+
+#include "multixact_old.h"
+#include "pg_upgrade.h"
+#include "slru_io.h"
+
+#include "access/multixact.h"
+#include "access/multixact_internal.h"
+
+typedef struct
+{
+	MultiXactId nextMXact;
+	MultiXactOffset nextOffset;
+
+	SlruSegState *offset;
+	SlruSegState *members;
+} MultiXactWriter;
+
+static MultiXactWriter *StartMultiXactWrite(MultiXactId firstMulti, MultiXactOffset firstOffset);
+static MultiXactId GetNewMultiXactId(MultiXactWriter *state, int nmembers, MultiXactOffset *offset);
+static void RecordNewMultiXact(MultiXactWriter *state,
+							   MultiXactOffset offset,
+							   MultiXactId multi,
+							   int nmembers, MultiXactMember *members);
+static void CloseMultiXactWrite(MultiXactWriter *state);
+
+
+/*
+ * Convert pg_multixact/offset and /members to new format with 64-bit offsets.
+ */
+void
+convert_multixacts(MultiXactId *new_nxtmulti, MultiXactOffset *new_nxtmxoff)
+{
+	MultiXactWriter	   *new_writer;
+	MultiXactId			oldest_multi = old_cluster.controldata.chkpnt_oldstMulti,
+						next_multi = old_cluster.controldata.chkpnt_nxtmulti,
+						multi;
+	OldMultiXactReader *old_reader;
+
+	if (next_multi < FirstMultiXactId)
+		next_multi = FirstMultiXactId;
+
+	old_reader = StartOldMultiXactRead();
+	new_writer = StartMultiXactWrite(oldest_multi, 1);
+
+	/*
+	 * Read multixids from old files one by one, and write them back in the
+	 * new format.
+	 *
+	 * The locking-only XIDs that may be part of multi-xids don't matter after
+	 * upgrade, as there can be no transactions running across upgrade. So as
+	 * a little optimization, we only read one member from each multixid: the
+	 * one updating one, or if there was no update, arbitrarily the first
+	 * locking xid.
+	 */
+	for (multi = oldest_multi; multi != next_multi;)
+	{
+		TransactionId xid;
+		bool		isupdate;
+		MultiXactMember member;
+		MultiXactId newmulti PG_USED_FOR_ASSERTS_ONLY;
+		MultiXactOffset offset;
+
+		/* Read the old multixid */
+		GetOldMultiXactIdSingleMember(old_reader, multi, &xid, &isupdate);
+
+		/* Write it out in new format */
+		member.xid = xid;
+		member.status = isupdate ? MultiXactStatusUpdate : MultiXactStatusForKeyShare;
+		newmulti = GetNewMultiXactId(new_writer, 1, &offset);
+		Assert(newmulti == multi);
+		RecordNewMultiXact(new_writer, offset, multi, 1, &member);
+
+		multi++;
+		if (multi < FirstMultiXactId)
+			multi = FirstMultiXactId;
+	}
+
+	/*
+	 * Update the nextMXact/Offset values in the control file to match what we
+	 * wrote. The nextMXact should be unchanged, but because we ignored the
+	 * locking XIDs members, the nextOffset will be different.
+	 */
+	Assert(new_writer->nextMXact == next_multi);
+	*new_nxtmulti = next_multi;
+	*new_nxtmxoff = new_writer->nextOffset;
+
+	/* Release resources */
+	CloseMultiXactWrite(new_writer);
+}
+
+/* Support routines for writing the new format */
+
+static MultiXactWriter *
+StartMultiXactWrite(MultiXactId firstMulti, MultiXactOffset firstOffset)
+{
+	MultiXactWriter *state;
+	char	   *dir;
+
+	state = pg_malloc(sizeof(MultiXactWriter));
+	state->nextMXact = firstMulti;
+	state->nextOffset = firstOffset;
+
+	dir = psprintf("%s/pg_multixact/offsets", new_cluster.pgdata);
+	state->offset = OpenSlruWrite(dir, MultiXactIdToOffsetPage(firstMulti));
+	pg_free(dir);
+
+	dir = psprintf("%s/pg_multixact/members", new_cluster.pgdata);
+	state->members = OpenSlruWrite(dir, MXOffsetToMemberPage(1));
+	pg_free(dir);
+
+	return state;
+}
+
+static void
+CloseMultiXactWrite(MultiXactWriter *state)
+{
+	CloseSlruWrite(state->offset);
+	CloseSlruWrite(state->members);
+	pg_free(state);
+}
+
+/*
+ * Simplified copy of the corresponding server function
+ */
+static MultiXactId
+GetNewMultiXactId(MultiXactWriter *state, int nmembers, MultiXactOffset *offset)
+{
+	MultiXactId		result;
+
+	/* Handle wraparound of the nextMXact counter */
+	if (state->nextMXact < FirstMultiXactId)
+		state->nextMXact = FirstMultiXactId;
+
+	/* Assign the MXID */
+	result = state->nextMXact;
+
+	/*
+	 * Reserve the members space, similarly to above.
+	 */
+	*offset = state->nextOffset;
+
+	/*
+	 * Advance counters.  As in GetNewTransactionId(), this must not happen
+	 * until after file extension has succeeded!
+	 *
+	 * We don't care about MultiXactId wraparound here; it will be handled by
+	 * the next iteration.  But note that nextMXact may be InvalidMultiXactId
+	 * or the first value on a segment-beginning page after this routine
+	 * exits, so anyone else looking at the variable must be prepared to deal
+	 * with either case.  Similarly, nextOffset may be zero, but we won't use
+	 * that as the actual start offset of the next multixact.
+	 */
+	(state->nextMXact)++;
+
+	state->nextOffset += nmembers;
+
+	return result;
+}
+
+/*
+ * Write a new multixact with members.
+ *
+ * Simplified version of the correspoding server function.
+ */
+static void
+RecordNewMultiXact(MultiXactWriter *state, MultiXactOffset offset,
+				   MultiXactId multi,
+				   int nmembers, MultiXactMember *members)
+{
+	int64		pageno;
+	int64		prev_pageno;
+	int			entryno;
+
+	char	   *buf;
+	MultiXactOffset *offptr;
+
+	pageno = MultiXactIdToOffsetPage(multi);
+	entryno = MultiXactIdToOffsetEntry(multi);
+
+	/*
+	 * Note: we pass the MultiXactId to SimpleLruReadPage as the "transaction"
+	 * to complain about if there's any I/O error.  This is kinda bogus, but
+	 * since the errors will always give the full pathname, it should be clear
+	 * enough that a MultiXactId is really involved.  Perhaps someday we'll
+	 * take the trouble to generalize the slru.c error reporting code.
+	 */
+	buf = SlruWriteSwitchPage(state->offset, pageno);
+	offptr = (MultiXactOffset *) buf;
+	offptr += entryno;
+
+	*offptr = offset;
+
+	prev_pageno = -1;
+
+	for (int i = 0; i < nmembers; i++, offset++)
+	{
+		TransactionId *memberptr;
+		uint32	   *flagsptr;
+		uint32		flagsval;
+		int			bshift;
+		int			flagsoff;
+		int			memberoff;
+
+		Assert(members[i].status <= MultiXactStatusUpdate);
+
+		pageno = MXOffsetToMemberPage(offset);
+		memberoff = MXOffsetToMemberOffset(offset);
+		flagsoff = MXOffsetToFlagsOffset(offset);
+		bshift = MXOffsetToFlagsBitShift(offset);
+
+		if (pageno != prev_pageno)
+		{
+			buf = SlruWriteSwitchPage(state->members, pageno);
+			prev_pageno = pageno;
+		}
+
+		memberptr = (TransactionId *) (buf + memberoff);
+
+		*memberptr = members[i].xid;
+
+		flagsptr = (uint32 *) (buf + flagsoff);
+
+		flagsval = *flagsptr;
+		flagsval &= ~(((1 << MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift);
+		flagsval |= (members[i].status << bshift);
+		*flagsptr = flagsval;
+	}
+}
diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c
index 36c7f3879d..9bf191b984 100644
--- a/src/bin/pg_upgrade/pg_upgrade.c
+++ b/src/bin/pg_upgrade/pg_upgrade.c
@@ -750,8 +750,27 @@ copy_xact_xlog_xid(void)
 	if (old_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER &&
 		new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER)
 	{
-		copy_subdir_files("pg_multixact/offsets", "pg_multixact/offsets");
-		copy_subdir_files("pg_multixact/members", "pg_multixact/members");
+		MultiXactId new_nxtmulti = old_cluster.controldata.chkpnt_nxtmulti;
+		MultiXactOffset new_nxtmxoff = old_cluster.controldata.chkpnt_nxtmxoff;
+
+		/*
+		 * If the old server is before the MULTIXACTOFFSET_FORMATCHANGE_CAT_VER
+		 * it must have 32-bit multixid offsets, thus it should be converted.
+		 */
+		if (old_cluster.controldata.cat_ver < MULTIXACTOFFSET_FORMATCHANGE_CAT_VER &&
+			new_cluster.controldata.cat_ver >= MULTIXACTOFFSET_FORMATCHANGE_CAT_VER)
+		{
+			remove_new_subdir("pg_multixact/members", false);
+			remove_new_subdir("pg_multixact/offsets", false);
+			prep_status("Converting pg_multixact/offsets to 64-bit");
+			convert_multixacts(&new_nxtmulti, &new_nxtmxoff);
+			check_ok();
+		}
+		else
+		{
+			copy_subdir_files("pg_multixact/offsets", "pg_multixact/offsets");
+			copy_subdir_files("pg_multixact/members", "pg_multixact/members");
+		}
 
 		prep_status("Setting next multixact ID and offset for new cluster");
 
@@ -760,10 +779,10 @@ copy_xact_xlog_xid(void)
 		 * counters here and the oldest multi present on system.
 		 */
 		exec_prog(UTILITY_LOG_FILE, NULL, true, true,
-				  "\"%s/pg_resetwal\" -O %u -m %u,%u \"%s\"",
+				  "\"%s/pg_resetwal\" -O %llu -m %u,%u \"%s\"",
 				  new_cluster.bindir,
-				  old_cluster.controldata.chkpnt_nxtmxoff,
-				  old_cluster.controldata.chkpnt_nxtmulti,
+				  (unsigned long long) new_nxtmxoff,
+				  new_nxtmulti,
 				  old_cluster.controldata.chkpnt_oldstMulti,
 				  new_cluster.pgdata);
 		check_ok();
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f..9b3d645b08 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -114,6 +114,13 @@ extern char *output_files[];
  */
 #define MULTIXACT_FORMATCHANGE_CAT_VER 201301231
 
+/*
+ * Swicth from 32-bit to 64-bit for multixid offsets.
+ *
+ * XXX: should be changed to the actual CATALOG_VERSION_NO on commit.
+ */
+#define MULTIXACTOFFSET_FORMATCHANGE_CAT_VER 202409041
+
 /*
  * large object chunk size added to pg_controldata,
  * commit 5f93c37805e7485488480916b4585e098d3cc883
@@ -230,7 +237,7 @@ typedef struct
 	uint32		chkpnt_nxtepoch;
 	uint32		chkpnt_nxtoid;
 	uint32		chkpnt_nxtmulti;
-	uint32		chkpnt_nxtmxoff;
+	uint64		chkpnt_nxtmxoff;
 	uint32		chkpnt_oldstMulti;
 	uint32		chkpnt_oldstxid;
 	uint32		align;
@@ -515,3 +522,7 @@ typedef struct
 	FILE	   *file;
 	char		path[MAXPGPATH];
 } UpgradeTaskReport;
+
+/* multixact_rewrite.c */
+
+void convert_multixacts(MultiXactId *new_nxtmulti, MultiXactOffset *new_nxtmxoff);
diff --git a/src/bin/pg_upgrade/slru_io.c b/src/bin/pg_upgrade/slru_io.c
new file mode 100644
index 0000000000..87acf16732
--- /dev/null
+++ b/src/bin/pg_upgrade/slru_io.c
@@ -0,0 +1,211 @@
+/*
+ *	slru_io.c
+ *
+ *	Routines for reading and writing SLRU files during upgrade.
+ *
+ *	Copyright (c) 2024, PostgreSQL Global Development Group
+ *	src/bin/pg_upgrade/slru_io.c
+ */
+
+#include "postgres_fe.h"
+
+#include <fcntl.h>
+
+#include "pg_upgrade.h"
+#include "slru_io.h"
+
+#include "common/file_perm.h"
+#include "common/file_utils.h"
+#include "port/pg_iovec.h"
+
+/*
+ * State for reading or writing an SLRU, with a one page buffer.
+ */
+typedef struct SlruSegState
+{
+	bool		writing;
+
+	char	   *dir;
+	char	   *fn;
+	int			fd;
+	int64		segno;
+	uint64		pageno;
+
+	PGAlignedBlock buf;
+} SlruSegState;
+
+static void SlruFlush(SlruSegState *state);
+
+
+SlruSegState *
+OpenSlruRead(char *dir)
+{
+	SlruSegState *state;
+
+	state = pg_malloc(sizeof(SlruSegState));
+	state->writing = false;
+	state->segno = -1;
+	state->pageno = 0;
+	state->dir = pstrdup(dir);
+	state->fd = -1;
+	state->fn = NULL;
+
+	return state;
+}
+
+void
+CloseSlruRead(SlruSegState *state)
+{
+	Assert(!state->writing);
+	close(state->fd);
+	pg_free(state);
+}
+
+SlruSegState *
+OpenSlruWrite(char *dir, int64 startPageno)
+{
+	SlruSegState *state;
+
+	state = pg_malloc(sizeof(SlruSegState));
+	state->writing = true;
+	state->segno = -1;
+	state->pageno = 0;
+	state->dir = pstrdup(dir);
+	state->fd = -1;
+	state->fn = NULL;
+
+	return state;
+}
+
+void
+CloseSlruWrite(SlruSegState *state)
+{
+	Assert(state->writing);
+	SlruFlush(state);
+
+	close(state->fd);
+	pg_free(state);
+}
+
+static void
+SlruFlush(SlruSegState *state)
+{
+	struct iovec iovec = {
+		.iov_base = &state->buf,
+		.iov_len = BLCKSZ,
+	};
+	off_t		offset;
+
+	if (state->segno == -1)
+		return;
+
+	offset = (state->pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+
+	if (pg_pwritev_with_retry(state->fd, &iovec, 1, offset) < 0)
+		pg_fatal("could not write file \"%s\": %m", state->fn);
+}
+
+/*
+ * Open the given page for writing.
+ *
+ * NOTE: This uses O_EXCL when stepping to a new segment, so this assumes that
+ * each segment is written in full before moving on to next one. This
+ * limitation would be easy to lift if needed, but it fits the usage pattern
+ * of current callers.
+ */
+char *
+SlruWriteSwitchPage(SlruSegState *state, uint64 pageno)
+{
+	int64		segno = pageno / SLRU_PAGES_PER_SEGMENT;
+	off_t		offset;
+
+	if (state->segno != -1 && pageno == state->pageno)
+		return state->buf.data;
+
+	segno = pageno / SLRU_PAGES_PER_SEGMENT;
+	offset = (pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+
+	SlruFlush(state);
+	memset(state->buf.data, 0, BLCKSZ);
+
+	if (segno != state->segno)
+	{
+		if (state->segno != -1)
+		{
+			close(state->fd);
+			state->fd = -1;
+			pg_free(state->fn);
+			state->fn = NULL;
+		}
+
+		/* Create the segment */
+		state->fn = psprintf("%s/%04X", state->dir, (unsigned int) segno);
+		if ((state->fd = open(state->fn, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
+							  pg_file_create_mode)) < 0)
+		{
+			pg_fatal("could not create file \"%s\": %m", state->fn);
+		}
+		state->segno = segno;
+
+		if (offset > 0)
+		{
+			if (pg_pwrite_zeros(state->fd, offset, 0) < 0)
+				pg_fatal("could not write file \"%s\": %m", state->fn);
+		}
+	}
+
+	state->pageno = pageno;
+	return state->buf.data;
+}
+
+/*
+ * Open given page for reading.
+ *
+ * Reading can be done in random order.
+ */
+char *
+SlruReadSwitchPage(SlruSegState *state, uint64 pageno)
+{
+	int64		segno;
+
+	if (state->segno != -1 && pageno == state->pageno)
+		return state->buf.data;
+
+	segno = pageno / SLRU_PAGES_PER_SEGMENT;
+
+	if (segno != state->segno)
+	{
+		if (state->segno != -1)
+		{
+			close(state->fd);
+			state->fd = -1;
+			pg_free(state->fn);
+			state->fn = NULL;
+		}
+
+		/* Open new segment */
+		state->fn = psprintf("%s/%04X", state->dir, (unsigned int) segno);
+		if ((state->fd = open(state->fn, O_RDONLY | PG_BINARY, 0)) < 0)
+		{
+			pg_fatal("could not open file \"%s\": %m", state->fn);
+		}
+		state->segno = segno;
+	}
+
+	{
+		struct iovec iovec = {
+			.iov_base = &state->buf,
+			.iov_len = BLCKSZ,
+		};
+		off_t		offset;
+
+		offset = (pageno % SLRU_PAGES_PER_SEGMENT) * BLCKSZ;
+
+		if (pg_preadv(state->fd, &iovec, 1, offset) < 0)
+			pg_fatal("could not read file \"%s\": %m", state->fn);
+
+		state->pageno = pageno;
+	}
+
+	return state->buf.data;
+}
diff --git a/src/bin/pg_upgrade/slru_io.h b/src/bin/pg_upgrade/slru_io.h
new file mode 100644
index 0000000000..e1a9c06313
--- /dev/null
+++ b/src/bin/pg_upgrade/slru_io.h
@@ -0,0 +1,23 @@
+/*
+ *	slru_io.h
+ *
+ *	Copyright (c) 2010-2024, PostgreSQL Global Development Group
+ *	src/bin/pg_upgrade/slru_io.h
+ */
+
+/* XXX: copied from slru.h */
+#define SLRU_PAGES_PER_SEGMENT	32
+
+/*
+ * Some kind of iterator associated with a particular SLRU segment.  The idea is
+ * to specify the segment and page number and then move through the pages.
+ */
+typedef struct SlruSegState SlruSegState;
+
+extern SlruSegState *OpenSlruRead(char *dir);
+extern void CloseSlruRead(SlruSegState *state);
+extern char *SlruReadSwitchPage(SlruSegState *state, uint64 pageno);
+
+extern SlruSegState *OpenSlruWrite(char *dir, int64 startPageno);
+extern void CloseSlruWrite(SlruSegState *state);
+extern char *SlruWriteSwitchPage(SlruSegState *state, uint64 pageno);
-- 
2.43.0