v9-0015-Add-new-vartag_external-for-8-byte-TOAST-values.patch

From ab4a39a70fba34762163208612809ee5246febd4 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Thu, 14 Aug 2025 14:10:36 +0900
Subject: [PATCH v9 15/15] Add new vartag_external for 8-byte TOAST values

This is a new type of external TOAST pointer, able to be fed 8-byte
TOAST values.  It uses a dedicated vartag_external, which is used when
a TOAST table uses bigint for its chunk_id.

The relevant callbacks are added to toast_external.c.
---
 src/include/access/heaptoast.h                |   8 +-
 src/include/varatt.h                          |  34 +++-
 src/backend/access/common/toast_external.c    | 145 ++++++++++++++++--
 src/backend/access/heap/heaptoast.c           |   1 +
 .../replication/logical/reorderbuffer.c       |  10 +-
 doc/src/sgml/storage.sgml                     |   6 +-
 contrib/amcheck/verify_heapam.c               |   2 +-
 7 files changed, 189 insertions(+), 17 deletions(-)

diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index 12c9702af689..0f65e076efe5 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -81,6 +81,12 @@
 
 #define EXTERN_TUPLE_MAX_SIZE	MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
 
+#define TOAST_OID8_MAX_CHUNK_SIZE	\
+	(EXTERN_TUPLE_MAX_SIZE -							\
+	 MAXALIGN(SizeofHeapTupleHeader) -					\
+	 (sizeof(uint32) * 2) -								\
+	 sizeof(int32) -									\
+	 VARHDRSZ)
 #define TOAST_OID_MAX_CHUNK_SIZE	\
 	(EXTERN_TUPLE_MAX_SIZE -							\
 	 MAXALIGN(SizeofHeapTupleHeader) -					\
@@ -89,7 +95,7 @@
 	 VARHDRSZ)
 
 /* Maximum size of chunk possible */
-#define TOAST_MAX_CHUNK_SIZE	TOAST_OID_MAX_CHUNK_SIZE
+#define TOAST_MAX_CHUNK_SIZE	Max(TOAST_OID_MAX_CHUNK_SIZE, TOAST_OID8_MAX_CHUNK_SIZE)
 
 /* ----------
  * heap_toast_insert_or_update -
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 035c0f95e5b6..de38d1cd1ce1 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -41,6 +41,27 @@ typedef struct varatt_external_oid
 	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
 }			varatt_external_oid;
 
+/*
+ * struct varatt_external_oid8 is a "larger" version of "TOAST pointer",
+ * that uses an 8-byte integer as value.
+ *
+ * This follows the same properties as varatt_external_oid, except that
+ * this is used in TOAST relations with oid8 as attribute for chunk_id.
+ */
+typedef struct varatt_external_oid8
+{
+	int32		va_rawsize;		/* Original data size (includes header) */
+	uint32		va_extinfo;		/* External saved size (without header) and
+								 * compression method */
+	/*
+	 * Unique ID of value within TOAST table, as two uint32 for alignment
+	 * and padding.
+	 */
+	uint32		va_valueid_lo;
+	uint32		va_valueid_hi;
+	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
+}			varatt_external_oid8;
+
 /*
  * These macros define the "saved size" portion of va_extinfo.  Its remaining
  * two high-order bits identify the compression method.
@@ -90,6 +111,7 @@ typedef enum vartag_external
 	VARTAG_INDIRECT = 1,
 	VARTAG_EXPANDED_RO = 2,
 	VARTAG_EXPANDED_RW = 3,
+	VARTAG_ONDISK_OID8 = 4,
 	VARTAG_ONDISK_OID = 18
 } vartag_external;
 
@@ -111,6 +133,8 @@ VARTAG_SIZE(vartag_external tag)
 		return sizeof(varatt_expanded);
 	else if (tag == VARTAG_ONDISK_OID)
 		return sizeof(varatt_external_oid);
+	else if (tag == VARTAG_ONDISK_OID8)
+		return sizeof(varatt_external_oid8);
 	else
 	{
 		Assert(false);
@@ -367,11 +391,19 @@ VARATT_IS_EXTERNAL_ONDISK_OID(const void *PTR)
 	return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID;
 }
 
+/* Is varlena datum a pointer to on-disk toasted data with OID8 value? */
+static inline bool
+VARATT_IS_EXTERNAL_ONDISK_OID8(const void *PTR)
+{
+	return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID8;
+}
+
 /* Is varlena datum a pointer to on-disk toasted data? */
 static inline bool
 VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
 {
-	return VARATT_IS_EXTERNAL_ONDISK_OID(PTR);
+	return VARATT_IS_EXTERNAL_ONDISK_OID(PTR) ||
+		VARATT_IS_EXTERNAL_ONDISK_OID8(PTR);
 }
 
 /* Is varlena datum an indirect pointer? */
diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c
index e2f0a9dc1c50..431258b2be96 100644
--- a/src/backend/access/common/toast_external.c
+++ b/src/backend/access/common/toast_external.c
@@ -18,8 +18,19 @@
 #include "postgres.h"
 
 #include "access/detoast.h"
+#include "access/genam.h"
 #include "access/heaptoast.h"
 #include "access/toast_external.h"
+#include "catalog/catalog.h"
+#include "miscadmin.h"
+#include "utils/fmgroids.h"
+#include "utils/lsyscache.h"
+
+
+/* Callbacks for VARTAG_ONDISK_OID8 */
+static void ondisk_oid8_to_external_data(struct varlena *attr,
+										 toast_external_data *data);
+static struct varlena *ondisk_oid8_create_external_data(toast_external_data data);
 
 /* Callbacks for VARTAG_ONDISK_OID */
 static void ondisk_oid_to_external_data(struct varlena *attr,
@@ -28,7 +39,7 @@ static struct varlena *ondisk_oid_create_external_data(toast_external_data data)
 
 /*
  * Fetch the possibly-unaligned contents of an on-disk external TOAST with
- * OID values into a local "varatt_external_oid" pointer.
+ * OID or OID8 values into a local "varatt_external_*" pointer.
  *
  * This should be just a memcpy, but some versions of gcc seem to produce
  * broken code that assumes the datum contents are aligned.  Introducing
@@ -45,9 +56,20 @@ varatt_external_oid_get_pointer(varatt_external_oid *toast_pointer,
 	memcpy(toast_pointer, VARDATA_EXTERNAL(attre), sizeof(varatt_external_oid));
 }
 
+static inline void
+varatt_external_oid8_get_pointer(varatt_external_oid8 *toast_pointer,
+								 struct varlena *attr)
+{
+	varattrib_1b_e *attre = (varattrib_1b_e *) attr;
+
+	Assert(VARATT_IS_EXTERNAL_ONDISK_OID8(attre));
+	Assert(VARSIZE_EXTERNAL(attre) == sizeof(varatt_external_oid8) + VARHDRSZ_EXTERNAL);
+	memcpy(toast_pointer, VARDATA_EXTERNAL(attre), sizeof(varatt_external_oid8));
+}
+
 /*
  * Decompressed size of an on-disk varlena; but note argument is a struct
- * varatt_external_oid.
+ * varatt_external_oid or varatt_external_oid8.
  */
 static inline Size
 varatt_external_oid_get_extsize(varatt_external_oid toast_pointer)
@@ -55,9 +77,15 @@ varatt_external_oid_get_extsize(varatt_external_oid toast_pointer)
 	return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
 }
 
+static inline Size
+varatt_external_oid8_get_extsize(varatt_external_oid8 toast_pointer)
+{
+	return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
+}
+
 /*
  * Compression method of an on-disk varlena; but note argument is a struct
- *  varatt_external_oid.
+ *  varatt_external_oid or varatt_external_oid8.
  */
 static inline uint32
 varatt_external_oid_get_compress_method(varatt_external_oid toast_pointer)
@@ -65,6 +93,12 @@ varatt_external_oid_get_compress_method(varatt_external_oid toast_pointer)
 	return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
 }
 
+static inline uint32
+varatt_external_oid8_get_compress_method(varatt_external_oid8 toast_pointer)
+{
+	return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
+}
+
 /*
  * Testing whether an externally-stored TOAST value is compressed now requires
  * comparing size stored in va_extinfo (the actual length of the external data)
@@ -79,6 +113,19 @@ varatt_external_oid_is_compressed(varatt_external_oid toast_pointer)
 		(Size) (toast_pointer.va_rawsize - VARHDRSZ);
 }
 
+static inline bool
+varatt_external_oid8_is_compressed(varatt_external_oid8 toast_pointer)
+{
+	return varatt_external_oid8_get_extsize(toast_pointer) <
+		(Size) (toast_pointer.va_rawsize - VARHDRSZ);
+}
+
+/*
+ * Size of an EXTERNAL datum that contains a standard TOAST pointer
+ * (oid8 value).
+ */
+#define TOAST_POINTER_OID8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8))
+
 /*
  * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID
  * value).
@@ -99,6 +146,12 @@ varatt_external_oid_is_compressed(varatt_external_oid toast_pointer)
  * individual fields.
  */
 static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = {
+	[VARTAG_ONDISK_OID8] = {
+		.toast_pointer_size = TOAST_POINTER_OID8_SIZE,
+		.maximum_chunk_size = TOAST_OID_MAX_CHUNK_SIZE,
+		.to_external_data = ondisk_oid8_to_external_data,
+		.create_external_data = ondisk_oid8_create_external_data,
+	},
 	[VARTAG_ONDISK_OID] = {
 		.toast_pointer_size = TOAST_OID_POINTER_SIZE,
 		.maximum_chunk_size = TOAST_OID_MAX_CHUNK_SIZE,
@@ -155,22 +208,33 @@ toast_external_info_get_pointer_size(uint8 tag)
 uint8
 toast_external_assign_vartag(Oid toastrelid, Oid8 valueid)
 {
+	Oid		toast_typid;
+
 	/*
-	 * If dealing with a code path where a TOAST relation may not be assigned,
-	 * like heap_toast_insert_or_update(), just use the legacy
-	 * vartag_external.
+	 * If dealing with a code path where a TOAST relation may not be assigned
+	 * like heap_toast_insert_or_update(), just use the default with an OID
+	 * type.
+	 *
+	 * In bootstrap mode, we should not do any kind of syscache lookups,
+	 * so also rely on OID.
 	 */
-	if (!OidIsValid(toastrelid))
+	if (!OidIsValid(toastrelid) || IsBootstrapProcessingMode())
 		return VARTAG_ONDISK_OID;
 
 	/*
-	 * Currently there is only one type of vartag_external supported: 4-byte
-	 * value with OID for the chunk_id type.
+	 * Two types of vartag_external are currently supported: OID and OID8,
+	 * which depend on the type assigned to "chunk_id" for the TOAST table.
 	 *
-	 * Note: This routine will be extended to be able to use multiple
-	 * vartag_external within a single TOAST relation type, that may change
-	 * depending on the value used.
+	 * XXX: Should we assign from the start an OID vartag if dealing with
+	 * a TOAST relation with OID8 as value if the value assigned is less
+	 * than UINT_MAX?  This just takes the "safe" approach of assigning
+	 * the larger vartag in all cases, but this can be made cheaper
+	 * depending on the OID consumption.
 	 */
+	toast_typid = get_atttype(toastrelid, 1);
+	if (toast_typid == OID8OID)
+		return VARTAG_ONDISK_OID8;
+
 	return VARTAG_ONDISK_OID;
 }
 
@@ -179,6 +243,63 @@ toast_external_assign_vartag(Oid toastrelid, Oid8 valueid)
  * the in-memory representation toast_external_data used in the backend.
  */
 
+/* Callbacks for VARTAG_ONDISK_OID8 */
+static void
+ondisk_oid8_to_external_data(struct varlena *attr, toast_external_data *data)
+{
+	varatt_external_oid8	external;
+
+	varatt_external_oid8_get_pointer(&external, attr);
+	data->rawsize = external.va_rawsize;
+
+	/* External size and compression methods are stored in the same field */
+	if (varatt_external_oid8_is_compressed(external))
+	{
+		data->extsize = varatt_external_oid8_get_extsize(external);
+		data->compression_method = varatt_external_oid8_get_compress_method(external);
+	}
+	else
+	{
+		data->extsize = external.va_extinfo;
+		data->compression_method = TOAST_INVALID_COMPRESSION_ID;
+	}
+
+	data->valueid = (((uint64) external.va_valueid_hi) << 32) |
+		external.va_valueid_lo;
+	data->toastrelid = external.va_toastrelid;
+
+}
+
+static struct varlena *
+ondisk_oid8_create_external_data(toast_external_data data)
+{
+	struct varlena *result = NULL;
+	varatt_external_oid8 external;
+
+	external.va_rawsize = data.rawsize;
+
+	if (data.compression_method != TOAST_INVALID_COMPRESSION_ID)
+	{
+		/* Set size and compression method, in a single field. */
+		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external,
+													 data.extsize,
+													 data.compression_method);
+	}
+	else
+		external.va_extinfo = data.extsize;
+
+	external.va_toastrelid = data.toastrelid;
+	external.va_valueid_hi = (((uint64) data.valueid) >> 32);
+	external.va_valueid_lo = (uint32) data.valueid;
+
+	result = (struct varlena *) palloc(TOAST_POINTER_OID8_SIZE);
+	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8);
+	memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external));
+
+	return result;
+}
+
+
 /* Callbacks for VARTAG_ONDISK_OID */
 
 /*
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index da4d9594b243..8434105ea790 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -32,6 +32,7 @@
 #include "access/toast_helper.h"
 #include "access/toast_internals.h"
 #include "utils/fmgroids.h"
+#include "utils/syscache.h"
 
 
 /* ----------
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index e64501c84de4..2668e924f224 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -5005,14 +5005,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	TupleDesc	desc = RelationGetDescr(relation);
 	Oid8		chunk_id;
 	int32		chunk_seq;
+	Oid			toast_typid;
 
 	if (txn->toast_hash == NULL)
 		ReorderBufferToastInitHash(rb, txn);
+	toast_typid = TupleDescAttr(desc, 0)->atttypid;
 
 	Assert(IsToastRelation(relation));
 
 	newtup = change->data.tp.newtuple;
-	chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	/* This depends on the type of TOAST value dealt with. */
+	if (toast_typid == OIDOID)
+		chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	else if (toast_typid == INT8OID)
+		chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull));
+	else
+		Assert(false);
 	Assert(!isnull);
 	chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull));
 	Assert(!isnull);
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index afddf663fec5..dbec30d48b4a 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -417,7 +417,11 @@ described in more detail below.
 
 <para>
 Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> bytes if the
+<acronym>TOAST</acronym> relation uses the <literal>oid</literal> type for
+<literal>chunk_id</literal>, or <symbol>TOAST_OID8_MAX_CHUNK_SIZE</symbol>
+bytes if the <acronym>TOAST</acronym> relation uses the <literal>oid8</literal>
+type for <literal>chunk_id</literal> (by default these values are chosen
 so that four chunk rows will fit on a page, making it about 2000 bytes).
 Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
 belonging to the owning table.  Every
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 80de3545e73a..b95adcb9b57c 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
 	{
 		uint8		va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
 
-		if (va_tag != VARTAG_ONDISK_OID)
+		if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8)
 		{
 			report_corruption(ctx,
 							  psprintf("toasted attribute has unexpected TOAST tag %u",
-- 
2.51.0