v4-0014-Add-new-vartag_external-for-8-byte-TOAST-values.patch

From b1be9990e57bc2e2c33ad7feb076a6d0cf521625 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Fri, 8 Aug 2025 16:19:33 +0900
Subject: [PATCH v4 14/15] Add new vartag_external for 8-byte TOAST values

This is a new type of external TOAST pointer, able to be fed 8-byte
TOAST values.  It uses a dedicated vartag_external, which is used when
a TOAST table uses bigint for its chunk_id.

The relevant callbacks are added to toast_external.c.
---
 src/include/access/heaptoast.h                |   8 +-
 src/include/varatt.h                          |  34 +++-
 src/backend/access/common/toast_external.c    | 182 ++++++++++++++++--
 src/backend/access/common/toast_internals.c   |   1 -
 src/backend/access/heap/heaptoast.c           |   2 +
 .../replication/logical/reorderbuffer.c       |  10 +-
 doc/src/sgml/storage.sgml                     |   6 +-
 contrib/amcheck/verify_heapam.c               |   2 +-
 8 files changed, 225 insertions(+), 20 deletions(-)

diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index 673e96f5488c..a39ad79a5ae9 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -81,6 +81,12 @@
 
 #define EXTERN_TUPLE_MAX_SIZE	MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
 
+#define TOAST_MAX_CHUNK_SIZE_INT8	\
+	(EXTERN_TUPLE_MAX_SIZE -							\
+	 MAXALIGN(SizeofHeapTupleHeader) -					\
+	 (sizeof(uint32) * 2) -								\
+	 sizeof(int32) -									\
+	 VARHDRSZ)
 #define TOAST_MAX_CHUNK_SIZE_OID	\
 	(EXTERN_TUPLE_MAX_SIZE -							\
 	 MAXALIGN(SizeofHeapTupleHeader) -					\
@@ -89,7 +95,7 @@
 	 VARHDRSZ)
 
 /* Maximum size of chunk possible */
-#define TOAST_MAX_CHUNK_SIZE	TOAST_MAX_CHUNK_SIZE_OID
+#define TOAST_MAX_CHUNK_SIZE	Max(TOAST_MAX_CHUNK_SIZE_INT8, TOAST_MAX_CHUNK_SIZE_OID)
 
 /* ----------
  * heap_toast_insert_or_update -
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 631aa2ecc494..a4b85031f06c 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -41,6 +41,27 @@ typedef struct varatt_external_oid
 	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
 }			varatt_external_oid;
 
+/*
+ * struct varatt_external_int8 is a "larger" version of "TOAST pointer",
+ * that uses an 8-byte integer as value.
+ *
+ * This follows the same properties as varatt_external_oid, except that
+ * this is used in TOAST relations with int8 as attribute for chunk_id.
+ */
+typedef struct varatt_external_int8
+{
+	int32		va_rawsize;		/* Original data size (includes header) */
+	uint32		va_extinfo;		/* External saved size (without header) and
+								 * compression method */
+	/*
+	 * Unique ID of value within TOAST table, as two uint32 for alignment
+	 * and padding.
+	 */
+	uint32		va_valueid_lo;
+	uint32		va_valueid_hi;
+	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
+}			varatt_external_int8;
+
 /*
  * These macros define the "saved size" portion of va_extinfo.  Its remaining
  * two high-order bits identify the compression method.
@@ -90,6 +111,7 @@ typedef enum vartag_external
 	VARTAG_INDIRECT = 1,
 	VARTAG_EXPANDED_RO = 2,
 	VARTAG_EXPANDED_RW = 3,
+	VARTAG_ONDISK_INT8 = 4,
 	VARTAG_ONDISK_OID = 18
 } vartag_external;
 
@@ -111,6 +133,8 @@ VARTAG_SIZE(vartag_external tag)
 		return sizeof(varatt_expanded);
 	else if (tag == VARTAG_ONDISK_OID)
 		return sizeof(varatt_external_oid);
+	else if (tag == VARTAG_ONDISK_INT8)
+		return sizeof(varatt_external_int8);
 	else
 	{
 		Assert(false);
@@ -367,11 +391,19 @@ VARATT_IS_EXTERNAL_ONDISK_OID(const void *PTR)
 	return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID;
 }
 
+/* Is varlena datum a pointer to on-disk toasted data with int8 value? */
+static inline bool
+VARATT_IS_EXTERNAL_ONDISK_INT8(const void *PTR)
+{
+	return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_INT8;
+}
+
 /* Is varlena datum a pointer to on-disk toasted data? */
 static inline bool
 VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
 {
-	return VARATT_IS_EXTERNAL_ONDISK_OID(PTR);
+	return VARATT_IS_EXTERNAL_ONDISK_OID(PTR) ||
+		VARATT_IS_EXTERNAL_ONDISK_INT8(PTR);
 }
 
 /* Is varlena datum an indirect pointer? */
diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c
index 5c36e5a11392..f02a707d7671 100644
--- a/src/backend/access/common/toast_external.c
+++ b/src/backend/access/common/toast_external.c
@@ -14,9 +14,24 @@
 #include "postgres.h"
 
 #include "access/detoast.h"
+#include "access/genam.h"
 #include "access/heaptoast.h"
+#include "access/toast_counter.h"
 #include "access/toast_external.h"
+#include "access/toast_type.h"
 #include "catalog/catalog.h"
+#include "miscadmin.h"
+#include "utils/fmgroids.h"
+#include "utils/snapmgr.h"
+#include "utils/lsyscache.h"
+
+
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void ondisk_int8_to_external_data(struct varlena *attr,
+										 toast_external_data *data);
+static struct varlena *ondisk_int8_create_external_data(toast_external_data data);
+static uint64 ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+										AttrNumber attnum);
 
 /* Callbacks for VARTAG_ONDISK_OID */
 static void ondisk_oid_to_external_data(struct varlena *attr,
@@ -27,7 +42,7 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid,
 
 /*
  * Decompressed size of an on-disk varlena; but note argument is a struct
- * varatt_external_oid.
+ * varatt_external_oid or varatt_external_int8.
  */
 static inline Size
 varatt_external_oid_get_extsize(struct varatt_external_oid toast_pointer)
@@ -35,9 +50,15 @@ varatt_external_oid_get_extsize(struct varatt_external_oid toast_pointer)
 	return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
 }
 
+static inline Size
+varatt_external_int8_get_extsize(struct varatt_external_int8 toast_pointer)
+{
+	return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
+}
+
 /*
  * Compression method of an on-disk varlena; but note argument is a struct
- *  varatt_external_oid.
+ *  varatt_external_oid or varatt_external_int8.
  */
 static inline uint32
 varatt_external_oid_get_compress_method(struct varatt_external_oid toast_pointer)
@@ -45,6 +66,12 @@ varatt_external_oid_get_compress_method(struct varatt_external_oid toast_pointer
 	return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
 }
 
+static inline uint32
+varatt_external_int8_get_compress_method(struct varatt_external_int8 toast_pointer)
+{
+	return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
+}
+
 /*
  * Testing whether an externally-stored TOAST value is compressed now requires
  * comparing size stored in va_extinfo (the actual length of the external data)
@@ -59,6 +86,19 @@ varatt_external_oid_is_compressed(struct varatt_external_oid toast_pointer)
 		(Size) (toast_pointer.va_rawsize - VARHDRSZ);
 }
 
+static inline bool
+varatt_external_int8_is_compressed(struct varatt_external_int8 toast_pointer)
+{
+	return varatt_external_int8_get_extsize(toast_pointer) <
+		(Size) (toast_pointer.va_rawsize - VARHDRSZ);
+}
+
+/*
+ * Size of an EXTERNAL datum that contains a standard TOAST pointer
+ * (int8 value).
+ */
+#define TOAST_POINTER_INT8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_int8))
+
 /*
  * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID
  * value).
@@ -79,6 +119,13 @@ varatt_external_oid_is_compressed(struct varatt_external_oid toast_pointer)
  * individual fields.
  */
 static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = {
+	[VARTAG_ONDISK_INT8] = {
+		.toast_pointer_size = TOAST_POINTER_INT8_SIZE,
+		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_INT8,
+		.to_external_data = ondisk_int8_to_external_data,
+		.create_external_data = ondisk_int8_create_external_data,
+		.get_new_value = ondisk_int8_get_new_value,
+	},
 	[VARTAG_ONDISK_OID] = {
 		.toast_pointer_size = TOAST_POINTER_OID_SIZE,
 		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID,
@@ -117,22 +164,31 @@ toast_external_info_get_pointer_size(uint8 tag)
 uint8
 toast_external_assign_vartag(Oid toastrelid, uint64 value)
 {
-	/*
-	 * If dealing with a code path where a TOAST relation may not be assigned,
-	 * like heap_toast_insert_or_update(), just use the legacy
-	 * vartag_external.
-	 */
-	if (!OidIsValid(toastrelid))
-		return VARTAG_ONDISK_OID;
+	Oid		toast_typid;
 
 	/*
-	 * Currently there is only one type of vartag_external supported: 4-byte
-	 * value with OID for the chunk_id type.
+	 * If dealing with a code path where a TOAST relation may not be assigned
+	 * like heap_toast_insert_or_update(), just use the vartag_external that
+	 * can be guessed based on the GUC default_toast_type.
 	 *
-	 * Note: This routine will be extended to be able to use multiple
-	 * vartag_external within a single TOAST relation type, that may change
-	 * depending on the value used.
+	 * In bootstrap mode, we should not do any kind of syscache lookups,
+	 * so do the same and rely on the value of default_toast_type.
 	 */
+	if (!OidIsValid(toastrelid) || IsBootstrapProcessingMode())
+	{
+		if (default_toast_type == TOAST_TYPE_INT8)
+			return VARTAG_ONDISK_INT8;
+		return VARTAG_ONDISK_OID;
+	}
+
+	/*
+	 * Two types of vartag_external are currently supported: OID and int8,
+	 * which depend on the type assigned to "chunk_id" for the TOAST table.
+	 */
+	toast_typid = get_atttype(toastrelid, 1);
+	if (toast_typid == INT8OID)
+		return VARTAG_ONDISK_INT8;
+
 	return VARTAG_ONDISK_OID;
 }
 
@@ -141,6 +197,104 @@ toast_external_assign_vartag(Oid toastrelid, uint64 value)
  * the in-memory representation toast_external_data used in the backend.
  */
 
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void
+ondisk_int8_to_external_data(struct varlena *attr, toast_external_data *data)
+{
+	varatt_external_int8	external;
+
+	VARATT_EXTERNAL_GET_POINTER(external, attr);
+	data->rawsize = external.va_rawsize;
+
+	/* External size and compression methods are stored in the same field */
+	if (varatt_external_int8_is_compressed(external))
+	{
+		data->extsize = varatt_external_int8_get_extsize(external);
+		data->compression_method = varatt_external_int8_get_compress_method(external);
+	}
+	else
+	{
+		data->extsize = external.va_extinfo;
+		data->compression_method = TOAST_INVALID_COMPRESSION_ID;
+	}
+
+	data->value = (((uint64) external.va_valueid_hi) << 32) |
+		external.va_valueid_lo;
+	data->toastrelid = external.va_toastrelid;
+
+}
+
+static struct varlena *
+ondisk_int8_create_external_data(toast_external_data data)
+{
+	struct varlena *result = NULL;
+	varatt_external_int8 external;
+
+	external.va_rawsize = data.rawsize;
+
+	if (data.compression_method != TOAST_INVALID_COMPRESSION_ID)
+	{
+		/* Set size and compression method, in a single field. */
+		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external,
+													 data.extsize,
+													 data.compression_method);
+	}
+	else
+		external.va_extinfo = data.extsize;
+
+	external.va_toastrelid = data.toastrelid;
+	external.va_valueid_hi = (((uint64) data.value) >> 32);
+	external.va_valueid_lo = (uint32) data.value;
+
+	result = (struct varlena *) palloc(TOAST_POINTER_INT8_SIZE);
+	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_INT8);
+	memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external));
+
+	return result;
+}
+
+static uint64
+ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+						  AttrNumber attnum)
+{
+	uint64		new_value;
+	SysScanDesc	scan;
+	ScanKeyData	key;
+	bool		collides = false;
+
+retry:
+	new_value = GetNewToastId();
+
+	/* No indexes in bootstrap mode, so leave */
+	if (IsBootstrapProcessingMode())
+		return new_value;
+
+	Assert(IsSystemRelation(toastrel));
+
+	CHECK_FOR_INTERRUPTS();
+
+	/*
+	 * Check if the new value picked already exists in the toast relation.
+	 * If there is a conflict, retry.
+	 */
+	ScanKeyInit(&key,
+				attnum,
+				BTEqualStrategyNumber, F_INT8EQ,
+				Int64GetDatum(new_value));
+
+	/* see notes in GetNewOidWithIndex() above about using SnapshotAny */
+	scan = systable_beginscan(toastrel, indexid, true,
+							  SnapshotAny, 1, &key);
+	collides = HeapTupleIsValid(systable_getnext(scan));
+	systable_endscan(scan);
+
+	if (collides)
+		goto retry;
+
+	return new_value;
+}
+
+
 /* Callbacks for VARTAG_ONDISK_OID */
 static void
 ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data)
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 98a240242127..ee7ef99181b3 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -18,7 +18,6 @@
 #include "access/heapam.h"
 #include "access/heaptoast.h"
 #include "access/table.h"
-#include "access/toast_counter.h"
 #include "access/toast_external.h"
 #include "access/toast_internals.h"
 #include "access/xact.h"
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index a01c1d627c6f..e3ace3acc9b9 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -31,7 +31,9 @@
 #include "access/toast_external.h"
 #include "access/toast_helper.h"
 #include "access/toast_internals.h"
+#include "access/toast_type.h"
 #include "utils/fmgroids.h"
+#include "utils/syscache.h"
 
 
 /* ----------
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index be33e7de6f8d..6c6526ab9c74 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -4971,14 +4971,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	TupleDesc	desc = RelationGetDescr(relation);
 	uint64		chunk_id;
 	int32		chunk_seq;
+	Oid			toast_typid;
 
 	if (txn->toast_hash == NULL)
 		ReorderBufferToastInitHash(rb, txn);
+	toast_typid = TupleDescAttr(desc, 0)->atttypid;
 
 	Assert(IsToastRelation(relation));
 
 	newtup = change->data.tp.newtuple;
-	chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	/* This depends on the type of TOAST value dealt with. */
+	if (toast_typid == OIDOID)
+		chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	else if (toast_typid == INT8OID)
+		chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull));
+	else
+		Assert(false);
 	Assert(!isnull);
 	chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull));
 	Assert(!isnull);
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 564783a1c559..29ba80e8423c 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -415,7 +415,11 @@ described in more detail below.
 
 <para>
 Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes if the
+<acronym>TOAST</acronym> relation uses the <literal>oid</literal> type for
+<literal>chunk_id</literal>, or <symbol>TOAST_MAX_CHUNK_SIZE_INT8</symbol>
+bytes if the <acronym>TOAST</acronym> relation uses the <literal>int8</literal>
+type for <literal>chunk_id</literal> (by default these values are chosen
 so that four chunk rows will fit on a page, making it about 2000 bytes).
 Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
 belonging to the owning table.  Every
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 833811c75437..958b1451b4ff 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
 	{
 		uint8		va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
 
-		if (va_tag != VARTAG_ONDISK_OID)
+		if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_INT8)
 		{
 			report_corruption(ctx,
 							  psprintf("toasted attribute has unexpected TOAST tag %u",
-- 
2.50.0