v3-0013-Add-new-vartag_external-for-8-byte-TOAST-values.patch

From bc34eae687be3a62dbd73a588e389001042f0ae5 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Tue, 8 Jul 2025 09:02:40 +0900
Subject: [PATCH v3 13/14] Add new vartag_external for 8-byte TOAST values

This is a new type of external TOAST pointer, able to be fed 8-byte
TOAST values.  It uses a dedicated vartag_external, which is used when
a TOAST table uses bigint for its chunk_id.

The relevant callbacks are added to toast_external.c.
---
 src/include/access/heaptoast.h                |   8 +-
 src/include/varatt.h                          |  31 +++-
 src/backend/access/common/toast_external.c    | 161 ++++++++++++++++--
 src/backend/access/common/toast_internals.c   |   1 -
 src/backend/access/heap/heaptoast.c           |   4 +-
 .../replication/logical/reorderbuffer.c       |  10 +-
 doc/src/sgml/storage.sgml                     |   6 +-
 contrib/amcheck/verify_heapam.c               |   2 +-
 8 files changed, 203 insertions(+), 20 deletions(-)

diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index 673e96f5488c..a39ad79a5ae9 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -81,6 +81,12 @@
 
 #define EXTERN_TUPLE_MAX_SIZE	MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
 
+#define TOAST_MAX_CHUNK_SIZE_INT8	\
+	(EXTERN_TUPLE_MAX_SIZE -							\
+	 MAXALIGN(SizeofHeapTupleHeader) -					\
+	 (sizeof(uint32) * 2) -								\
+	 sizeof(int32) -									\
+	 VARHDRSZ)
 #define TOAST_MAX_CHUNK_SIZE_OID	\
 	(EXTERN_TUPLE_MAX_SIZE -							\
 	 MAXALIGN(SizeofHeapTupleHeader) -					\
@@ -89,7 +95,7 @@
 	 VARHDRSZ)
 
 /* Maximum size of chunk possible */
-#define TOAST_MAX_CHUNK_SIZE	TOAST_MAX_CHUNK_SIZE_OID
+#define TOAST_MAX_CHUNK_SIZE	Max(TOAST_MAX_CHUNK_SIZE_INT8, TOAST_MAX_CHUNK_SIZE_OID)
 
 /* ----------
  * heap_toast_insert_or_update -
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 126e5e112a17..46c831092887 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -41,6 +41,29 @@ typedef struct varatt_external_oid
 	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
 }			varatt_external_oid;
 
+/*
+ * struct varatt_external_int8 is a "larger" version of "TOAST pointer",
+ * that uses an 8-byte integer as value.
+ *
+ * This follows the same properties as varatt_external_oid, except that
+ * this is used in TOAST relations with int8 as attribute for chunk_id.
+ */
+typedef struct varatt_external_int8
+{
+	int32		va_rawsize;		/* Original data size (includes header) */
+	uint32		va_extinfo;		/* External saved size (without header) and
+								 * compression method */
+	/*
+	 * Unique ID of value within TOAST table, as two uint32 for alignment
+	 * and padding.
+	 * XXX: think for example about the addition of an extra field for
+	 * meta-data and/or more compression data, even if it's OK here).
+	 */
+	uint32		va_valueid_lo;
+	uint32		va_valueid_hi;
+	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
+}			varatt_external_int8;
+
 /*
  * These macros define the "saved size" portion of va_extinfo.  Its remaining
  * two high-order bits identify the compression method.
@@ -90,6 +113,7 @@ typedef enum vartag_external
 	VARTAG_INDIRECT = 1,
 	VARTAG_EXPANDED_RO = 2,
 	VARTAG_EXPANDED_RW = 3,
+	VARTAG_ONDISK_INT8 = 4,
 	VARTAG_ONDISK_OID = 18
 } vartag_external;
 
@@ -101,6 +125,7 @@ typedef enum vartag_external
 	((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
 	 VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
 	 (tag) == VARTAG_ONDISK_OID ? sizeof(varatt_external_oid) : \
+	 (tag) == VARTAG_ONDISK_INT8 ? sizeof(varatt_external_int8) : \
 	 (AssertMacro(false), 0))
 
 /*
@@ -293,8 +318,10 @@ typedef struct
 #define VARATT_IS_EXTERNAL(PTR)				VARATT_IS_1B_E(PTR)
 #define VARATT_IS_EXTERNAL_ONDISK_OID(PTR) \
 	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID)
+#define VARATT_IS_EXTERNAL_ONDISK_INT8(PTR) \
+	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_INT8)
 #define VARATT_IS_EXTERNAL_ONDISK(PTR) \
-	(VARATT_IS_EXTERNAL_ONDISK_OID(PTR))
+	(VARATT_IS_EXTERNAL_ONDISK_OID(PTR) || VARATT_IS_EXTERNAL_ONDISK_INT8(PTR))
 #define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
 	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
 #define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
@@ -338,7 +365,7 @@ typedef struct
 
 /*
  * Same for external Datums; but note argument is a struct
- * varatt_external_oid.
+ * varatt_external_oid or varatt_external_int8.
  */
 #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
 	((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c
index a17de2fa8b3d..57c3f8084548 100644
--- a/src/backend/access/common/toast_external.c
+++ b/src/backend/access/common/toast_external.c
@@ -14,9 +14,24 @@
 #include "postgres.h"
 
 #include "access/detoast.h"
+#include "access/genam.h"
 #include "access/heaptoast.h"
+#include "access/toast_counter.h"
 #include "access/toast_external.h"
+#include "access/toast_type.h"
 #include "catalog/catalog.h"
+#include "miscadmin.h"
+#include "utils/fmgroids.h"
+#include "utils/snapmgr.h"
+#include "utils/lsyscache.h"
+
+
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void ondisk_int8_to_external_data(struct varlena *attr,
+										 toast_external_data *data);
+static struct varlena *ondisk_int8_create_external_data(toast_external_data data);
+static uint64 ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+										AttrNumber attnum);
 
 /* Callbacks for VARTAG_ONDISK_OID */
 static void ondisk_oid_to_external_data(struct varlena *attr,
@@ -26,6 +41,12 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid,
 									   AttrNumber attnum);
 
 
+/*
+ * Size of an EXTERNAL datum that contains a standard TOAST pointer
+ * (int8 value).
+ */
+#define TOAST_POINTER_INT8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_int8))
+
 /*
  * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID
  * value).
@@ -46,6 +67,13 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid,
  * individual fields.
  */
 static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = {
+	[VARTAG_ONDISK_INT8] = {
+		.toast_pointer_size = TOAST_POINTER_INT8_SIZE,
+		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_INT8,
+		.to_external_data = ondisk_int8_to_external_data,
+		.create_external_data = ondisk_int8_create_external_data,
+		.get_new_value = ondisk_int8_get_new_value,
+	},
 	[VARTAG_ONDISK_OID] = {
 		.toast_pointer_size = TOAST_POINTER_OID_SIZE,
 		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID,
@@ -80,21 +108,32 @@ toast_external_info_get_pointer_size(uint8 tag)
 uint8
 toast_external_assign_vartag(Oid toastrelid, uint64 value)
 {
-	/*
-	 * If dealing with a code path where a TOAST relation may not be
-	 * assigned, like heap_toast_insert_or_update(), just use the legacy
-	 * vartag_external.
-	 */
-	if (!OidIsValid(toastrelid))
-		return VARTAG_ONDISK_OID;
+	Oid		toast_typid;
 
 	/*
-	 * Currently there is only one type of vartag_external supported:
-	 * 4-byte value with OID for the chunk_id type.  This routine will
-	 * be extended to be able to use multiple vartag_external within
-	 * a single TOAST relation type, that may change depending on the
-	 * value used.
+	 * If dealing with a code path where a TOAST relation may not be
+	 * assigned, like heap_toast_insert_or_update(), just use the
+	 * vartag_external that can be guessed based on the GUC
+	 * default_toast_type.
+	 *
+	 * In bootstrap mode, we should not do any kind of syscache lookups,
+	 * so do the same and rely on the value of default_toast_type.
 	 */
+	if (!OidIsValid(toastrelid) || IsBootstrapProcessingMode())
+	{
+		if (default_toast_type == TOAST_TYPE_INT8)
+			return VARTAG_ONDISK_INT8;
+		return VARTAG_ONDISK_OID;
+	}
+
+	/*
+	 * Two types of vartag_external are currently supported: OID and int8,
+	 * which depend on the type assigned to "chunk_id" for the TOAST table.
+	 */
+	toast_typid = get_atttype(toastrelid, 1);
+	if (toast_typid == INT8OID)
+		return VARTAG_ONDISK_INT8;
+
 	return VARTAG_ONDISK_OID;
 }
 
@@ -103,6 +142,104 @@ toast_external_assign_vartag(Oid toastrelid, uint64 value)
  * the in-memory representation toast_external_data used in the backend.
  */
 
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void
+ondisk_int8_to_external_data(struct varlena *attr, toast_external_data *data)
+{
+	varatt_external_int8	external;
+
+	VARATT_EXTERNAL_GET_POINTER(external, attr);
+	data->rawsize = external.va_rawsize;
+
+	/* External size and compression methods are stored in the same field */
+	if (VARATT_EXTERNAL_IS_COMPRESSED(external))
+	{
+		data->extsize = VARATT_EXTERNAL_GET_EXTSIZE(external);
+		data->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(external);
+	}
+	else
+	{
+		data->extsize = external.va_extinfo;
+		data->compression_method = TOAST_INVALID_COMPRESSION_ID;
+	}
+
+	data->value = (((uint64) external.va_valueid_hi) << 32) |
+		external.va_valueid_lo;
+	data->toastrelid = external.va_toastrelid;
+
+}
+
+static struct varlena *
+ondisk_int8_create_external_data(toast_external_data data)
+{
+	struct varlena *result = NULL;
+	varatt_external_int8 external;
+
+	external.va_rawsize = data.rawsize;
+
+	if (data.compression_method != TOAST_INVALID_COMPRESSION_ID)
+	{
+		/* Set size and compression method, in a single field. */
+		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external,
+													 data.extsize,
+													 data.compression_method);
+	}
+	else
+		external.va_extinfo = data.extsize;
+
+	external.va_toastrelid = data.toastrelid;
+	external.va_valueid_hi = (((uint64) data.value) >> 32);
+	external.va_valueid_lo = (uint32) data.value;
+
+	result = (struct varlena *) palloc(TOAST_POINTER_INT8_SIZE);
+	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_INT8);
+	memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external));
+
+	return result;
+}
+
+static uint64
+ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+						  AttrNumber attnum)
+{
+	uint64		new_value;
+	SysScanDesc	scan;
+	ScanKeyData	key;
+	bool		collides = false;
+
+retry:
+	new_value = GetNewToastId();
+
+	/* No indexes in bootstrap mode, so leave */
+	if (IsBootstrapProcessingMode())
+		return new_value;
+
+	Assert(IsSystemRelation(toastrel));
+
+	CHECK_FOR_INTERRUPTS();
+
+	/*
+	 * Check if the new value picked already exists in the toast relation.
+	 * If there is a conflict, retry.
+	 */
+	ScanKeyInit(&key,
+				attnum,
+				BTEqualStrategyNumber, F_INT8EQ,
+				Int64GetDatum(new_value));
+
+	/* see notes in GetNewOidWithIndex() above about using SnapshotAny */
+	scan = systable_beginscan(toastrel, indexid, true,
+							  SnapshotAny, 1, &key);
+	collides = HeapTupleIsValid(systable_getnext(scan));
+	systable_endscan(scan);
+
+	if (collides)
+		goto retry;
+
+	return new_value;
+}
+
+
 /* Callbacks for VARTAG_ONDISK_OID */
 static void
 ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data)
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 663af02e4634..4f71f9ee399a 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -18,7 +18,6 @@
 #include "access/heapam.h"
 #include "access/heaptoast.h"
 #include "access/table.h"
-#include "access/toast_counter.h"
 #include "access/toast_external.h"
 #include "access/toast_internals.h"
 #include "access/xact.h"
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index 3238d21830ff..3ecd3993aea4 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -31,7 +31,9 @@
 #include "access/toast_external.h"
 #include "access/toast_helper.h"
 #include "access/toast_internals.h"
+#include "access/toast_type.h"
 #include "utils/fmgroids.h"
+#include "utils/syscache.h"
 
 
 /* ----------
@@ -654,7 +656,7 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize,
 	int32		max_chunk_size;
 	const toast_external_info *info;
 	uint8		tag = VARTAG_INDIRECT;  /* init value does not matter */
-	Oid			toast_typid;
+	Oid			toast_typid = InvalidOid;
 
 	/* Look for the valid index of toast relation */
 	validIndex = toast_open_indexes(toastrel,
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 26508bd01c86..b8932700ce6e 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -4971,14 +4971,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	TupleDesc	desc = RelationGetDescr(relation);
 	uint64		chunk_id;
 	int32		chunk_seq;
+	Oid			toast_typid;
 
 	if (txn->toast_hash == NULL)
 		ReorderBufferToastInitHash(rb, txn);
+	toast_typid = TupleDescAttr(desc, 0)->atttypid;
 
 	Assert(IsToastRelation(relation));
 
 	newtup = change->data.tp.newtuple;
-	chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	/* This depends on the type of TOAST value dealt with. */
+	if (toast_typid == OIDOID)
+		chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	else if (toast_typid == INT8OID)
+		chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull));
+	else
+		Assert(false);
 	Assert(!isnull);
 	chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull));
 	Assert(!isnull);
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 564783a1c559..29ba80e8423c 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -415,7 +415,11 @@ described in more detail below.
 
 <para>
 Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes if the
+<acronym>TOAST</acronym> relation uses the <literal>oid</literal> type for
+<literal>chunk_id</literal>, or <symbol>TOAST_MAX_CHUNK_SIZE_INT8</symbol>
+bytes if the <acronym>TOAST</acronym> relation uses the <literal>int8</literal>
+type for <literal>chunk_id</literal> (by default these values are chosen
 so that four chunk rows will fit on a page, making it about 2000 bytes).
 Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
 belonging to the owning table.  Every
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 833811c75437..958b1451b4ff 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
 	{
 		uint8		va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
 
-		if (va_tag != VARTAG_ONDISK_OID)
+		if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_INT8)
 		{
 			report_corruption(ctx,
 							  psprintf("toasted attribute has unexpected TOAST tag %u",
-- 
2.50.0