v2-0012-Add-new-vartag_external-for-8-byte-TOAST-values.patch

From 392fe1b61eb7dbe84d4ad65f6749e74d075d370f Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Tue, 8 Jul 2025 08:20:31 +0900
Subject: [PATCH v2 12/13] Add new vartag_external for 8-byte TOAST values

This is a new type of external TOAST pointer, able to be fed 8-byte
TOAST values.  It uses a dedicated vartag_external, which is used when
a TOAST table uses bigint for its chunk_id.

The relevant callbacks are added to toast_external.c.
---
 src/include/access/heaptoast.h                |   8 +-
 src/include/varatt.h                          |  31 ++++-
 src/backend/access/common/toast_external.c    | 124 ++++++++++++++++++
 src/backend/access/common/toast_internals.c   |  11 +-
 src/backend/access/heap/heaptoast.c           |  69 ++++++++--
 .../replication/logical/reorderbuffer.c       |  10 +-
 doc/src/sgml/storage.sgml                     |   6 +-
 contrib/amcheck/verify_heapam.c               |   2 +-
 8 files changed, 239 insertions(+), 22 deletions(-)

diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index 673e96f5488c..a39ad79a5ae9 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -81,6 +81,12 @@
 
 #define EXTERN_TUPLE_MAX_SIZE	MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
 
+#define TOAST_MAX_CHUNK_SIZE_INT8	\
+	(EXTERN_TUPLE_MAX_SIZE -							\
+	 MAXALIGN(SizeofHeapTupleHeader) -					\
+	 (sizeof(uint32) * 2) -								\
+	 sizeof(int32) -									\
+	 VARHDRSZ)
 #define TOAST_MAX_CHUNK_SIZE_OID	\
 	(EXTERN_TUPLE_MAX_SIZE -							\
 	 MAXALIGN(SizeofHeapTupleHeader) -					\
@@ -89,7 +95,7 @@
 	 VARHDRSZ)
 
 /* Maximum size of chunk possible */
-#define TOAST_MAX_CHUNK_SIZE	TOAST_MAX_CHUNK_SIZE_OID
+#define TOAST_MAX_CHUNK_SIZE	Max(TOAST_MAX_CHUNK_SIZE_INT8, TOAST_MAX_CHUNK_SIZE_OID)
 
 /* ----------
  * heap_toast_insert_or_update -
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 793030dae932..aa36e8e1f561 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -41,6 +41,29 @@ typedef struct varatt_external_oid
 	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
 }			varatt_external_oid;
 
+/*
+ * struct varatt_external_int8 is a "larger" version of "TOAST pointer",
+ * that uses an 8-byte integer as value.
+ *
+ * This follows the same properties as varatt_external_oid, except that
+ * this is used in TOAST relations with int8 as attribute for chunk_id.
+ */
+typedef struct varatt_external_int8
+{
+	int32		va_rawsize;		/* Original data size (includes header) */
+	uint32		va_extinfo;		/* External saved size (without header) and
+								 * compression method */
+	/*
+	 * Unique ID of value within TOAST table, as two uint32 for alignment
+	 * and padding.
+	 * XXX: think for example about the addition of an extra field for
+	 * meta-data and/or more compression data, even if it's OK here).
+	 */
+	uint32		va_valueid_lo;
+	uint32		va_valueid_hi;
+	Oid			va_toastrelid;	/* RelID of TOAST table containing it */
+}			varatt_external_int8;
+
 
 /*
  * These macros define the "saved size" portion of va_extinfo.  Its remaining
@@ -91,6 +114,7 @@ typedef enum vartag_external
 	VARTAG_INDIRECT = 1,
 	VARTAG_EXPANDED_RO = 2,
 	VARTAG_EXPANDED_RW = 3,
+	VARTAG_ONDISK_INT8 = 4,
 	VARTAG_ONDISK_OID = 18
 } vartag_external;
 
@@ -102,6 +126,7 @@ typedef enum vartag_external
 	((tag) == VARTAG_INDIRECT ? sizeof(varatt_indirect) : \
 	 VARTAG_IS_EXPANDED(tag) ? sizeof(varatt_expanded) : \
 	 (tag) == VARTAG_ONDISK_OID ? sizeof(varatt_external_oid) : \
+	 (tag) == VARTAG_ONDISK_INT8 ? sizeof(varatt_external_int8) : \
 	 (AssertMacro(false), 0))
 
 /*
@@ -294,8 +319,10 @@ typedef struct
 #define VARATT_IS_EXTERNAL(PTR)				VARATT_IS_1B_E(PTR)
 #define VARATT_IS_EXTERNAL_ONDISK_OID(PTR) \
 	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID)
+#define VARATT_IS_EXTERNAL_ONDISK_INT8(PTR) \
+	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_INT8)
 #define VARATT_IS_EXTERNAL_ONDISK(PTR) \
-	(VARATT_IS_EXTERNAL_ONDISK_OID(PTR))
+	(VARATT_IS_EXTERNAL_ONDISK_OID(PTR) || VARATT_IS_EXTERNAL_ONDISK_INT8(PTR))
 #define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
 	(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
 #define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
@@ -339,7 +366,7 @@ typedef struct
 
 /*
  * Same for external Datums; but note argument is a struct
- * varatt_external_oid.
+ * varatt_external_oid or varatt_external_int8.
  */
 #define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
 	((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c
index d179f0143803..0e79ac8acae8 100644
--- a/src/backend/access/common/toast_external.c
+++ b/src/backend/access/common/toast_external.c
@@ -14,9 +14,22 @@
 #include "postgres.h"
 
 #include "access/detoast.h"
+#include "access/genam.h"
 #include "access/heaptoast.h"
+#include "access/toast_counter.h"
 #include "access/toast_external.h"
 #include "catalog/catalog.h"
+#include "miscadmin.h"
+#include "utils/fmgroids.h"
+#include "utils/snapmgr.h"
+
+
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void ondisk_int8_to_external_data(struct varlena *attr,
+										 toast_external_data *data);
+static struct varlena *ondisk_int8_create_external_data(toast_external_data data);
+static uint64 ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+										AttrNumber attnum);
 
 /* Callbacks for VARTAG_ONDISK_OID */
 static void ondisk_oid_to_external_data(struct varlena *attr,
@@ -26,6 +39,12 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid,
 									   AttrNumber attnum);
 
 
+/*
+ * Size of an EXTERNAL datum that contains a standard TOAST pointer
+ * (int8 value).
+ */
+#define TOAST_POINTER_INT8_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_int8))
+
 /*
  * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID
  * value).
@@ -46,6 +65,13 @@ static uint64 ondisk_oid_get_new_value(Relation toastrel, Oid indexid,
  * individual fields.
  */
 static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = {
+	[VARTAG_ONDISK_INT8] = {
+		.toast_pointer_size = TOAST_POINTER_INT8_SIZE,
+		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_INT8,
+		.to_external_data = ondisk_int8_to_external_data,
+		.create_external_data = ondisk_int8_create_external_data,
+		.get_new_value = ondisk_int8_get_new_value,
+	},
 	[VARTAG_ONDISK_OID] = {
 		.toast_pointer_size = TOAST_POINTER_OID_SIZE,
 		.maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID,
@@ -78,6 +104,104 @@ toast_external_info_get_pointer_size(uint8 tag)
  * the in-memory representation toast_external_data used in the backend.
  */
 
+/* Callbacks for VARTAG_ONDISK_INT8 */
+static void
+ondisk_int8_to_external_data(struct varlena *attr, toast_external_data *data)
+{
+	varatt_external_int8	external;
+
+	VARATT_EXTERNAL_GET_POINTER(external, attr);
+	data->rawsize = external.va_rawsize;
+
+	/* External size and compression methods are stored in the same field */
+	if (VARATT_EXTERNAL_IS_COMPRESSED(external))
+	{
+		data->extsize = VARATT_EXTERNAL_GET_EXTSIZE(external);
+		data->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(external);
+	}
+	else
+	{
+		data->extsize = external.va_extinfo;
+		data->compression_method = TOAST_INVALID_COMPRESSION_ID;
+	}
+
+	data->value = (((uint64) external.va_valueid_hi) << 32) |
+		external.va_valueid_lo;
+	data->toastrelid = external.va_toastrelid;
+
+}
+
+static struct varlena *
+ondisk_int8_create_external_data(toast_external_data data)
+{
+	struct varlena *result = NULL;
+	varatt_external_int8 external;
+
+	external.va_rawsize = data.rawsize;
+
+	if (data.compression_method != TOAST_INVALID_COMPRESSION_ID)
+	{
+		/* Set size and compression method, in a single field. */
+		VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external,
+													 data.extsize,
+													 data.compression_method);
+	}
+	else
+		external.va_extinfo = data.extsize;
+
+	external.va_toastrelid = data.toastrelid;
+	external.va_valueid_hi = (((uint64) data.value) >> 32);
+	external.va_valueid_lo = (uint32) data.value;
+
+	result = (struct varlena *) palloc(TOAST_POINTER_INT8_SIZE);
+	SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_INT8);
+	memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external));
+
+	return result;
+}
+
+static uint64
+ondisk_int8_get_new_value(Relation toastrel, Oid indexid,
+						  AttrNumber attnum)
+{
+	uint64		new_value;
+	SysScanDesc	scan;
+	ScanKeyData	key;
+	bool		collides = false;
+
+retry:
+	new_value = GetNewToastId();
+
+	/* No indexes in bootstrap mode, so leave */
+	if (IsBootstrapProcessingMode())
+		return new_value;
+
+	Assert(IsSystemRelation(toastrel));
+
+	CHECK_FOR_INTERRUPTS();
+
+	/*
+	 * Check if the new value picked already exists in the toast relation.
+	 * If there is a conflict, retry.
+	 */
+	ScanKeyInit(&key,
+				attnum,
+				BTEqualStrategyNumber, F_INT8EQ,
+				Int64GetDatum(new_value));
+
+	/* see notes in GetNewOidWithIndex() above about using SnapshotAny */
+	scan = systable_beginscan(toastrel, indexid, true,
+							  SnapshotAny, 1, &key);
+	collides = HeapTupleIsValid(systable_getnext(scan));
+	systable_endscan(scan);
+
+	if (collides)
+		goto retry;
+
+	return new_value;
+}
+
+
 /* Callbacks for VARTAG_ONDISK_OID */
 static void
 ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data)
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 28e2867f8209..c6b2d1522cef 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -18,7 +18,6 @@
 #include "access/heapam.h"
 #include "access/heaptoast.h"
 #include "access/table.h"
-#include "access/toast_counter.h"
 #include "access/toast_external.h"
 #include "access/toast_internals.h"
 #include "access/xact.h"
@@ -158,6 +157,8 @@ toast_save_datum(Relation rel, Datum value,
 	 */
 	toastrel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock);
 	toasttupDesc = toastrel->rd_att;
+	toast_typid = TupleDescAttr(toasttupDesc, 0)->atttypid;
+	Assert(toast_typid == OIDOID || toast_typid == INT8OID);
 
 	/*
 	 * Grab the information for toast_external_data.
@@ -165,12 +166,12 @@ toast_save_datum(Relation rel, Datum value,
 	 * Note: if we support multiple external vartags for a single value
 	 * type, we would need to be smarter in the vartag selection.
 	 */
-	tag = VARTAG_ONDISK_OID;
+	if (toast_typid == OIDOID)
+		tag = VARTAG_ONDISK_OID;
+	else if (toast_typid == INT8OID)
+		tag = VARTAG_ONDISK_INT8;
 	info = toast_external_get_info(tag);
 
-	toast_typid = TupleDescAttr(toasttupDesc, 0)->atttypid;
-	Assert(toast_typid == OIDOID || toast_typid == INT8OID);
-
 	/* Open all the toast indexes and look for the valid one */
 	validIndex = toast_open_indexes(toastrel,
 									RowExclusiveLock,
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index 7a54cf7fae34..2121eb02848e 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -31,7 +31,9 @@
 #include "access/toast_external.h"
 #include "access/toast_helper.h"
 #include "access/toast_internals.h"
+#include "access/toast_type.h"
 #include "utils/fmgroids.h"
+#include "utils/syscache.h"
 
 
 /* ----------
@@ -145,12 +147,52 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	/*
 	 * Retrieve the toast pointer size based on the type of external TOAST
 	 * pointer assumed to be used.
-	 *
-	 * Only one format of external TOAST pointer is supported currently,
-	 * making this code simple, based on a single vartag.
 	 */
-	ttc.ttc_toast_pointer_size =
-		toast_external_info_get_pointer_size(VARTAG_ONDISK_OID);
+	if (OidIsValid(rel->rd_rel->reltoastrelid))
+	{
+		HeapTuple	atttuple;
+		Form_pg_attribute atttoast;
+		uint8		vartag;
+
+		/*
+		 * XXX: This is very unlikely efficient, but it is not possible to
+		 * rely on the relation cache to retrieve this information as syscache
+		 * lookups should not happen when loading critical entries.
+		 */
+		atttuple = SearchSysCacheAttNum(rel->rd_rel->reltoastrelid, 1);
+		if (!HeapTupleIsValid(atttuple))
+			elog(ERROR, "cache lookup failed for relation %u",
+				 rel->rd_rel->reltoastrelid);
+		atttoast = (Form_pg_attribute) GETSTRUCT(atttuple);
+
+		if (atttoast->atttypid == OIDOID)
+			vartag = VARTAG_ONDISK_OID;
+		else if (atttoast->atttypid == INT8OID)
+			vartag = VARTAG_ONDISK_INT8;
+		else
+			Assert(false);
+		ttc.ttc_toast_pointer_size =
+			toast_external_info_get_pointer_size(vartag);
+		ReleaseSysCache(atttuple);
+	}
+	else
+	{
+		/*
+		 * No TOAST relation to rely on, which is a case possible when
+		 * dealing with partitioned tables, for example.  Hence, do a best
+		 * guess based on the GUC default_toast_type.
+		 */
+		uint8	vartag = VARTAG_ONDISK_OID;
+
+		if (default_toast_type == TOAST_TYPE_INT8)
+			vartag = VARTAG_ONDISK_INT8;
+		else if (default_toast_type == TOAST_TYPE_OID)
+			vartag = VARTAG_ONDISK_OID;
+		else
+			Assert(false);
+		ttc.ttc_toast_pointer_size =
+			toast_external_info_get_pointer_size(vartag);
+	}
 
 	ttc.ttc_rel = rel;
 	ttc.ttc_values = toast_values;
@@ -654,7 +696,7 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize,
 	int32		max_chunk_size;
 	const toast_external_info *info;
 	uint8		tag = VARTAG_INDIRECT;  /* init value does not matter */
-	Oid			toast_typid;
+	Oid			toast_typid = InvalidOid;
 
 	/* Look for the valid index of toast relation */
 	validIndex = toast_open_indexes(toastrel,
@@ -673,14 +715,19 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize,
 	 * to pass down this information from the upper callers.  This is
 	 * currently on required for the maximum chunk_size.
 	 */
-	tag = VARTAG_ONDISK_OID;
-	info = toast_external_get_info(tag);
-
-	max_chunk_size = info->maximum_chunk_size;
-
 	toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid;
 	Assert(toast_typid == OIDOID || toast_typid == INT8OID);
 
+	if (toast_typid == OIDOID)
+		tag = VARTAG_ONDISK_OID;
+	else if (toast_typid == INT8OID)
+		tag = VARTAG_ONDISK_INT8;
+	else
+		Assert(false);
+
+	info = toast_external_get_info(tag);
+	max_chunk_size = info->maximum_chunk_size;
+
 	totalchunks = ((attrsize - 1) / max_chunk_size) + 1;
 	startchunk = sliceoffset / max_chunk_size;
 	endchunk = (sliceoffset + slicelength - 1) / max_chunk_size;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 77939c7f849c..834040395e62 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -4971,14 +4971,22 @@ ReorderBufferToastAppendChunk(ReorderBuffer *rb, ReorderBufferTXN *txn,
 	TupleDesc	desc = RelationGetDescr(relation);
 	uint64		chunk_id;
 	int32		chunk_seq;
+	Oid			toast_typid;
 
 	if (txn->toast_hash == NULL)
 		ReorderBufferToastInitHash(rb, txn);
+	toast_typid = TupleDescAttr(desc, 0)->atttypid;
 
 	Assert(IsToastRelation(relation));
 
 	newtup = change->data.tp.newtuple;
-	chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	/* This depends on the type of TOAST value dealt with. */
+	if (toast_typid == OIDOID)
+		chunk_id = DatumGetObjectId(fastgetattr(newtup, 1, desc, &isnull));
+	else if (toast_typid == INT8OID)
+		chunk_id = DatumGetUInt64(fastgetattr(newtup, 1, desc, &isnull));
+	else
+		Assert(false);
 	Assert(!isnull);
 	chunk_seq = DatumGetInt32(fastgetattr(newtup, 2, desc, &isnull));
 	Assert(!isnull);
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 564783a1c559..29ba80e8423c 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -415,7 +415,11 @@ described in more detail below.
 
 <para>
 Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes if the
+<acronym>TOAST</acronym> relation uses the <literal>oid</literal> type for
+<literal>chunk_id</literal>, or <symbol>TOAST_MAX_CHUNK_SIZE_INT8</symbol>
+bytes if the <acronym>TOAST</acronym> relation uses the <literal>int8</literal>
+type for <literal>chunk_id</literal> (by default these values are chosen
 so that four chunk rows will fit on a page, making it about 2000 bytes).
 Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
 belonging to the owning table.  Every
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 833811c75437..958b1451b4ff 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1733,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
 	{
 		uint8		va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
 
-		if (va_tag != VARTAG_ONDISK_OID)
+		if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_INT8)
 		{
 			report_corruption(ctx,
 							  psprintf("toasted attribute has unexpected TOAST tag %u",
-- 
2.50.0