v3-0004-Refactor-external-TOAST-pointer-code-for-better-p.patch
text/x-diff
Filename: v3-0004-Refactor-external-TOAST-pointer-code-for-better-p.patch
Type: text/x-diff
Part: 3
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: format-patch
Series: patch v3-0004
Subject: Refactor external TOAST pointer code for better pluggability
| File | + | − |
|---|---|---|
| contrib/amcheck/verify_heapam.c | 20 | 15 |
| doc/src/sgml/storage.sgml | 1 | 1 |
| src/backend/access/common/detoast.c | 28 | 29 |
| src/backend/access/common/Makefile | 1 | 0 |
| src/backend/access/common/meson.build | 1 | 0 |
| src/backend/access/common/toast_compression.c | 2 | 8 |
| src/backend/access/common/toast_external.c | 156 | 0 |
| src/backend/access/common/toast_internals.c | 53 | 31 |
| src/backend/access/heap/heaptoast.c | 19 | 1 |
| src/backend/access/table/toast_helper.c | 9 | 3 |
| src/backend/replication/logical/reorderbuffer.c | 7 | 6 |
| src/backend/utils/adt/varlena.c | 4 | 3 |
| src/include/access/detoast.h | 5 | 7 |
| src/include/access/heaptoast.h | 4 | 1 |
| src/include/access/toast_external.h | 170 | 0 |
| src/include/access/toast_helper.h | 1 | 0 |
| src/include/varatt.h | 19 | 8 |
| src/tools/pgindent/typedefs.list | 2 | 0 |
From 8b179d9a443f033136ea2b6dee2ae2474ceb0c05 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Fri, 1 Aug 2025 16:22:13 +0900
Subject: [PATCH v3 04/14] Refactor external TOAST pointer code for better
pluggability
This commit introduces a new interface for external TOAST pointers,
which is able to make a translation of the varlena pointers stored on
disk to/from an new in-memory structure called toast_external. The
types of varatt_external supported on disk need to be registered into a
new subsystem in a new file, called toast_external.[c|h], then define a
set of callbacks to allow the toasting and detoasting code to use it.
A follow-up change will rely on this refactoring to introduce new
vartag_external values with an associated varatt_external_* that is
able, which would be used in int8 TOAST tables.
---
src/include/access/detoast.h | 12 +-
src/include/access/heaptoast.h | 5 +-
src/include/access/toast_external.h | 170 ++++++++++++++++++
src/include/access/toast_helper.h | 1 +
src/include/varatt.h | 27 ++-
src/backend/access/common/Makefile | 1 +
src/backend/access/common/detoast.c | 57 +++---
src/backend/access/common/meson.build | 1 +
src/backend/access/common/toast_compression.c | 10 +-
src/backend/access/common/toast_external.c | 156 ++++++++++++++++
src/backend/access/common/toast_internals.c | 84 +++++----
src/backend/access/heap/heaptoast.c | 20 ++-
src/backend/access/table/toast_helper.c | 12 +-
.../replication/logical/reorderbuffer.c | 13 +-
src/backend/utils/adt/varlena.c | 7 +-
doc/src/sgml/storage.sgml | 2 +-
contrib/amcheck/verify_heapam.c | 35 ++--
src/tools/pgindent/typedefs.list | 2 +
18 files changed, 502 insertions(+), 113 deletions(-)
create mode 100644 src/include/access/toast_external.h
create mode 100644 src/backend/access/common/toast_external.c
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index d80a62e64fd5..4195f7b5bdfd 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -14,10 +14,11 @@
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
- * into a local "struct varatt_external_oid" toast pointer. This should be
- * just a memcpy, but some versions of gcc seem to produce broken code
- * that assumes the datum contents are aligned. Introducing an explicit
- * intermediate "varattrib_1b_e *" variable seems to fix it.
+ * into a local "struct varatt_external_*" toast pointer, as supported
+ * in toast_external.c and varatt.h. This should be just a memcpy, but
+ * some versions of gcc seem to produce broken code that assumes the datum
+ * contents are aligned. Introducing an explicit intermediate
+ * "varattrib_1b_e *" variable seems to fix it.
*/
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
do { \
@@ -27,9 +28,6 @@ do { \
memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)
-/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
-#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid))
-
/* Size of an EXTERNAL datum that contains an indirection pointer */
#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index 6e3558cbd6d2..673e96f5488c 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -81,13 +81,16 @@
#define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE)
-#define TOAST_MAX_CHUNK_SIZE \
+#define TOAST_MAX_CHUNK_SIZE_OID \
(EXTERN_TUPLE_MAX_SIZE - \
MAXALIGN(SizeofHeapTupleHeader) - \
sizeof(Oid) - \
sizeof(int32) - \
VARHDRSZ)
+/* Maximum size of chunk possible */
+#define TOAST_MAX_CHUNK_SIZE TOAST_MAX_CHUNK_SIZE_OID
+
/* ----------
* heap_toast_insert_or_update -
*
diff --git a/src/include/access/toast_external.h b/src/include/access/toast_external.h
new file mode 100644
index 000000000000..849cb1779b67
--- /dev/null
+++ b/src/include/access/toast_external.h
@@ -0,0 +1,170 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_external.h
+ * Support for on-disk external TOAST pointers
+ *
+ *
+ * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1995, Regents of the University of California
+ *
+ * src/include/access/toast_external.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TOAST_EXTERNAL_H
+#define TOAST_EXTERNAL_H
+
+#include "access/toast_compression.h"
+#include "varatt.h"
+
+/*
+ * Intermediate in-memory structure used when creating on-disk
+ * varatt_external_* or when deserializing varlena contents.
+ */
+typedef struct toast_external_data
+{
+ /* Original data size (includes header) */
+ int32 rawsize;
+
+ /* External saved size (without header) */
+ uint32 extsize;
+
+ /* compression method */
+ ToastCompressionId compression_method;
+
+ /* Relation OID of TOAST table containing the value */
+ Oid toastrelid;
+
+ /*
+ * Unique ID of value within TOAST table. This could be an OID or an
+ * int8 value. This field is large enough to be able to store any of
+ * them.
+ */
+ uint64 value;
+} toast_external_data;
+
+/*
+ * Metadata for external TOAST pointer kinds, separated based on their
+ * vartag_external.
+ */
+typedef struct toast_external_info
+{
+ /*
+ * Maximum chunk of data authorized for this type of external TOAST
+ * pointer, when dividing an entry by chunks. Sized depending on
+ * the size of its varatt_external_* structure.
+ */
+ int32 maximum_chunk_size;
+
+ /*
+ * Size of an external TOAST pointer of this type, typically
+ * (VARHDRSZ_EXTERNAL + sizeof(varatt_external_struct)).
+ */
+ int32 toast_pointer_size;
+
+ /*
+ * Map an input varlena to a toast_external_data, for consumption
+ * in the backend code. "data" is an input/output result.
+ */
+ void (*to_external_data) (struct varlena *attr,
+ toast_external_data *data);
+
+ /*
+ * Create a varlena that will be used on-disk for the given TOAST
+ * type, based on the given input data.
+ *
+ * The result is the varlena created, for on-disk insertion.
+ */
+ struct varlena *(*create_external_data) (toast_external_data data);
+
+} toast_external_info;
+
+/* Retrieve a toast_external_info from a vartag */
+extern const toast_external_info *toast_external_get_info(uint8 tag);
+
+/* Retrieve toast_pointer_size using a TOAST attribute type */
+extern int32 toast_external_info_get_pointer_size(uint8 tag);
+
+/* Retrieve the vartag to assign to a TOAST typle */
+extern uint8 toast_external_assign_vartag(Oid toastrelid, uint64 value);
+
+/*
+ * Testing whether an externally-stored value is compressed now requires
+ * comparing size stored in extsize (the actual length of the external data)
+ * to rawsize (the original uncompressed datum's size). The latter includes
+ * VARHDRSZ overhead, the former doesn't. We never use compression unless it
+ * actually saves space, so we expect either equality or less-than.
+ */
+#define TOAST_EXTERNAL_IS_COMPRESSED(data) \
+ ((data).extsize < (data).rawsize - VARHDRSZ)
+
+/* Full data structure */
+static inline void
+toast_external_info_get_data(struct varlena *attr, toast_external_data *data)
+{
+ uint8 tag = VARTAG_EXTERNAL(attr);
+ const toast_external_info *info = toast_external_get_info(tag);
+
+ info->to_external_data(attr, data);
+}
+
+/*
+ * Helper routines to recover specific fields in toast_external_data. Most
+ * code paths doing work with on-disk external TOAST pointers care about
+ * these.
+ */
+
+/* Detoasted "raw" size */
+static inline Size
+toast_external_info_get_rawsize(struct varlena *attr)
+{
+ uint8 tag = VARTAG_EXTERNAL(attr);
+ const toast_external_info *info = toast_external_get_info(tag);
+ toast_external_data data;
+
+ info->to_external_data(attr, &data);
+
+ return data.rawsize;
+}
+
+/* External saved size */
+static inline Size
+toast_external_info_get_extsize(struct varlena *attr)
+{
+ uint8 tag = VARTAG_EXTERNAL(attr);
+ const toast_external_info *info = toast_external_get_info(tag);
+ toast_external_data data;
+
+ info->to_external_data(attr, &data);
+
+ return data.extsize;
+}
+
+/* Compression method ID */
+static inline ToastCompressionId
+toast_external_info_get_compression_method(struct varlena *attr)
+{
+ uint8 tag = VARTAG_EXTERNAL(attr);
+ const toast_external_info *info = toast_external_get_info(tag);
+ toast_external_data data;
+
+ info->to_external_data(attr, &data);
+
+ return data.compression_method;
+}
+
+/* Value ID */
+static inline Size
+toast_external_info_get_value(struct varlena *attr)
+{
+ uint8 tag = VARTAG_EXTERNAL(attr);
+ const toast_external_info *info = toast_external_get_info(tag);
+ toast_external_data data;
+
+ info->to_external_data(attr, &data);
+
+ return data.value;
+}
+
+#endif /* TOAST_EXTERNAL_H */
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index e6ab8afffb67..729c593afebd 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -47,6 +47,7 @@ typedef struct
* should be NULL in the case of an insert.
*/
Relation ttc_rel; /* the relation that contains the tuple */
+ int32 ttc_toast_pointer_size; /* size of external TOAST pointer */
Datum *ttc_values; /* values from the tuple columns */
bool *ttc_isnull; /* null flags for the tuple columns */
Datum *ttc_oldvalues; /* values from previous tuple */
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 8cd6312df432..126e5e112a17 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -21,6 +21,9 @@
* The data is compressed if and only if the external size stored in
* va_extinfo is less than va_rawsize - VARHDRSZ.
*
+ * The value ID is an OID, used for TOAST relations with OID as attribute
+ * for chunk_id.
+ *
* This struct must not contain any padding, because we sometimes compare
* these pointers using memcmp.
*
@@ -51,7 +54,7 @@ typedef struct varatt_external_oid
* The creator of such a Datum is entirely responsible that the referenced
* storage survives for as long as referencing pointer Datums can exist.
*
- * Note that just as for struct varatt_external_oid, this struct is stored
+ * Note that just as for struct varatt_external_*, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct varatt_indirect
@@ -66,7 +69,7 @@ typedef struct varatt_indirect
* storage. APIs for this, in particular the definition of struct
* ExpandedObjectHeader, are in src/include/utils/expandeddatum.h.
*
- * Note that just as for struct varatt_external_oid, this struct is stored
+ * Note that just as for struct varatt_external_*, this struct is stored
* unaligned within any containing tuple.
*/
typedef struct ExpandedObjectHeader ExpandedObjectHeader;
@@ -78,8 +81,9 @@ typedef struct varatt_expanded
/*
* Type tag for the various sorts of "TOAST pointer" datums. The peculiar
- * value for VARTAG_ONDISK_OID comes from a requirement for on-disk compatibility
- * with a previous notion that the tag field was the pointer datum's length.
+ * value for VARTAG_ONDISK_OID comes from a requirement for on-disk
+ * compatibility with a previous notion that the tag field was the pointer
+ * datum's length.
*/
typedef enum vartag_external
{
@@ -287,8 +291,10 @@ typedef struct
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
-#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
+#define VARATT_IS_EXTERNAL_ONDISK_OID(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID)
+#define VARATT_IS_EXTERNAL_ONDISK(PTR) \
+ (VARATT_IS_EXTERNAL_ONDISK_OID(PTR))
#define VARATT_IS_EXTERNAL_INDIRECT(PTR) \
(VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_INDIRECT)
#define VARATT_IS_EXTERNAL_EXPANDED_RO(PTR) \
@@ -330,7 +336,10 @@ typedef struct
#define VARDATA_COMPRESSED_GET_COMPRESS_METHOD(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_EXTSIZE_BITS)
-/* Same for external Datums; but note argument is a struct varatt_external_oid */
+/*
+ * Same for external Datums; but note argument is a struct
+ * varatt_external_oid.
+ */
#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
((toast_pointer).va_extinfo & VARLENA_EXTSIZE_MASK)
#define VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) \
@@ -351,8 +360,10 @@ typedef struct
* VARHDRSZ overhead, the former doesn't. We never use compression unless it
* actually saves space, so we expect either equality or less-than.
*/
+
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
- (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
- (toast_pointer).va_rawsize - VARHDRSZ)
+ (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
+ (toast_pointer).va_rawsize - VARHDRSZ)
+
#endif
diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile
index e78de312659e..1ef86a245886 100644
--- a/src/backend/access/common/Makefile
+++ b/src/backend/access/common/Makefile
@@ -27,6 +27,7 @@ OBJS = \
syncscan.o \
tidstore.o \
toast_compression.o \
+ toast_external.o \
toast_internals.o \
tupconvert.o \
tupdesc.o
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 04eedb474c74..6a9b5200203c 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -16,6 +16,7 @@
#include "access/detoast.h"
#include "access/table.h"
#include "access/tableam.h"
+#include "access/toast_external.h"
#include "access/toast_internals.h"
#include "common/int.h"
#include "common/pg_lzcompress.h"
@@ -225,12 +226,12 @@ detoast_attr_slice(struct varlena *attr,
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get_data(attr, &toast_pointer);
/* fast path for non-compressed external datums */
- if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (!TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer))
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/*
@@ -240,7 +241,7 @@ detoast_attr_slice(struct varlena *attr,
*/
if (slicelimit >= 0)
{
- int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ int32 max_size = toast_pointer.extsize;
/*
* Determine maximum amount of compressed data needed for a prefix
@@ -251,8 +252,7 @@ detoast_attr_slice(struct varlena *attr,
* determine how much compressed data we need to be sure of being
* able to decompress the required slice.
*/
- if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
- TOAST_PGLZ_COMPRESSION_ID)
+ if (toast_pointer.compression_method == TOAST_PGLZ_COMPRESSION_ID)
max_size = pglz_maximum_compressed_size(slicelimit, max_size);
/*
@@ -344,20 +344,21 @@ toast_fetch_datum(struct varlena *attr)
{
Relation toastrel;
struct varlena *result;
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
int32 attrsize;
+ uint64 valueid;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
/* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get_data(attr, &toast_pointer);
- attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ attrsize = toast_pointer.extsize;
result = (struct varlena *) palloc(attrsize + VARHDRSZ);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer))
SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
else
SET_VARSIZE(result, attrsize + VARHDRSZ);
@@ -365,14 +366,15 @@ toast_fetch_datum(struct varlena *attr)
if (attrsize == 0)
return result; /* Probably shouldn't happen, but just in
* case. */
+ valueid = toast_pointer.value;
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toast_pointer.toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, 0, attrsize, result);
/* Close toast table */
@@ -398,23 +400,26 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
{
Relation toastrel;
struct varlena *result;
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
int32 attrsize;
+ uint64 valueid;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
/* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get_data(attr, &toast_pointer);
+
+ valueid = toast_pointer.value;
/*
* It's nonsense to fetch slices of a compressed datum unless when it's a
* prefix -- this isn't lo_* we can't return a compressed datum which is
* meaningful to toast later.
*/
- Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
+ Assert(!TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
- attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ attrsize = toast_pointer.extsize;
if (sliceoffset >= attrsize)
{
@@ -427,7 +432,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
* space required by va_tcinfo, which is stored at the beginning as an
* int32 value.
*/
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0)
+ if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer) && slicelength > 0)
slicelength = slicelength + sizeof(int32);
/*
@@ -440,7 +445,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
result = (struct varlena *) palloc(slicelength + VARHDRSZ);
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer))
SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
else
SET_VARSIZE(result, slicelength + VARHDRSZ);
@@ -449,10 +454,11 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
return result; /* Can save a lot of work at this point! */
/* Open the toast relation */
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toast_pointer.toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel,
+ valueid,
attrsize, sliceoffset, slicelength,
result);
@@ -549,11 +555,7 @@ toast_raw_datum_size(Datum value)
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- /* va_rawsize is the size of the original datum -- including header */
- struct varatt_external_oid toast_pointer;
-
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_rawsize;
+ result = toast_external_info_get_rawsize(attr);
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
@@ -610,10 +612,7 @@ toast_datum_size(Datum value)
* compressed or not. We do not count the size of the toast pointer
* ... should we?
*/
- struct varatt_external_oid toast_pointer;
-
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
+ result = toast_external_info_get_extsize(attr);
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/meson.build b/src/backend/access/common/meson.build
index e3cdbe7a22e1..c20f2e88921e 100644
--- a/src/backend/access/common/meson.build
+++ b/src/backend/access/common/meson.build
@@ -15,6 +15,7 @@ backend_sources += files(
'syncscan.c',
'tidstore.c',
'toast_compression.c',
+ 'toast_external.c',
'toast_internals.c',
'tupconvert.c',
'tupdesc.c',
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 26aad84d367a..94606a58c8fb 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -19,6 +19,7 @@
#include "access/detoast.h"
#include "access/toast_compression.h"
+#include "access/toast_external.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"
@@ -261,14 +262,7 @@ toast_get_compression_id(struct varlena *attr)
* toast compression header.
*/
if (VARATT_IS_EXTERNAL_ONDISK(attr))
- {
- struct varatt_external_oid toast_pointer;
-
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
-
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
- cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
- }
+ cmid = toast_external_info_get_compression_method(attr);
else if (VARATT_IS_COMPRESSED(attr))
cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
diff --git a/src/backend/access/common/toast_external.c b/src/backend/access/common/toast_external.c
new file mode 100644
index 000000000000..5bfb6a6d434f
--- /dev/null
+++ b/src/backend/access/common/toast_external.c
@@ -0,0 +1,156 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_external.c
+ * Functions for the support of external on-disk TOAST pointers.
+ *
+ * Copyright (c) 2025, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/access/common/toast_external.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/detoast.h"
+#include "access/heaptoast.h"
+#include "access/toast_external.h"
+
+/* Callbacks for VARTAG_ONDISK_OID */
+static void ondisk_oid_to_external_data(struct varlena *attr,
+ toast_external_data *data);
+static struct varlena *ondisk_oid_create_external_data(toast_external_data data);
+
+
+/*
+ * Size of an EXTERNAL datum that contains a standard TOAST pointer (OID
+ * value).
+ */
+#define TOAST_POINTER_OID_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid))
+
+/*
+ * For now there are only two types, all defined in this file. For now this
+ * is the maximum value of vartag_external, which is a historical choice.
+ */
+#define TOAST_EXTERNAL_INFO_SIZE (VARTAG_ONDISK_OID + 1)
+
+/*
+ * The different kinds of on-disk external TOAST pointers, divided by
+ * vartag_external.
+ *
+ * See comments for struct toast_external_info about the details of the
+ * individual fields.
+ */
+static const toast_external_info toast_external_infos[TOAST_EXTERNAL_INFO_SIZE] = {
+ [VARTAG_ONDISK_OID] = {
+ .toast_pointer_size = TOAST_POINTER_OID_SIZE,
+ .maximum_chunk_size = TOAST_MAX_CHUNK_SIZE_OID,
+ .to_external_data = ondisk_oid_to_external_data,
+ .create_external_data = ondisk_oid_create_external_data,
+ },
+};
+
+
+/* Get toast_external_info of the defined vartag_external */
+const toast_external_info *
+toast_external_get_info(uint8 tag)
+{
+ return &toast_external_infos[tag];
+}
+
+/*
+ * Get external TOAST pointer size based on the attribute type of a TOAST
+ * value.
+ */
+int32
+toast_external_info_get_pointer_size(uint8 tag)
+{
+ return toast_external_infos[tag].toast_pointer_size;
+}
+
+/*
+ * Assign the vartag_external of a TOAST tuple, based on the TOAST relation
+ * it uses and its value.
+ */
+uint8
+toast_external_assign_vartag(Oid toastrelid, uint64 value)
+{
+ /*
+ * If dealing with a code path where a TOAST relation may not be
+ * assigned, like heap_toast_insert_or_update(), just use the legacy
+ * vartag_external.
+ */
+ if (!OidIsValid(toastrelid))
+ return VARTAG_ONDISK_OID;
+
+ /*
+ * Currently there is only one type of vartag_external supported:
+ * 4-byte value with OID for the chunk_id type. This routine will
+ * be extended to be able to use multiple vartag_external within
+ * a single TOAST relation type, that may change depending on the
+ * value used.
+ */
+ return VARTAG_ONDISK_OID;
+}
+
+/*
+ * Helper routines able to translate the various varatt_external_* from/to
+ * the in-memory representation toast_external_data used in the backend.
+ */
+
+/* Callbacks for VARTAG_ONDISK_OID */
+static void
+ondisk_oid_to_external_data(struct varlena *attr, toast_external_data *data)
+{
+ varatt_external_oid external;
+
+ VARATT_EXTERNAL_GET_POINTER(external, attr);
+ data->rawsize = external.va_rawsize;
+
+ /*
+ * External size and compression methods are stored in the same field,
+ * extract.
+ */
+ if (VARATT_EXTERNAL_IS_COMPRESSED(external))
+ {
+ data->extsize = VARATT_EXTERNAL_GET_EXTSIZE(external);
+ data->compression_method = VARATT_EXTERNAL_GET_COMPRESS_METHOD(external);
+ }
+ else
+ {
+ data->extsize = external.va_extinfo;
+ data->compression_method = TOAST_INVALID_COMPRESSION_ID;
+ }
+
+ data->value = (uint64) external.va_valueid;
+ data->toastrelid = external.va_toastrelid;
+}
+
+static struct varlena *
+ondisk_oid_create_external_data(toast_external_data data)
+{
+ struct varlena *result = NULL;
+ varatt_external_oid external;
+
+ external.va_rawsize = data.rawsize;
+
+ if (data.compression_method != TOAST_INVALID_COMPRESSION_ID)
+ {
+ /* Set size and compression method, in a single field. */
+ VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(external,
+ data.extsize,
+ data.compression_method);
+ }
+ else
+ external.va_extinfo = data.extsize;
+
+ external.va_toastrelid = data.toastrelid;
+ external.va_valueid = (Oid) data.value;
+
+ result = (struct varlena *) palloc(TOAST_POINTER_OID_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
+ memcpy(VARDATA_EXTERNAL(result), &external, sizeof(external));
+
+ return result;
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 0fbefd412783..80bda9034cdf 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -18,6 +18,7 @@
#include "access/heapam.h"
#include "access/heaptoast.h"
#include "access/table.h"
+#include "access/toast_external.h"
#include "access/toast_internals.h"
#include "access/xact.h"
#include "catalog/catalog.h"
@@ -127,7 +128,7 @@ toast_save_datum(Relation rel, Datum value,
bool t_isnull[3];
CommandId mycid = GetCurrentCommandId(true);
struct varlena *result;
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
union
{
struct varlena hdr;
@@ -143,6 +144,8 @@ toast_save_datum(Relation rel, Datum value,
Pointer dval = DatumGetPointer(value);
int num_indexes;
int validIndex;
+ const toast_external_info *info;
+ uint8 tag = VARTAG_INDIRECT; /* init value does not matter */
Assert(!VARATT_IS_EXTERNAL(value));
@@ -174,28 +177,41 @@ toast_save_datum(Relation rel, Datum value,
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
- toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
- toast_pointer.va_extinfo = data_todo;
+ toast_pointer.rawsize = data_todo + VARHDRSZ; /* as if not short */
+ toast_pointer.extsize = data_todo;
+
+ /*
+ * Note: we set compression_method to be able to build a correct
+ * on-disk TOAST pointer.
+ */
+ toast_pointer.compression_method = TOAST_INVALID_COMPRESSION_ID;
}
else if (VARATT_IS_COMPRESSED(dval))
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
- toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
+ toast_pointer.rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
/* set external size and compression method */
- VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
- VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+ toast_pointer.extsize = data_todo;
+ toast_pointer.compression_method = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+
/* Assert that the numbers look like it's compressed */
- Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
+ Assert(TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
- toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extinfo = data_todo;
+ toast_pointer.rawsize = VARSIZE(dval);
+ toast_pointer.extsize = data_todo;
+
+ /*
+ * Note: we set compression_method to be able to build a correct
+ * on-disk TOAST pointer.
+ */
+ toast_pointer.compression_method = TOAST_INVALID_COMPRESSION_ID;
}
/*
@@ -207,9 +223,9 @@ toast_save_datum(Relation rel, Datum value,
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
- toast_pointer.va_toastrelid = rel->rd_toastoid;
+ toast_pointer.toastrelid = rel->rd_toastoid;
else
- toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
+ toast_pointer.toastrelid = RelationGetRelid(toastrel);
/*
* Choose an OID to use as the value ID for this toast value.
@@ -226,7 +242,7 @@ toast_save_datum(Relation rel, Datum value,
if (!OidIsValid(rel->rd_toastoid))
{
/* normal case: just choose an unused OID */
- toast_pointer.va_valueid =
+ toast_pointer.value =
GetNewOidWithIndex(toastrel,
RelationGetRelid(toastidxs[validIndex]),
(AttrNumber) 1);
@@ -234,18 +250,18 @@ toast_save_datum(Relation rel, Datum value,
else
{
/* rewrite case: check to see if value was in old toast table */
- toast_pointer.va_valueid = InvalidOid;
+ toast_pointer.value = InvalidOid;
if (oldexternal != NULL)
{
- struct varatt_external_oid old_toast_pointer;
+ struct toast_external_data old_toast_pointer;
Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
- if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
+ toast_external_info_get_data(oldexternal, &old_toast_pointer);
+
+ if (old_toast_pointer.toastrelid == rel->rd_toastoid)
{
/* This value came from the old toast table; reuse its OID */
- toast_pointer.va_valueid = old_toast_pointer.va_valueid;
+ toast_pointer.value = old_toast_pointer.value;
/*
* There is a corner case here: the table rewrite might have
@@ -265,14 +281,14 @@ toast_save_datum(Relation rel, Datum value,
* be reclaimed by VACUUM.
*/
if (toastrel_valueid_exists(toastrel,
- toast_pointer.va_valueid))
+ toast_pointer.value))
{
/* Match, so short-circuit the data storage loop below */
data_todo = 0;
}
}
}
- if (toast_pointer.va_valueid == InvalidOid)
+ if (toast_pointer.value == InvalidOid)
{
/*
* new value; must choose an OID that doesn't conflict in either
@@ -280,24 +296,32 @@ toast_save_datum(Relation rel, Datum value,
*/
do
{
- toast_pointer.va_valueid =
+ toast_pointer.value =
GetNewOidWithIndex(toastrel,
RelationGetRelid(toastidxs[validIndex]),
(AttrNumber) 1);
} while (toastid_valueid_exists(rel->rd_toastoid,
- toast_pointer.va_valueid));
+ toast_pointer.value));
}
}
/*
* Initialize constant parts of the tuple data
*/
- t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
+ t_values[0] = ObjectIdGetDatum(toast_pointer.value);
t_values[2] = PointerGetDatum(&chunk_data);
t_isnull[0] = false;
t_isnull[1] = false;
t_isnull[2] = false;
+ /*
+ * Retrieve the vartag that can be assigned for the new TOAST tuple.
+ * This depends on the type of TOAST table and its assigned value.
+ */
+ tag = toast_external_assign_vartag(toast_pointer.toastrelid,
+ toast_pointer.value);
+ info = toast_external_get_info(tag);
+
/*
* Split up the item into chunks
*/
@@ -310,7 +334,7 @@ toast_save_datum(Relation rel, Datum value,
/*
* Calculate the size of this chunk
*/
- chunk_size = Min(TOAST_MAX_CHUNK_SIZE, data_todo);
+ chunk_size = Min(info->maximum_chunk_size, data_todo);
/*
* Build a tuple and store it
@@ -368,9 +392,7 @@ toast_save_datum(Relation rel, Datum value,
/*
* Create the TOAST pointer value that we'll return
*/
- result = (struct varlena *) palloc(TOAST_POINTER_SIZE);
- SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
- memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ result = info->create_external_data(toast_pointer);
return PointerGetDatum(result);
}
@@ -385,7 +407,7 @@ void
toast_delete_datum(Relation rel, Datum value, bool is_speculative)
{
struct varlena *attr = (struct varlena *) DatumGetPointer(value);
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
Relation toastrel;
Relation *toastidxs;
ScanKeyData toastkey;
@@ -398,12 +420,12 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
return;
/* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get_data(attr, &toast_pointer);
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
+ toastrel = table_open(toast_pointer.toastrelid, RowExclusiveLock);
/* Fetch valid relation used for process */
validIndex = toast_open_indexes(toastrel,
@@ -417,7 +439,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
- ObjectIdGetDatum(toast_pointer.va_valueid));
+ ObjectIdGetDatum(toast_pointer.value));
/*
* Find all the chunks. (We don't actually care whether we see them in
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index ae8d502ddcd3..47cc49be2aba 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -28,6 +28,7 @@
#include "access/genam.h"
#include "access/heapam.h"
#include "access/heaptoast.h"
+#include "access/toast_external.h"
#include "access/toast_helper.h"
#include "access/toast_internals.h"
#include "utils/fmgroids.h"
@@ -109,6 +110,7 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
Datum toast_oldvalues[MaxHeapAttributeNumber];
ToastAttrInfo toast_attr[MaxHeapAttributeNumber];
ToastTupleContext ttc;
+ uint8 tag;
/*
* Ignore the INSERT_SPECULATIVE option. Speculative insertions/super
@@ -140,6 +142,16 @@ heap_toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
* Prepare for toasting
* ----------
*/
+
+ /*
+ * Retrieve the toast pointer size based on the type of external TOAST
+ * pointer assumed to be used.
+ */
+
+ /* The default value is invalid, to work as a default. */
+ tag = toast_external_assign_vartag(rel->rd_rel->reltoastrelid, InvalidOid);
+ ttc.ttc_toast_pointer_size = toast_external_info_get_pointer_size(tag);
+
ttc.ttc_rel = rel;
ttc.ttc_values = toast_values;
ttc.ttc_isnull = toast_isnull;
@@ -640,6 +652,8 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize,
int num_indexes;
int validIndex;
int32 max_chunk_size;
+ const toast_external_info *info;
+ uint8 tag = VARTAG_INDIRECT; /* init value does not matter */
/* Look for the valid index of toast relation */
validIndex = toast_open_indexes(toastrel,
@@ -647,7 +661,11 @@ heap_fetch_toast_slice(Relation toastrel, uint64 valueid, int32 attrsize,
&toastidxs,
&num_indexes);
- max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+ /* Grab the information for toast_external_data */
+ tag = toast_external_assign_vartag(RelationGetRelid(toastrel), valueid);
+ info = toast_external_get_info(tag);
+
+ max_chunk_size = info->maximum_chunk_size;
totalchunks = ((attrsize - 1) / max_chunk_size) + 1;
startchunk = sliceoffset / max_chunk_size;
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index b60fab0a4d29..a2b44e093d79 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -171,8 +171,10 @@ toast_tuple_init(ToastTupleContext *ttc)
* The column must have attstorage EXTERNAL or EXTENDED if check_main is
* false, and must have attstorage MAIN if check_main is true.
*
- * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
- * if not, no benefit is to be expected by compressing it.
+ * The column must have a minimum size of MAXALIGN(tcc_toast_pointer_size);
+ * if not, no benefit is to be expected by compressing it. The TOAST
+ * pointer size is given by the caller, depending on the type of TOAST
+ * table we are dealing with.
*
* The return value is the index of the biggest suitable column, or
* -1 if there is none.
@@ -184,10 +186,14 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
int numAttrs = tupleDesc->natts;
int biggest_attno = -1;
- int32 biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
+ int32 biggest_size = 0;
int32 skip_colflags = TOASTCOL_IGNORE;
int i;
+ /* Define the lower-bound */
+ biggest_size = MAXALIGN(ttc->ttc_toast_pointer_size);
+ Assert(biggest_size != 0);
+
if (for_compression)
skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 3542696f0d99..26508bd01c86 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -92,6 +92,7 @@
#include "access/detoast.h"
#include "access/heapam.h"
#include "access/rewriteheap.h"
+#include "access/toast_external.h"
#include "access/transam.h"
#include "access/xact.h"
#include "access/xlog_internal.h"
@@ -5102,7 +5103,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
struct varlena *varlena;
/* va_rawsize is the size of the original datum -- including header */
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
struct varatt_indirect redirect_pointer;
struct varlena *new_datum = NULL;
struct varlena *reconstructed;
@@ -5132,8 +5133,8 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
if (!VARATT_IS_EXTERNAL(varlena))
continue;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena);
- toast_valueid = toast_pointer.va_valueid;
+ toast_external_info_get_data(varlena, &toast_pointer);
+ toast_valueid = toast_pointer.value;
/*
* Check whether the toast tuple changed, replace if so.
@@ -5151,7 +5152,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
free[natt] = true;
- reconstructed = palloc0(toast_pointer.va_rawsize);
+ reconstructed = palloc0(toast_pointer.rawsize);
ent->reconstructed = reconstructed;
@@ -5176,10 +5177,10 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer));
+ Assert(data_done == toast_pointer.extsize);
/* make sure its marked as compressed or not */
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer))
SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
else
SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 38d4e6d45f28..d76386407a08 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -19,6 +19,7 @@
#include "access/detoast.h"
#include "access/toast_compression.h"
+#include "access/toast_external.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
#include "common/hashfn.h"
@@ -4219,7 +4220,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
{
int typlen;
struct varlena *attr;
- struct varatt_external_oid toast_pointer;
+ uint64 toast_valueid;
/* On first call, get the input type's typlen, and save at *fn_extra */
if (fcinfo->flinfo->fn_extra == NULL)
@@ -4246,9 +4247,9 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
PG_RETURN_NULL();
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_valueid = toast_external_info_get_value(attr);
- PG_RETURN_OID(toast_pointer.va_valueid);
+ PG_RETURN_OID(toast_valueid);
}
/*
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 61250799ec07..f3c6cd8860b5 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -415,7 +415,7 @@ described in more detail below.
<para>
Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_MAX_CHUNK_SIZE</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_MAX_CHUNK_SIZE_OID</symbol> bytes (by default this value is chosen
so that four chunk rows will fit on a page, making it about 2000 bytes).
Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
belonging to the owning table. Every
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 030f9fb64b51..11c4507ae6e2 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -16,6 +16,7 @@
#include "access/multixact.h"
#include "access/relation.h"
#include "access/table.h"
+#include "access/toast_external.h"
#include "access/toast_internals.h"
#include "access/visibilitymap.h"
#include "access/xact.h"
@@ -73,7 +74,8 @@ typedef enum SkipPages
*/
typedef struct ToastedAttribute
{
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
+ const toast_external_info *info;
BlockNumber blkno; /* block in main table */
OffsetNumber offnum; /* offset in main table */
AttrNumber attnum; /* attribute in main table */
@@ -1564,9 +1566,9 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
uint64 toast_valueid;
int32 max_chunk_size;
- toast_valueid = ta->toast_pointer.va_valueid;
+ toast_valueid = ta->toast_pointer.value;
- max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+ max_chunk_size = ta->info->maximum_chunk_size;
last_chunk_seq = (extsize - 1) / max_chunk_size;
/* Sanity-check the sequence number. */
@@ -1672,7 +1674,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
uint16 infomask;
uint64 toast_pointer_valueid;
CompactAttribute *thisatt;
- struct varatt_external_oid toast_pointer;
+ struct toast_external_data toast_pointer;
infomask = ctx->tuphdr->t_infomask;
thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
@@ -1774,28 +1776,28 @@ check_tuple_attribute(HeapCheckContext *ctx)
return true;
/* It is external, and we're looking at a page on disk */
- toast_pointer_valueid = toast_pointer.va_valueid;
/*
* Must copy attr into toast_pointer for alignment considerations
*/
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get_data(attr, &toast_pointer);
+ toast_pointer_valueid = toast_pointer.value;
/* Toasted attributes too large to be untoasted should never be stored */
- if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
+ if (toast_pointer.rawsize > VARLENA_SIZE_LIMIT)
report_corruption(ctx,
psprintf("toast value %" PRIu64 " rawsize %d exceeds limit %d",
toast_pointer_valueid,
- toast_pointer.va_rawsize,
+ toast_pointer.rawsize,
VARLENA_SIZE_LIMIT));
- if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ if (TOAST_EXTERNAL_IS_COMPRESSED(toast_pointer))
{
ToastCompressionId cmid;
bool valid = false;
/* Compressed attributes should have a valid compression method */
- cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+ cmid = toast_pointer.compression_method;
switch (cmid)
{
/* List of all valid compression method IDs */
@@ -1849,7 +1851,8 @@ check_tuple_attribute(HeapCheckContext *ctx)
ta = (ToastedAttribute *) palloc0(sizeof(ToastedAttribute));
- VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+ toast_external_info_get_data(attr, &ta->toast_pointer);
+ ta->info = toast_external_get_info(VARTAG_EXTERNAL(attr));
ta->blkno = ctx->blkno;
ta->offnum = ctx->offnum;
ta->attnum = ctx->attnum;
@@ -1876,9 +1879,11 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
int32 expected_chunk_seq = 0;
int32 last_chunk_seq;
uint64 toast_valueid;
- int32 max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+ int32 max_chunk_size;
- extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer);
+ extsize = ta->toast_pointer.extsize;
+
+ max_chunk_size = ta->info->maximum_chunk_size;
last_chunk_seq = (extsize - 1) / max_chunk_size;
/*
@@ -1887,7 +1892,7 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
- ObjectIdGetDatum(ta->toast_pointer.va_valueid));
+ ObjectIdGetDatum(ta->toast_pointer.value));
/*
* Check if any chunks for this toasted object exist in the toast table,
@@ -1907,7 +1912,7 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
}
systable_endscan_ordered(toastscan);
- toast_valueid = ta->toast_pointer.va_valueid;
+ toast_valueid = ta->toast_pointer.value;
if (!found_toasttup)
report_toast_corruption(ctx, ta,
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e6f2e93b2d6f..995dc1f28208 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -4143,6 +4143,8 @@ timeout_params
timerCA
tlist_vinfo
toast_compress_header
+toast_external_data
+toast_external_info
tokenize_error_callback_arg
transferMode
transfer_thread_arg
--
2.50.0