v1-0011-Add-support-for-64-bit-TOAST-pointers.patch
text/plain
Filename: v1-0011-Add-support-for-64-bit-TOAST-pointers.patch
Type: text/plain
Part: 10
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: format-patch
Series: patch v1-0011
Subject: Add support for 64-bit TOAST pointers
| File | + | − |
|---|---|---|
| contrib/amcheck/verify_heapam.c | 73 | 29 |
| doc/src/sgml/storage.sgml | 7 | 3 |
| src/backend/access/common/detoast.c | 123 | 48 |
| src/backend/access/common/toast_internals.c | 212 | 104 |
| src/backend/access/heap/heaptoast.c | 4 | 1 |
| src/backend/replication/logical/reorderbuffer.c | 36 | 8 |
| src/include/access/detoast.h | 3 | 0 |
| src/include/access/heaptoast.h | 12 | 4 |
| src/include/varatt.h | 67 | 11 |
From e69450b3ee2511ea36552b0b065387280f5b52c3 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Fri, 8 May 2026 14:58:10 +0900
Subject: [PATCH v1 11/11] Add support for 64-bit TOAST pointers
This introduces a new varlena external pointer structure,
varatt_external_oid8, that carries a 64-bit value ID (Oid8) for
TOAST tables using oid8 as their chunk_id type.
XXX: Catalog version bump required.
---
src/include/access/detoast.h | 3 +
src/include/access/heaptoast.h | 16 +-
src/include/varatt.h | 78 ++++-
src/backend/access/common/detoast.c | 171 +++++++---
src/backend/access/common/toast_internals.c | 316 ++++++++++++------
src/backend/access/heap/heaptoast.c | 5 +-
.../replication/logical/reorderbuffer.c | 44 ++-
doc/src/sgml/storage.sgml | 10 +-
contrib/amcheck/verify_heapam.c | 102 ++++--
9 files changed, 537 insertions(+), 208 deletions(-)
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index 2b1428b1e81c..f065ea034e98 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -30,6 +30,9 @@ do { \
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
#define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid))
+/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */
+#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8))
+
/* Size of an EXTERNAL datum that contains an indirection pointer */
#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index aaae6e0fef69..b7ef69a8f56b 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -69,13 +69,14 @@
/*
* When we store an oversize datum externally, we divide it into chunks
- * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must*
- * be small enough that the completed toast-table tuple (including the
- * ID and sequence fields and all overhead) will fit on a page.
+ * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE
+ * data bytes, depending on the chunk_id type of the TOAST table. These
+ * numbers *must* be small enough that the completed toast-table tuple
+ * (including the ID and sequence fields and all overhead) will fit on a page.
* The coding here sets the size on the theory that we want to fit
* EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
*
- * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb.
+ * NB: Changing these values requires an initdb.
*/
#define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
@@ -88,6 +89,13 @@
sizeof(int32) - \
VARHDRSZ)
+#define TOAST_OID8_MAX_CHUNK_SIZE \
+ (EXTERN_TUPLE_MAX_SIZE - \
+ MAXALIGN(SizeofHeapTupleHeader) - \
+ sizeof(Oid8) - \
+ sizeof(int32) - \
+ VARHDRSZ)
+
/* ----------
* heap_toast_insert_or_update -
*
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 33979bbaaad1..6d7ed246fe2f 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -38,6 +38,34 @@ typedef struct varatt_external_oid
Oid va_toastrelid; /* RelID of TOAST table containing it */
} varatt_external_oid;
+/*
+ * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use
+ * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid
+ * except that va_valueid is split into two uint32 halves to enforce a
+ * stricter structure size without padding.
+ *
+ * This struct must not contain any padding, because we sometimes compare
+ * these pointers using memcmp.
+ */
+typedef struct varatt_external_oid8
+{
+ int32 va_rawsize; /* Original data size (includes header) */
+ uint32 va_extinfo; /* External saved size (without header) and
+ * compression method */
+ uint32 va_valueid_lo; /* Low 32 bits of value ID */
+ uint32 va_valueid_hi; /* High 32 bits of value ID */
+ Oid va_toastrelid; /* RelID of TOAST table containing it */
+} varatt_external_oid8;
+
+/* Helper macros to get/set the 64-bit value ID from the lo/hi fields */
+#define VARATT_EXTERNAL_OID8_GET_VALUEID(tp) \
+ ((Oid8) (tp).va_valueid_lo | ((Oid8) (tp).va_valueid_hi << 32))
+#define VARATT_EXTERNAL_OID8_SET_VALUEID(tp, id) \
+ do { \
+ (tp).va_valueid_lo = (uint32) (id); \
+ (tp).va_valueid_hi = (uint32) ((id) >> 32); \
+ } while (0)
+
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
@@ -87,6 +115,7 @@ typedef enum vartag_external
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
+ VARTAG_ONDISK_OID8 = 4,
VARTAG_ONDISK_OID = 18
} vartag_external;
@@ -108,6 +137,8 @@ VARTAG_SIZE(vartag_external tag)
return sizeof(varatt_expanded);
else if (tag == VARTAG_ONDISK_OID)
return sizeof(varatt_external_oid);
+ else if (tag == VARTAG_ONDISK_OID8)
+ return sizeof(varatt_external_oid8);
else
{
Assert(false);
@@ -361,7 +392,19 @@ VARATT_IS_EXTERNAL(const void *PTR)
static inline bool
VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
{
- return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID;
+ vartag_external tag;
+
+ if (!VARATT_IS_EXTERNAL(PTR))
+ return false;
+ tag = VARTAG_EXTERNAL(PTR);
+ return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8);
+}
+
+/* Is varlena datum a pointer to on-disk toasted data with 8-byte value ID? */
+static inline bool
+VARATT_IS_EXTERNAL_ONDISK_OID8(const void *PTR)
+{
+ return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID8;
}
/* Is varlena datum an indirect pointer? */
@@ -519,16 +562,6 @@ VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer)
return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
}
-/* Set size and compress method of an externally-stored varlena datum */
-/* This has to remain a macro; beware multiple evaluations! */
-#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
- do { \
- Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
- } while (0)
-
/*
* Testing whether an externally-stored value is compressed now requires
* comparing size stored in va_extinfo (the actual length of the external data)
@@ -543,4 +576,27 @@ VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer)
(Size) (toast_pointer.va_rawsize - VARHDRSZ);
}
+/*
+ * Same for external Datums; but note argument is a struct
+ * varatt_external_oid8.
+ */
+static inline Size
+VARATT_EXTERNAL_OID8_GET_EXTSIZE(varatt_external_oid8 toast_pointer)
+{
+ return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
+}
+
+static inline uint32
+VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(varatt_external_oid8 toast_pointer)
+{
+ return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
+}
+
+static inline bool
+VARATT_EXTERNAL_OID8_IS_COMPRESSED(varatt_external_oid8 toast_pointer)
+{
+ return VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer) <
+ (Size) (toast_pointer.va_rawsize - VARHDRSZ);
+}
+
#endif
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index a50aea7b6b12..e1919809751c 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -225,40 +225,51 @@ detoast_attr_slice(varlena *attr,
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- varatt_external_oid toast_pointer;
+ int32 extsize;
+ uint32 compress_method;
+ bool is_compressed;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /* Extract extsize and compression info from the appropriate pointer */
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ extsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer);
+ compress_method = VARATT_EXTERNAL_OID8_GET_COMPRESS_METHOD(toast_pointer);
+ is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer);
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ compress_method = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer);
+ is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer);
+ }
/* fast path for non-compressed external datums */
- if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (!is_compressed)
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/*
- * For compressed values, we need to fetch enough slices to decompress
- * at least the requested part (when a prefix is requested).
- * Otherwise, just fetch all slices.
+ * For compressed values, we need to fetch enough slices to
+ * decompress at least the requested part (when a prefix is
+ * requested). Otherwise, just fetch all slices.
+ *
+ * At least for now, if it's LZ4 data, we'll have to fetch the
+ * whole thing, because there doesn't seem to be an API call to
+ * determine how much compressed data we need to be sure of being
+ * able to decompress the required slice.
*/
if (slicelimit >= 0)
{
- int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ int32 max_size = extsize;
- /*
- * Determine maximum amount of compressed data needed for a prefix
- * of a given length (after decompression).
- *
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
- */
- if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) ==
- TOAST_PGLZ_COMPRESSION_ID)
+ if (compress_method == TOAST_PGLZ_COMPRESSION_ID)
max_size = pglz_maximum_compressed_size(slicelimit, max_size);
- /*
- * Fetch enough compressed slices (compressed marker will get set
- * automatically).
- */
preslice = toast_fetch_datum_slice(attr, 0, max_size);
}
else
@@ -344,23 +355,47 @@ toast_fetch_datum(varlena *attr)
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer);
- result = (varlena *) palloc(attrsize + VARHDRSZ);
+ result = (varlena *) palloc(attrsize + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
- SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
+ if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer))
+ SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
+ else
+ SET_VARSIZE(result, attrsize + VARHDRSZ);
+ }
else
- SET_VARSIZE(result, attrsize + VARHDRSZ);
+ {
+ varatt_external_oid toast_pointer;
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = toast_pointer.va_valueid;
+
+ result = (varlena *) palloc(attrsize + VARHDRSZ);
+
+ if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
+ else
+ SET_VARSIZE(result, attrsize + VARHDRSZ);
+ }
if (attrsize == 0)
return result; /* Probably shouldn't happen, but just in
@@ -369,10 +404,10 @@ toast_fetch_datum(varlena *attr)
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, 0, attrsize, result);
/* Close toast table */
@@ -398,23 +433,43 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
+ bool is_compressed;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer);
+ is_compressed = VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer);
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toastrelid = toast_pointer.va_toastrelid;
+ valueid = toast_pointer.va_valueid;
+ is_compressed = VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer);
+ }
/*
* It's nonsense to fetch slices of a compressed datum unless when it's a
* prefix -- this isn't lo_* we can't return a compressed datum which is
* meaningful to toast later.
*/
- Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
-
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ Assert(!is_compressed || 0 == sliceoffset);
if (sliceoffset >= attrsize)
{
@@ -427,7 +482,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
* space required by va_tcinfo, which is stored at the beginning as an
* int32 value.
*/
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0)
+ if (is_compressed && slicelength > 0)
slicelength = slicelength + sizeof(int32);
/*
@@ -440,7 +495,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
result = (varlena *) palloc(slicelength + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
else
SET_VARSIZE(result, slicelength + VARHDRSZ);
@@ -449,10 +504,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
return result; /* Can save a lot of work at this point! */
/* Open the toast relation */
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, sliceoffset, slicelength,
result);
@@ -550,10 +605,20 @@ toast_raw_datum_size(Datum value)
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/* va_rawsize is the size of the original datum -- including header */
- varatt_external_oid toast_pointer;
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_rawsize;
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ result = toast_pointer.va_rawsize;
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ result = toast_pointer.va_rawsize;
+ }
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
@@ -610,10 +675,20 @@ toast_datum_size(Datum value)
* compressed or not. We do not count the size of the toast pointer
* ... should we?
*/
- varatt_external_oid toast_pointer;
+ if (VARTAG_EXTERNAL(attr) == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ result = VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer);
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ }
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index e9ced27baf7e..a4d56ae3a967 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -125,7 +125,6 @@ toast_save_datum(Relation rel, Datum value,
TupleDesc toasttupDesc;
CommandId mycid = GetCurrentCommandId(true);
varlena *result;
- varatt_external_oid toast_pointer;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
@@ -133,9 +132,17 @@ toast_save_datum(Relation rel, Datum value,
int num_indexes;
int validIndex;
Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1);
+ int32 max_chunk_size;
+
+ /* Fields that will be assembled into the TOAST pointer at the end */
+ int32 va_rawsize;
+ uint32 va_extinfo;
+ Oid8 va_valueid;
+ Oid va_toastrelid;
Assert(!VARATT_IS_EXTERNAL(dval));
Assert(OidIsValid(toast_typid));
+ Assert(toast_typid == OIDOID || toast_typid == OID8OID);
/*
* Open the toast relation and its indexes. We can use the index to check
@@ -165,28 +172,32 @@ toast_save_datum(Relation rel, Datum value,
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
- toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = data_todo + VARHDRSZ; /* as if not short */
+ va_extinfo = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
+ uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
- toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
+ va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
/* set external size and compression method */
- VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
- VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+ Assert(cmid == TOAST_PGLZ_COMPRESSION_ID ||
+ cmid == TOAST_LZ4_COMPRESSION_ID);
+ va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS);
+
/* Assert that the numbers look like it's compressed */
- Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer));
+ Assert((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
- toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = VARSIZE(dval);
+ va_extinfo = data_todo;
}
/*
@@ -198,13 +209,14 @@ toast_save_datum(Relation rel, Datum value,
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
- toast_pointer.va_toastrelid = rel->rd_toastoid;
+ va_toastrelid = rel->rd_toastoid;
else
- toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
+ va_toastrelid = RelationGetRelid(toastrel);
/*
- * Choose a new value to use as the value ID for this toast value, be it
- * for OID or int8-based TOAST relations.
+ * Choose a new value to use as the value ID for this toast value, and
+ * determine the maximum chunk size based on the TOAST table's chunk_id
+ * type.
*
* Normally we just choose an unused value within the toast table. But
* during table-rewriting operations where we are preserving an existing
@@ -213,84 +225,125 @@ toast_save_datum(Relation rel, Datum value,
* toast table, re-use its value ID. If we didn't have a prior external
* value (which is a corner case, but possible if the table's attstorage
* options have been changed), we have to pick a value ID that doesn't
- * conflict with either new or existing toast value IDs. If the TOAST
- * table uses 8-byte value IDs, we should not really care much about
- * that.
+ * conflict with either new or existing toast value IDs. For Oid8 tables,
+ * value conflicts are not a concern.
*/
- if (!OidIsValid(rel->rd_toastoid))
+ if (toast_typid == OID8OID)
{
- /* normal case: just choose an unused OID */
- if (toast_typid == OIDOID)
- toast_pointer.va_valueid =
- GetNewOidWithIndex(toastrel,
- RelationGetRelid(toastidxs[validIndex]),
- (AttrNumber) 1);
- else if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
+ if (!OidIsValid(rel->rd_toastoid))
+ {
+ /* normal case: just choose a new Oid8 */
+ va_valueid = GetNewObjectId8();
+ }
else
- Assert(false);
+ {
+ /* rewrite case: check to see if value was in old toast table */
+ va_valueid = InvalidOid8;
+ if (oldexternal != NULL)
+ {
+ varatt_external_oid8 old_toast_pointer;
+
+ Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
+ if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
+ {
+ /* This value came from the old toast table; reuse its ID */
+ va_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(old_toast_pointer);
+
+ /*
+ * Check if this value already exists in the new toast
+ * table (corner case during table rewrite with multiple
+ * versions of the same row).
+ */
+ if (toastrel_valueid_exists(toastrel, va_valueid))
+ {
+ /* Match, so short-circuit the data storage loop below */
+ data_todo = 0;
+ }
+ }
+ }
+ if (va_valueid == InvalidOid8)
+ {
+ /*
+ * New value. For Oid8 we just pick a new ID without worrying
+ * about conflicts.
+ */
+ va_valueid = GetNewObjectId8();
+ }
+ }
+
+ max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE;
}
else
{
- /* rewrite case: check to see if value was in old toast table */
- toast_pointer.va_valueid = InvalidOid;
- if (oldexternal != NULL)
+ if (!OidIsValid(rel->rd_toastoid))
{
- varatt_external_oid old_toast_pointer;
-
- Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
- if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
+ /* normal case: just choose an unused OID */
+ va_valueid =
+ GetNewOidWithIndex(toastrel,
+ RelationGetRelid(toastidxs[validIndex]),
+ (AttrNumber) 1);
+ }
+ else
+ {
+ /* rewrite case: check to see if value was in old toast table */
+ va_valueid = InvalidOid;
+ if (oldexternal != NULL)
{
- /* This value came from the old toast table; reuse its OID */
- toast_pointer.va_valueid = old_toast_pointer.va_valueid;
+ varatt_external_oid old_toast_pointer;
- /*
- * There is a corner case here: the table rewrite might have
- * to copy both live and recently-dead versions of a row, and
- * those versions could easily reference the same toast value.
- * When we copy the second or later version of such a row,
- * reusing the OID will mean we select an OID that's already
- * in the new toast table. Check for that, and if so, just
- * fall through without writing the data again.
- *
- * While annoying and ugly-looking, this is a good thing
- * because it ensures that we wind up with only one copy of
- * the toast value when there is only one copy in the old
- * toast table. Before we detected this case, we'd have made
- * multiple copies, wasting space; and what's worse, the
- * copies belonging to already-deleted heap tuples would not
- * be reclaimed by VACUUM.
- */
- if (toastrel_valueid_exists(toastrel,
- toast_pointer.va_valueid))
+ Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
+ if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
{
- /* Match, so short-circuit the data storage loop below */
- data_todo = 0;
+ /* This value came from the old toast table; reuse its OID */
+ va_valueid = old_toast_pointer.va_valueid;
+
+ /*
+ * There is a corner case here: the table rewrite might
+ * have to copy both live and recently-dead versions of a
+ * row, and those versions could easily reference the same
+ * toast value. When we copy the second or later version
+ * of such a row, reusing the OID will mean we select an
+ * OID that's already in the new toast table. Check for
+ * that, and if so, just fall through without writing the
+ * data again.
+ *
+ * While annoying and ugly-looking, this is a good thing
+ * because it ensures that we wind up with only one copy
+ * of the toast value when there is only one copy in the
+ * old toast table. Before we detected this case, we'd
+ * have made multiple copies, wasting space; and what's
+ * worse, the copies belonging to already-deleted heap
+ * tuples would not be reclaimed by VACUUM.
+ */
+ if (toastrel_valueid_exists(toastrel, va_valueid))
+ {
+ /* Match, so short-circuit the data storage loop below */
+ data_todo = 0;
+ }
}
}
- }
- if (toast_pointer.va_valueid == InvalidOid)
- {
- /*
- * new value; must choose a value that doesn't conflict in either
- * old or new toast table.
- */
- if (toast_typid == OIDOID)
+ if (va_valueid == InvalidOid)
{
+ /*
+ * new value; must choose a value that doesn't conflict in
+ * either old or new toast table.
+ */
do
{
- toast_pointer.va_valueid =
+ va_valueid =
GetNewOidWithIndex(toastrel,
RelationGetRelid(toastidxs[validIndex]),
(AttrNumber) 1);
} while (toastid_valueid_exists(rel->rd_toastoid,
- toast_pointer.va_valueid));
+ va_valueid));
}
- else if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
}
+
+ max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
}
/*
@@ -314,15 +367,15 @@ toast_save_datum(Relation rel, Datum value,
/*
* Calculate the size of this chunk
*/
- chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo);
+ chunk_size = Min(max_chunk_size, data_todo);
/*
* Build a tuple and store it
*/
- if (toast_typid == OIDOID)
- t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
- else if (toast_typid == OID8OID)
- t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid);
+ if (toast_typid == OID8OID)
+ t_values[0] = ObjectId8GetDatum(va_valueid);
+ else
+ t_values[0] = ObjectIdGetDatum((Oid) va_valueid);
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
@@ -376,11 +429,34 @@ toast_save_datum(Relation rel, Datum value,
table_close(toastrel, NoLock);
/*
- * Create the TOAST pointer value that we'll return
+ * Create the TOAST pointer value that we'll return.
*/
- result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
- SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
- memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ if (toast_typid == OID8OID)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ VARATT_EXTERNAL_OID8_SET_VALUEID(toast_pointer, va_valueid);
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ toast_pointer.va_valueid = (Oid) va_valueid;
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
return PointerGetDatum(result);
}
@@ -395,7 +471,6 @@ void
toast_delete_datum(Relation rel, Datum value, bool is_speculative)
{
varlena *attr = (varlena *) DatumGetPointer(value);
- varatt_external_oid toast_pointer;
Relation toastrel;
Relation *toastidxs;
ScanKeyData toastkey;
@@ -403,42 +478,75 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
HeapTuple toasttup;
int num_indexes;
int validIndex;
- Oid toast_typid;
+ Oid toastrelid;
+ vartag_external tag;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
return;
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
-
/*
- * Open the toast relation and its indexes
+ * Determine the pointer type from the datum's vartag and extract the
+ * toast relation OID and value ID accordingly. The vartag tells us
+ * everything we need — no TOAST table schema lookup required.
*/
- toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
- toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid;
- Assert(toast_typid == OIDOID || toast_typid == OID8OID);
+ tag = VARTAG_EXTERNAL(attr);
- /* Fetch valid relation used for process */
- validIndex = toast_open_indexes(toastrel,
- RowExclusiveLock,
- &toastidxs,
- &num_indexes);
+ if (tag == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer8;
- /*
- * Setup a scan key to find chunks with matching va_valueid
- */
- if (toast_typid == OIDOID)
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr);
+ toastrelid = toast_pointer8.va_toastrelid;
+
+ /*
+ * Open the toast relation and its indexes
+ */
+ toastrel = table_open(toastrelid, RowExclusiveLock);
+
+ /* Fetch valid relation used for process */
+ validIndex = toast_open_indexes(toastrel,
+ RowExclusiveLock,
+ &toastidxs,
+ &num_indexes);
+
+ /*
+ * Setup a scan key to find chunks with matching va_valueid
+ */
+ ScanKeyInit(&toastkey,
+ (AttrNumber) 1,
+ BTEqualStrategyNumber, F_OID8EQ,
+ ObjectId8GetDatum(VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8)));
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ Assert(tag == VARTAG_ONDISK_OID);
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toastrelid = toast_pointer.va_toastrelid;
+
+ /*
+ * Open the toast relation and its indexes
+ */
+ toastrel = table_open(toastrelid, RowExclusiveLock);
+
+ /* Fetch valid relation used for process */
+ validIndex = toast_open_indexes(toastrel,
+ RowExclusiveLock,
+ &toastidxs,
+ &num_indexes);
+
+ /*
+ * Setup a scan key to find chunks with matching va_valueid
+ */
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(toast_pointer.va_valueid));
- else if (toast_typid == OID8OID)
- ScanKeyInit(&toastkey,
- (AttrNumber) 1,
- BTEqualStrategyNumber, F_OID8EQ,
- ObjectId8GetDatum(toast_pointer.va_valueid));
- else
- Assert(false);
+ }
/*
* Find all the chunks. (We don't actually care whether we see them in
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index a93e98788da7..364aec11fc88 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -650,7 +650,10 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize,
toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid;
Assert(toast_typid == OIDOID || toast_typid == OID8OID);
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ if (toast_typid == OID8OID)
+ max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE;
+ else
+ max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
totalchunks = ((attrsize - 1) / max_chunk_size) + 1;
startchunk = sliceoffset / max_chunk_size;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index a7b524906995..6c71fcee5816 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -5135,12 +5135,14 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
/* va_rawsize is the size of the original datum -- including header */
varatt_external_oid toast_pointer;
+ varatt_external_oid8 toast_pointer8;
varatt_indirect redirect_pointer;
varlena *new_datum = NULL;
varlena *reconstructed;
dlist_iter it;
Size data_done = 0;
Oid8 toast_valueid;
+ int32 rawsize;
if (attr->attisdropped)
continue;
@@ -5160,8 +5162,19 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
if (!VARATT_IS_EXTERNAL(varlena_pointer))
continue;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer);
- toast_valueid = toast_pointer.va_valueid;
+ /* Branch on vartag to handle both pointer types */
+ if (VARTAG_EXTERNAL(varlena_pointer) == VARTAG_ONDISK_OID8)
+ {
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer8, varlena_pointer);
+ toast_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8);
+ rawsize = toast_pointer8.va_rawsize;
+ }
+ else
+ {
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer);
+ toast_valueid = toast_pointer.va_valueid;
+ rawsize = toast_pointer.va_rawsize;
+ }
/*
* Check whether the toast tuple changed, replace if so.
@@ -5179,7 +5192,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
free[natt] = true;
- reconstructed = palloc0(toast_pointer.va_rawsize);
+ reconstructed = palloc0(rawsize);
ent->reconstructed = reconstructed;
@@ -5204,13 +5217,28 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer));
- /* make sure its marked as compressed or not */
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
- SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
+ /* Verify size and set compression status based on pointer type */
+ if (VARTAG_EXTERNAL(varlena_pointer) == VARTAG_ONDISK_OID8)
+ {
+ Assert(data_done == VARATT_EXTERNAL_OID8_GET_EXTSIZE(toast_pointer8));
+
+ /* make sure its marked as compressed or not */
+ if (VARATT_EXTERNAL_OID8_IS_COMPRESSED(toast_pointer8))
+ SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
+ else
+ SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
+ }
else
- SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
+ {
+ Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer));
+
+ /* make sure its marked as compressed or not */
+ if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
+ else
+ SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
+ }
memset(&redirect_pointer, 0, sizeof(redirect_pointer));
redirect_pointer.pointer = reconstructed;
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 7afe435747d2..996c2f26d737 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -417,7 +417,9 @@ described in more detail below.
<para>
Out-of-line values are divided (after compression if used) into chunks of at
-most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> bytes (by default this value is chosen
+most <symbol>TOAST_OID_MAX_CHUNK_SIZE</symbol> or
+<symbol>TOAST_OID8_MAX_CHUNK_SIZE</symbol> bytes depending on the
+<structfield>chunk_id</structfield> type (by default this value is chosen
so that four chunk rows will fit on a page, making it about 2000 bytes).
Each chunk is stored as a separate row in the <acronym>TOAST</acronym> table
belonging to the owning table. Every
@@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk
logical datum size (original uncompressed data length), physical stored size
(different if compression was applied), and the compression method used, if
any. Allowing for the varlena header bytes,
-the total size of an on-disk <acronym>TOAST</acronym> pointer datum is therefore 18
-bytes regardless of the actual size of the represented value.
+the total size of an on-disk <acronym>TOAST</acronym> pointer datum is 18
+bytes when using an OID as <structfield>chunk_id</structfield>, or 22 bytes
+when using an 8-byte integer, regardless of the actual size of the represented
+value.
</para>
<para>
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index ad6c37eefae6..0e1c83958af3 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -75,7 +75,9 @@ typedef enum SkipPages
*/
typedef struct ToastedAttribute
{
- varatt_external_oid toast_pointer;
+ vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */
+ Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */
+ uint32 va_extinfo; /* external size and compression method */
BlockNumber blkno; /* block in main table */
OffsetNumber offnum; /* offset in main table */
AttrNumber attnum; /* attribute in main table */
@@ -1566,9 +1568,11 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
Oid8 toast_valueid;
int32 max_chunk_size;
- toast_valueid = ta->toast_pointer.va_valueid;
+ toast_valueid = ta->va_valueid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ max_chunk_size = ta->tag == VARTAG_ONDISK_OID8
+ ? TOAST_OID8_MAX_CHUNK_SIZE
+ : TOAST_OID_MAX_CHUNK_SIZE;
last_chunk_seq = (extsize - 1) / max_chunk_size;
/* Sanity-check the sequence number. */
@@ -1673,8 +1677,10 @@ check_tuple_attribute(HeapCheckContext *ctx)
char *tp; /* pointer to the tuple data */
uint16 infomask;
Oid8 toast_pointer_valueid;
+ int32 va_rawsize;
+ uint32 va_extinfo;
CompactAttribute *thisatt;
- varatt_external_oid toast_pointer;
+ vartag_external va_tag_value;
infomask = ctx->tuphdr->t_infomask;
thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
@@ -1733,7 +1739,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
{
uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
- if (va_tag != VARTAG_ONDISK_OID)
+ if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8)
{
report_corruption(ctx,
psprintf("toasted attribute has unexpected TOAST tag %u",
@@ -1778,26 +1784,46 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* It is external, and we're looking at a page on disk */
/*
- * Must copy attr into toast_pointer for alignment considerations
+ * Must copy attr into toast_pointer for alignment considerations.
+ * Branch on the tag to determine which pointer type to extract.
*/
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- toast_pointer_valueid = toast_pointer.va_valueid;
+ va_tag_value = VARTAG_EXTERNAL(attr);
+ if (va_tag_value == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer8;
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer8, attr);
+ toast_pointer_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer8);
+ va_rawsize = toast_pointer8.va_rawsize;
+ va_extinfo = toast_pointer8.va_extinfo;
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_pointer_valueid = toast_pointer.va_valueid;
+ va_rawsize = toast_pointer.va_rawsize;
+ va_extinfo = toast_pointer.va_extinfo;
+ }
/* Toasted attributes too large to be untoasted should never be stored */
- if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
+ if (va_rawsize > VARLENA_SIZE_LIMIT)
report_corruption(ctx,
psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d",
toast_pointer_valueid,
- toast_pointer.va_rawsize,
+ va_rawsize,
VARLENA_SIZE_LIMIT));
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if ((va_extinfo & VARLENA_EXTSIZE_MASK) < (Size) (va_rawsize - VARHDRSZ))
{
ToastCompressionId cmid;
bool valid = false;
/* Compressed attributes should have a valid compression method */
- cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+ cmid = va_extinfo >> VARLENA_EXTSIZE_BITS;
switch (cmid)
{
/* List of all valid compression method IDs */
@@ -1851,7 +1877,23 @@ check_tuple_attribute(HeapCheckContext *ctx)
ta = palloc0_object(ToastedAttribute);
- VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+ ta->tag = va_tag_value;
+ if (va_tag_value == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 tp;
+
+ VARATT_EXTERNAL_GET_POINTER(tp, attr);
+ ta->va_valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(tp);
+ ta->va_extinfo = tp.va_extinfo;
+ }
+ else
+ {
+ varatt_external_oid tp;
+
+ VARATT_EXTERNAL_GET_POINTER(tp, attr);
+ ta->va_valueid = tp.va_valueid;
+ ta->va_extinfo = tp.va_extinfo;
+ }
ta->blkno = ctx->blkno;
ta->offnum = ctx->offnum;
ta->attnum = ctx->attnum;
@@ -1879,29 +1921,33 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
int32 last_chunk_seq;
Oid8 toast_valueid;
int32 max_chunk_size;
- Oid toast_typid;
- toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
-
- extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer);
- last_chunk_seq = (extsize - 1) / max_chunk_size;
+ toast_valueid = ta->va_valueid;
+ extsize = ta->va_extinfo & VARLENA_EXTSIZE_MASK;
/*
* Setup a scan key to find chunks in toast table with matching va_valueid
*/
- if (toast_typid == OIDOID)
- ScanKeyInit(&toastkey,
- (AttrNumber) 1,
- BTEqualStrategyNumber, F_OIDEQ,
- ObjectIdGetDatum(ta->toast_pointer.va_valueid));
- else if (toast_typid == OID8OID)
+ if (ta->tag == VARTAG_ONDISK_OID8)
+ {
+ max_chunk_size = TOAST_OID8_MAX_CHUNK_SIZE;
+
ScanKeyInit(&toastkey,
(AttrNumber) 1,
BTEqualStrategyNumber, F_OID8EQ,
- ObjectId8GetDatum(ta->toast_pointer.va_valueid));
+ ObjectId8GetDatum(toast_valueid));
+ }
else
- Assert(false);
+ {
+ max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+
+ ScanKeyInit(&toastkey,
+ (AttrNumber) 1,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum((Oid) toast_valueid));
+ }
+
+ last_chunk_seq = (extsize - 1) / max_chunk_size;
/*
* Check if any chunks for this toasted object exist in the toast table,
@@ -1921,8 +1967,6 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
}
systable_endscan_ordered(toastscan);
- toast_valueid = ta->toast_pointer.va_valueid;
-
if (!found_toasttup)
report_toast_corruption(ctx, ta,
psprintf("toast value " OID8_FORMAT " not found in toast table",
--
2.54.0