v4-0003-amcheck-avoid-failing-on-oversized-tuples.patch

From aa611847e8fee0994b35cda881058cc74a7a5ab0 Mon Sep 17 00:00:00 2001
From: "Andrey M. Borodin" <x4mmm@night.local>
Date: Tue, 23 Jan 2024 23:03:28 +0500
Subject: [PATCH v4 3/3] amcheck: avoid failing on oversized tuples

Due to changes in toast policies, some heap tuples might become too
big index tuples. This commit prevents ERRORs in this case, because
this anomaly does not create dangerous conditions to DMS. However,
the database cannot be dumped-restored, so we emit a NOTICE.

Reported-by: Alexander Lakhin
---
 contrib/amcheck/expected/check_btree.out | 14 +++++++
 contrib/amcheck/sql/check_btree.sql      | 11 +++++
 contrib/amcheck/verify_nbtree.c          | 53 +++++++++++++++++++++---
 3 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/contrib/amcheck/expected/check_btree.out b/contrib/amcheck/expected/check_btree.out
index f8638582180..0ea2e3e1f38 100644
--- a/contrib/amcheck/expected/check_btree.out
+++ b/contrib/amcheck/expected/check_btree.out
@@ -268,6 +268,20 @@ SELECT bt_index_check('tbl_idx', true);
  
 (1 row)
 
+DROP TABLE tbl;
+-- Check oversized datums that cannot be inserted into index
+CREATE TABLE t(f1 text);
+CREATE INDEX t_idx ON t(f1);
+INSERT INTO t VALUES(repeat('1234567890', 1000));
+ALTER TABLE t ALTER COLUMN f1 SET STORAGE plain;
+SELECT bt_index_check('t_idx', true);
+NOTICE:  Index contain tuples that cannot fit into index page, if toasted with current toast policy
+ bt_index_check 
+----------------
+ 
+(1 row)
+
+DROP TABLE t;
 -- cleanup
 DROP TABLE bttest_a;
 DROP TABLE bttest_b;
diff --git a/contrib/amcheck/sql/check_btree.sql b/contrib/amcheck/sql/check_btree.sql
index 5da49ea94fd..ceeb7258754 100644
--- a/contrib/amcheck/sql/check_btree.sql
+++ b/contrib/amcheck/sql/check_btree.sql
@@ -171,6 +171,17 @@ INSERT INTO tbl VALUES (repeat('Test', 250));
 ALTER TABLE tbl ALTER COLUMN t SET STORAGE extended;
 
 SELECT bt_index_check('tbl_idx', true);
+DROP TABLE tbl;
+
+-- Check oversized datums that cannot be inserted into index
+
+CREATE TABLE t(f1 text);
+CREATE INDEX t_idx ON t(f1);
+INSERT INTO t VALUES(repeat('1234567890', 1000));
+ALTER TABLE t ALTER COLUMN f1 SET STORAGE plain;
+
+SELECT bt_index_check('t_idx', true);
+DROP TABLE t;
 
 -- cleanup
 DROP TABLE bttest_a;
diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c
index 6654b5afe73..e30f162e0fb 100644
--- a/contrib/amcheck/verify_nbtree.c
+++ b/contrib/amcheck/verify_nbtree.c
@@ -128,6 +128,11 @@ typedef struct BtreeCheckState
 	bloom_filter *filter;
 	/* Debug counter */
 	int64		heaptuplespresent;
+	/*
+	 * During check we might find tuples that due to current TOAST policies
+	 * should not reside in index, but still are there.
+	 */
+	bool has_oversized_tuples;
 } BtreeCheckState;
 
 /*
@@ -1624,10 +1629,17 @@ bt_target_page_check(BtreeCheckState *state)
 
 					logtuple = bt_posting_plain_tuple(itup, i);
 					norm = bt_normalize_tuple(state, logtuple);
-					bloom_add_element(state->filter, (unsigned char *) norm,
+					if (norm == NULL)
+					{
+						if (!state->has_oversized_tuples)
+							elog(NOTICE, "Index contain tuples that cannot fit into index page, if toasted with current toast policy");
+						state->has_oversized_tuples = true;
+					}
+					else
+						bloom_add_element(state->filter, (unsigned char *) norm,
 									  IndexTupleSize(norm));
 					/* Be tidy */
-					if (norm != logtuple)
+					if (norm != logtuple && norm != NULL)
 						pfree(norm);
 					pfree(logtuple);
 				}
@@ -1635,10 +1647,17 @@ bt_target_page_check(BtreeCheckState *state)
 			else
 			{
 				norm = bt_normalize_tuple(state, itup);
-				bloom_add_element(state->filter, (unsigned char *) norm,
-								  IndexTupleSize(norm));
+				if (norm == NULL)
+				{
+					if (!state->has_oversized_tuples)
+						elog(NOTICE, "Index contain tuples that cannot fit into index page, if toasted with current toast policy");
+					state->has_oversized_tuples = true;
+				}
+				else
+					bloom_add_element(state->filter, (unsigned char *) norm,
+								IndexTupleSize(norm));
 				/* Be tidy */
-				if (norm != itup)
+				if (norm != itup && norm != NULL)
 					pfree(norm);
 			}
 		}
@@ -2883,9 +2902,19 @@ bt_tuple_present_callback(Relation index, ItemPointer tid, Datum *values,
 	itup = index_form_tuple(RelationGetDescr(index), values, isnull);
 	itup->t_tid = *tid;
 	norm = bt_normalize_tuple(state, itup);
+	if (norm == NULL)
+	{
+		if (state->has_oversized_tuples)
+		{
+			/* exempt this oversized tuple */
+			state->heaptuplespresent++;
+			pfree(itup);
+			return;
+		}
+	}
 
 	/* Probe Bloom filter -- tuple should be present */
-	if (bloom_lacks_element(state->filter, (unsigned char *) norm,
+	if ((norm == NULL) || bloom_lacks_element(state->filter, (unsigned char *) norm,
 							IndexTupleSize(norm)))
 		ereport(ERROR,
 				(errcode(ERRCODE_DATA_CORRUPTED),
@@ -2936,6 +2965,9 @@ bt_tuple_present_callback(Relation index, ItemPointer tid, Datum *values,
  * Caller does normalization for non-pivot tuples that have a posting list,
  * since dummy CREATE INDEX callback code generates new tuples with the same
  * normalized representation.
+ * 
+ * If the tuple is exampt from checking due to has_oversized_tuples this function
+ * returns NULL.
  */
 static IndexTuple
 bt_normalize_tuple(BtreeCheckState *state, IndexTuple itup)
@@ -2947,6 +2979,7 @@ bt_normalize_tuple(BtreeCheckState *state, IndexTuple itup)
 	bool		formnewtup = false;
 	IndexTuple	reformed;
 	int			i;
+	Size		data_size;
 
 	/* Caller should only pass "logical" non-pivot tuples here */
 	Assert(!BTreeTupleIsPosting(itup) && !BTreeTupleIsPivot(itup));
@@ -3026,6 +3059,14 @@ bt_normalize_tuple(BtreeCheckState *state, IndexTuple itup)
 	if (!formnewtup)
 		return itup;
 
+	data_size = MAXALIGN(heap_compute_data_size(tupleDescriptor,
+									   normalized, isnull)
+				+ MAXALIGN(sizeof(IndexTupleData) + sizeof(IndexAttributeBitMapData)));
+	if ((data_size & INDEX_SIZE_MASK) != data_size)
+	{
+		return NULL;
+	}
+
 	/*
 	 * Hard case: Tuple had compressed varlena datums that necessitate
 	 * creating normalized version of the tuple from uncompressed input datums
-- 
2.43.0