0001-Fix-btree_gist-strategy-on-internal-index-pages.patch

From a53b3f59a7d01f01d21d7f19fd6d34a9a3860228 Mon Sep 17 00:00:00 2001
From: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
Date: Wed, 17 Jun 2026 22:52:36 +0530
Subject: [PATCH v1] Fix btree_gist <> strategy on internal index pages

gbt_var_consistent() handled the <> (BtreeGistNotEqual) strategy without
distinguishing leaf from internal pages, unlike every other strategy.
Internal keys are lossy, truncated prefixes, and for bit/varbit the
leaf-to-node transform drops the VarBit length header, so such a key is
not a valid value of the indexed type.

Passing it to the type's equality function is a type confusion.  For
bit/varbit it reaches bit_cmp(), where VARBITBYTES() computes VARSIZE - 8;
a key with VARSIZE < 8 yields a negative length that, passed to memcmp()
as size_t, becomes huge and reads out of bounds (eventually crashing the
backend).  Even without a crash it can give wrong equality results and
defeat an exclusion constraint declared with <>.

Make the <> branch distinguish leaf and internal pages: on a leaf, negate
the equality test against the stored value; on an internal page, recurse
unconditionally.  The fix is in the shared gbt_var_consistent() and so
covers every variable-length type.  Add a regression test.
---
 contrib/btree_gist/btree_utils_var.c      | 14 +++++++++++--
 contrib/btree_gist/expected/not_equal.out | 24 +++++++++++++++++++++++
 contrib/btree_gist/sql/not_equal.sql      | 18 +++++++++++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/contrib/btree_gist/btree_utils_var.c b/contrib/btree_gist/btree_utils_var.c
index 25c3bbe8eac..6043dd811bc 100644
--- a/contrib/btree_gist/btree_utils_var.c
+++ b/contrib/btree_gist/btree_utils_var.c
@@ -611,8 +611,18 @@ gbt_var_consistent(GBT_VARKEY_R *key,
 					|| gbt_var_node_pf_match(key, query, tinfo);
 			break;
 		case BtreeGistNotEqualStrategyNumber:
-			retval = !(tinfo->f_eq(query, key->lower, collation, flinfo) &&
-					   tinfo->f_eq(query, key->upper, collation, flinfo));
+
+			/*
+			 * Must distinguish leaf and internal pages.  Internal keys are
+			 * lossy, truncated prefixes whose representation may differ from
+			 * the leaf form (e.g. bit/varbit), so f_eq() on them can give
+			 * wrong answers or read out of bounds.  An internal page may
+			 * always hold a non-equal value, so just recurse into it.
+			 */
+			if (is_leaf)
+				retval = !tinfo->f_eq(query, key->lower, collation, flinfo);
+			else
+				retval = true;
 			break;
 		default:
 			retval = false;
diff --git a/contrib/btree_gist/expected/not_equal.out b/contrib/btree_gist/expected/not_equal.out
index 85b1e868a87..922950e0c2f 100644
--- a/contrib/btree_gist/expected/not_equal.out
+++ b/contrib/btree_gist/expected/not_equal.out
@@ -39,3 +39,27 @@ INSERT INTO zoo VALUES(123, 'lion');
 ERROR:  conflicting key value violates exclusion constraint "zoo_cage_animal_excl"
 DETAIL:  Key (cage, animal)=(123, lion) conflicts with existing key (cage, animal)=(123, zebra).
 INSERT INTO zoo VALUES(124, 'lion');
+-- A <> search must descend internal pages instead of comparing the query to a
+-- truncated internal key.  Use a multi-level index and check it agrees with seq scan.
+CREATE TABLE bitne (a bit(8));
+INSERT INTO bitne SELECT (g % 256)::bit(8) FROM generate_series(1, 20000) g;
+CREATE INDEX bitne_idx ON bitne USING gist (a);
+SET enable_seqscan to on;
+SET enable_indexscan to off;
+SET enable_bitmapscan to off;
+SELECT count(*) FROM bitne WHERE a <> B'00000000';
+ count 
+-------
+ 19922
+(1 row)
+
+SET enable_seqscan to off;
+SET enable_indexscan to on;
+SET enable_bitmapscan to on;
+SELECT count(*) FROM bitne WHERE a <> B'00000000';
+ count 
+-------
+ 19922
+(1 row)
+
+DROP TABLE bitne;
diff --git a/contrib/btree_gist/sql/not_equal.sql b/contrib/btree_gist/sql/not_equal.sql
index 6dfac5d0aae..ecb135f47d2 100644
--- a/contrib/btree_gist/sql/not_equal.sql
+++ b/contrib/btree_gist/sql/not_equal.sql
@@ -34,3 +34,21 @@ INSERT INTO zoo VALUES(123, 'zebra');
 INSERT INTO zoo VALUES(123, 'zebra');
 INSERT INTO zoo VALUES(123, 'lion');
 INSERT INTO zoo VALUES(124, 'lion');
+
+-- A <> search must descend internal pages instead of comparing the query to a
+-- truncated internal key.  Use a multi-level index and check it agrees with seq scan.
+CREATE TABLE bitne (a bit(8));
+INSERT INTO bitne SELECT (g % 256)::bit(8) FROM generate_series(1, 20000) g;
+CREATE INDEX bitne_idx ON bitne USING gist (a);
+
+SET enable_seqscan to on;
+SET enable_indexscan to off;
+SET enable_bitmapscan to off;
+SELECT count(*) FROM bitne WHERE a <> B'00000000';
+
+SET enable_seqscan to off;
+SET enable_indexscan to on;
+SET enable_bitmapscan to on;
+SELECT count(*) FROM bitne WHERE a <> B'00000000';
+
+DROP TABLE bitne;
-- 
2.34.1