0001-Fix-int32-overflow-in-ltree_compare.patch

From 5af86ee09cc3a28b22067ad543919f0cbcc31157 Mon Sep 17 00:00:00 2001
From: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
Date: Sat, 13 Jun 2026 11:35:20 +0530
Subject: [PATCH v1] Fix int32 overflow in ltree_compare()

The expression (len_diff * 10 * (an + 1)) used as the return value of
ltree_compare() is computed at int32 width.  With LTREE_MAX_LEVELS =
65535, the product can exceed INT32_MAX once an ltree has more than
~14,653 levels, which causes the result to wrap and invert its sign.
That corrupts btree ordering as well as the magnitude consumed by
ltree_penalty() for GiST page splits.

Widen the multiplication to int64 and saturate the result to
PG_INT32_MIN / PG_INT32_MAX before returning.  Saturation preserves both
the sign needed by btree comparators and the "deeper mismatch yields
larger magnitude" property used by ltree_penalty().  Returning INT_MIN
from a comparator has been explicitly allowed since commit 6e63e069751.

ltree_penalty() sums the two comparator results, so a pair of saturated
INT32_MAX values would re-overflow int32 and yield a negative penalty
once assigned to its float output.  Widen that sum to int64 before
converting to float.

Existing btree or GiST indexes on ltree columns containing values with
more than ~14,653 levels may be corrupt and should be REINDEXed.

Add a regression test based on the reporter's PoC.
---
 contrib/ltree/expected/ltree.out | 13 +++++++++++++
 contrib/ltree/ltree_gist.c       |  2 +-
 contrib/ltree/ltree_op.c         | 29 ++++++++++++++++++++++++++---
 contrib/ltree/sql/ltree.sql      |  9 +++++++++
 4 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/contrib/ltree/expected/ltree.out b/contrib/ltree/expected/ltree.out
index 15b9131a750..9d83aa2fa84 100644
--- a/contrib/ltree/expected/ltree.out
+++ b/contrib/ltree/expected/ltree.out
@@ -8226,3 +8226,16 @@ DETAIL:  Total size of level exceeds the maximum allowed (65535 bytes).
 SELECT (repeat('a|', 65535) || 'a')::lquery;
 ERROR:  lquery level has too many variants
 DETAIL:  Number of variants exceeds the maximum allowed (65535).
+-- Test that ltree_compare() does not overflow int32 with very deep paths.
+-- Without saturation, the product (len_diff * 10 * (an + 1)) wraps past
+-- INT32_MAX above ~14653 levels and the sign of the result is inverted,
+-- which corrupts btree ordering and GiST page splits.
+WITH s AS (SELECT 'a'::ltree AS v),
+     l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v)
+SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok
+  FROM s, l;
+ gt_ok | lt_ok | eq_ok 
+-------+-------+-------
+ t     | f     | f
+(1 row)
+
diff --git a/contrib/ltree/ltree_gist.c b/contrib/ltree/ltree_gist.c
index 78c95052990..433597de2da 100644
--- a/contrib/ltree/ltree_gist.c
+++ b/contrib/ltree/ltree_gist.c
@@ -270,7 +270,7 @@ ltree_penalty(PG_FUNCTION_ARGS)
 	cmpl = ltree_compare(LTG_GETLNODE(origval, siglen), LTG_GETLNODE(newval, siglen));
 	cmpr = ltree_compare(LTG_GETRNODE(newval, siglen), LTG_GETRNODE(origval, siglen));
 
-	*penalty = Max(cmpl, 0) + Max(cmpr, 0);
+	*penalty = (float) ((int64) Max(cmpl, 0) + (int64) Max(cmpr, 0));
 
 	PG_RETURN_POINTER(penalty);
 }
diff --git a/contrib/ltree/ltree_op.c b/contrib/ltree/ltree_op.c
index c1fc77fc804..f2f5ad9fd0f 100644
--- a/contrib/ltree/ltree_op.c
+++ b/contrib/ltree/ltree_op.c
@@ -57,15 +57,30 @@ ltree_compare(const ltree *a, const ltree *b)
 		if ((res = memcmp(al->name, bl->name, Min(al->len, bl->len))) == 0)
 		{
 			if (al->len != bl->len)
-				return (al->len - bl->len) * 10 * (an + 1);
+			{
+				int64		v = (int64) (al->len - bl->len) * 10 * (an + 1);
+
+				if (v > PG_INT32_MAX)
+					return PG_INT32_MAX;
+				if (v < PG_INT32_MIN)
+					return PG_INT32_MIN;
+				return (int32) v;
+			}
 		}
 		else
 		{
+			int64		v;
+
 			if (res < 0)
 				res = -1;
 			else
 				res = 1;
-			return res * 10 * (an + 1);
+			v = (int64) res * 10 * (an + 1);
+			if (v > PG_INT32_MAX)
+				return PG_INT32_MAX;
+			if (v < PG_INT32_MIN)
+				return PG_INT32_MIN;
+			return (int32) v;
 		}
 
 		an--;
@@ -74,7 +89,15 @@ ltree_compare(const ltree *a, const ltree *b)
 		bl = LEVEL_NEXT(bl);
 	}
 
-	return (a->numlevel - b->numlevel) * 10 * (an + 1);
+	{
+		int64		v = (int64) (a->numlevel - b->numlevel) * 10 * (an + 1);
+
+		if (v > PG_INT32_MAX)
+			return PG_INT32_MAX;
+		if (v < PG_INT32_MIN)
+			return PG_INT32_MIN;
+		return (int32) v;
+	}
 }
 
 #define RUNCMP						\
diff --git a/contrib/ltree/sql/ltree.sql b/contrib/ltree/sql/ltree.sql
index d0fade9d17d..3e9c1ff79ee 100644
--- a/contrib/ltree/sql/ltree.sql
+++ b/contrib/ltree/sql/ltree.sql
@@ -477,3 +477,12 @@ SELECT (repeat('x', 255) || repeat('|' || repeat('x', 255), 256))::lquery;
 --- Test for overflow of lquery_level.numvar, with a set of single-char
 --- variants in one level.
 SELECT (repeat('a|', 65535) || 'a')::lquery;
+
+-- Test that ltree_compare() does not overflow int32 with very deep paths.
+-- Without saturation, the product (len_diff * 10 * (an + 1)) wraps past
+-- INT32_MAX above ~14653 levels and the sign of the result is inverted,
+-- which corrupts btree ordering and GiST page splits.
+WITH s AS (SELECT 'a'::ltree AS v),
+     l AS (SELECT (repeat('a.', 14999) || 'a')::ltree AS v)
+SELECT (l.v > s.v) AS gt_ok, (l.v < s.v) AS lt_ok, (l.v = s.v) AS eq_ok
+  FROM s, l;
-- 
2.34.1