v7-0003-Fix-varlen-keys-sorting.patch

From 424058ee8d1ab2641bdaac6d0f2ce67d818d4430 Mon Sep 17 00:00:00 2001
From: Andrey Borodin <amborodin@acm.org>
Date: Fri, 7 May 2021 17:47:23 +0500
Subject: [PATCH v7 3/3] Fix varlen keys sorting

---
 contrib/btree_gist/btree_bit.c     | 6 ++++--
 contrib/btree_gist/btree_bytea.c   | 6 ++++--
 contrib/btree_gist/btree_numeric.c | 6 ++++--
 contrib/btree_gist/btree_text.c    | 6 ++++--
 4 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/contrib/btree_gist/btree_bit.c b/contrib/btree_gist/btree_bit.c
index 61b2eecfd5..947e3b63ac 100644
--- a/contrib/btree_gist/btree_bit.c
+++ b/contrib/btree_gist/btree_bit.c
@@ -214,10 +214,12 @@ gbt_bit_penalty(PG_FUNCTION_ARGS)
 static int
 gbt_bit_sort_build_cmp(Datum a, Datum b, SortSupport ssup)
 {
+	GBT_VARKEY_R ra = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(a));
+	GBT_VARKEY_R rb = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(b));
 	/* Use byteacmp(), like gbt_bitcmp() does */
 	return DatumGetInt32(DirectFunctionCall2(byteacmp,
-											 PointerGetDatum(a),
-											 PointerGetDatum(b)));
+											 PointerGetDatum(ra.lower),
+											 PointerGetDatum(rb.lower)));
 }
 
 /*
diff --git a/contrib/btree_gist/btree_bytea.c b/contrib/btree_gist/btree_bytea.c
index a2abfb7d7c..9ffb3b9a5d 100644
--- a/contrib/btree_gist/btree_bytea.c
+++ b/contrib/btree_gist/btree_bytea.c
@@ -173,9 +173,11 @@ gbt_bytea_penalty(PG_FUNCTION_ARGS)
 static int
 gbt_bytea_sort_build_cmp(Datum a, Datum b, SortSupport ssup)
 {
+	GBT_VARKEY_R ra = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(a));
+	GBT_VARKEY_R rb = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(b));
 	return DatumGetInt32(DirectFunctionCall2(byteacmp,
-											 PointerGetDatum(a),
-											 PointerGetDatum(b)));
+												 PointerGetDatum(ra.lower),
+												 PointerGetDatum(rb.lower)));
 }
 
 /*
diff --git a/contrib/btree_gist/btree_numeric.c b/contrib/btree_gist/btree_numeric.c
index face4e2b3a..76cd5d7380 100644
--- a/contrib/btree_gist/btree_numeric.c
+++ b/contrib/btree_gist/btree_numeric.c
@@ -232,9 +232,11 @@ gbt_numeric_picksplit(PG_FUNCTION_ARGS)
 static int
 gbt_numeric_sort_build_cmp(Datum a, Datum b, SortSupport ssup)
 {
+	GBT_VARKEY_R ra = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(a));
+	GBT_VARKEY_R rb = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(b));
 	return DatumGetInt32(DirectFunctionCall2(numeric_cmp,
-											 PointerGetDatum(a),
-											 PointerGetDatum(b)));
+											 PointerGetDatum(ra.lower),
+											 PointerGetDatum(rb.lower)));
 }
 
 /*
diff --git a/contrib/btree_gist/btree_text.c b/contrib/btree_gist/btree_text.c
index 01b1bda2f6..2deb8dd76f 100644
--- a/contrib/btree_gist/btree_text.c
+++ b/contrib/btree_gist/btree_text.c
@@ -244,10 +244,12 @@ gbt_text_penalty(PG_FUNCTION_ARGS)
 static int
 gbt_text_sort_build_cmp(Datum a, Datum b, SortSupport ssup)
 {
+	GBT_VARKEY_R ra = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(a));
+	GBT_VARKEY_R rb = gbt_var_key_readable((GBT_VARKEY *) PG_DETOAST_DATUM(b));
 	return DatumGetInt32(DirectFunctionCall2Coll(bttextcmp,
 												 ssup->ssup_collation,
-												 PointerGetDatum(a),
-												 PointerGetDatum(b)));
+												 PointerGetDatum(ra.lower),
+												 PointerGetDatum(rb.lower)));
 }
 
 /*
-- 
2.24.3 (Apple Git-128)