v7-0002-Add-test-module-for-icu-collation-provider-hook.patch

From 02ee737d4ef9ddcea7c1469fc94320273d448878 Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Wed, 21 Dec 2022 19:50:10 -0800
Subject: [PATCH v7 2/4] Add test module for icu collation provider hook.

---
 src/test/modules/Makefile                     |   1 +
 src/test/modules/meson.build                  |   1 +
 .../modules/test_collator_lib_hooks/Makefile  |  24 ++
 .../test_collator_lib_hooks/meson.build       |  37 +++
 .../test_collator_lib_hooks/t/001_icu.pl      | 153 ++++++++++++
 .../test_collator_lib_hooks.c                 |  43 ++++
 .../test_collator_lib_hooks.control           |   4 +
 .../test_collator_lib_hooks.h                 |  28 +++
 .../test_collator_lib_hooks/test_icu_hook.c   | 228 ++++++++++++++++++
 9 files changed, 519 insertions(+)
 create mode 100644 src/test/modules/test_collator_lib_hooks/Makefile
 create mode 100644 src/test/modules/test_collator_lib_hooks/meson.build
 create mode 100644 src/test/modules/test_collator_lib_hooks/t/001_icu.pl
 create mode 100644 src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.c
 create mode 100644 src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.control
 create mode 100644 src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.h
 create mode 100644 src/test/modules/test_collator_lib_hooks/test_icu_hook.c

diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index c629cbe383..8a90b206a1 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -15,6 +15,7 @@ SUBDIRS = \
 		  snapshot_too_old \
 		  spgist_name_ops \
 		  test_bloomfilter \
+		  test_collator_lib_hooks \
 		  test_copy_callbacks \
 		  test_custom_rmgrs \
 		  test_ddl_deparse \
diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build
index 1baa6b558d..a59cc9a89f 100644
--- a/src/test/modules/meson.build
+++ b/src/test/modules/meson.build
@@ -11,6 +11,7 @@ subdir('snapshot_too_old')
 subdir('spgist_name_ops')
 subdir('ssl_passphrase_callback')
 subdir('test_bloomfilter')
+subdir('test_collator_lib_hooks')
 subdir('test_copy_callbacks')
 subdir('test_custom_rmgrs')
 subdir('test_ddl_deparse')
diff --git a/src/test/modules/test_collator_lib_hooks/Makefile b/src/test/modules/test_collator_lib_hooks/Makefile
new file mode 100644
index 0000000000..b44c8f5914
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/Makefile
@@ -0,0 +1,24 @@
+# src/test/modules/test_collator_lib_hooks/Makefile
+
+MODULE_big = test_collator_lib_hooks
+OBJS = \
+	$(WIN32RES) \
+	test_collator_lib_hooks.o test_icu_hook.o
+PGFILEDESC = "test_collator_lib_hooks - test collator provider library hooks"
+
+EXTENSION = test_collator_lib_hooks
+
+TAP_TESTS = 1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_collator_lib_hooks
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+export with_icu
diff --git a/src/test/modules/test_collator_lib_hooks/meson.build b/src/test/modules/test_collator_lib_hooks/meson.build
new file mode 100644
index 0000000000..0190a856e0
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/meson.build
@@ -0,0 +1,37 @@
+# FIXME: prevent install during main install, but not during test :/
+
+test_collator_lib_hooks_sources = files(
+  'test_collator_lib_hooks.c',
+  'test_icu_hook.c',
+)
+
+if host_system == 'windows'
+  test_collator_lib_hooks_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
+    '--NAME', 'test_collator_lib_hooks',
+    '--FILEDESC', 'test_collator_lib_hooks - test collation provider library hooks',])
+endif
+
+test_collator_lib_hooks = shared_module('test_collator_lib_hooks',
+  test_collator_lib_hooks_sources,
+  kwargs: pg_mod_args,
+)
+testprep_targets += test_collator_lib_hooks
+
+install_data(
+  'test_collator_lib_hooks.control',
+  kwargs: contrib_data_args,
+)
+
+tests += {
+  'name': 'test_collator_lib_hooks',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'tap': {
+    'tests': [
+      't/001_icu.pl',
+    ],
+    'env': {
+      'with_icu': icu.found() ? 'yes' : 'no',
+    },
+  },
+}
diff --git a/src/test/modules/test_collator_lib_hooks/t/001_icu.pl b/src/test/modules/test_collator_lib_hooks/t/001_icu.pl
new file mode 100644
index 0000000000..d3b5729527
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/t/001_icu.pl
@@ -0,0 +1,153 @@
+# Copyright (c) 2021-2022, PostgreSQL Global Development Group
+
+use strict;
+use warnings;
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+if ($ENV{with_icu} ne 'yes')
+{
+	plan skip_all => 'ICU not supported by this build';
+}
+
+my $node = PostgreSQL::Test::Cluster->new('main');
+
+$node->init;
+$node->append_conf(
+	'postgresql.conf', q{
+shared_preload_libraries = 'test_collator_lib_hooks'
+});
+$node->start;
+
+$node->safe_psql('postgres',
+	q{CREATE DATABASE dbicu LOCALE_PROVIDER icu LOCALE 'C' ICU_LOCALE 'DESC' ENCODING 'UTF8' TEMPLATE template0}
+);
+
+# setup
+$node->safe_psql('dbicu',
+	qq[CREATE COLLATION test_asc (PROVIDER=icu, LOCALE='ASC')]);
+$node->safe_psql('dbicu',
+	qq[CREATE COLLATION test_desc (PROVIDER=icu, LOCALE='DESC')]);
+
+$node->safe_psql('dbicu', qq[CREATE TABLE strings(t text)]);
+$node->safe_psql('dbicu',
+	qq[INSERT INTO strings VALUES ('aBcD'), ('fGhI'), ('wXyZ')]);
+
+# check versions
+
+my $version_db =
+  $node->safe_psql('dbicu',
+	  qq[SELECT datcollversion FROM pg_database WHERE datname='dbicu']);
+is($version_db, '2.72',
+	'database "dbicu" has correct version 2.72'
+);
+
+my $version_asc =
+  $node->safe_psql('dbicu',
+	  qq[SELECT collversion FROM pg_collation WHERE collname='test_asc']);
+is($version_asc, '2.72',
+	'collation "test_asc" has correct version 2.72'
+);
+
+my $version_desc =
+  $node->safe_psql('dbicu',
+	  qq[SELECT collversion FROM pg_collation WHERE collname='test_desc']);
+is($version_desc, '2.72',
+	'collation "test_desc" has correct version 2.72'
+);
+
+my $res_sort_expected = "aBcD
+fGhI
+wXyZ";
+
+my $res_reversesort_expected = "wXyZ
+fGhI
+aBcD";
+
+# test comparison
+
+my $comparison =
+  $node->safe_psql('dbicu',
+	  qq[SELECT 'aBcD' COLLATE test_asc < 'wXyZ' COLLATE test_asc]);
+is($comparison, 't',
+	'correct comparison'
+);
+
+# test reverse comparison (database)
+
+my $dbcomparison_reverse =
+  $node->safe_psql('dbicu', qq[SELECT 'aBcD' < 'wXyZ']);
+is($dbcomparison_reverse, 'f',
+	'correct reverse comparison (database)'
+);
+
+# test reverse comparison
+
+my $comparison_reverse =
+  $node->safe_psql('dbicu',
+	  qq[SELECT 'aBcD' COLLATE test_desc < 'wXyZ' COLLATE test_desc]);
+is($comparison_reverse, 'f',
+	'correct reverse comparison'
+);
+
+# test asc sort
+
+my $res_sort =
+  $node->safe_psql('dbicu',
+	  qq[SELECT t FROM strings ORDER BY t COLLATE test_asc]);
+is($res_sort, $res_sort_expected,
+	'correct ascending sort'
+);
+
+# test desc sort
+
+my $res_db_reversesort =
+  $node->safe_psql('dbicu',
+	  qq[SELECT t FROM strings ORDER BY t]);
+is($res_db_reversesort, $res_reversesort_expected,
+	'correct descending sort (database)'
+);
+
+# test desc sort
+
+my $res_reversesort =
+  $node->safe_psql('dbicu',
+	  qq[SELECT t FROM strings ORDER BY t COLLATE test_desc]);
+is($res_reversesort, $res_reversesort_expected,
+	'correct descending sort'
+);
+
+# test lower/upper
+
+my $tcase =
+  $node->safe_psql('dbicu',
+	  qq[SELECT lower('aBcDfgHiwXyZ' collate test_asc),
+                upper('aBcDfgHiwXyZ' collate test_asc)]);
+is($tcase, 'abcdfghiwxyz|ABCDFGHIWXYZ',
+	'correct lowercase and uppercase'
+);
+
+# test reverse lower/upper (database)
+
+my $tcase_db_reverse =
+  $node->safe_psql('dbicu',
+	  qq[SELECT lower('aBcDfgHiwXyZ'),
+                upper('aBcDfgHiwXyZ')]);
+is($tcase_db_reverse, 'ABCDFGHIWXYZ|abcdfghiwxyz',
+	'correct reverse lowercase and uppercase (database)'
+);
+
+# test reverse lower/upper
+
+my $tcase_reverse =
+  $node->safe_psql('dbicu',
+	  qq[SELECT lower('aBcDfgHiwXyZ' collate test_desc),
+                upper('aBcDfgHiwXyZ' collate test_desc)]);
+is($tcase_reverse, 'ABCDFGHIWXYZ|abcdfghiwxyz',
+	'correct reverse lowercase and uppercase'
+);
+
+$node->stop;
+done_testing();
diff --git a/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.c b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.c
new file mode 100644
index 0000000000..45a849b52d
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.c
@@ -0,0 +1,43 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_collator_lib_hooks.c
+ *		Code for testing collation provider library hooks
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.c
+ *
+ * Test implementation of icu-like collation provider.
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "miscadmin.h"
+
+#include "test_collator_lib_hooks.h"
+
+#ifdef USE_ICU
+static get_icu_library_hook_type prev_get_icu_library_hook = NULL;
+#endif
+
+PG_MODULE_MAGIC;
+
+/*
+ * Module load callback
+ */
+void
+_PG_init(void)
+{
+	if (!process_shared_preload_libraries_in_progress)
+		ereport(ERROR, (errmsg("test_collator_lib_hooks must be loaded via shared_preload_libraries")));
+
+#ifdef USE_ICU
+	prev_get_icu_library_hook = get_icu_library_hook;
+	get_icu_library_hook = test_get_icu_library;
+#endif
+}
diff --git a/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.control b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.control
new file mode 100644
index 0000000000..a0b8e031a4
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.control
@@ -0,0 +1,4 @@
+comment = 'Test code for collation provider library hooks'
+default_version = '1.0'
+module_pathname = '$libdir/test_collation_lib_hooks'
+
diff --git a/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.h b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.h
new file mode 100644
index 0000000000..c3a9ff45e2
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.h
@@ -0,0 +1,28 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_collator_lib_hooks.h
+ *		Definitions for collation library hooks.
+ *
+ * Copyright (c) 2015-2022, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_collator_lib_hooks/test_collator_lib_hooks.h
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#ifndef TEST_COLLATOR_LIB_HOOKS_H
+#define TEST_COLLATOR_LIB_HOOKS_H
+
+#include "postgres.h"
+
+#include "utils/memutils.h"
+#include "utils/pg_locale.h"
+#include "utils/pg_locale_internal.h"
+
+#ifdef USE_ICU
+extern pg_icu_library *test_get_icu_library(Oid collid, const char *locale,
+											const char *version);
+#endif
+
+#endif
diff --git a/src/test/modules/test_collator_lib_hooks/test_icu_hook.c b/src/test/modules/test_collator_lib_hooks/test_icu_hook.c
new file mode 100644
index 0000000000..423ff30b5b
--- /dev/null
+++ b/src/test/modules/test_collator_lib_hooks/test_icu_hook.c
@@ -0,0 +1,228 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_icu_hook.c
+ *		Code for testing collation provider icu hook.
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_collator_lib_hooks/test_icu_hook.c
+ *
+ * Implements a custom icu-like collation provider library for testing the
+ * hooks. It accepts any collator name requested. All behave exactly like the
+ * "en_US" locale, except for the locale named "DESC", which reverses the sort
+ * order and reverses uppercase/lowercase behavior.
+ *
+ * The version is always reported as 2.72, so loading it will cause a version
+ * mismatch warning.
+ *
+ * -------------------------------------------------------------------------
+ */
+
+#include "test_collator_lib_hooks.h"
+
+#ifdef USE_ICU
+
+#include <unicode/ucnv.h>
+#include <unicode/ulocdata.h>
+#include <unicode/ustring.h>
+
+#define TEST_LOCALE "en_US"
+
+typedef struct TestUCollator {
+	UCollator	*ucol;
+	bool		 reverse;
+} TestUCollator;
+
+static pg_icu_library *test_icu_library = NULL;
+static const UVersionInfo test_icu_version = { 2, 72 };
+
+static bool
+locale_is_reverse(const char *locale)
+{
+	if (strcmp(locale, "DESC") == 0)
+		return true;
+	else
+		return false;
+}
+
+static UCollator *
+test_openCollator(const char *loc, UErrorCode *status)
+{
+	TestUCollator *testcol = MemoryContextAlloc(TopMemoryContext, sizeof(TestUCollator));
+	UCollator *ucol = ucol_open(TEST_LOCALE, status);
+	testcol->ucol = ucol;
+	testcol->reverse = locale_is_reverse(loc);
+	return (UCollator *)testcol;
+}
+
+static void
+test_closeCollator(UCollator *coll)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	ucol_close(testcol->ucol);
+	pfree(testcol);
+}
+
+static void
+test_setAttribute(UCollator *coll, UColAttribute attr,
+				  UColAttributeValue value, UErrorCode *status)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	ucol_setAttribute(testcol->ucol, attr, value, status);
+}
+
+static void
+test_getCollatorVersion(const UCollator *coll, UVersionInfo info)
+{
+	memcpy(info, test_icu_version, sizeof(UVersionInfo));
+}
+
+static UCollationResult
+test_strcoll(const UCollator *coll, const UChar *source, int32_t sourceLength,
+			 const UChar *target, int32_t targetLength)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	UCollationResult ret = ucol_strcoll(testcol->ucol, source, sourceLength,
+										target, targetLength);
+	if (testcol->reverse)
+		return -ret;
+	else
+		return ret;
+}
+
+static UCollationResult
+test_strcollUTF8(const UCollator *coll, const char *source,
+				 int32_t sourceLength, const char *target,
+				 int32_t targetLength, UErrorCode *status)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	UCollationResult ret = ucol_strcollUTF8(testcol->ucol, source,
+											sourceLength, target,
+											targetLength, status);
+	if (testcol->reverse)
+		return -ret;
+	else
+		return ret;
+}
+
+static int32_t
+test_getSortKey(const UCollator *coll, const UChar *source,
+				int32_t sourceLength, uint8_t *result, int32_t resultLength)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	int32_t ret = ucol_getSortKey(testcol->ucol, source, sourceLength,
+								  result, resultLength);
+	size_t result_size = ret + 1;
+
+	if (resultLength >= result_size)
+	{
+		result[resultLength] = '\0';
+
+		if (testcol->reverse)
+			for (int i = 0; i < result_size; i++)
+				*((unsigned char *) result + i) ^= (unsigned char) 0xff;
+	}
+
+	return result_size;
+}
+
+static int32_t
+test_nextSortKeyPart(const UCollator *coll, UCharIterator *iter,
+					 uint32_t state[2], uint8_t *dest, int32_t count,
+					 UErrorCode *status)
+{
+	TestUCollator *testcol = (TestUCollator *) coll;
+	int32_t ret = ucol_nextSortKeyPart(testcol->ucol, iter, state, dest,
+									   count, status);
+
+	if (testcol->reverse)
+		for (int i = 0; i < ret; i++)
+			*((unsigned char *) dest + i) ^= (unsigned char) 0xff;
+
+	/*
+	 * The following is not correct for cases where we finish precisely on the
+	 * boundary (i.e. count is exactly enough). To fix this we'd need to track
+	 * additional state across calls, which doesn't seem worth it for a test
+	 * case.
+	 */
+	if (count >= ret && ret > 0)
+	{
+		if (testcol->reverse)
+			dest[ret] = 0xff;
+		else
+			dest[ret] = '\0';
+		return ret + 1;
+	}
+
+	return ret;
+}
+
+static int32_t
+test_strToUpper(UChar *dest, int32_t destCapacity, const UChar *src,
+				int32_t srcLength, const char *locale, UErrorCode *pErrorCode)
+{
+	if (locale_is_reverse(locale))
+		return u_strToLower(dest, destCapacity, src, srcLength,
+							TEST_LOCALE, pErrorCode);
+	else
+		return u_strToUpper(dest, destCapacity, src, srcLength,
+							TEST_LOCALE, pErrorCode);
+}
+
+static int32_t
+test_strToLower(UChar *dest, int32_t destCapacity, const UChar *src,
+				int32_t srcLength, const char *locale, UErrorCode *pErrorCode)
+{
+	if (locale_is_reverse(locale))
+		return u_strToUpper(dest, destCapacity, src, srcLength,
+							TEST_LOCALE, pErrorCode);
+	else
+		return u_strToLower(dest, destCapacity, src, srcLength,
+							TEST_LOCALE, pErrorCode);
+}
+
+pg_icu_library *
+test_get_icu_library(Oid collid, const char *locale, const char *version)
+{
+	pg_icu_library *lib;
+
+	if (test_icu_library != NULL)
+		return test_icu_library;
+
+	ereport(LOG, (errmsg("loading custom ICU provider for test_collator_lib_hooks")));
+
+	lib = MemoryContextAlloc(TopMemoryContext, sizeof(pg_icu_library));
+	lib->getICUVersion = u_getVersion;
+	lib->getUnicodeVersion = u_getUnicodeVersion;
+	lib->getCLDRVersion = ulocdata_getCLDRVersion;
+	lib->openCollator = test_openCollator;
+	lib->closeCollator = test_closeCollator;
+	lib->getCollatorVersion = test_getCollatorVersion;
+	lib->getUCAVersion = ucol_getUCAVersion;
+	lib->versionToString = u_versionToString;
+	lib->strcoll = test_strcoll;
+	lib->strcollUTF8 = test_strcollUTF8;
+	lib->getSortKey = test_getSortKey;
+	lib->nextSortKeyPart = test_nextSortKeyPart;
+	lib->setUTF8 = uiter_setUTF8;
+	lib->errorName = u_errorName;
+	lib->strToUpper = test_strToUpper;
+	lib->strToLower = test_strToLower;
+	lib->strToTitle = u_strToTitle;
+	lib->setAttribute = test_setAttribute;
+	lib->openConverter = ucnv_open;
+	lib->closeConverter = ucnv_close;
+	lib->fromUChars = ucnv_fromUChars;
+	lib->toUChars = ucnv_toUChars;
+	lib->toLanguageTag = uloc_toLanguageTag;
+	lib->getDisplayName = uloc_getDisplayName;
+	lib->countAvailable = uloc_countAvailable;
+	lib->getAvailable = uloc_getAvailable;
+
+	test_icu_library = lib;
+	return lib;
+}
+
+#endif				/* USE_ICU */
-- 
2.34.1