v5-0006-Control-case-mapping-behavior-with-a-method-table.patch
text/x-patch
Filename: v5-0006-Control-case-mapping-behavior-with-a-method-table.patch
Type: text/x-patch
Part: 2
Patch
Format: format-patch
Series: patch v5-0006
Subject: Control case mapping behavior with a method table.
| File | + | − |
|---|---|---|
| src/backend/utils/adt/formatting.c | 57 | 388 |
| src/backend/utils/adt/pg_locale.c | 102 | 0 |
| src/backend/utils/adt/pg_locale_icu.c | 137 | 5 |
| src/backend/utils/adt/pg_locale_libc.c | 219 | 0 |
| src/include/utils/pg_locale.h | 24 | 5 |
From eec6c3eb165f85614780374cb83ec98f09d3bd3e Mon Sep 17 00:00:00 2001
From: Jeff Davis <jeff@j-davis.com>
Date: Thu, 26 Sep 2024 12:12:51 -0700
Subject: [PATCH v5 6/8] Control case mapping behavior with a method table.
Previously, case mapping (LOWER(), INITCAP(), UPPER()) behavior
branched based on the provider.
A method table is less error-prone and easier to hook.
---
src/backend/utils/adt/formatting.c | 445 ++++---------------------
src/backend/utils/adt/pg_locale.c | 102 ++++++
src/backend/utils/adt/pg_locale_icu.c | 142 +++++++-
src/backend/utils/adt/pg_locale_libc.c | 219 ++++++++++++
src/include/utils/pg_locale.h | 29 +-
5 files changed, 539 insertions(+), 398 deletions(-)
diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c
index 85a7dd45619..6a0571f93e6 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -1570,52 +1570,6 @@ str_numth(char *dest, char *num, int type)
* upper/lower/initcap functions
*****************************************************************************/
-#ifdef USE_ICU
-
-typedef int32_t (*ICU_Convert_Func) (UChar *dest, int32_t destCapacity,
- const UChar *src, int32_t srcLength,
- const char *locale,
- UErrorCode *pErrorCode);
-
-static int32_t
-icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale,
- UChar **buff_dest, UChar *buff_source, int32_t len_source)
-{
- UErrorCode status;
- int32_t len_dest;
-
- len_dest = len_source; /* try first with same length */
- *buff_dest = palloc(len_dest * sizeof(**buff_dest));
- status = U_ZERO_ERROR;
- len_dest = func(*buff_dest, len_dest, buff_source, len_source,
- mylocale->info.icu.locale, &status);
- if (status == U_BUFFER_OVERFLOW_ERROR)
- {
- /* try again with adjusted length */
- pfree(*buff_dest);
- *buff_dest = palloc(len_dest * sizeof(**buff_dest));
- status = U_ZERO_ERROR;
- len_dest = func(*buff_dest, len_dest, buff_source, len_source,
- mylocale->info.icu.locale, &status);
- }
- if (U_FAILURE(status))
- ereport(ERROR,
- (errmsg("case conversion failed: %s", u_errorName(status))));
- return len_dest;
-}
-
-static int32_t
-u_strToTitle_default_BI(UChar *dest, int32_t destCapacity,
- const UChar *src, int32_t srcLength,
- const char *locale,
- UErrorCode *pErrorCode)
-{
- return u_strToTitle(dest, destCapacity, src, srcLength,
- NULL, locale, pErrorCode);
-}
-
-#endif /* USE_ICU */
-
/*
* If the system provides the needed functions for wide-character manipulation
* (which are all standardized by C99), then we implement upper/lower/initcap
@@ -1663,101 +1617,28 @@ str_tolower(const char *buff, size_t nbytes, Oid collid)
}
else
{
-#ifdef USE_ICU
- if (mylocale->provider == COLLPROVIDER_ICU)
+ const char *src = buff;
+ size_t srclen = nbytes;
+ size_t dstsize;
+ char *dst;
+ size_t needed;
+
+ /* first try buffer of equal size plus terminating NUL */
+ dstsize = srclen + 1;
+ dst = palloc(dstsize);
+
+ needed = pg_strlower(dst, dstsize, src, srclen, mylocale);
+ if (needed + 1 > dstsize)
{
- int32_t len_uchar;
- int32_t len_conv;
- UChar *buff_uchar;
- UChar *buff_conv;
-
- len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes);
- len_conv = icu_convert_case(u_strToLower, mylocale,
- &buff_conv, buff_uchar, len_uchar);
- icu_from_uchar(&result, buff_conv, len_conv);
- pfree(buff_uchar);
- pfree(buff_conv);
+ /* grow buffer if needed and retry */
+ dstsize = needed + 1;
+ dst = repalloc(dst, dstsize);
+ needed = pg_strlower(dst, dstsize, src, srclen, mylocale);
+ Assert(needed + 1 <= dstsize);
}
- else
-#endif
- if (mylocale->provider == COLLPROVIDER_BUILTIN)
- {
- const char *src = buff;
- size_t srclen = nbytes;
- size_t dstsize;
- char *dst;
- size_t needed;
-
- Assert(GetDatabaseEncoding() == PG_UTF8);
-
- /* first try buffer of equal size plus terminating NUL */
- dstsize = srclen + 1;
- dst = palloc(dstsize);
-
- needed = unicode_strlower(dst, dstsize, src, srclen);
- if (needed + 1 > dstsize)
- {
- /* grow buffer if needed and retry */
- dstsize = needed + 1;
- dst = repalloc(dst, dstsize);
- needed = unicode_strlower(dst, dstsize, src, srclen);
- Assert(needed + 1 == dstsize);
- }
-
- Assert(dst[needed] == '\0');
- result = dst;
- }
- else
- {
- Assert(mylocale->provider == COLLPROVIDER_LIBC);
-
- if (pg_database_encoding_max_length() > 1)
- {
- wchar_t *workspace;
- size_t curr_char;
- size_t result_size;
-
- /* Overflow paranoia */
- if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t)))
- ereport(ERROR,
- (errcode(ERRCODE_OUT_OF_MEMORY),
- errmsg("out of memory")));
-
- /* Output workspace cannot have more codes than input bytes */
- workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t));
-
- char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale);
-
- for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
- workspace[curr_char] = towlower_l(workspace[curr_char], mylocale->info.lt);
- /*
- * Make result large enough; case change might change number
- * of bytes
- */
- result_size = curr_char * pg_database_encoding_max_length() + 1;
- result = palloc(result_size);
-
- wchar2char(result, workspace, result_size, mylocale);
- pfree(workspace);
- }
- else
- {
- char *p;
-
- result = pnstrdup(buff, nbytes);
-
- /*
- * Note: we assume that tolower_l() will not be so broken as
- * to need an isupper_l() guard test. When using the default
- * collation, we apply the traditional Postgres behavior that
- * forces ASCII-style treatment of I/i, but in non-default
- * collations you get exactly what the collation says.
- */
- for (p = result; *p; p++)
- *p = tolower_l((unsigned char) *p, mylocale->info.lt);
- }
- }
+ Assert(dst[needed] == '\0');
+ result = dst;
}
return result;
@@ -1800,147 +1681,33 @@ str_toupper(const char *buff, size_t nbytes, Oid collid)
}
else
{
-#ifdef USE_ICU
- if (mylocale->provider == COLLPROVIDER_ICU)
- {
- int32_t len_uchar,
- len_conv;
- UChar *buff_uchar;
- UChar *buff_conv;
-
- len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes);
- len_conv = icu_convert_case(u_strToUpper, mylocale,
- &buff_conv, buff_uchar, len_uchar);
- icu_from_uchar(&result, buff_conv, len_conv);
- pfree(buff_uchar);
- pfree(buff_conv);
- }
- else
-#endif
- if (mylocale->provider == COLLPROVIDER_BUILTIN)
+ const char *src = buff;
+ size_t srclen = nbytes;
+ size_t dstsize;
+ char *dst;
+ size_t needed;
+
+ /* first try buffer of equal size plus terminating NUL */
+ dstsize = srclen + 1;
+ dst = palloc(dstsize);
+
+ needed = pg_strupper(dst, dstsize, src, srclen, mylocale);
+ if (needed + 1 > dstsize)
{
- const char *src = buff;
- size_t srclen = nbytes;
- size_t dstsize;
- char *dst;
- size_t needed;
-
- Assert(GetDatabaseEncoding() == PG_UTF8);
-
- /* first try buffer of equal size plus terminating NUL */
- dstsize = srclen + 1;
- dst = palloc(dstsize);
-
- needed = unicode_strupper(dst, dstsize, src, srclen);
- if (needed + 1 > dstsize)
- {
- /* grow buffer if needed and retry */
- dstsize = needed + 1;
- dst = repalloc(dst, dstsize);
- needed = unicode_strupper(dst, dstsize, src, srclen);
- Assert(needed + 1 == dstsize);
- }
-
- Assert(dst[needed] == '\0');
- result = dst;
+ /* grow buffer if needed and retry */
+ dstsize = needed + 1;
+ dst = repalloc(dst, dstsize);
+ needed = pg_strupper(dst, dstsize, src, srclen, mylocale);
+ Assert(needed + 1 <= dstsize);
}
- else
- {
- Assert(mylocale->provider == COLLPROVIDER_LIBC);
-
- if (pg_database_encoding_max_length() > 1)
- {
- wchar_t *workspace;
- size_t curr_char;
- size_t result_size;
-
- /* Overflow paranoia */
- if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t)))
- ereport(ERROR,
- (errcode(ERRCODE_OUT_OF_MEMORY),
- errmsg("out of memory")));
-
- /* Output workspace cannot have more codes than input bytes */
- workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t));
-
- char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale);
-
- for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
- workspace[curr_char] = towupper_l(workspace[curr_char], mylocale->info.lt);
- /*
- * Make result large enough; case change might change number
- * of bytes
- */
- result_size = curr_char * pg_database_encoding_max_length() + 1;
- result = palloc(result_size);
-
- wchar2char(result, workspace, result_size, mylocale);
- pfree(workspace);
- }
- else
- {
- char *p;
-
- result = pnstrdup(buff, nbytes);
-
- /*
- * Note: we assume that toupper_l() will not be so broken as
- * to need an islower_l() guard test. When using the default
- * collation, we apply the traditional Postgres behavior that
- * forces ASCII-style treatment of I/i, but in non-default
- * collations you get exactly what the collation says.
- */
- for (p = result; *p; p++)
- *p = toupper_l((unsigned char) *p, mylocale->info.lt);
- }
- }
+ Assert(dst[needed] == '\0');
+ result = dst;
}
return result;
}
-struct WordBoundaryState
-{
- const char *str;
- size_t len;
- size_t offset;
- bool init;
- bool prev_alnum;
-};
-
-/*
- * Simple word boundary iterator that draws boundaries each time the result of
- * pg_u_isalnum() changes.
- */
-static size_t
-initcap_wbnext(void *state)
-{
- struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
-
- while (wbstate->offset < wbstate->len &&
- wbstate->str[wbstate->offset] != '\0')
- {
- pg_wchar u = utf8_to_unicode((unsigned char *) wbstate->str +
- wbstate->offset);
- bool curr_alnum = pg_u_isalnum(u, true);
-
- if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
- {
- size_t prev_offset = wbstate->offset;
-
- wbstate->init = true;
- wbstate->offset += unicode_utf8len(u);
- wbstate->prev_alnum = curr_alnum;
- return prev_offset;
- }
-
- wbstate->offset += unicode_utf8len(u);
- }
-
- return wbstate->len;
-}
-
/*
* collation-aware, wide-character-aware initcap function
*
@@ -1951,7 +1718,6 @@ char *
str_initcap(const char *buff, size_t nbytes, Oid collid)
{
char *result;
- int wasalnum = false;
pg_locale_t mylocale;
if (!buff)
@@ -1979,125 +1745,28 @@ str_initcap(const char *buff, size_t nbytes, Oid collid)
}
else
{
-#ifdef USE_ICU
- if (mylocale->provider == COLLPROVIDER_ICU)
+ const char *src = buff;
+ size_t srclen = nbytes;
+ size_t dstsize;
+ char *dst;
+ size_t needed;
+
+ /* first try buffer of equal size plus terminating NUL */
+ dstsize = srclen + 1;
+ dst = palloc(dstsize);
+
+ needed = pg_strtitle(dst, dstsize, src, srclen, mylocale);
+ if (needed + 1 > dstsize)
{
- int32_t len_uchar,
- len_conv;
- UChar *buff_uchar;
- UChar *buff_conv;
-
- len_uchar = icu_to_uchar(&buff_uchar, buff, nbytes);
- len_conv = icu_convert_case(u_strToTitle_default_BI, mylocale,
- &buff_conv, buff_uchar, len_uchar);
- icu_from_uchar(&result, buff_conv, len_conv);
- pfree(buff_uchar);
- pfree(buff_conv);
+ /* grow buffer if needed and retry */
+ dstsize = needed + 1;
+ dst = repalloc(dst, dstsize);
+ needed = pg_strtitle(dst, dstsize, src, srclen, mylocale);
+ Assert(needed + 1 <= dstsize);
}
- else
-#endif
- if (mylocale->provider == COLLPROVIDER_BUILTIN)
- {
- const char *src = buff;
- size_t srclen = nbytes;
- size_t dstsize;
- char *dst;
- size_t needed;
- struct WordBoundaryState wbstate = {
- .str = src,
- .len = srclen,
- .offset = 0,
- .init = false,
- .prev_alnum = false,
- };
-
- Assert(GetDatabaseEncoding() == PG_UTF8);
-
- /* first try buffer of equal size plus terminating NUL */
- dstsize = srclen + 1;
- dst = palloc(dstsize);
-
- needed = unicode_strtitle(dst, dstsize, src, srclen,
- initcap_wbnext, &wbstate);
- if (needed + 1 > dstsize)
- {
- /* reset iterator */
- wbstate.offset = 0;
- wbstate.init = false;
-
- /* grow buffer if needed and retry */
- dstsize = needed + 1;
- dst = repalloc(dst, dstsize);
- needed = unicode_strtitle(dst, dstsize, src, srclen,
- initcap_wbnext, &wbstate);
- Assert(needed + 1 == dstsize);
- }
-
- result = dst;
- }
- else
- {
- Assert(mylocale->provider == COLLPROVIDER_LIBC);
-
- if (pg_database_encoding_max_length() > 1)
- {
- wchar_t *workspace;
- size_t curr_char;
- size_t result_size;
-
- /* Overflow paranoia */
- if ((nbytes + 1) > (INT_MAX / sizeof(wchar_t)))
- ereport(ERROR,
- (errcode(ERRCODE_OUT_OF_MEMORY),
- errmsg("out of memory")));
-
- /* Output workspace cannot have more codes than input bytes */
- workspace = (wchar_t *) palloc((nbytes + 1) * sizeof(wchar_t));
-
- char2wchar(workspace, nbytes + 1, buff, nbytes, mylocale);
-
- for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
- {
- if (wasalnum)
- workspace[curr_char] = towlower_l(workspace[curr_char], mylocale->info.lt);
- else
- workspace[curr_char] = towupper_l(workspace[curr_char], mylocale->info.lt);
- wasalnum = iswalnum_l(workspace[curr_char], mylocale->info.lt);
- }
-
- /*
- * Make result large enough; case change might change number
- * of bytes
- */
- result_size = curr_char * pg_database_encoding_max_length() + 1;
- result = palloc(result_size);
-
- wchar2char(result, workspace, result_size, mylocale);
- pfree(workspace);
- }
- else
- {
- char *p;
- result = pnstrdup(buff, nbytes);
-
- /*
- * Note: we assume that toupper_l()/tolower_l() will not be so
- * broken as to need guard tests. When using the default
- * collation, we apply the traditional Postgres behavior that
- * forces ASCII-style treatment of I/i, but in non-default
- * collations you get exactly what the collation says.
- */
- for (p = result; *p; p++)
- {
- if (wasalnum)
- *p = tolower_l((unsigned char) *p, mylocale->info.lt);
- else
- *p = toupper_l((unsigned char) *p, mylocale->info.lt);
- wasalnum = isalnum_l((unsigned char) *p, mylocale->info.lt);
- }
- }
- }
+ Assert(dst[needed] == '\0');
+ result = dst;
}
return result;
diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c
index dfb9c3bd952..a106478b119 100644
--- a/src/backend/utils/adt/pg_locale.c
+++ b/src/backend/utils/adt/pg_locale.c
@@ -58,6 +58,8 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_database.h"
#include "common/hashfn.h"
+#include "common/unicode_case.h"
+#include "common/unicode_category.h"
#include "mb/pg_wchar.h"
#include "miscadmin.h"
#include "utils/builtins.h"
@@ -170,6 +172,83 @@ static pg_locale_t last_collation_cache_locale = NULL;
static char *IsoLocaleName(const char *);
#endif
+struct WordBoundaryState
+{
+ const char *str;
+ size_t len;
+ size_t offset;
+ bool init;
+ bool prev_alnum;
+};
+
+/*
+ * Simple word boundary iterator that draws boundaries each time the result of
+ * pg_u_isalnum() changes.
+ */
+static size_t
+initcap_wbnext(void *state)
+{
+ struct WordBoundaryState *wbstate = (struct WordBoundaryState *) state;
+
+ while (wbstate->offset < wbstate->len &&
+ wbstate->str[wbstate->offset] != '\0')
+ {
+ pg_wchar u = utf8_to_unicode((unsigned char *) wbstate->str +
+ wbstate->offset);
+ bool curr_alnum = pg_u_isalnum(u, true);
+
+ if (!wbstate->init || curr_alnum != wbstate->prev_alnum)
+ {
+ size_t prev_offset = wbstate->offset;
+
+ wbstate->init = true;
+ wbstate->offset += unicode_utf8len(u);
+ wbstate->prev_alnum = curr_alnum;
+ return prev_offset;
+ }
+
+ wbstate->offset += unicode_utf8len(u);
+ }
+
+ return wbstate->len;
+}
+
+static size_t
+strlower_builtin(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ return unicode_strlower(dest, destsize, src, srclen);
+}
+
+static size_t
+strtitle_builtin(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ struct WordBoundaryState wbstate = {
+ .str = src,
+ .len = srclen,
+ .offset = 0,
+ .init = false,
+ .prev_alnum = false,
+ };
+
+ return unicode_strtitle(dest, destsize, src, srclen,
+ initcap_wbnext, &wbstate);
+}
+
+static size_t
+strupper_builtin(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ return unicode_strupper(dest, destsize, src, srclen);
+}
+
+static struct casemap_methods casemap_methods_builtin = {
+ .strlower = strlower_builtin,
+ .strtitle = strtitle_builtin,
+ .strupper = strupper_builtin,
+};
+
/*
* POSIX doesn't define _l-variants of these functions, but several systems
* have them. We provide our own replacements here.
@@ -1239,6 +1318,7 @@ dat_create_locale_builtin(HeapTuple dattuple)
result->deterministic = true;
result->collate_is_c = true;
result->ctype_is_c = (strcmp(locstr, "C") == 0);
+ result->casemap = &casemap_methods_builtin;
return result;
}
@@ -1265,6 +1345,7 @@ coll_create_locale_builtin(HeapTuple colltuple, MemoryContext context)
result->deterministic = collform->collisdeterministic;
result->collate_is_c = true;
result->ctype_is_c = (strcmp(locstr, "C") == 0);
+ result->casemap = &casemap_methods_builtin;
return result;
}
@@ -1537,6 +1618,27 @@ get_collation_actual_version(char collprovider, const char *collcollate)
return collversion;
}
+size_t
+pg_strlower(char *dst, size_t dstsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ return locale->casemap->strlower(dst, dstsize, src, srclen, locale);
+}
+
+size_t
+pg_strtitle(char *dst, size_t dstsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ return locale->casemap->strtitle(dst, dstsize, src, srclen, locale);
+}
+
+size_t
+pg_strupper(char *dst, size_t dstsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ return locale->casemap->strupper(dst, dstsize, src, srclen, locale);
+}
+
/*
* pg_strcoll
*
diff --git a/src/backend/utils/adt/pg_locale_icu.c b/src/backend/utils/adt/pg_locale_icu.c
index e53bf2d4b33..97e96d5b9fb 100644
--- a/src/backend/utils/adt/pg_locale_icu.c
+++ b/src/backend/utils/adt/pg_locale_icu.c
@@ -51,6 +51,11 @@ static size_t strnxfrm_prefix_icu(char *dest, size_t destsize,
const char *src, ssize_t srclen,
pg_locale_t locale);
+typedef int32_t (*ICU_Convert_Func) (UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ const char *locale,
+ UErrorCode *pErrorCode);
+
/*
* Converter object for converting between ICU's UChar strings and C strings
* in database encoding. Since the database encoding doesn't change, we only
@@ -60,9 +65,20 @@ static UConverter *icu_converter = NULL;
static UCollator *make_icu_collator(const char *iculocstr,
const char *icurules);
+
+static size_t strlower_icu(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+static size_t strtitle_icu(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+static size_t strupper_icu(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
static int strncoll_icu_no_utf8(const char *arg1, ssize_t len1,
const char *arg2, ssize_t len2,
pg_locale_t locale);
+
static size_t strnxfrm_prefix_icu_no_utf8(char *dest, size_t destsize,
const char *src, ssize_t srclen,
pg_locale_t locale);
@@ -72,8 +88,19 @@ static size_t uchar_length(UConverter *converter,
static int32_t uchar_convert(UConverter *converter,
UChar *dest, int32_t destlen,
const char *src, int32_t srclen);
+static int32_t icu_to_uchar(UChar **buff_uchar, const char *buff,
+ size_t nbytes);
+static size_t icu_from_uchar(char *dest, size_t destsize,
+ const UChar *buff_uchar, int32_t len_uchar);
static void icu_set_collation_attributes(UCollator *collator, const char *loc,
UErrorCode *status);
+static int32_t icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale,
+ UChar **buff_dest, UChar *buff_source,
+ int32_t len_source);
+static int32_t u_strToTitle_default_BI(UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ const char *locale,
+ UErrorCode *pErrorCode);
static struct collate_methods collate_methods_icu = {
.strncoll = strncoll_icu,
@@ -82,6 +109,11 @@ static struct collate_methods collate_methods_icu = {
.strxfrm_is_safe = true,
};
+static struct casemap_methods casemap_methods_icu = {
+ .strlower = strlower_icu,
+ .strtitle = strtitle_icu,
+ .strupper = strupper_icu,
+};
#endif
pg_locale_t
@@ -118,6 +150,7 @@ dat_create_locale_icu(HeapTuple dattuple)
result->collate_is_c = false;
result->ctype_is_c = false;
result->collate = &collate_methods_icu;
+ result->casemap = &casemap_methods_icu;
return result;
#else
@@ -163,6 +196,7 @@ coll_create_locale_icu(HeapTuple colltuple, MemoryContext context)
result->collate_is_c = false;
result->ctype_is_c = false;
result->collate = &collate_methods_icu;
+ result->casemap = &casemap_methods_icu;
return result;
#else
@@ -336,6 +370,66 @@ make_icu_collator(const char *iculocstr, const char *icurules)
}
}
+static size_t
+strlower_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ int32_t len_uchar;
+ int32_t len_conv;
+ UChar *buff_uchar;
+ UChar *buff_conv;
+ size_t result_len;
+
+ len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
+ len_conv = icu_convert_case(u_strToLower, locale,
+ &buff_conv, buff_uchar, len_uchar);
+ result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
+ pfree(buff_uchar);
+ pfree(buff_conv);
+
+ return result_len;
+}
+
+static size_t
+strtitle_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ int32_t len_uchar;
+ int32_t len_conv;
+ UChar *buff_uchar;
+ UChar *buff_conv;
+ size_t result_len;
+
+ len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
+ len_conv = icu_convert_case(u_strToTitle_default_BI, locale,
+ &buff_conv, buff_uchar, len_uchar);
+ result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
+ pfree(buff_uchar);
+ pfree(buff_conv);
+
+ return result_len;
+}
+
+static size_t
+strupper_icu(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ int32_t len_uchar;
+ int32_t len_conv;
+ UChar *buff_uchar;
+ UChar *buff_conv;
+ size_t result_len;
+
+ len_uchar = icu_to_uchar(&buff_uchar, src, srclen);
+ len_conv = icu_convert_case(u_strToUpper, locale,
+ &buff_conv, buff_uchar, len_uchar);
+ result_len = icu_from_uchar(dest, destsize, buff_conv, len_conv);
+ pfree(buff_uchar);
+ pfree(buff_conv);
+
+ return result_len;
+}
+
/*
* strncoll_icu
*
@@ -470,7 +564,7 @@ strnxfrm_prefix_icu(char *dest, size_t destsize,
* The result string is nul-terminated, though most callers rely on the
* result length instead.
*/
-int32_t
+static int32_t
icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes)
{
int32_t len_uchar;
@@ -497,8 +591,8 @@ icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes)
*
* The result string is nul-terminated.
*/
-int32_t
-icu_from_uchar(char **result, const UChar *buff_uchar, int32_t len_uchar)
+static size_t
+icu_from_uchar(char *dest, size_t destsize, const UChar *buff_uchar, int32_t len_uchar)
{
UErrorCode status;
int32_t len_result;
@@ -513,10 +607,11 @@ icu_from_uchar(char **result, const UChar *buff_uchar, int32_t len_uchar)
(errmsg("%s failed: %s", "ucnv_fromUChars",
u_errorName(status))));
- *result = palloc(len_result + 1);
+ if (len_result + 1 > destsize)
+ return len_result;
status = U_ZERO_ERROR;
- len_result = ucnv_fromUChars(icu_converter, *result, len_result + 1,
+ len_result = ucnv_fromUChars(icu_converter, dest, len_result + 1,
buff_uchar, len_uchar, &status);
if (U_FAILURE(status) ||
status == U_STRING_NOT_TERMINATED_WARNING)
@@ -527,6 +622,43 @@ icu_from_uchar(char **result, const UChar *buff_uchar, int32_t len_uchar)
return len_result;
}
+static int32_t
+icu_convert_case(ICU_Convert_Func func, pg_locale_t mylocale,
+ UChar **buff_dest, UChar *buff_source, int32_t len_source)
+{
+ UErrorCode status;
+ int32_t len_dest;
+
+ len_dest = len_source; /* try first with same length */
+ *buff_dest = palloc(len_dest * sizeof(**buff_dest));
+ status = U_ZERO_ERROR;
+ len_dest = func(*buff_dest, len_dest, buff_source, len_source,
+ mylocale->info.icu.locale, &status);
+ if (status == U_BUFFER_OVERFLOW_ERROR)
+ {
+ /* try again with adjusted length */
+ pfree(*buff_dest);
+ *buff_dest = palloc(len_dest * sizeof(**buff_dest));
+ status = U_ZERO_ERROR;
+ len_dest = func(*buff_dest, len_dest, buff_source, len_source,
+ mylocale->info.icu.locale, &status);
+ }
+ if (U_FAILURE(status))
+ ereport(ERROR,
+ (errmsg("case conversion failed: %s", u_errorName(status))));
+ return len_dest;
+}
+
+static int32_t
+u_strToTitle_default_BI(UChar *dest, int32_t destCapacity,
+ const UChar *src, int32_t srcLength,
+ const char *locale,
+ UErrorCode *pErrorCode)
+{
+ return u_strToTitle(dest, destCapacity, src, srcLength,
+ NULL, locale, pErrorCode);
+}
+
/*
* strncoll_icu_no_utf8
*
diff --git a/src/backend/utils/adt/pg_locale_libc.c b/src/backend/utils/adt/pg_locale_libc.c
index b8ccd24715d..79828ab3524 100644
--- a/src/backend/utils/adt/pg_locale_libc.c
+++ b/src/backend/utils/adt/pg_locale_libc.c
@@ -11,6 +11,9 @@
#include "postgres.h"
+#include <limits.h>
+#include <wctype.h>
+
#include "access/htup_details.h"
#include "catalog/pg_database.h"
#include "catalog/pg_collation.h"
@@ -48,6 +51,16 @@ static int strncoll_libc_win32_utf8(const char *arg1, ssize_t len1,
pg_locale_t locale);
#endif
+static size_t strlower_libc(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+static size_t strtitle_libc(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+static size_t strupper_libc(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+
static struct collate_methods collate_methods_libc = {
.strncoll = strncoll_libc,
.strnxfrm = strnxfrm_libc,
@@ -69,6 +82,208 @@ static struct collate_methods collate_methods_libc = {
#endif
};
+static struct casemap_methods casemap_methods_libc = {
+ .strlower = strlower_libc,
+ .strtitle = strtitle_libc,
+ .strupper = strupper_libc,
+};
+
+static size_t
+strlower_libc(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ locale_t loc = locale->info.lt;
+ size_t result_size;
+
+ if (pg_database_encoding_max_length() > 1)
+ {
+ wchar_t *workspace;
+ size_t curr_char;
+
+ /* Overflow paranoia */
+ if ((srclen + 1) > (INT_MAX / sizeof(wchar_t)))
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory")));
+
+ /* Output workspace cannot have more codes than input bytes */
+ workspace = (wchar_t *) palloc((srclen + 1) * sizeof(wchar_t));
+
+ char2wchar(workspace, srclen + 1, src, srclen, locale);
+
+ for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
+ workspace[curr_char] = towlower_l(workspace[curr_char], loc);
+
+ /*
+ * Make result large enough; case change might change number of bytes
+ */
+ result_size = curr_char * pg_database_encoding_max_length();
+ if (result_size + 1 > destsize)
+ return result_size;
+
+ wchar2char(dest, workspace, result_size + 1, locale);
+ pfree(workspace);
+ }
+ else
+ {
+ char *p;
+
+ result_size = srclen;
+ if (result_size + 1 > destsize)
+ return result_size;
+
+ strlcpy(dest, src, result_size + 1);
+
+ /*
+ * Note: we assume that tolower_l() will not be so broken as to need
+ * an isupper_l() guard test. When using the default collation, we
+ * apply the traditional Postgres behavior that forces ASCII-style
+ * treatment of I/i, but in non-default collations you get exactly
+ * what the collation says.
+ */
+ for (p = dest; *p; p++)
+ *p = tolower_l((unsigned char) *p, loc);
+ }
+
+ result_size = strlen(dest);
+ return result_size;
+}
+
+static size_t
+strtitle_libc(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ locale_t loc = locale->info.lt;
+ int wasalnum = false;
+ size_t result_size;
+
+ if (pg_database_encoding_max_length() > 1)
+ {
+ wchar_t *workspace;
+ size_t curr_char;
+
+ /* Overflow paranoia */
+ if ((srclen + 1) > (INT_MAX / sizeof(wchar_t)))
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory")));
+
+ /* Output workspace cannot have more codes than input bytes */
+ workspace = (wchar_t *) palloc((srclen + 1) * sizeof(wchar_t));
+
+ char2wchar(workspace, srclen + 1, src, srclen, locale);
+
+ for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
+ {
+ if (wasalnum)
+ workspace[curr_char] = towlower_l(workspace[curr_char], loc);
+ else
+ workspace[curr_char] = towupper_l(workspace[curr_char], loc);
+ wasalnum = iswalnum_l(workspace[curr_char], loc);
+ }
+
+ /*
+ * Make result large enough; case change might change number of bytes
+ */
+ result_size = curr_char * pg_database_encoding_max_length();
+
+ if (result_size + 1 > destsize)
+ return result_size;
+
+ wchar2char(dest, workspace, result_size + 1, locale);
+ pfree(workspace);
+ }
+ else
+ {
+ char *p;
+
+ if (srclen + 1 > destsize)
+ return srclen;
+
+ strlcpy(dest, src, srclen + 1);
+
+ /*
+ * Note: we assume that toupper_l()/tolower_l() will not be so broken
+ * as to need guard tests. When using the default collation, we apply
+ * the traditional Postgres behavior that forces ASCII-style treatment
+ * of I/i, but in non-default collations you get exactly what the
+ * collation says.
+ */
+ for (p = dest; *p; p++)
+ {
+ if (wasalnum)
+ *p = tolower_l((unsigned char) *p, loc);
+ else
+ *p = toupper_l((unsigned char) *p, loc);
+ wasalnum = isalnum_l((unsigned char) *p, loc);
+ }
+ }
+
+ result_size = strlen(dest);
+ return result_size;
+}
+
+static size_t
+strupper_libc(char *dest, size_t destsize, const char *src, ssize_t srclen,
+ pg_locale_t locale)
+{
+ locale_t loc = locale->info.lt;
+ size_t result_size;
+
+ if (pg_database_encoding_max_length() > 1)
+ {
+ wchar_t *workspace;
+ size_t curr_char;
+
+ /* Overflow paranoia */
+ if ((srclen + 1) > (INT_MAX / sizeof(wchar_t)))
+ ereport(ERROR,
+ (errcode(ERRCODE_OUT_OF_MEMORY),
+ errmsg("out of memory")));
+
+ /* Output workspace cannot have more codes than input bytes */
+ workspace = (wchar_t *) palloc((srclen + 1) * sizeof(wchar_t));
+
+ char2wchar(workspace, srclen + 1, src, srclen, locale);
+
+ for (curr_char = 0; workspace[curr_char] != 0; curr_char++)
+ workspace[curr_char] = towupper_l(workspace[curr_char], loc);
+
+ /*
+ * Make result large enough; case change might change number of bytes
+ */
+ result_size = curr_char * pg_database_encoding_max_length();
+ if (result_size + 1 > destsize)
+ return result_size;
+
+ wchar2char(dest, workspace, result_size + 1, locale);
+ pfree(workspace);
+ }
+ else
+ {
+ char *p;
+
+ result_size = srclen;
+ if (result_size + 1 > destsize)
+ return result_size;
+
+ strlcpy(dest, src, srclen + 1);
+
+ /*
+ * Note: we assume that toupper_l() will not be so broken as to need
+ * an islower_l() guard test. When using the default collation, we
+ * apply the traditional Postgres behavior that forces ASCII-style
+ * treatment of I/i, but in non-default collations you get exactly
+ * what the collation says.
+ */
+ for (p = dest; *p; p++)
+ *p = toupper_l((unsigned char) *p, loc);
+ }
+
+ result_size = strlen(dest);
+ return result_size;
+}
+
pg_locale_t
dat_create_locale_libc(HeapTuple dattuple)
{
@@ -102,6 +317,8 @@ dat_create_locale_libc(HeapTuple dattuple)
result->info.lt = loc;
if (!result->collate_is_c)
result->collate = &collate_methods_libc;
+ if (!result->ctype_is_c)
+ result->casemap = &casemap_methods_libc;
return result;
}
@@ -137,6 +354,8 @@ coll_create_locale_libc(HeapTuple colltuple, MemoryContext context)
result->info.lt = loc;
if (!result->collate_is_c)
result->collate = &collate_methods_libc;
+ if (!result->ctype_is_c)
+ result->casemap = &casemap_methods_libc;
return result;
}
diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h
index 6c2a0456f22..4bd9e6de7a3 100644
--- a/src/include/utils/pg_locale.h
+++ b/src/include/utils/pg_locale.h
@@ -93,6 +93,20 @@ struct collate_methods
bool strxfrm_is_safe;
};
+/* methods that define string case mapping behavior */
+struct casemap_methods
+{
+ size_t (*strlower) (char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+ size_t (*strtitle) (char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+ size_t (*strupper) (char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+};
+
/*
* We use a discriminated union to hold either a locale_t or an ICU collator.
* pg_locale_t is occasionally checked for truth, so make it a pointer.
@@ -117,6 +131,7 @@ struct pg_locale_struct
bool ctype_is_c;
struct collate_methods *collate; /* NULL if collate_is_c */
+ struct casemap_methods *casemap; /* NULL if ctype_is_c */
union
{
@@ -141,6 +156,15 @@ extern void init_database_collation(void);
extern pg_locale_t pg_newlocale_from_collation(Oid collid);
extern char *get_collation_actual_version(char collprovider, const char *collcollate);
+extern size_t pg_strlower(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+extern size_t pg_strtitle(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
+extern size_t pg_strupper(char *dest, size_t destsize,
+ const char *src, ssize_t srclen,
+ pg_locale_t locale);
extern int pg_strcoll(const char *arg1, const char *arg2, pg_locale_t locale);
extern int pg_strncoll(const char *arg1, ssize_t len1,
const char *arg2, ssize_t len2, pg_locale_t locale);
@@ -160,11 +184,6 @@ extern const char *builtin_validate_locale(int encoding, const char *locale);
extern void icu_validate_locale(const char *loc_str);
extern char *icu_language_tag(const char *loc_str, int elevel);
-#ifdef USE_ICU
-extern int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes);
-extern int32_t icu_from_uchar(char **result, const UChar *buff_uchar, int32_t len_uchar);
-#endif
-
/* These functions convert from/to libc's wchar_t, *not* pg_wchar_t */
extern size_t wchar2char(char *to, const wchar_t *from, size_t tolen,
pg_locale_t locale);
--
2.34.1