From bf07696da676482d874602bd0f267b979dea5b82 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Wed, 26 Nov 2025 10:28:45 -0800 Subject: [PATCH v11 6/9] Use multibyte-aware extraction of pattern prefixes. Previously, like_fixed_prefix() used char-at-a-time logic, which forced it to be too conservative for case-insensitive matching. Now, use pg_wchar-at-a-time loop for text types, along with proper detection of cased characters; and preserve and char-at-a-time logic for bytea. Removes the pg_locale_t char_is_cased() single-byte method and replaces it with a proper multibyte pg_iswcased() method. Discussion: https://postgr.es/m/450ceb6260cad30d7afdf155d991a9caafee7c0d.camel@j-davis.com --- src/backend/utils/adt/like.c | 9 +- src/backend/utils/adt/like_support.c | 128 ++++++++++++---------- src/backend/utils/adt/pg_locale.c | 15 --- src/backend/utils/adt/pg_locale_builtin.c | 8 -- src/backend/utils/adt/pg_locale_icu.c | 8 -- src/backend/utils/adt/pg_locale_libc.c | 14 --- src/include/utils/pg_locale.h | 3 - 7 files changed, 76 insertions(+), 109 deletions(-) diff --git a/src/backend/utils/adt/like.c b/src/backend/utils/adt/like.c index 4a7fc583c71..772879f0a81 100644 --- a/src/backend/utils/adt/like.c +++ b/src/backend/utils/adt/like.c @@ -118,7 +118,7 @@ wchareq(const char *p1, const char *p2) #include "like_match.c" -/* setup to compile like_match.c for single byte case insensitive matches */ +/* setup to compile like_match.c for case-insensitive matches in C locale */ #define MATCH_LOWER #define NextChar(p, plen) NextByte((p), (plen)) #define MatchText C_IMatchText @@ -190,8 +190,11 @@ Generic_Text_IC_like(text *str, text *pat, Oid collation) errmsg("nondeterministic collations are not supported for ILIKE"))); /* - * For efficiency reasons, in the C locale we don't call lower() on the - * pattern and text, but instead call SB_lower_char on each character. + * For efficiency reasons, in the C locale lowercase each character + * lazily. Otherwise, we lowercase the entire pattern and text strings + * prior to matching. + * + * XXX: use casefolding instead? */ if (locale->ctype_is_c) diff --git a/src/backend/utils/adt/like_support.c b/src/backend/utils/adt/like_support.c index 999f23f86d5..2db06bd1728 100644 --- a/src/backend/utils/adt/like_support.c +++ b/src/backend/utils/adt/like_support.c @@ -99,8 +99,6 @@ static Selectivity like_selectivity(const char *patt, int pattlen, static Selectivity regex_selectivity(const char *patt, int pattlen, bool case_insensitive, int fixed_prefix_len); -static int pattern_char_isalpha(char c, bool is_multibyte, - pg_locale_t locale); static Const *make_greater_string(const Const *str_const, FmgrInfo *ltproc, Oid collation); static Datum string_to_datum(const char *str, Oid datatype); @@ -989,13 +987,11 @@ static Pattern_Prefix_Status like_fixed_prefix(Const *patt_const, bool case_insensitive, Oid collation, Const **prefix_const, Selectivity *rest_selec) { - char *match; char *patt; int pattlen; Oid typeid = patt_const->consttype; - int pos, - match_pos; - bool is_multibyte = (pg_database_encoding_max_length() > 1); + int pos; + int match_pos = 0; pg_locale_t locale = 0; /* the right-hand const is type text or bytea */ @@ -1023,60 +1019,94 @@ like_fixed_prefix(Const *patt_const, bool case_insensitive, Oid collation, locale = pg_newlocale_from_collation(collation); } + /* for text types, use pg_wchar; for BYTEA, use char */ if (typeid != BYTEAOID) { - patt = TextDatumGetCString(patt_const->constvalue); - pattlen = strlen(patt); + text *val = DatumGetTextPP(patt_const->constvalue); + pg_wchar *wpatt; + pg_wchar *wmatch; + char *match; + int match_mblen; + + patt = VARDATA_ANY(val); + pattlen = VARSIZE_ANY_EXHDR(val); + wpatt = palloc((pattlen + 1) * sizeof(pg_wchar)); + pg_mb2wchar_with_len(patt, wpatt, pattlen); + + wmatch = palloc((pattlen + 1) * sizeof(pg_wchar)); + for (pos = 0; pos < pattlen; pos++) + { + /* % and _ are wildcard characters in LIKE */ + if (wpatt[pos] == '%' || + wpatt[pos] == '_') + break; + + /* Backslash escapes the next character */ + if (wpatt[pos] == '\\') + { + pos++; + if (pos >= pattlen) + break; + } + + /* + * For ILIKE, stop if it's a case-varying character (it's sort of + * a wildcard). + */ + if (case_insensitive && pg_iswcased(wpatt[pos], locale)) + break; + + wmatch[match_pos++] = wpatt[pos]; + } + + wmatch[match_pos] = '\0'; + + match_mblen = pg_database_encoding_max_length() * match_pos + 1; + match = palloc(match_mblen); + pg_wchar2mb_with_len(wmatch, match, match_pos); + + pfree(wpatt); + pfree(wmatch); + + *prefix_const = string_to_const(match, typeid); + pfree(match); } else { bytea *bstr = DatumGetByteaPP(patt_const->constvalue); + char *match; + patt = VARDATA_ANY(bstr); pattlen = VARSIZE_ANY_EXHDR(bstr); - patt = (char *) palloc(pattlen); - memcpy(patt, VARDATA_ANY(bstr), pattlen); - Assert((Pointer) bstr == DatumGetPointer(patt_const->constvalue)); - } - - match = palloc(pattlen + 1); - match_pos = 0; - for (pos = 0; pos < pattlen; pos++) - { - /* % and _ are wildcard characters in LIKE */ - if (patt[pos] == '%' || - patt[pos] == '_') - break; - /* Backslash escapes the next character */ - if (patt[pos] == '\\') + match = palloc(pattlen + 1); + for (pos = 0; pos < pattlen; pos++) { - pos++; - if (pos >= pattlen) + /* % and _ are wildcard characters in LIKE */ + if (patt[pos] == '%' || + patt[pos] == '_') break; - } - /* Stop if case-varying character (it's sort of a wildcard) */ - if (case_insensitive && - pattern_char_isalpha(patt[pos], is_multibyte, locale)) - break; - - match[match_pos++] = patt[pos]; - } + /* Backslash escapes the next character */ + if (patt[pos] == '\\') + { + pos++; + if (pos >= pattlen) + break; + } - match[match_pos] = '\0'; + match[match_pos++] = patt[pos]; + } - if (typeid != BYTEAOID) - *prefix_const = string_to_const(match, typeid); - else *prefix_const = string_to_bytea_const(match, match_pos); + pfree(match); + } + if (rest_selec != NULL) *rest_selec = like_selectivity(&patt[pos], pattlen - pos, case_insensitive); - pfree(patt); - pfree(match); - /* in LIKE, an empty pattern is an exact match! */ if (pos == pattlen) return Pattern_Prefix_Exact; /* reached end of pattern, so exact */ @@ -1481,24 +1511,6 @@ regex_selectivity(const char *patt, int pattlen, bool case_insensitive, return sel; } -/* - * Check whether char is a letter (and, hence, subject to case-folding) - * - * In multibyte character sets or with ICU, we can't use isalpha, and it does - * not seem worth trying to convert to wchar_t to use iswalpha or u_isalpha. - * Instead, just assume any non-ASCII char is potentially case-varying, and - * hard-wire knowledge of which ASCII chars are letters. - */ -static int -pattern_char_isalpha(char c, bool is_multibyte, - pg_locale_t locale) -{ - if (locale->ctype_is_c) - return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); - else - return char_is_cased(c, locale); -} - /* * For bytea, the increment function need only increment the current byte diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index e5e75ca2c2c..c4e89502f85 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -1625,21 +1625,6 @@ pg_towlower(pg_wchar wc, pg_locale_t locale) return locale->ctype->wc_tolower(wc, locale); } -/* - * char_is_cased() - * - * Fuzzy test of whether the given char is case-varying or not. The argument - * is a single byte, so in a multibyte encoding, just assume any non-ASCII - * char is case-varying. - */ -bool -char_is_cased(char ch, pg_locale_t locale) -{ - if (locale->ctype == NULL) - return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); - return locale->ctype->char_is_cased(ch, locale); -} - /* * Return required encoding ID for the given locale, or -1 if any encoding is * valid for the locale. diff --git a/src/backend/utils/adt/pg_locale_builtin.c b/src/backend/utils/adt/pg_locale_builtin.c index 0d4c754a267..0c2920112bb 100644 --- a/src/backend/utils/adt/pg_locale_builtin.c +++ b/src/backend/utils/adt/pg_locale_builtin.c @@ -191,13 +191,6 @@ wc_iscased_builtin(pg_wchar wc, pg_locale_t locale) return pg_u_prop_cased(to_char32(wc)); } -static bool -char_is_cased_builtin(char ch, pg_locale_t locale) -{ - return IS_HIGHBIT_SET(ch) || - (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); -} - static pg_wchar wc_toupper_builtin(pg_wchar wc, pg_locale_t locale) { @@ -225,7 +218,6 @@ static const struct ctype_methods ctype_methods_builtin = { .wc_ispunct = wc_ispunct_builtin, .wc_isspace = wc_isspace_builtin, .wc_isxdigit = wc_isxdigit_builtin, - .char_is_cased = char_is_cased_builtin, .wc_iscased = wc_iscased_builtin, .wc_tolower = wc_tolower_builtin, .wc_toupper = wc_toupper_builtin, diff --git a/src/backend/utils/adt/pg_locale_icu.c b/src/backend/utils/adt/pg_locale_icu.c index e8820666b2d..18d026deda8 100644 --- a/src/backend/utils/adt/pg_locale_icu.c +++ b/src/backend/utils/adt/pg_locale_icu.c @@ -121,13 +121,6 @@ static int32_t u_strFoldCase_default(UChar *dest, int32_t destCapacity, const char *locale, UErrorCode *pErrorCode); -static bool -char_is_cased_icu(char ch, pg_locale_t locale) -{ - return IS_HIGHBIT_SET(ch) || - (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z'); -} - /* * XXX: many of the functions below rely on casts directly from pg_wchar to * UChar32, which is correct for the UTF-8 encoding, but not in general. @@ -244,7 +237,6 @@ static const struct ctype_methods ctype_methods_icu = { .wc_ispunct = wc_ispunct_icu, .wc_isspace = wc_isspace_icu, .wc_isxdigit = wc_isxdigit_icu, - .char_is_cased = char_is_cased_icu, .wc_iscased = wc_iscased_icu, .wc_toupper = toupper_icu, .wc_tolower = tolower_icu, diff --git a/src/backend/utils/adt/pg_locale_libc.c b/src/backend/utils/adt/pg_locale_libc.c index cd54198f0c7..4cb3c64b4a6 100644 --- a/src/backend/utils/adt/pg_locale_libc.c +++ b/src/backend/utils/adt/pg_locale_libc.c @@ -262,17 +262,6 @@ wc_iscased_libc_mb(pg_wchar wc, pg_locale_t locale) iswlower_l((wint_t) wc, locale->lt); } -static bool -char_is_cased_libc(char ch, pg_locale_t locale) -{ - bool is_multibyte = pg_database_encoding_max_length() > 1; - - if (is_multibyte && IS_HIGHBIT_SET(ch)) - return true; - else - return isalpha_l((unsigned char) ch, locale->lt); -} - static pg_wchar toupper_libc_sb(pg_wchar wc, pg_locale_t locale) { @@ -345,7 +334,6 @@ static const struct ctype_methods ctype_methods_libc_sb = { .wc_ispunct = wc_ispunct_libc_sb, .wc_isspace = wc_isspace_libc_sb, .wc_isxdigit = wc_isxdigit_libc_sb, - .char_is_cased = char_is_cased_libc, .wc_iscased = wc_iscased_libc_sb, .wc_toupper = toupper_libc_sb, .wc_tolower = tolower_libc_sb, @@ -371,7 +359,6 @@ static const struct ctype_methods ctype_methods_libc_other_mb = { .wc_ispunct = wc_ispunct_libc_sb, .wc_isspace = wc_isspace_libc_sb, .wc_isxdigit = wc_isxdigit_libc_sb, - .char_is_cased = char_is_cased_libc, .wc_iscased = wc_iscased_libc_sb, .wc_toupper = toupper_libc_sb, .wc_tolower = tolower_libc_sb, @@ -393,7 +380,6 @@ static const struct ctype_methods ctype_methods_libc_utf8 = { .wc_ispunct = wc_ispunct_libc_mb, .wc_isspace = wc_isspace_libc_mb, .wc_isxdigit = wc_isxdigit_libc_mb, - .char_is_cased = char_is_cased_libc, .wc_iscased = wc_iscased_libc_mb, .wc_toupper = toupper_libc_mb, .wc_tolower = tolower_libc_mb, diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index 832007385d8..01f891def7a 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -125,9 +125,6 @@ struct ctype_methods bool (*wc_iscased) (pg_wchar wc, pg_locale_t locale); pg_wchar (*wc_toupper) (pg_wchar wc, pg_locale_t locale); pg_wchar (*wc_tolower) (pg_wchar wc, pg_locale_t locale); - - /* required */ - bool (*char_is_cased) (char ch, pg_locale_t locale); }; /* -- 2.43.0