From 6d0de521a41f32037f13d242f30cac7a2c16ae53 Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Fri, 17 Mar 2023 09:55:31 -0700 Subject: [PATCH v6 6/6] Validate ICU locales. Ensure that it can be transformed into a language tag in "strict" mode (which validates the attributes), and also that the language exists in ICU. Basic validation helps avoid minor mistakes and misspellings, which often fall back to the root locale instead of the intended locale. It's even more important in ICU versions 54 and earlier, where the same (misspelled) locale string could fall back to different locales depending on the environment. Discussion: https://postgr.es/m/11b1eeb7e7667fdd4178497aeb796c48d26e69b9.camel@j-davis.com Discussion: https://postgr.es/m/df2efad0cae7c65180df8e5ebb709e5eb4f2a82b.camel@j-davis.com --- doc/src/sgml/config.sgml | 17 +++++ src/backend/commands/collationcmds.c | 8 +-- src/backend/commands/dbcommands.c | 8 +-- src/backend/utils/adt/pg_locale.c | 64 +++++++++++++++++++ src/backend/utils/misc/guc_tables.c | 10 +++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/utils/pg_locale.h | 1 + .../regress/expected/collate.icu.utf8.out | 10 ++- src/test/regress/sql/collate.icu.utf8.sql | 6 +- 9 files changed, 112 insertions(+), 14 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e5c41cc6c6..44e9924f6b 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -9776,6 +9776,23 @@ SET XML OPTION { DOCUMENT | CONTENT }; + + icu_locale_validation (boolean) + + icu_locale_validation configuration parameter + + + + + Validation is performed on an ICU locale specified for a new collation + or database. If this parameter is set to true, an + error is raised for a validation failure; if set to + false, a warning is issued. The default is + false. + + + + default_text_search_config (string) diff --git a/src/backend/commands/collationcmds.c b/src/backend/commands/collationcmds.c index bc0e48d507..592718fb50 100644 --- a/src/backend/commands/collationcmds.c +++ b/src/backend/commands/collationcmds.c @@ -284,12 +284,8 @@ DefineCollation(ParseState *pstate, List *names, List *parameters, bool if_not_e colliculocale = langtag; } - else - { - ereport(WARNING, - (errmsg("could not convert locale \"%s\" to language tag", - colliculocale))); - } + + icu_validate_locale(colliculocale); } #else ereport(ERROR, diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 5935477f44..600b3a0f61 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1079,12 +1079,8 @@ createdb(ParseState *pstate, const CreatedbStmt *stmt) dbiculocale = langtag; } - else - { - ereport(WARNING, - (errmsg("could not convert locale \"%s\" to language tag", - dbiculocale))); - } + + icu_validate_locale(dbiculocale); } #else ereport(ERROR, diff --git a/src/backend/utils/adt/pg_locale.c b/src/backend/utils/adt/pg_locale.c index 45a7f77b2d..b655bdc142 100644 --- a/src/backend/utils/adt/pg_locale.c +++ b/src/backend/utils/adt/pg_locale.c @@ -88,6 +88,7 @@ #define MAX_L10N_DATA 80 +extern bool icu_locale_validation; /* GUC settings */ char *locale_messages; @@ -2876,6 +2877,69 @@ icu_language_tag(const char *loc_str, bool noError) return langtag; } +/* + * Perform best-effort check that the locale is a valid one. + */ +void +icu_validate_locale(const char *loc_str) +{ + UErrorCode status; + int elevel = icu_locale_validation ? ERROR : WARNING; + char *langtag = icu_language_tag(loc_str, true); + char lang[ULOC_LANG_CAPACITY]; + + /* check that it can be converted to a language tag */ + if (langtag == NULL) + { + ereport(elevel, + (errmsg("could not convert locale \"%s\" to language tag", + loc_str))); + return; + } + pfree(langtag); + + /* validate that we can extract the language */ + status = U_ZERO_ERROR; + uloc_getLanguage(loc_str, lang, ULOC_LANG_CAPACITY, &status); + if (U_FAILURE(status)) + { + ereport(elevel, + (errmsg("could not get language from locale \"%s\": %s", + loc_str, u_errorName(status)))); + return; + } + + /* check for special languages */ + if (strcmp(lang, "") == 0 || + strcmp(lang, "root") == 0 || strcmp(lang, "und") == 0 || + strcmp(lang, "c") == 0 || strcmp(lang, "posix") == 0) + return; + + /* search for matching language within ICU */ + for (int32_t i = 0; i < uloc_countAvailable(); i++) + { + const char *otherloc = uloc_getAvailable(i); + char otherlang[ULOC_LANG_CAPACITY]; + + status = U_ZERO_ERROR; + uloc_getLanguage(otherloc, otherlang, ULOC_LANG_CAPACITY, &status); + if (U_FAILURE(status)) + { + ereport(elevel, + (errmsg("could not get language from locale \"%s\": %s", + loc_str, u_errorName(status)))); + continue; + } + + if (strcmp(lang, otherlang) == 0) + return; + } + + ereport(elevel, + (errmsg("language \"%s\" of locale \"%s\" not found", + lang, loc_str))); +} + #endif /* USE_ICU */ /* diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1c0583fe26..1c63ed0d21 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -481,6 +481,7 @@ char *event_source; bool row_security; bool check_function_bodies = true; +bool icu_locale_validation = false; /* * This GUC exists solely for backward compatibility, check its definition for @@ -1586,6 +1587,15 @@ struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"icu_locale_validation", PGC_USERSET, CLIENT_CONN_LOCALE, + gettext_noop("Raise an error for invalid ICU locale strings."), + NULL + }, + &icu_locale_validation, + false, + NULL, NULL, NULL + }, { {"array_nulls", PGC_USERSET, COMPAT_OPTIONS_PREVIOUS, gettext_noop("Enable input of NULL elements in arrays."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index d06074b86f..cff927e8be 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -730,6 +730,8 @@ #lc_numeric = 'C' # locale for number formatting #lc_time = 'C' # locale for time formatting +#icu_locale_validation = off # validate ICU locale strings + # default configuration for text search #default_text_search_config = 'pg_catalog.simple' diff --git a/src/include/utils/pg_locale.h b/src/include/utils/pg_locale.h index 999122b6f4..125fa7f4a9 100644 --- a/src/include/utils/pg_locale.h +++ b/src/include/utils/pg_locale.h @@ -120,6 +120,7 @@ extern size_t pg_strnxfrm_prefix(char *dest, size_t destsize, const char *src, extern int32_t icu_to_uchar(UChar **buff_uchar, const char *buff, size_t nbytes); extern int32_t icu_from_uchar(char **result, const UChar *buff_uchar, int32_t len_uchar); extern char *icu_language_tag(const char *loc_str, bool noError); +extern void icu_validate_locale(const char *loc_str); #endif extern void check_icu_locale(const char *icu_locale); diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out index 9715c12c93..83253b9666 100644 --- a/src/test/regress/expected/collate.icu.utf8.out +++ b/src/test/regress/expected/collate.icu.utf8.out @@ -1092,8 +1092,16 @@ $$; RESET client_min_messages; CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, needs "locale" ERROR: parameter "locale" must be specified -CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; +SET icu_locale_validation = true; +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); -- fails NOTICE: using language tag "nonsense" for locale "nonsense" +ERROR: language "nonsense" of locale "nonsense" not found +CREATE COLLATION testx (provider = icu, locale = '@colStrength=primary;nonsense=yes'); -- fails +ERROR: could not convert locale "@colStrength=primary;nonsense=yes" to language tag +RESET icu_locale_validation; +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); DROP COLLATION testx; +NOTICE: using language tag "nonsense" for locale "nonsense" +WARNING: language "nonsense" of locale "nonsense" not found CREATE COLLATION test4 FROM nonsense; ERROR: collation "nonsense" for encoding "UTF8" does not exist CREATE COLLATION test5 FROM test0; diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql index ddd1ab7e66..a8d9917208 100644 --- a/src/test/regress/sql/collate.icu.utf8.sql +++ b/src/test/regress/sql/collate.icu.utf8.sql @@ -386,7 +386,11 @@ $$; RESET client_min_messages; CREATE COLLATION test3 (provider = icu, lc_collate = 'en_US.utf8'); -- fail, needs "locale" -CREATE COLLATION testx (provider = icu, locale = 'nonsense'); /* never fails with ICU */ DROP COLLATION testx; +SET icu_locale_validation = true; +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); -- fails +CREATE COLLATION testx (provider = icu, locale = '@colStrength=primary;nonsense=yes'); -- fails +RESET icu_locale_validation; +CREATE COLLATION testx (provider = icu, locale = 'nonsense'); DROP COLLATION testx; CREATE COLLATION test4 FROM nonsense; CREATE COLLATION test5 FROM test0; -- 2.34.1