From f7a7f21ce173fb149c9d1fbaa4ac13a5c0b9bf6d Mon Sep 17 00:00:00 2001 From: Jeff Davis Date: Sat, 2 Mar 2024 11:27:23 -0800 Subject: [PATCH v20 6/6] Inline basic UTF-8 functions. Shows a measurable speedup when processing UTF-8 data, such as with the new builtin collation provider. --- src/common/wchar.c | 92 +------------------------------------ src/include/mb/pg_wchar.h | 96 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 91 deletions(-) diff --git a/src/common/wchar.c b/src/common/wchar.c index a238c0106c..17788ac8b8 100644 --- a/src/common/wchar.c +++ b/src/common/wchar.c @@ -476,39 +476,6 @@ pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len) } -/* - * Map a Unicode code point to UTF-8. utf8string must have at least - * unicode_utf8len(c) bytes available. - */ -unsigned char * -unicode_to_utf8(pg_wchar c, unsigned char *utf8string) -{ - if (c <= 0x7F) - { - utf8string[0] = c; - } - else if (c <= 0x7FF) - { - utf8string[0] = 0xC0 | ((c >> 6) & 0x1F); - utf8string[1] = 0x80 | (c & 0x3F); - } - else if (c <= 0xFFFF) - { - utf8string[0] = 0xE0 | ((c >> 12) & 0x0F); - utf8string[1] = 0x80 | ((c >> 6) & 0x3F); - utf8string[2] = 0x80 | (c & 0x3F); - } - else - { - utf8string[0] = 0xF0 | ((c >> 18) & 0x07); - utf8string[1] = 0x80 | ((c >> 12) & 0x3F); - utf8string[2] = 0x80 | ((c >> 6) & 0x3F); - utf8string[3] = 0x80 | (c & 0x3F); - } - - return utf8string; -} - /* * Trivial conversion from pg_wchar to UTF-8. * caller should allocate enough space for "to" @@ -535,39 +502,10 @@ pg_wchar2utf_with_len(const pg_wchar *from, unsigned char *to, int len) return cnt; } -/* - * Return the byte length of a UTF8 character pointed to by s - * - * Note: in the current implementation we do not support UTF8 sequences - * of more than 4 bytes; hence do NOT return a value larger than 4. - * We return "1" for any leading byte that is either flat-out illegal or - * indicates a length larger than we support. - * - * pg_utf2wchar_with_len(), utf8_to_unicode(), pg_utf8_islegal(), and perhaps - * other places would need to be fixed to change this. - */ int pg_utf_mblen(const unsigned char *s) { - int len; - - if ((*s & 0x80) == 0) - len = 1; - else if ((*s & 0xe0) == 0xc0) - len = 2; - else if ((*s & 0xf0) == 0xe0) - len = 3; - else if ((*s & 0xf8) == 0xf0) - len = 4; -#ifdef NOT_USED - else if ((*s & 0xfc) == 0xf8) - len = 5; - else if ((*s & 0xfe) == 0xfc) - len = 6; -#endif - else - len = 1; - return len; + return utf8_mblen(s); } /* @@ -670,34 +608,6 @@ ucs_wcwidth(pg_wchar ucs) return 1; } -/* - * Convert a UTF-8 character to a Unicode code point. - * This is a one-character version of pg_utf2wchar_with_len. - * - * No error checks here, c must point to a long-enough string. - */ -pg_wchar -utf8_to_unicode(const unsigned char *c) -{ - if ((*c & 0x80) == 0) - return (pg_wchar) c[0]; - else if ((*c & 0xe0) == 0xc0) - return (pg_wchar) (((c[0] & 0x1f) << 6) | - (c[1] & 0x3f)); - else if ((*c & 0xf0) == 0xe0) - return (pg_wchar) (((c[0] & 0x0f) << 12) | - ((c[1] & 0x3f) << 6) | - (c[2] & 0x3f)); - else if ((*c & 0xf8) == 0xf0) - return (pg_wchar) (((c[0] & 0x07) << 18) | - ((c[1] & 0x3f) << 12) | - ((c[2] & 0x3f) << 6) | - (c[3] & 0x3f)); - else - /* that is an invalid code on purpose */ - return 0xffffffff; -} - static int pg_utf_dsplen(const unsigned char *s) { diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 69a55b66f4..f0ce47977c 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -555,6 +555,102 @@ surrogate_pair_to_codepoint(pg_wchar first, pg_wchar second) return ((first & 0x3FF) << 10) + 0x10000 + (second & 0x3FF); } +/* + * Return the byte length of a UTF8 character pointed to by s + * + * Note: in the current implementation we do not support UTF8 sequences + * of more than 4 bytes; hence do NOT return a value larger than 4. + * We return "1" for any leading byte that is either flat-out illegal or + * indicates a length larger than we support. + * + * pg_utf2wchar_with_len(), utf8_to_unicode(), pg_utf8_islegal(), and perhaps + * other places would need to be fixed to change this. + */ +static inline int +utf8_mblen(const unsigned char *s) +{ + int len; + + if ((*s & 0x80) == 0) + len = 1; + else if ((*s & 0xe0) == 0xc0) + len = 2; + else if ((*s & 0xf0) == 0xe0) + len = 3; + else if ((*s & 0xf8) == 0xf0) + len = 4; +#ifdef NOT_USED + else if ((*s & 0xfc) == 0xf8) + len = 5; + else if ((*s & 0xfe) == 0xfc) + len = 6; +#endif + else + len = 1; + return len; +} + +/* + * Convert a UTF-8 character to a Unicode code point. + * This is a one-character version of pg_utf2wchar_with_len. + * + * No error checks here, c must point to a long-enough string. + */ +static inline pg_wchar +utf8_to_unicode(const unsigned char *c) +{ + if ((*c & 0x80) == 0) + return (pg_wchar) c[0]; + else if ((*c & 0xe0) == 0xc0) + return (pg_wchar) (((c[0] & 0x1f) << 6) | + (c[1] & 0x3f)); + else if ((*c & 0xf0) == 0xe0) + return (pg_wchar) (((c[0] & 0x0f) << 12) | + ((c[1] & 0x3f) << 6) | + (c[2] & 0x3f)); + else if ((*c & 0xf8) == 0xf0) + return (pg_wchar) (((c[0] & 0x07) << 18) | + ((c[1] & 0x3f) << 12) | + ((c[2] & 0x3f) << 6) | + (c[3] & 0x3f)); + else + /* that is an invalid code on purpose */ + return 0xffffffff; +} + +/* + * Map a Unicode code point to UTF-8. utf8string must have at least + * unicode_utf8len(c) bytes available. + */ +static inline unsigned char * +unicode_to_utf8(pg_wchar c, unsigned char *utf8string) +{ + if (c <= 0x7F) + { + utf8string[0] = c; + } + else if (c <= 0x7FF) + { + utf8string[0] = 0xC0 | ((c >> 6) & 0x1F); + utf8string[1] = 0x80 | (c & 0x3F); + } + else if (c <= 0xFFFF) + { + utf8string[0] = 0xE0 | ((c >> 12) & 0x0F); + utf8string[1] = 0x80 | ((c >> 6) & 0x3F); + utf8string[2] = 0x80 | (c & 0x3F); + } + else + { + utf8string[0] = 0xF0 | ((c >> 18) & 0x07); + utf8string[1] = 0x80 | ((c >> 12) & 0x3F); + utf8string[2] = 0x80 | ((c >> 6) & 0x3F); + utf8string[3] = 0x80 | (c & 0x3F); + } + + return utf8string; +} + /* * Number of bytes needed to represent the given char in UTF8. */ -- 2.34.1