From e642b6d32d4715c988b6b93d57385a7c0779182d Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Thu, 4 Sep 2025 12:23:24 +0530 Subject: [PATCH v6 1/3] NEON support for hex coding --- src/backend/utils/adt/Makefile | 1 + src/backend/utils/adt/encode.c | 6 +- src/backend/utils/adt/encode_aarch64.c | 195 +++++++++++++++++++++++++ src/backend/utils/adt/meson.build | 1 + src/include/utils/builtins.h | 57 +++++++- 5 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 src/backend/utils/adt/encode_aarch64.c diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index cc68ac545a5..40eaee14899 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -33,6 +33,7 @@ OBJS = \ dbsize.o \ domains.o \ encode.o \ + encode_aarch64.o \ enum.o \ expandeddatum.o \ expandedrecord.o \ diff --git a/src/backend/utils/adt/encode.c b/src/backend/utils/adt/encode.c index 4ccaed815d1..fa62ce3107d 100644 --- a/src/backend/utils/adt/encode.c +++ b/src/backend/utils/adt/encode.c @@ -178,7 +178,7 @@ static const int8 hexlookup[128] = { }; uint64 -hex_encode(const char *src, size_t len, char *dst) +hex_encode_scalar(const char *src, size_t len, char *dst) { const char *end = src + len; @@ -208,13 +208,13 @@ get_hex(const char *cp, char *out) } uint64 -hex_decode(const char *src, size_t len, char *dst) +hex_decode_scalar(const char *src, size_t len, char *dst) { return hex_decode_safe(src, len, dst, NULL); } uint64 -hex_decode_safe(const char *src, size_t len, char *dst, Node *escontext) +hex_decode_safe_scalar(const char *src, size_t len, char *dst, Node *escontext) { const char *s, *srcend; diff --git a/src/backend/utils/adt/encode_aarch64.c b/src/backend/utils/adt/encode_aarch64.c new file mode 100644 index 00000000000..7b0412dc255 --- /dev/null +++ b/src/backend/utils/adt/encode_aarch64.c @@ -0,0 +1,195 @@ +/*------------------------------------------------------------------------- + * + * encode_aarch64.c + * Holds the AArch64 hex encode/decode implementations. + * + * Copyright (c) 2025, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/utils/adt/encode_aarch64.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "utils/builtins.h" + +#ifdef HEX_CODING_AARCH64 + +#include + +uint64 +hex_encode_optimized(const char *src, size_t len, char *dst) +{ + const char hextbl[] = "0123456789abcdef"; + uint8x16_t hextbl_vec = vld1q_u8((uint8 *) hextbl); + uint8x16x2_t zipped; + uint32 vec_len = sizeof(uint8x16_t); + size_t chunks_len = len & ~(2 * vec_len - 1); + + for (size_t i = 0; i < chunks_len; i += 2 * vec_len) + { + uint8x16_t bytes = vld1q_u8((uint8 *) src); + + /* Right-shift by 4 to get the high nibble */ + uint8x16_t high = vshrq_n_u8(bytes, 4); + + /* Mask the high nibble to get the low nibble */ + uint8x16_t low = vandq_u8(bytes, vdupq_n_u8(0xF)); + + /* + * Convert the high and low nibbles to hexadecimal digits using a table + * lookup, and then zip (interleave) the resulting digits. + */ + zipped.val[0] = vqtbl1q_u8(hextbl_vec, high); + zipped.val[1] = vqtbl1q_u8(hextbl_vec, low); + vst2q_u8((uint8 *) dst, zipped); + + src += vec_len; + dst += 2 * vec_len; + + /* unrolled */ + bytes = vld1q_u8((uint8 *) src); + high = vshrq_n_u8(bytes, 4); + low = vandq_u8(bytes, vdupq_n_u8(0xF)); + + zipped.val[0] = vqtbl1q_u8(hextbl_vec, high); + zipped.val[1] = vqtbl1q_u8(hextbl_vec, low); + vst2q_u8((uint8 *) dst, zipped); + + src += vec_len; + dst += 2 * vec_len; + } + + + + if (len > chunks_len) + hex_encode_scalar(src, len - chunks_len, dst); + + return (uint64) len * 2; +} + +/* + * get_hex_neon + * Returns true if the hexadecimal digits are successfully converted + * to nibbles and stored in 'res'; otherwise, returns false. + */ +static inline bool +get_hex_neon(uint8x16_t vec, uint8x16_t *res) +{ + /* + * Convert ASCII of '0'-'9' to integers 0-9 by subtracting 48 (ASCII of '0'). + * Similarly, convert letters 'A'–'F' and 'a'–'f' to integers 10–15 by + * subtracting 55 ('A' - 10) and 87 ('a' - 10). + */ + uint8x16_t digit = vsubq_u8(vec, vdupq_n_u8('0')); + uint8x16_t upper = vsubq_u8(vec, vdupq_n_u8('A' - 10)); + uint8x16_t lower = vsubq_u8(vec, vdupq_n_u8('a' - 10)); + + /* + * Identify valid values in digit, upper, and lower vectors. + * Values 0-9 are valid in digits, while values 10-15 are valid + * in upper and lower. + * + * Example: + * vec: '0' '9' 'A' 'F' 'a' 'f' + * vec (in ASCII): 48 57 65 70 97 102 + * + * digit: 0 9 17 22 49 54 + * valid_digit: 1 1 0 0 0 0 + * + * upper: 249 2 10 15 42 47 + * valid_upper: 0 1 1 1 0 0 + * + * lower: 217 226 234 239 10 15 + * + * Note that values 0-9 are also marked valid in valid_upper, this will be + * handled later. + */ + + uint8x16_t valid_digit = vcltq_u8(digit, vdupq_n_u8(10)); + uint8x16_t valid_upper = vcltq_u8(upper, vdupq_n_u8(16)); + + /* + * Merge upper and lower vector using the logic: pick the element from + * upper if it's true in valid_upper else pick the element in lower + * + * Mark the valid range i.e. 10-15 in letter vector + * + * letter: 217 2 10 15 10 15 + * valid_letter: 0 0 1 1 1 1 + */ + + uint8x16_t letter = vbslq_u8(valid_upper, upper, lower); + uint8x16_t valid_letter = vandq_u8(vcgtq_u8(letter, vdupq_n_u8(9)), + vcltq_u8(letter, vdupq_n_u8(16))); + + /* + * Check for invalid hexadecimal digit. Each value must fall within + * the range 0-9 (true in valid_digit) or 10-15 (true in valid_letter) i.e. + * the OR of valid_digit and valid_letter should be all true. + */ + uint8x16_t invalid_mask = vmvnq_u8(vorrq_u8(valid_digit, valid_letter)); + + if (vmaxvq_u8(invalid_mask) != 0) + return false; + + /* + * Finally, combine digit and letter vectors using the logic: + * pick the element from digit if it's true in valid_digit else pick the + * element in letter. + * + * res: 0 9 10 15 10 15 + */ + *res = vbslq_u8(valid_digit, digit, letter); + return true; +} + +uint64 +hex_decode_optimized(const char *src, size_t len, char *dst) +{ + return hex_decode_safe_optimized(src, len, dst, NULL); +} + +uint64 +hex_decode_safe_optimized(const char *src, size_t len, char *dst, Node *escontext) +{ + uint32 vec_len = sizeof(uint8x16_t); + size_t i = 0; + size_t chunks_len = len & ~(2 * vec_len - 1); /* process 2 x vec_len per iteration */ + uint8x16_t ascii_zero = vdupq_n_u8('0'); + const char *p = dst; + + while (i < chunks_len) + { + uint8x16x2_t bytes = vld2q_u8((uint8 *) src); + uint8x16_t high = bytes.val[0]; /* hex digits for high nibble */ + uint8x16_t low = bytes.val[1]; /* hex digits for low nibble */ + + /* fallback for characters with ASCII values below '0' */ + uint8x16_t is_below_zero = vorrq_u8(vcltq_u8(high, ascii_zero), + vcltq_u8(low, ascii_zero)); + if (vmaxvq_u8(is_below_zero) != 0) + break; + + /* fallback if an invalid hexadecimal digit is found */ + if (!get_hex_neon(high, &high) || !get_hex_neon(low, &low)) + break; + + /* form the byte by left-shifting the high nibble and OR-ing it with the low nibble */ + vst1q_u8((uint8 *) dst, vorrq_u8(vshlq_n_u8(high, 4), low)); + + i += 2 * vec_len; + src += 2 * vec_len; + dst += vec_len; + } + + if (len > i) /* fallback */ + return dst - p + hex_decode_safe_scalar(src, len - i, dst, escontext); + + return dst - p; +} + +#endif /* HEX_CODING_AARCH64 */ diff --git a/src/backend/utils/adt/meson.build b/src/backend/utils/adt/meson.build index dac372c3bea..8b106d03d33 100644 --- a/src/backend/utils/adt/meson.build +++ b/src/backend/utils/adt/meson.build @@ -22,6 +22,7 @@ backend_sources += files( 'dbsize.c', 'domains.c', 'encode.c', + 'encode_aarch64.c', 'enum.c', 'expandeddatum.c', 'expandedrecord.c', diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 1c98c7d2255..a809fad2771 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -35,11 +35,62 @@ extern int errdatatype(Oid datatypeOid); extern int errdomainconstraint(Oid datatypeOid, const char *conname); /* encode.c */ -extern uint64 hex_encode(const char *src, size_t len, char *dst); -extern uint64 hex_decode(const char *src, size_t len, char *dst); -extern uint64 hex_decode_safe(const char *src, size_t len, char *dst, +extern uint64 hex_encode_scalar(const char *src, size_t len, char *dst); +extern uint64 hex_decode_scalar(const char *src, size_t len, char *dst); +extern uint64 hex_decode_safe_scalar(const char *src, size_t len, char *dst, Node *escontext); +/* + * On AArch64, we can use Neon instructions if the compiler provides access to + * them (as indicated by __ARM_NEON). As in simd.h, we assume that all + * available 64-bit hardware has Neon support. + */ +#if defined(__aarch64__) && defined(__ARM_NEON) +#define HEX_CODING_AARCH64 1 +#endif + +#ifdef HEX_CODING_AARCH64 +extern uint64 hex_encode_optimized(const char *src, size_t len, char *dst); +extern uint64 hex_decode_optimized(const char *src, size_t len, char *dst); +extern uint64 hex_decode_safe_optimized(const char *src, size_t len, char *dst, Node *escontext); +#endif + +static inline uint64 +hex_encode(const char *src, size_t len, char *dst) +{ +#ifdef HEX_CODING_AARCH64 + int threshold = 32; + + if (len >= threshold) + return hex_encode_optimized(src, len, dst); +#endif + return hex_encode_scalar(src, len, dst); +} + +static inline uint64 +hex_decode(const char *src, size_t len, char *dst) +{ +#ifdef HEX_CODING_AARCH64 + int threshold = 32; + + if (len >= threshold) + return hex_decode_optimized(src, len, dst); +#endif + return hex_decode_scalar(src, len, dst); +} + +static inline uint64 +hex_decode_safe(const char *src, size_t len, char *dst, Node *escontext) +{ +#ifdef HEX_CODING_AARCH64 + int threshold = 32; + + if (len >= threshold) + return hex_decode_safe_optimized(src, len, dst, escontext); +#endif + return hex_decode_safe_scalar(src, len, dst, escontext); +} + /* int.c */ extern int2vector *buildint2vector(const int16 *int2s, int n); -- 2.34.1