From 558da005e91b71517d788498c36d66c900366bfe Mon Sep 17 00:00:00 2001 From: Paul Amonson Date: Tue, 27 Aug 2024 08:26:19 -0700 Subject: [PATCH v6 4/6] New COMP_CRC32C macro for AVX512, simplify code some. Signed-off-by: Paul Amonson Signed-off-by: Raghuveer Devulapalli --- configure | 2 +- configure.ac | 2 +- src/include/port/pg_crc32c.h | 17 ++++++------- src/port/meson.build | 1 + src/port/pg_crc32c_avx512.c | 46 ++---------------------------------- 5 files changed, 12 insertions(+), 56 deletions(-) diff --git a/configure b/configure index 474282e8ba..8af995e48f 100755 --- a/configure +++ b/configure @@ -17880,7 +17880,7 @@ else $as_echo "#define USE_AVX512_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_avx512_choose.o" + PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_sse42.o pg_crc32c_avx512_choose.o" { $as_echo "$as_me:${as_lineno-$LINENO}: result: AVX 512 with runtime check" >&5 $as_echo "AVX 512 with runtime check" >&6; } else diff --git a/configure.ac b/configure.ac index 5d7ececfbc..a8c7911754 100644 --- a/configure.ac +++ b/configure.ac @@ -2197,7 +2197,7 @@ if test x"$USE_SSE42_CRC32C" = x"1"; then else if test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then AC_DEFINE(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel AVX 512 CRC instructions with a runtime check.]) - PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_avx512_choose.o" + PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_sse42.o pg_crc32c_avx512_choose.o" AC_MSG_RESULT(AVX 512 with runtime check) else if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h index ade06dbcab..3f83d9f815 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h @@ -40,12 +40,12 @@ typedef uint32 pg_crc32c; /* The INIT and EQ macros are the same for all implementations. */ #define INIT_CRC32C(crc) ((crc) = 0xFFFFFFFF) #define EQ_CRC32C(c1, c2) ((c1) == (c2)) +#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) #if defined(USE_SSE42_CRC32C) /* Use Intel SSE4.2 instructions. */ #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_sse42((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); @@ -53,7 +53,6 @@ extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t le /* Use Intel AVX512 instructions. */ #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_avx512((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len); @@ -62,7 +61,6 @@ extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t l #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_armv8((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); @@ -71,7 +69,6 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_loongarch((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len); @@ -82,14 +79,17 @@ extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_ * they are available. */ #define COMP_CRC32C(crc, data, len) \ - ((crc) = pg_comp_crc32c((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + ((crc) = ((len) < 256 ? \ + pg_comp_crc32c_sse42((crc), (data), (len)) : \ + pg_comp_crc32c((crc), (data), (len)))) extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); extern pg_crc32c (*pg_comp_crc32c)(pg_crc32c crc, const void *data, size_t len); extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len); +extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #elif defined(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK) || defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK) /* @@ -98,7 +98,6 @@ extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t l */ #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c((crc), (data), (len))) -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); @@ -121,13 +120,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_sb8((crc), (data), (len))) #ifdef WORDS_BIGENDIAN +#undef FIN_CRC32C #define FIN_CRC32C(crc) ((crc) = pg_bswap32(crc) ^ 0xFFFFFFFF) -#else -#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) #endif extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); - #endif #endif /* PG_CRC32C_H */ diff --git a/src/port/meson.build b/src/port/meson.build index e3b05622d1..b53c33c8eb 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -86,6 +86,7 @@ replace_funcs_pos = [ ['pg_crc32c_avx512', 'USE_AVX512_CRC32C'], ['pg_crc32c_avx512', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'crc'], ['pg_crc32c_avx512_choose', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK'], + ['pg_crc32c_sse42', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'crc'], ['pg_crc32c_sb8', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK'], ['pg_crc32c_sb8', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK'], ['pg_popcount_avx512', 'USE_AVX512_POPCNT_WITH_RUNTIME_CHECK', 'popcnt'], diff --git a/src/port/pg_crc32c_avx512.c b/src/port/pg_crc32c_avx512.c index be42a34a73..98353f7e1d 100644 --- a/src/port/pg_crc32c_avx512.c +++ b/src/port/pg_crc32c_avx512.c @@ -18,48 +18,6 @@ #include "port/pg_crc32c.h" -/* - * Process eight bytes of data at a time. - * - * NB: We do unaligned accesses here. The Intel architecture allows that, - * and performance testing didn't show any performance gain from aligning - * the begin address. - */ -pg_attribute_no_sanitize_alignment() -inline static pg_crc32c -crc32c_fallback(pg_crc32c crc, const uint8 *p, size_t length) -{ - const unsigned char *pend = p + length; - - /* - * Process eight bytes of data at a time. - * - * NB: We do unaligned accesses here. The Intel architecture allows that, - * and performance testing didn't show any performance gain from aligning - * the begin address. - */ - while (p + 8 <= pend) - { - crc = (uint32)_mm_crc32_u64(crc, *((const uint64 *)p)); - p += 8; - } - - /* Process remaining full four bytes if any */ - if (p + 4 <= pend) - { - crc = _mm_crc32_u32(crc, *((const unsigned int *)p)); - p += 4; - } - - /* Process any remaining bytes one at a time. */ - while (p < pend) - { - crc = _mm_crc32_u8(crc, *p); - p++; - } - - return crc; -} /******************************************************************* * pg_crc32c_avx512(): compute the crc32c of the buffer, where the @@ -233,7 +191,7 @@ pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t length) } /* - * Finish any remaining bytes. + * Finish any remaining bytes with legacy AVX algorithm. */ - return crc32c_fallback(crc, input, length); + return pg_comp_crc32c_sse42(crc, input, length); } -- 2.43.0