From 7ac29b452f024e43cb34e77600f3d232d4a80874 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Mon, 21 Oct 2024 14:26:22 -0700 Subject: [PATCH v6 6/6] Use __attribute__(target(...)) for SSE42 and AVX512 CRC32C --- config/c-compiler.m4 | 88 ++--- configure | 350 ++++++------------ configure.ac | 130 +++---- meson.build | 30 +- src/include/pg_config.h.in | 6 +- src/include/pg_cpu.h | 23 ++ src/include/port/pg_crc32c.h | 71 +--- src/port/Makefile | 10 - src/port/meson.build | 24 +- src/port/pg_crc32c_avx512.c | 5 + src/port/pg_crc32c_avx512_choose.c | 42 --- src/port/pg_crc32c_sse42.c | 4 + ..._sse42_choose.c => pg_crc32c_x86_choose.c} | 27 +- src/port/pg_hw_feat_check.c | 3 + src/port/pg_popcount_avx512.c | 78 +--- 15 files changed, 297 insertions(+), 594 deletions(-) create mode 100644 src/include/pg_cpu.h delete mode 100644 src/port/pg_crc32c_avx512_choose.c rename src/port/{pg_crc32c_sse42_choose.c => pg_crc32c_x86_choose.c} (58%) diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 33df694ae7..d7b3ceeb60 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -608,21 +608,22 @@ fi])# PGAC_HAVE_GCC__ATOMIC_INT64_CAS # An optional compiler flag can be passed as argument (e.g. -msse4.2). If the # intrinsics are supported, sets pgac_sse42_crc32_intrinsics, and CFLAGS_CRC. AC_DEFUN([PGAC_SSE42_CRC32_INTRINSICS], -[define([Ac_cachevar], [AS_TR_SH([pgac_cv_sse42_crc32_intrinsics_$1])])dnl -AC_CACHE_CHECK([for _mm_crc32_u8 and _mm_crc32_u32 with CFLAGS=$1], [Ac_cachevar], -[pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS $1" -AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [unsigned int crc = 0; - crc = _mm_crc32_u8(crc, 0); - crc = _mm_crc32_u32(crc, 0); - /* return computed value, to prevent the above being optimized away */ - return crc == 0;])], +[define([Ac_cachevar], [AS_TR_SH([pgac_cv_sse42_crc32_intrinsics])])dnl +AC_CACHE_CHECK([for _mm_crc32_u8 and _mm_crc32_u32 with function attribute], [Ac_cachevar], +[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include + __attribute__((target("sse4.2"))) + static int crc32_sse42_test(void) + { + unsigned int crc = 0; + crc = _mm_crc32_u8(crc, 0); + crc = _mm_crc32_u32(crc, 0); + /* return computed value, to prevent the above being optimized away */ + return crc == 0; + }], + [return crc32_sse42_test();])], [Ac_cachevar=yes], - [Ac_cachevar=no]) -CFLAGS="$pgac_save_CFLAGS"]) + [Ac_cachevar=no])]) if test x"$Ac_cachevar" = x"yes"; then - CFLAGS_CRC="$1" pgac_sse42_crc32_intrinsics=yes fi undefine([Ac_cachevar])dnl @@ -639,44 +640,45 @@ undefine([Ac_cachevar])dnl # An optional compiler flag can be passed as arguments (e.g. -msse4.2 # -mavx512vl -mvpclmulqdq). If the intrinsics are supported, sets # pgac_avx512_crc32_intrinsics, and CFLAGS_CRC. + AC_DEFUN([PGAC_AVX512_CRC32_INTRINSICS], -[define([Ac_cachevar], [AS_TR_SH([pgac_cv_avx512_crc32_intrinsics_$1])])dnl -AC_CACHE_CHECK([for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=$1], [Ac_cachevar], -[pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS $1" -AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [const unsigned long k1k2[[8]] = { - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; - unsigned char buffer[[512]]; - unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L); - unsigned long val; - __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; - __m128i a1, a2; - unsigned int crc = 0xffffffff; - y8 = _mm512_load_si512((__m512i *)aligned); - x0 = _mm512_loadu_si512((__m512i *)k1k2); - x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00)); - x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); - x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); - x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); - a1 = _mm512_extracti32x4_epi32(x1, 3); - a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); - x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); - val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); - crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); - return crc != 0;])], +[define([Ac_cachevar], [AS_TR_SH([pgac_cv_avx512_crc32_intrinsics])])dnl +AC_CACHE_CHECK([for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128 with function attribute], [Ac_cachevar], +[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include + __attribute__((target("avx512f","avx512vl","vpclmulqdq"))) + static int crc32_avx512_test(void) + { + const unsigned long k1k2[[8]] = { + 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, + 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; + unsigned char buffer[[512]]; + unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L); + unsigned long val; + __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + __m128i a1, a2; + unsigned int crc = 0xffffffff; + y8 = _mm512_load_si512((__m512i *)aligned); + x0 = _mm512_loadu_si512((__m512i *)k1k2); + x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00)); + x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); + a1 = _mm512_extracti32x4_epi32(x1, 3); + a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); + x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); + val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); + crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); + return crc != 0; + }], + [return crc32_avx512_test();])], [Ac_cachevar=yes], - [Ac_cachevar=no]) -CFLAGS="$pgac_save_CFLAGS"]) + [Ac_cachevar=no])]) if test x"$Ac_cachevar" = x"yes"; then - CFLAGS_CRC="$1" pgac_avx512_crc32_intrinsics=yes fi undefine([Ac_cachevar])dnl ])# PGAC_AVX512_CRC32_INTRINSICS - # PGAC_ARMV8_CRC32C_INTRINSICS # ---------------------------- # Check if the compiler supports the CRC32C instructions using the __crc32cb, diff --git a/configure b/configure index 38e7b1889b..99bbeaf5c5 100755 --- a/configure +++ b/configure @@ -14728,7 +14728,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -14774,7 +14774,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -14798,7 +14798,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -14843,7 +14843,7 @@ else We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -14867,7 +14867,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext We can't simply define LARGE_OFF_T to be 9223372036854775807, since some C++ compilers masquerading as C compilers incorrectly reject 9223372036854775807. */ -#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31)) +#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62)) int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721 && LARGE_OFF_T % 2147483647 == 1) ? 1 : -1]; @@ -17360,206 +17360,111 @@ $as_echo "#define USE_AVX512_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h fi fi -# Check for Intel SSE 4.2 intrinsics to do CRC calculations. -# -# First check if the _mm_crc32_u8 and _mm_crc32_u64 intrinsics can be used -# with the default compiler flags. If not, check if adding the -msse4.2 -# flag helps. CFLAGS_CRC is set to -msse4.2 if that's required. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm_crc32_u8 and _mm_crc32_u32 with CFLAGS=" >&5 -$as_echo_n "checking for _mm_crc32_u8 and _mm_crc32_u32 with CFLAGS=... " >&6; } -if ${pgac_cv_sse42_crc32_intrinsics_+:} false; then : - $as_echo_n "(cached) " >&6 -else - pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS " -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -unsigned int crc = 0; - crc = _mm_crc32_u8(crc, 0); - crc = _mm_crc32_u32(crc, 0); - /* return computed value, to prevent the above being optimized away */ - return crc == 0; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_sse42_crc32_intrinsics_=yes -else - pgac_cv_sse42_crc32_intrinsics_=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_sse42_crc32_intrinsics_" >&5 -$as_echo "$pgac_cv_sse42_crc32_intrinsics_" >&6; } -if test x"$pgac_cv_sse42_crc32_intrinsics_" = x"yes"; then - CFLAGS_CRC="" - pgac_sse42_crc32_intrinsics=yes -fi - -if test x"$pgac_sse42_crc32_intrinsics" != x"yes"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm_crc32_u8 and _mm_crc32_u32 with CFLAGS=-msse4.2" >&5 -$as_echo_n "checking for _mm_crc32_u8 and _mm_crc32_u32 with CFLAGS=-msse4.2... " >&6; } -if ${pgac_cv_sse42_crc32_intrinsics__msse4_2+:} false; then : - $as_echo_n "(cached) " >&6 -else - pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS -msse4.2" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -unsigned int crc = 0; - crc = _mm_crc32_u8(crc, 0); - crc = _mm_crc32_u32(crc, 0); - /* return computed value, to prevent the above being optimized away */ - return crc == 0; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_sse42_crc32_intrinsics__msse4_2=yes -else - pgac_cv_sse42_crc32_intrinsics__msse4_2=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_sse42_crc32_intrinsics__msse4_2" >&5 -$as_echo "$pgac_cv_sse42_crc32_intrinsics__msse4_2" >&6; } -if test x"$pgac_cv_sse42_crc32_intrinsics__msse4_2" = x"yes"; then - CFLAGS_CRC="-msse4.2" - pgac_sse42_crc32_intrinsics=yes -fi - -fi - # Check for Intel AVX-512 intrinsics to do CRC calculations. # # First check if the _mm512_clmulepi64_epi128 and more intrinsics can # be used with the default compiler flags. If not, check if adding -# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. CFLAGS_CRC -# is set to -msse4.2, -mavx512vl and -mvpclmulqdqif that's required. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=" >&5 -$as_echo_n "checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=... " >&6; } -if ${pgac_cv_avx512_crc32_intrinsics_+:} false; then : +# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128 with function attribute" >&5 +$as_echo_n "checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128 with function attribute... " >&6; } +if ${pgac_cv_avx512_crc32_intrinsics+:} false; then : $as_echo_n "(cached) " >&6 else - pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS " -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include + __attribute__((target("avx512f","avx512vl","vpclmulqdq"))) + static int crc32_avx512_test(void) + { + const unsigned long k1k2[8] = { + 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, + 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; + unsigned char buffer[512]; + unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L); + unsigned long val; + __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + __m128i a1, a2; + unsigned int crc = 0xffffffff; + y8 = _mm512_load_si512((__m512i *)aligned); + x0 = _mm512_loadu_si512((__m512i *)k1k2); + x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00)); + x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); + x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); + x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); + a1 = _mm512_extracti32x4_epi32(x1, 3); + a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); + x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); + val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); + crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); + return crc != 0; + } int main () { -const unsigned long k1k2[8] = { - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; - unsigned char buffer[512]; - unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L); - unsigned long val; - __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; - __m128i a1, a2; - unsigned int crc = 0xffffffff; - y8 = _mm512_load_si512((__m512i *)aligned); - x0 = _mm512_loadu_si512((__m512i *)k1k2); - x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00)); - x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); - x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); - x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); - a1 = _mm512_extracti32x4_epi32(x1, 3); - a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); - x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); - val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); - crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); - return crc != 0; +return crc32_avx512_test(); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_avx512_crc32_intrinsics_=yes + pgac_cv_avx512_crc32_intrinsics=yes else - pgac_cv_avx512_crc32_intrinsics_=no + pgac_cv_avx512_crc32_intrinsics=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_avx512_crc32_intrinsics_" >&5 -$as_echo "$pgac_cv_avx512_crc32_intrinsics_" >&6; } -if test x"$pgac_cv_avx512_crc32_intrinsics_" = x"yes"; then - CFLAGS_CRC="" +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_avx512_crc32_intrinsics" >&5 +$as_echo "$pgac_cv_avx512_crc32_intrinsics" >&6; } +if test x"$pgac_cv_avx512_crc32_intrinsics" = x"yes"; then pgac_avx512_crc32_intrinsics=yes fi -if test x"$pgac_avx512_crc32_intrinsics" != x"yes"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=-msse4.2 -mavx512vl -mvpclmulqdq" >&5 -$as_echo_n "checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=-msse4.2 -mavx512vl -mvpclmulqdq... " >&6; } -if ${pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq+:} false; then : + +# Check for Intel SSE 4.2 intrinsics to do CRC calculations. +# +# First check if the _mm_crc32_u8 and _mm_crc32_u64 intrinsics can be used +# with the default compiler flags. If not, check if adding the -msse4.2 +# flag helps. CFLAGS_CRC is set to -msse4.2 if that's required. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm_crc32_u8 and _mm_crc32_u32 with function attribute" >&5 +$as_echo_n "checking for _mm_crc32_u8 and _mm_crc32_u32 with function attribute... " >&6; } +if ${pgac_cv_sse42_crc32_intrinsics+:} false; then : $as_echo_n "(cached) " >&6 else - pgac_save_CFLAGS=$CFLAGS -CFLAGS="$pgac_save_CFLAGS -msse4.2 -mavx512vl -mvpclmulqdq" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include +#include + __attribute__((target("sse4.2"))) + static int crc32_sse42_test(void) + { + unsigned int crc = 0; + crc = _mm_crc32_u8(crc, 0); + crc = _mm_crc32_u32(crc, 0); + /* return computed value, to prevent the above being optimized away */ + return crc == 0; + } int main () { -const unsigned long k1k2[8] = { - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, - 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86}; - unsigned char buffer[512]; - unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L); - unsigned long val; - __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; - __m128i a1, a2; - unsigned int crc = 0xffffffff; - y8 = _mm512_load_si512((__m512i *)aligned); - x0 = _mm512_loadu_si512((__m512i *)k1k2); - x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00)); - x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc))); - x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00); - x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96); - a1 = _mm512_extracti32x4_epi32(x1, 3); - a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0)); - x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E); - val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0)); - crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1)); - return crc != 0; +return crc32_sse42_test(); ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq=yes + pgac_cv_sse42_crc32_intrinsics=yes else - pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq=no + pgac_cv_sse42_crc32_intrinsics=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" >&5 -$as_echo "$pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" >&6; } -if test x"$pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" = x"yes"; then - CFLAGS_CRC="-msse4.2 -mavx512vl -mvpclmulqdq" - pgac_avx512_crc32_intrinsics=yes +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_sse42_crc32_intrinsics" >&5 +$as_echo "$pgac_cv_sse42_crc32_intrinsics" >&6; } +if test x"$pgac_cv_sse42_crc32_intrinsics" = x"yes"; then + pgac_sse42_crc32_intrinsics=yes fi -fi # Are we targeting a processor that supports SSE 4.2? gcc, clang and icc all # define __SSE4_2__ in that case. @@ -17714,6 +17619,7 @@ fi + # Select CRC-32C implementation. # # If we are targeting a processor that has Intel SSE 4.2 instructions, we can @@ -17733,108 +17639,72 @@ fi # # If we are targeting a LoongArch processor, CRC instructions are # always available (at least on 64 bit), so no runtime check is needed. -if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then - # Use Intel AVX 512 if available. - if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && test x"$AVX512_TARGETED" = x"1" ; then - USE_AVX512_CRC32C=1 - else - # Use Intel SSE 4.2 if available. - if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then - USE_SSE42_CRC32C=1 - else - # Intel AVX 512, with runtime check? The CPUID instruction is needed for - # the runtime check. - if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then - USE_AVX512_CRC32C_WITH_RUNTIME_CHECK=1 - else - # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for - # the runtime check. - if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then - USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1 - else - # Use ARM CRC Extension if available. - if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then - USE_ARMV8_CRC32C=1 - else - # ARM CRC Extension, with runtime check? - if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then - USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1 - else - # LoongArch CRCC instructions. - if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then - USE_LOONGARCH_CRC32C=1 - else - # fall back to slicing-by-8 algorithm, which doesn't require any - # special CPU support. - USE_SLICING_BY_8_CRC32C=1 - fi - fi - fi - fi - fi - fi - fi -fi - -# Set PG_CRC32C_OBJS appropriately depending on the selected implementation. { $as_echo "$as_me:${as_lineno-$LINENO}: checking which CRC-32C implementation to use" >&5 $as_echo_n "checking which CRC-32C implementation to use... " >&6; } -if test x"$USE_SSE42_CRC32C" = x"1"; then +if test x"$host_cpu" = x"x86_64"; then + PG_CRC32C_OBJS="pg_crc32c_sb8.o pg_crc32c_x86_choose.o" + if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then $as_echo "#define USE_SSE42_CRC32C 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_sse42.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: SSE 4.2" >&5 -$as_echo "SSE 4.2" >&6; } -else - if test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then + PG_CRC32C_OBJS+=" pg_crc32c_sse42.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: CRC32C baseline feature SSE 4.2" >&5 +$as_echo "CRC32C baseline feature SSE 4.2" >&6; } + else + if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then -$as_echo "#define USE_AVX512_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h +$as_echo "#define USE_SSE42_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_sse42.o pg_crc32c_avx512_choose.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: AVX 512 with runtime check" >&5 -$as_echo "AVX 512 with runtime check" >&6; } - else - if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then + PG_CRC32C_OBJS+=" pg_crc32c_sse42.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: CRC32C SSE42 with runtime check" >&5 +$as_echo "CRC32C SSE42 with runtime check" >&6; } + fi + fi + if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then -$as_echo "#define USE_SSE42_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h +$as_echo "#define USE_AVX512_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: SSE 4.2 with runtime check" >&5 -$as_echo "SSE 4.2 with runtime check" >&6; } - else - if test x"$USE_ARMV8_CRC32C" = x"1"; then + PG_CRC32C_OBJS+=" pg_crc32c_avx512.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: CRC32C AVX-512 with runtime check" >&5 +$as_echo "CRC32C AVX-512 with runtime check" >&6; } + fi +else + # non x86 code: + # Use ARM CRC Extension if available. + if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then $as_echo "#define USE_ARMV8_CRC32C 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_armv8.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions" >&5 + PG_CRC32C_OBJS="pg_crc32c_armv8.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions" >&5 $as_echo "ARMv8 CRC instructions" >&6; } - else - if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then + else + # ARM CRC Extension, with runtime check? + if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then $as_echo "#define USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions with runtime check" >&5 + PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions with runtime check" >&5 $as_echo "ARMv8 CRC instructions with runtime check" >&6; } - else - if test x"$USE_LOONGARCH_CRC32C" = x"1"; then + else + # LoongArch CRCC instructions. + if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then $as_echo "#define USE_LOONGARCH_CRC32C 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_loongarch.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: LoongArch CRCC instructions" >&5 + PG_CRC32C_OBJS="pg_crc32c_loongarch.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: LoongArch CRCC instructions" >&5 $as_echo "LoongArch CRCC instructions" >&6; } - else + else + # fall back to slicing-by-8 algorithm, which doesn't require any + # special CPU support. $as_echo "#define USE_SLICING_BY_8_CRC32C 1" >>confdefs.h - PG_CRC32C_OBJS="pg_crc32c_sb8.o" - { $as_echo "$as_me:${as_lineno-$LINENO}: result: slicing-by-8" >&5 + PG_CRC32C_OBJS="pg_crc32c_sb8.o" + { $as_echo "$as_me:${as_lineno-$LINENO}: result: slicing-by-8" >&5 $as_echo "slicing-by-8" >&6; } - fi - fi fi fi fi diff --git a/configure.ac b/configure.ac index 70c78d11fa..c2d516adae 100644 --- a/configure.ac +++ b/configure.ac @@ -2066,26 +2066,19 @@ if test x"$host_cpu" = x"x86_64"; then fi fi +# Check for Intel AVX-512 intrinsics to do CRC calculations. +# +# First check if the _mm512_clmulepi64_epi128 and more intrinsics can +# be used with the default compiler flags. If not, check if adding +# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. +PGAC_AVX512_CRC32_INTRINSICS() + # Check for Intel SSE 4.2 intrinsics to do CRC calculations. # # First check if the _mm_crc32_u8 and _mm_crc32_u64 intrinsics can be used # with the default compiler flags. If not, check if adding the -msse4.2 # flag helps. CFLAGS_CRC is set to -msse4.2 if that's required. -PGAC_SSE42_CRC32_INTRINSICS([]) -if test x"$pgac_sse42_crc32_intrinsics" != x"yes"; then - PGAC_SSE42_CRC32_INTRINSICS([-msse4.2]) -fi - -# Check for Intel AVX-512 intrinsics to do CRC calculations. -# -# First check if the _mm512_clmulepi64_epi128 and more intrinsics can -# be used with the default compiler flags. If not, check if adding -# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. CFLAGS_CRC -# is set to -msse4.2, -mavx512vl and -mvpclmulqdqif that's required. -PGAC_AVX512_CRC32_INTRINSICS([]) -if test x"$pgac_avx512_crc32_intrinsics" != x"yes"; then - PGAC_AVX512_CRC32_INTRINSICS([-msse4.2 -mavx512vl -mvpclmulqdq]) -fi +PGAC_SSE42_CRC32_INTRINSICS() # Are we targeting a processor that supports SSE 4.2? gcc, clang and icc all # define __SSE4_2__ in that case. @@ -2113,6 +2106,7 @@ PGAC_LOONGARCH_CRC32C_INTRINSICS() AC_SUBST(CFLAGS_CRC) + # Select CRC-32C implementation. # # If we are targeting a processor that has Intel SSE 4.2 instructions, we can @@ -2132,86 +2126,50 @@ AC_SUBST(CFLAGS_CRC) # # If we are targeting a LoongArch processor, CRC instructions are # always available (at least on 64 bit), so no runtime check is needed. -if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then - # Use Intel AVX 512 if available. - if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && test x"$AVX512_TARGETED" = x"1" ; then - USE_AVX512_CRC32C=1 - else - # Use Intel SSE 4.2 if available. +AC_MSG_CHECKING([which CRC-32C implementation to use]) +if test x"$host_cpu" = x"x86_64"; then + PG_CRC32C_OBJS="pg_crc32c_sb8.o pg_crc32c_x86_choose.o" if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then - USE_SSE42_CRC32C=1 + AC_DEFINE(USE_SSE42_CRC32C, 1, [Define to 1 use Intel SSE 4.2 CRC instructions.]) + PG_CRC32C_OBJS+=" pg_crc32c_sse42.o" + AC_MSG_RESULT(CRC32C baseline feature SSE 4.2) else - # Intel AVX 512, with runtime check? The CPUID instruction is needed for - # the runtime check. - if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then - USE_AVX512_CRC32C_WITH_RUNTIME_CHECK=1 - else - # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for - # the runtime check. if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then - USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1 - else - # Use ARM CRC Extension if available. - if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then - USE_ARMV8_CRC32C=1 - else - # ARM CRC Extension, with runtime check? - if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then - USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1 - else - # LoongArch CRCC instructions. - if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then - USE_LOONGARCH_CRC32C=1 - else - # fall back to slicing-by-8 algorithm, which doesn't require any - # special CPU support. - USE_SLICING_BY_8_CRC32C=1 - fi - fi - fi + AC_DEFINE(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check.]) + PG_CRC32C_OBJS+=" pg_crc32c_sse42.o" + AC_MSG_RESULT(CRC32C SSE42 with runtime check) fi - fi fi - fi -fi - -# Set PG_CRC32C_OBJS appropriately depending on the selected implementation. -AC_MSG_CHECKING([which CRC-32C implementation to use]) -if test x"$USE_SSE42_CRC32C" = x"1"; then - AC_DEFINE(USE_SSE42_CRC32C, 1, [Define to 1 use Intel SSE 4.2 CRC instructions.]) - PG_CRC32C_OBJS="pg_crc32c_sse42.o" - AC_MSG_RESULT(SSE 4.2) + if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then + AC_DEFINE(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel AVX 512 CRC instructions with a runtime check.]) + PG_CRC32C_OBJS+=" pg_crc32c_avx512.o" + AC_MSG_RESULT(CRC32C AVX-512 with runtime check) + fi else - if test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then - AC_DEFINE(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel AVX 512 CRC instructions with a runtime check.]) - PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_sse42.o pg_crc32c_avx512_choose.o" - AC_MSG_RESULT(AVX 512 with runtime check) + # non x86 code: + # Use ARM CRC Extension if available. + if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then + AC_DEFINE(USE_ARMV8_CRC32C, 1, [Define to 1 to use ARMv8 CRC Extension.]) + PG_CRC32C_OBJS="pg_crc32c_armv8.o" + AC_MSG_RESULT(ARMv8 CRC instructions) else - if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then - AC_DEFINE(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check.]) - PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o" - AC_MSG_RESULT(SSE 4.2 with runtime check) + # ARM CRC Extension, with runtime check? + if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then + AC_DEFINE(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARMv8 CRC Extension with a runtime check.]) + PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o" + AC_MSG_RESULT(ARMv8 CRC instructions with runtime check) else - if test x"$USE_ARMV8_CRC32C" = x"1"; then - AC_DEFINE(USE_ARMV8_CRC32C, 1, [Define to 1 to use ARMv8 CRC Extension.]) - PG_CRC32C_OBJS="pg_crc32c_armv8.o" - AC_MSG_RESULT(ARMv8 CRC instructions) + # LoongArch CRCC instructions. + if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then + AC_DEFINE(USE_LOONGARCH_CRC32C, 1, [Define to 1 to use LoongArch CRCC instructions.]) + PG_CRC32C_OBJS="pg_crc32c_loongarch.o" + AC_MSG_RESULT(LoongArch CRCC instructions) else - if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then - AC_DEFINE(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARMv8 CRC Extension with a runtime check.]) - PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o" - AC_MSG_RESULT(ARMv8 CRC instructions with runtime check) - else - if test x"$USE_LOONGARCH_CRC32C" = x"1"; then - AC_DEFINE(USE_LOONGARCH_CRC32C, 1, [Define to 1 to use LoongArch CRCC instructions.]) - PG_CRC32C_OBJS="pg_crc32c_loongarch.o" - AC_MSG_RESULT(LoongArch CRCC instructions) - else - AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).]) - PG_CRC32C_OBJS="pg_crc32c_sb8.o" - AC_MSG_RESULT(slicing-by-8) - fi - fi + # fall back to slicing-by-8 algorithm, which doesn't require any + # special CPU support. + AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).]) + PG_CRC32C_OBJS="pg_crc32c_sb8.o" + AC_MSG_RESULT(slicing-by-8) fi fi fi diff --git a/meson.build b/meson.build index aefb64c094..5ec7975108 100644 --- a/meson.build +++ b/meson.build @@ -2233,9 +2233,10 @@ if host_cpu == 'x86' or host_cpu == 'x86_64' cdata.set('USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 1) have_optimized_crc = true else - avx_prog = ''' + avx512_crc_prog = ''' #include +__attribute__((target("avx512vl","vpclmulqdq"))) int main(void) { const unsigned long k1k2[8] = { @@ -2262,9 +2263,12 @@ int main(void) } ''' - prog = ''' + sse42_crc_prog = ''' #include +#ifdef TEST_SSE42_WITH_ATTRIBUTE +__attribute__((target("sse4.2"))) +#endif int main(void) { unsigned int crc = 0; @@ -2274,29 +2278,25 @@ int main(void) return crc == 0; } ''' - - if cc.links(avx_prog, - name: '_mm512_clmulepi64_epi128 ... with -msse4.2 -mavx512vl -mvpclmulqdq', - args: test_c_args + ['-msse4.2', '-mavx512vl', '-mvpclmulqdq']) - cflags_crc += ['-msse4.2','-mavx512vl','-mvpclmulqdq'] - cdata.set('USE_AVX512_CRC32C', false) - cdata.set('USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 1) - have_optimized_crc = true - endif - if have_optimized_crc == false and cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 without -msse4.2', + if cc.links(sse42_crc_prog, name: 'CRC32C baseline feature SSE4.2 ', args: test_c_args) # Use Intel SSE 4.2 unconditionally. cdata.set('USE_SSE42_CRC32C', 1) have_optimized_crc = true - elif have_optimized_crc == false and cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 with -msse4.2', - args: test_c_args + ['-msse4.2']) + elif cc.links(sse42_crc_prog, name: 'SSE4.2 CRC32C with function attributes', + args: test_c_args + ['-D TEST_SSE42_WITH_ATTRIBUTE']) # Use Intel SSE 4.2, with runtime check. The CPUID instruction is needed for # the runtime check. - cflags_crc += '-msse4.2' cdata.set('USE_SSE42_CRC32C', false) cdata.set('USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 1) have_optimized_crc = true endif + if cc.links(avx512_crc_prog, + name: 'AVX512 CRC32C with function attributes', + args: test_c_args) + cdata.set('USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 1) + have_optimized_crc = true + endif endif diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 65623df7f9..2c9278329b 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -663,6 +663,9 @@ /* Define to 1 to build with assertion checks. (--enable-cassert) */ #undef USE_ASSERT_CHECKING +/* Define to 1 to use Intel AVX 512 CRC instructions with a runtime check. */ +#undef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK + /* Define to 1 to use AVX-512 popcount instructions with a runtime check. */ #undef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK @@ -712,9 +715,6 @@ /* Define to 1 use Intel SSE 4.2 CRC instructions. */ #undef USE_SSE42_CRC32C -/* Define to 1 to use Intel AVX 512 CRC instructions with a runtime check. */ -#undef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK - /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK diff --git a/src/include/pg_cpu.h b/src/include/pg_cpu.h new file mode 100644 index 0000000000..223994cb0d --- /dev/null +++ b/src/include/pg_cpu.h @@ -0,0 +1,23 @@ +/* + * pg_cpu.h + * Useful macros to determine CPU types + */ + +#ifndef PG_CPU_H_ +#define PG_CPU_H_ +#if defined( __i386__ ) || defined(i386) || defined(_M_IX86) + /* + * __i386__ is defined by gcc and Intel compiler on Linux, + * _M_IX86 by VS compiler, + * i386 by Sun compilers on opensolaris at least + */ + #define PG_CPU_X86 +#elif defined(__x86_64__) || defined(__amd64__) || defined(__x86_64) || defined(_M_AMD64) + /* + * both __x86_64__ and __amd64__ are defined by gcc + * __x86_64 defined by sun compiler on opensolaris at least + * _M_AMD64 defined by MS compiler + */ + #define PG_CPU_x86_64 +#endif +#endif // PG_CPU_H_ diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h index 3f83d9f815..935c089eb6 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h @@ -33,6 +33,7 @@ #ifndef PG_CRC32C_H #define PG_CRC32C_H +#include "pg_cpu.h" #include "port/pg_bswap.h" typedef uint32 pg_crc32c; @@ -42,73 +43,35 @@ typedef uint32 pg_crc32c; #define EQ_CRC32C(c1, c2) ((c1) == (c2)) #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) -#if defined(USE_SSE42_CRC32C) -/* Use Intel SSE4.2 instructions. */ -#define COMP_CRC32C(crc, data, len) \ - ((crc) = pg_comp_crc32c_sse42((crc), (data), (len))) - +/* x86 */ +#if defined(PG_CPU_X86) || defined(PG_CPU_x86_64) +extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - -#elif defined (USE_AVX512_CRC32) -/* Use Intel AVX512 instructions. */ -#define COMP_CRC32C(crc, data, len) \ - ((crc) = pg_comp_crc32c_avx512((crc), (data), (len))) - extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len); +extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +#define COMP_CRC32C(crc, data, len) \ + ((crc) = pg_comp_crc32c((crc), (data), (len))) +/* ARMV8 */ #elif defined(USE_ARMV8_CRC32C) -/* Use ARMv8 CRC Extension instructions. */ - +extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_armv8((crc), (data), (len))) +/* ARMV8 with runtime check */ +#elif defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK) +extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); +extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +#define COMP_CRC32C(crc, data, len) \ + ((crc) = pg_comp_crc32c((crc), (data), (len))) +/* LoongArch */ #elif defined(USE_LOONGARCH_CRC32C) -/* Use LoongArch CRCC instructions. */ - +extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len); #define COMP_CRC32C(crc, data, len) \ ((crc) = pg_comp_crc32c_loongarch((crc), (data), (len))) -extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len); - -#elif defined(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK) - -/* - * Use Intel AVX-512 instructions, but perform a runtime check first to check that - * they are available. - */ -#define COMP_CRC32C(crc, data, len) \ - ((crc) = ((len) < 256 ? \ - pg_comp_crc32c_sse42((crc), (data), (len)) : \ - pg_comp_crc32c((crc), (data), (len)))) - -extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); -extern pg_crc32c (*pg_comp_crc32c)(pg_crc32c crc, const void *data, size_t len); - -extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len); - -extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - -#elif defined(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK) || defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK) - -/* - * Use Intel SSE 4.2 or ARMv8 instructions, but perform a runtime check first - * to check that they are available. - */ -#define COMP_CRC32C(crc, data, len) \ - ((crc) = pg_comp_crc32c((crc), (data), (len))) - -extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); -extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - -#ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK -extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); -#endif -#ifdef USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK -extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); -#endif - #else /* * Use slicing-by-8 algorithm. diff --git a/src/port/Makefile b/src/port/Makefile index 42c02f1b3d..805509b830 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -83,16 +83,6 @@ libpgport.a: $(OBJS) rm -f $@ $(AR) $(AROPT) $@ $^ -# all versions of pg_crc32c_sse42.o need CFLAGS_CRC -pg_crc32c_sse42.o: CFLAGS+=$(CFLAGS_CRC) -pg_crc32c_sse42_shlib.o: CFLAGS+=$(CFLAGS_CRC) -pg_crc32c_sse42_srv.o: CFLAGS+=$(CFLAGS_CRC) - -# all versions of pg_crc32c_avx512.o need CFLAGS_CRC -pg_crc32c_avx512.o: CFLAGS+=$(CFLAGS_CRC) -pg_crc32c_avx512_shlib.o: CFLAGS+=$(CFLAGS_CRC) -pg_crc32c_avx512_srv.o: CFLAGS+=$(CFLAGS_CRC) - # all versions of pg_crc32c_armv8.o need CFLAGS_CRC pg_crc32c_armv8.o: CFLAGS+=$(CFLAGS_CRC) pg_crc32c_armv8_shlib.o: CFLAGS+=$(CFLAGS_CRC) diff --git a/src/port/meson.build b/src/port/meson.build index 3f17cd2f8d..13c4be8ce2 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -7,7 +7,6 @@ pgport_sources = [ 'noblock.c', 'path.c', 'pg_bitutils.c', - 'pg_popcount_avx512.c', 'pg_strong_random.c', 'pgcheckdir.c', 'pgmkdirp.c', @@ -23,6 +22,17 @@ pgport_sources = [ 'tar.c', ] +if host_cpu == 'x86' or host_cpu == 'x86_64' + pgport_sources += files( + 'pg_hw_feat_check.c', + 'pg_popcount_avx512.c', + 'pg_crc32c_x86_choose.c', + 'pg_crc32c_avx512.c', + 'pg_crc32c_sse42.c', + 'pg_crc32c_sb8.c', + ) +endif + if host_system == 'windows' pgport_sources += files( 'dirmod.c', @@ -80,18 +90,6 @@ endif # Replacement functionality to be built if corresponding configure symbol # is true replace_funcs_pos = [ - # x86/x64 - ['pg_crc32c_sse42', 'USE_SSE42_CRC32C'], - ['pg_crc32c_sse42', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 'crc'], - ['pg_crc32c_sse42_choose', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK'], - ['pg_crc32c_avx512', 'USE_AVX512_CRC32C'], - ['pg_crc32c_avx512', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'crc'], - ['pg_crc32c_avx512_choose', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK'], - ['pg_crc32c_sse42', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'crc'], - ['pg_crc32c_sb8', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK'], - ['pg_crc32c_sb8', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK'], - ['pg_hw_feat_check', 'HAVE_XSAVE_INTRINSICS', 'xsave'], - # arm / aarch64 ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C'], ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 'crc'], diff --git a/src/port/pg_crc32c_avx512.c b/src/port/pg_crc32c_avx512.c index 98353f7e1d..3687f69da2 100644 --- a/src/port/pg_crc32c_avx512.c +++ b/src/port/pg_crc32c_avx512.c @@ -57,7 +57,11 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + +#if defined(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK) + pg_attribute_no_sanitize_alignment() +pg_attribute_target("avx512vl", "vpclmulqdq") inline pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t length) { @@ -195,3 +199,4 @@ pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t length) */ return pg_comp_crc32c_sse42(crc, input, length); } +#endif // AVX512_CRC32 diff --git a/src/port/pg_crc32c_avx512_choose.c b/src/port/pg_crc32c_avx512_choose.c deleted file mode 100644 index 4f11c278be..0000000000 --- a/src/port/pg_crc32c_avx512_choose.c +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_crc32c_avx512_choose.c - * Choose between Intel AVX-512 and software CRC-32C implementation. - * - * On first call, checks if the CPU we're running on supports Intel AVX- - * 512. If it does, use the special AVX-512 instructions for CRC-32C - * computation. Otherwise, fall back to the pure software implementation - * (slicing-by-8). - * - * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * - * IDENTIFICATION - * src/port/pg_crc32c_avx512_choose.c - * - *------------------------------------------------------------------------- - */ - -#include "c.h" - -#include "port/pg_crc32c.h" -#include "port/pg_hw_feat_check.h" - - -/* - * This gets called on the first call. It replaces the function pointer - * so that subsequent calls are routed directly to the chosen implementation. - */ -static pg_crc32c -pg_comp_avx512_choose(pg_crc32c crc, const void *data, size_t len) -{ - if (pg_crc32c_avx512_available()) - pg_comp_crc32c = pg_comp_crc32c_avx512; - else - pg_comp_crc32c = pg_comp_crc32c_sb8; - - return pg_comp_crc32c(crc, data, len); -} - -pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len) = pg_comp_avx512_choose; diff --git a/src/port/pg_crc32c_sse42.c b/src/port/pg_crc32c_sse42.c index 7f88c11480..0d6829af5c 100644 --- a/src/port/pg_crc32c_sse42.c +++ b/src/port/pg_crc32c_sse42.c @@ -18,7 +18,10 @@ #include "port/pg_crc32c.h" +#if defined(USE_SSE42_CRC32C) || defined(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK) + pg_attribute_no_sanitize_alignment() +pg_attribute_target("sse4.2") pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len) { @@ -67,3 +70,4 @@ pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len) return crc; } +#endif // SSE42_CRC32 diff --git a/src/port/pg_crc32c_sse42_choose.c b/src/port/pg_crc32c_x86_choose.c similarity index 58% rename from src/port/pg_crc32c_sse42_choose.c rename to src/port/pg_crc32c_x86_choose.c index 36e6949362..fa028327fb 100644 --- a/src/port/pg_crc32c_sse42_choose.c +++ b/src/port/pg_crc32c_x86_choose.c @@ -1,19 +1,18 @@ /*------------------------------------------------------------------------- * - * pg_crc32c_sse42_choose.c - * Choose between Intel SSE 4.2 and software CRC-32C implementation. + * pg_crc32c_x86_choose.c + * Choose between Intel AVX-512, SSE 4.2 and software CRC-32C implementation. * - * On first call, checks if the CPU we're running on supports Intel SSE - * 4.2. If it does, use the special SSE instructions for CRC-32C - * computation. Otherwise, fall back to the pure software implementation - * (slicing-by-8). + * On first call, checks if the CPU we're running on supports Intel AVX-512. If + * it does, use the special SSE instructions for CRC-32C computation. + * Otherwise, fall back to the pure software implementation (slicing-by-8). * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION - * src/port/pg_crc32c_sse42_choose.c + * src/port/pg_crc32c_x86_choose.c * *------------------------------------------------------------------------- */ @@ -30,11 +29,17 @@ static pg_crc32c pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) { - if (pg_crc32c_sse42_available()) + pg_comp_crc32c = pg_comp_crc32c_sb8; +#ifdef USE_SSE42_CRC32C + pg_comp_crc32c = pg_comp_crc32c_sse42; +#elif USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + if (pg_crc32c_sse42_available()) pg_comp_crc32c = pg_comp_crc32c_sse42; - else - pg_comp_crc32c = pg_comp_crc32c_sb8; - +#endif +#ifdef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK + if (pg_crc32c_avx512_available()) + pg_comp_crc32c = pg_comp_crc32c_avx512; +#endif return pg_comp_crc32c(crc, data, len); } diff --git a/src/port/pg_hw_feat_check.c b/src/port/pg_hw_feat_check.c index 35d6f9cdb1..c697d25b76 100644 --- a/src/port/pg_hw_feat_check.c +++ b/src/port/pg_hw_feat_check.c @@ -96,6 +96,9 @@ osxsave_available(void) * NB: Caller is responsible for verifying that osxsave_available() returns true * before calling this. */ +#ifdef HAVE_XSAVE_INTRINSICS +pg_attribute_target("xsave") +#endif inline static bool zmm_regs_available(void) { diff --git a/src/port/pg_popcount_avx512.c b/src/port/pg_popcount_avx512.c index b598e86554..6f18561cfb 100644 --- a/src/port/pg_popcount_avx512.c +++ b/src/port/pg_popcount_avx512.c @@ -12,19 +12,12 @@ */ #include "c.h" -#if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT) -#include -#endif - #ifdef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK #include #endif -#if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX) -#include -#endif - #include "port/pg_bitutils.h" +#include "port/pg_hw_feat_check.h" /* * It's probably unlikely that TRY_POPCNT_FAST won't be set if we are able to @@ -33,75 +26,6 @@ */ #if defined(TRY_POPCNT_FAST) && defined(USE_AVX512_POPCNT_WITH_RUNTIME_CHECK) -/* - * Does CPUID say there's support for XSAVE instructions? - */ -static inline bool -xsave_available(void) -{ - unsigned int exx[4] = {0, 0, 0, 0}; - -#if defined(HAVE__GET_CPUID) - __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); -#elif defined(HAVE__CPUID) - __cpuid(exx, 1); -#else -#error cpuid instruction not available -#endif - return (exx[2] & (1 << 27)) != 0; /* osxsave */ -} - -/* - * Does XGETBV say the ZMM registers are enabled? - * - * NB: Caller is responsible for verifying that xsave_available() returns true - * before calling this. - */ -#ifdef HAVE_XSAVE_INTRINSICS -pg_attribute_target("xsave") -#endif -static inline bool -zmm_regs_available(void) -{ -#ifdef HAVE_XSAVE_INTRINSICS - return (_xgetbv(0) & 0xe6) == 0xe6; -#else - return false; -#endif -} - -/* - * Does CPUID say there's support for AVX-512 popcount and byte-and-word - * instructions? - */ -static inline bool -avx512_popcnt_available(void) -{ - unsigned int exx[4] = {0, 0, 0, 0}; - -#if defined(HAVE__GET_CPUID_COUNT) - __get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); -#elif defined(HAVE__CPUIDEX) - __cpuidex(exx, 7, 0); -#else -#error cpuid instruction not available -#endif - return (exx[2] & (1 << 14)) != 0 && /* avx512-vpopcntdq */ - (exx[1] & (1 << 30)) != 0; /* avx512-bw */ -} - -/* - * Returns true if the CPU supports the instructions required for the AVX-512 - * pg_popcount() implementation. - */ -bool -pg_popcount_avx512_available(void) -{ - return xsave_available() && - zmm_regs_available() && - avx512_popcnt_available(); -} - /* * pg_popcount_avx512 * Returns the number of 1-bits in buf -- 2.43.0