From 04d2100e41f9e282c3f1cee7867f8c18841414d1 Mon Sep 17 00:00:00 2001 From: rajatma Date: Wed, 6 Nov 2024 15:50:04 +0530 Subject: [PATCH v1 1/6] Added new File of popcount SVE Implementation --- src/port/popcount_sve.c | 44 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/port/popcount_sve.c diff --git a/src/port/popcount_sve.c b/src/port/popcount_sve.c new file mode 100644 index 0000000000..22185dda3b --- /dev/null +++ b/src/port/popcount_sve.c @@ -0,0 +1,44 @@ +#include +#include +#include + +typedef uint8_t uint8; +typedef uint32_t uint32; +typedef uint64_t uint64; + +uint64 pg_popcount_sve(const char *buf, uint32 bytes) +{ + // Version 2.3: 64bit value size, no loop predicate + svbool_t predicate = svptrue_b64(); + svuint64_t segment, accum = svdup_u64(0); + + uint64 popcnt; + uint32 i = 0, segment_len = svcntb(); + uint32 num_vals_segment = svlen_u64(segment); // no. of 64bit values in a segment + uint32 num_vals_full_segment = (bytes / segment_len) * num_vals_segment; // no. of 64bit values in all full segments + + // Process full segments, predicate is not needed here + for (; i < num_vals_full_segment; i += num_vals_segment) + { + segment = svld1(predicate, (const uint64 *) buf); + accum = svadd_x(predicate, accum, svcnt_x(predicate, segment)); + buf += segment_len; + printf("i: %u, buf: %lu, accum: %lu\n", i, (uint64) buf, svaddv(predicate, accum)); + } + + popcnt = svaddv(predicate, accum); // horizontal reduce + + // process the last segment, no. of bytes in the last segment may be odd, so we use 8bit values + predicate = svwhilelt_b8(i * 8, bytes); // i * 8 because, each value was of size 64bits in the loop + popcnt += svaddv(predicate, svcnt_z(predicate, svld1(predicate, (const uint8 *) buf))); + + return popcnt; +} + +int main() { + char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount should be 258 + // char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount - 86 + printf("\nTotal Bit Count: %lu\n", pg_popcount_sve(buf, strlen(buf))); + return 0; +} + -- 2.34.1 From 3e6229e8a85eb4a365aa9436b30c4162a8a8688b Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Mon, 11 Nov 2024 15:55:12 +0530 Subject: [PATCH v1 2/6] Files added for popcount sve impl --- src/port/pg_popcount_sve.c | 1 + src/port/pg_popcount_sve_choose.c | 1 + src/port/popcount_sve.c | 44 ------------------------------- 3 files changed, 2 insertions(+), 44 deletions(-) create mode 100644 src/port/pg_popcount_sve.c create mode 100644 src/port/pg_popcount_sve_choose.c delete mode 100644 src/port/popcount_sve.c diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c new file mode 100644 index 0000000000..aad59755f9 --- /dev/null +++ b/src/port/pg_popcount_sve.c @@ -0,0 +1 @@ +// DUMMY \ No newline at end of file diff --git a/src/port/pg_popcount_sve_choose.c b/src/port/pg_popcount_sve_choose.c new file mode 100644 index 0000000000..aad59755f9 --- /dev/null +++ b/src/port/pg_popcount_sve_choose.c @@ -0,0 +1 @@ +// DUMMY \ No newline at end of file diff --git a/src/port/popcount_sve.c b/src/port/popcount_sve.c deleted file mode 100644 index 22185dda3b..0000000000 --- a/src/port/popcount_sve.c +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include -#include - -typedef uint8_t uint8; -typedef uint32_t uint32; -typedef uint64_t uint64; - -uint64 pg_popcount_sve(const char *buf, uint32 bytes) -{ - // Version 2.3: 64bit value size, no loop predicate - svbool_t predicate = svptrue_b64(); - svuint64_t segment, accum = svdup_u64(0); - - uint64 popcnt; - uint32 i = 0, segment_len = svcntb(); - uint32 num_vals_segment = svlen_u64(segment); // no. of 64bit values in a segment - uint32 num_vals_full_segment = (bytes / segment_len) * num_vals_segment; // no. of 64bit values in all full segments - - // Process full segments, predicate is not needed here - for (; i < num_vals_full_segment; i += num_vals_segment) - { - segment = svld1(predicate, (const uint64 *) buf); - accum = svadd_x(predicate, accum, svcnt_x(predicate, segment)); - buf += segment_len; - printf("i: %u, buf: %lu, accum: %lu\n", i, (uint64) buf, svaddv(predicate, accum)); - } - - popcnt = svaddv(predicate, accum); // horizontal reduce - - // process the last segment, no. of bytes in the last segment may be odd, so we use 8bit values - predicate = svwhilelt_b8(i * 8, bytes); // i * 8 because, each value was of size 64bits in the loop - popcnt += svaddv(predicate, svcnt_z(predicate, svld1(predicate, (const uint8 *) buf))); - - return popcnt; -} - -int main() { - char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount should be 258 - // char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount - 86 - printf("\nTotal Bit Count: %lu\n", pg_popcount_sve(buf, strlen(buf))); - return 0; -} - -- 2.34.1 From c1df7d044f7068285e6e5b7acf4e33afeb710107 Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Wed, 13 Nov 2024 12:34:55 +0530 Subject: [PATCH v1 3/6] Optimizing pg_popcount() with SVE intrinsics. --- config/c-compiler.m4 | 44 +++++++++++++ configure | 103 ++++++++++++++++++++++++++++-- configure.ac | 36 +++++++++++ meson.build | 31 +++++++++ src/Makefile.global.in | 2 + src/include/pg_config.h.in | 3 + src/include/port/pg_bitutils.h | 16 +++++ src/makefiles/meson.build | 3 +- src/port/Makefile | 11 ++++ src/port/meson.build | 4 +- src/port/pg_bitutils.c | 11 +++- src/port/pg_popcount_sve.c | 89 +++++++++++++++++++++++++- src/port/pg_popcount_sve_choose.c | 33 +++++++++- 13 files changed, 377 insertions(+), 9 deletions(-) diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 10f8c7bd0a..500c4e6662 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -752,3 +752,47 @@ if test x"$Ac_cachevar" = x"yes"; then fi undefine([Ac_cachevar])dnl ])# PGAC_AVX512_POPCNT_INTRINSICS + + +# PGAC_ARM_SVE_POPCNT_INTRINSICS +# ------------------------------ +# Check if the compiler supports the ARM SVE popcount instructions using the +# svdup_u64, svptrue_b64, svcnt_z, svcnt_x, svadd_x, svaddv, and svwhilelt_b8 +# intrinsic functions. +# +# Optional compiler flags can be passed as arguments (e.g., -march=armv8-a+sve). +AC_DEFUN([PGAC_ARM_SVE_POPCNT_INTRINSICS], +[ + AC_CACHE_CHECK([for svdup_u64 and other intrinsics with CFLAGS=$1], + [pgac_cv_arm_sve_popcnt_intrinsics], + [ + pgac_save_CFLAGS=$CFLAGS + CFLAGS="$pgac_save_CFLAGS $1" + + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [svbool_t predicate = svptrue_b64(); + svuint64_t segment = svdup_u64(0), accum = svdup_u64(0); + const char *buf = NULL; // Simulating a buffer pointer + uint32_t num_vals_segment = svlen_u64(segment); // 64-bit values per segment + + // Using other intrinsics as per the updated code + predicate = svwhilelt_b8(0, 128); // Simulating a conditional predicate + segment = svld1(predicate, (const uint64_t *)buf); + accum = svadd_x(predicate, accum, svcnt_x(predicate, segment)); + uint64_t popcnt = svaddv(predicate, accum); + + /* Return computed value, to prevent the above being optimized away */ + return popcnt;])], + [pgac_cv_arm_sve_popcnt_intrinsics=yes], + [pgac_cv_arm_sve_popcnt_intrinsics=no]) + + CFLAGS="$pgac_save_CFLAGS" + ]) + + if test x"$pgac_cv_arm_sve_popcnt_intrinsics" = x"yes"; then + CFLAGS_POPCNT_ARM="$1" + pgac_arm_sve_popcnt_intrinsics=yes + fi +]) + + diff --git a/configure b/configure index 53c8a1f2ba..0cd8f2879f 100755 --- a/configure +++ b/configure @@ -648,7 +648,9 @@ MSGFMT PG_CRC32C_OBJS CFLAGS_CRC PG_POPCNT_OBJS +PG_POPCNT_OBJS_ARM CFLAGS_POPCNT +CFLAGS_POPCNT_ARM CFLAGS_XSAVE LIBOBJS OPENSSL @@ -2787,10 +2789,12 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu - - - - +# Set architecture flags for ARM SVE +if [ "$target" = "aarch64" ]; then + CFLAGS="$CFLAGS -march=armv8-a+sve" +elif [ "$target" = "arm" ]; then + CFLAGS="$CFLAGS -march=armv8-a+sve" +fi ac_aux_dir= for ac_dir in config "$srcdir"/config; do @@ -17582,6 +17586,97 @@ $as_echo "#define HAVE_XSAVE_INTRINSICS 1" >>confdefs.h fi +# Check for ARM SVE popcount intrinsics +CFLAGS_POPCNT_ARM="" +PG_POPCNT_OBJS_ARM="" + +if test x"$host_cpu" = x"aarch64"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5 +$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; } +if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_save_CFLAGS=$CFLAGS + CFLAGS="$pgac_save_CFLAGS " + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ + svbool_t predicate = svptrue_b64(); + svuint64_t segment, accum = svdup_u64(0); + uint64_t numVals = svlen_u64(segment); // 64-bit count check + + svuint64_t counts = svcnt_u64_z(predicate, segment); + accum = svadd_u64_m(predicate, accum, counts); + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_arm_sve_popcnt_intrinsics_=yes +else + pgac_cv_arm_sve_popcnt_intrinsics_=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +CFLAGS="$pgac_save_CFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5 +$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; } +if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then + CFLAGS_POPCNT_ARM="" + pgac_arm_sve_popcnt_intrinsics=yes +fi + +if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5 +$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; } +if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_save_CFLAGS=$CFLAGS + CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ + svbool_t predicate = svptrue_b64(); + svuint64_t segment, accum = svdup_u64(0); + uint64_t numVals = svlen_u64(segment); // 64-bit count check + + svuint64_t counts = svcnt_u64_z(predicate, segment); + accum = svadd_u64_m(predicate, accum, counts); + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes +else + pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +CFLAGS="$pgac_save_CFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5 +$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; } +if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then + CFLAGS_POPCNT_ARM="-march=armv8-a+sve" + pgac_arm_sve_popcnt_intrinsics=yes +fi + +fi +if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then + PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" + + $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h + +fi +fi + # Check for AVX-512 popcount intrinsics # diff --git a/configure.ac b/configure.ac index 6a35b2880b..2b67cd2753 100644 --- a/configure.ac +++ b/configure.ac @@ -2089,6 +2089,41 @@ fi AC_SUBST(CFLAGS_POPCNT) AC_SUBST(PG_POPCNT_OBJS) +# Check for ARM popcount intrinsics +CFLAGS_POPCNT_ARM="" +PG_POPCNT_OBJS_ARM="" + +# Debug: Print the host CPU type +{ $as_echo "Host CPU: $host_cpu" >&5; } + +if test x"$host_cpu" = x"aarch64"; then + #PGAC_ARM_SVE_POPCNT_INTRINSICS([]) + + # Debug: Check if SVE intrinsics were found + { $as_echo "pgac_arm_sve_popcnt_intrinsics: $pgac_arm_sve_popcnt_intrinsics" >&5; } + + #if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then + PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve]) + + # Debug: Check if SVE intrinsics were found after trying with -march flag + { $as_echo "pgac_arm_sve_popcnt_intrinsics after -march flag: $pgac_arm_sve_popcnt_intrinsics" >&5; } + + #fi + if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then + PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation + AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.]) + + # Debug: Print the objects being used + { $as_echo "Using ARM popcount objects: $PG_POPCNT_OBJS_ARM" >&5; } + + fi +fi +# Debug: Print final values of flags +{ $as_echo "CFLAGS_POPCNT_ARM: $CFLAGS_POPCNT_ARM" >&5; } +{ $as_echo "PG_POPCNT_OBJS_ARM: $PG_POPCNT_OBJS_ARM" >&5; } +AC_SUBST(CFLAGS_POPCNT_ARM) +AC_SUBST(PG_POPCNT_OBJS_ARM) + # Check for Intel SSE 4.2 intrinsics to do CRC calculations. # # First check if the _mm_crc32_u8 and _mm_crc32_u64 intrinsics can be used @@ -2115,6 +2150,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ PGAC_ARMV8_CRC32C_INTRINSICS([]) if test x"$pgac_armv8_crc32c_intrinsics" != x"yes"; then PGAC_ARMV8_CRC32C_INTRINSICS([-march=armv8-a+crc]) + { $as_echo "pgac_armv8_crc32_intrinsics after -march flag: $pgac_armv8_crc32_intrinsics" >&5; } fi # Check for LoongArch CRC intrinsics to do CRC calculations. diff --git a/meson.build b/meson.build index 7150f85e0f..ba3bc4cd2b 100644 --- a/meson.build +++ b/meson.build @@ -2217,6 +2217,37 @@ int main(void) endif +############################################################### +# Check for the availability of ARM SVE popcount intrinsics. +############################################################### + +cflags_popcnt_arm = [] +if host_cpu == 'aarch64' + + prog = ''' +#include + +int main(void) +{ + const svuint64_t val = svdup_u64(0xFFFFFFFFFFFFFFFF); // Example value + svuint64_t popcnt = svcntb(val); // Count the number of 1 bits + /* return computed value, to prevent the above being optimized away */ + return popcnt == 0; // Ensure to return a valid value +} +''' + + if cc.links(prog, name: 'ARM SVE popcount without -march=armv8-a+sve', + args: test_c_args + ['-DSVINT64=@0@'.format(cdata.get('SV_INT64_TYPE'))]) + cdata.set('USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 1) + elif cc.links(prog, name: 'ARM SVE popcount with -march=armv8-a+sve', + args: test_c_args + ['-DSVINT64=@0@'.format(cdata.get('SV_INT64_TYPE'))] + ['-march=armv8-a+sve']) + cdata.set('USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 1) + cflags_popcnt_arm += ['-march=armv8-a+sve'] + endif + +endif + + ############################################################### # Select CRC-32C implementation. # diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 42f50b4976..2e79a24a2d 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -263,6 +263,7 @@ CXXFLAGS_SL_MODULE = @CXXFLAGS_SL_MODULE@ CFLAGS_UNROLL_LOOPS = @CFLAGS_UNROLL_LOOPS@ CFLAGS_VECTORIZE = @CFLAGS_VECTORIZE@ CFLAGS_POPCNT = @CFLAGS_POPCNT@ +CFLAGS_POPCNT_ARM = @CFLAGS_POPCNT_ARM@ CFLAGS_CRC = @CFLAGS_CRC@ CFLAGS_XSAVE = @CFLAGS_XSAVE@ PERMIT_DECLARATION_AFTER_STATEMENT = @PERMIT_DECLARATION_AFTER_STATEMENT@ @@ -764,6 +765,7 @@ PG_CRC32C_OBJS = @PG_CRC32C_OBJS@ # files needed for the chosen popcount implementation PG_POPCNT_OBJS = @PG_POPCNT_OBJS@ +PG_POPCNT_OBJS_ARM = @PG_POPCNT_OBJS_ARM@ LIBS := -lpgcommon -lpgport $(LIBS) diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 38006367a4..f2573ad4d1 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -678,6 +678,9 @@ /* Define to 1 to use AVX-512 popcount instructions with a runtime check. */ #undef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK +/* Define to 1 to use SVE popcount instructions with a runtime check. */ +#undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK + /* Define to 1 to build with Bonjour support. (--with-bonjour) */ #undef USE_BONJOUR diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 4d88478c9c..811b1cafe5 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -298,6 +298,16 @@ pg_ceil_log2_64(uint64 num) #endif #endif +/* + * On ARM builds, try SVE popcount instructions. + */ +#if defined(__aarch64__) || defined(__arm__) +#define ARM_BUILD 1 +#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK) +#define TRY_POPCNT_FAST 1 +#endif +#endif + #ifdef TRY_POPCNT_FAST /* Attempt to use the POPCNT instruction, but perform a runtime check first */ extern PGDLLIMPORT int (*pg_popcount32) (uint32 word); @@ -317,6 +327,12 @@ extern uint64 pg_popcount_avx512(const char *buf, int bytes); extern uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask); #endif +#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK +extern bool pg_popcount_sve_available(void); +extern uint64 pg_popcount_sve(const char *buf, int bytes); +extern uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask); +#endif + #else /* Use a portable implementation -- no need for a function pointer. */ extern int pg_popcount32(uint32 word); diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 850e927584..0ff4e05993 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -103,6 +103,7 @@ pgxs_kv = { 'CFLAGS_CRC': ' '.join(cflags_crc), 'CFLAGS_POPCNT': ' '.join(cflags_popcnt), + 'CFLAGS_POPCNT_ARM': ' '.join(cflags_popcnt_arm) 'CFLAGS_UNROLL_LOOPS': ' '.join(unroll_loops_cflags), 'CFLAGS_VECTORIZE': ' '.join(vectorize_cflags), 'CFLAGS_XSAVE': ' '.join(cflags_xsave), @@ -181,7 +182,7 @@ pgxs_empty = [ 'WANTED_LANGUAGES', # Not needed because we don't build the server / PLs with the generated makefile - 'LIBOBJS', 'PG_CRC32C_OBJS', 'PG_POPCNT_OBJS', 'TAS', + 'LIBOBJS', 'PG_CRC32C_OBJS', 'PG_POPCNT_OBJS', 'PG_POPCNT_OBJS_ARM', 'TAS', 'DTRACEFLAGS', # only server has dtrace probes 'perl_archlibexp', 'perl_embed_ccflags', 'perl_embed_ldflags', 'perl_includespec', 'perl_privlibexp', diff --git a/src/port/Makefile b/src/port/Makefile index 9324ec2d9f..73ec5847d7 100644 --- a/src/port/Makefile +++ b/src/port/Makefile @@ -39,6 +39,7 @@ OBJS = \ $(LIBOBJS) \ $(PG_CRC32C_OBJS) \ $(PG_POPCNT_OBJS) \ + $(PG_POPCNT_OBJS_ARM) \ bsearch_arg.o \ chklocale.o \ inet_net_ntop.o \ @@ -102,6 +103,16 @@ pg_popcount_avx512.o: CFLAGS+=$(CFLAGS_POPCNT) pg_popcount_avx512_shlib.o: CFLAGS+=$(CFLAGS_POPCNT) pg_popcount_avx512_srv.o: CFLAGS+=$(CFLAGS_POPCNT) +# all version of pg_popcount_sve.o need CFLAGS_POPCNT_ARM +pg_popcount_sve.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) +pg_popcount_sve_shlib.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) +pg_popcount_sve_srv.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) + +# all versions of pg_popcount_sve_choose.o need CFLAGS_POPCNT_ARM +pg_popcount_sve_choose.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) +pg_popcount_sve_choose_shlib.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) +pg_popcount_sve_choose_srv.o: CFLAGS+=$(CFLAGS_POPCNT_ARM) + # # Shared library versions of object files # diff --git a/src/port/meson.build b/src/port/meson.build index 1150966ab7..5308898bb4 100644 --- a/src/port/meson.build +++ b/src/port/meson.build @@ -91,6 +91,8 @@ replace_funcs_pos = [ ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C'], ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 'crc'], ['pg_crc32c_armv8_choose', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'], + ['pg_popcount_sve', 'USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 'popcnt'], + ['pg_popcount_sve_choose', 'USE_SVE_POPCNT_WITH_RUNTIME_CHECK'], ['pg_crc32c_sb8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'], # loongarch @@ -100,7 +102,7 @@ replace_funcs_pos = [ ['pg_crc32c_sb8', 'USE_SLICING_BY_8_CRC32C'], ] -pgport_cflags = {'crc': cflags_crc, 'popcnt': cflags_popcnt, 'xsave': cflags_xsave} +pgport_cflags = {'crc': cflags_crc, 'popcnt': cflags_popcnt + cflags_popcnt_arm, 'xsave': cflags_xsave} pgport_sources_cflags = {'crc': [], 'popcnt': [], 'xsave': []} foreach f : replace_funcs_neg diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c index 87f56e82b8..15b3ea2e05 100644 --- a/src/port/pg_bitutils.c +++ b/src/port/pg_bitutils.c @@ -129,13 +129,16 @@ uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) /* * Return true if CPUID indicates that the POPCNT instruction is available. + * Return false for ARM builds. */ static bool pg_popcount_available(void) { unsigned int exx[4] = {0, 0, 0, 0}; -#if defined(HAVE__GET_CPUID) +#if defined(ARM_BUILD) + return false; +#elif defined(HAVE__GET_CPUID) __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); #elif defined(HAVE__CPUID) __cpuid(exx, 1); @@ -176,6 +179,12 @@ choose_popcount_functions(void) pg_popcount_optimized = pg_popcount_avx512; pg_popcount_masked_optimized = pg_popcount_masked_avx512; } +#elif USE_SVE_POPCNT_WITH_RUNTIME_CHECK + if (pg_popcount_sve_available()) + { + pg_popcount_optimized = pg_popcount_sve; + pg_popcount_masked_optimized = pg_popcount_masked_sve; + } #endif } diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c index aad59755f9..94e97eaff9 100644 --- a/src/port/pg_popcount_sve.c +++ b/src/port/pg_popcount_sve.c @@ -1 +1,88 @@ -// DUMMY \ No newline at end of file +/*------------------------------------------------------------------------- + * + * pg_popcount_sve.c + * Holds the SVE pg_popcount() implementation. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_popcount_sve.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" +#include "port/pg_bitutils.h" + +#include + +#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK + +/* + * pg_popcount_sve + * Returns the number of 1-bits in buf + */ +uint64 +pg_popcount_sve(const char *buf, int bytes) +{ + svbool_t pred = svptrue_b64(); + svuint64_t vec, accum = svdup_u64(0); + + uint32 i = 0, vec_len = svcntb(); + uint32 num_vals_vec = svlen_u64(vec); + uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec; + uint64 popcnt; + + /* Process full vectors */ + for (; i < num_vals_full_vec; i += num_vals_vec) + { + vec = svld1(pred, (const uint64 *) buf); + accum = svadd_x(pred, accum, svcnt_x(pred, vec)); + buf += vec_len; + } + + popcnt = svaddv(pred, accum); + + /* Process the last incomplete vector */ + if(i < bytes) + { + pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */ + popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf))); + } + + return popcnt; +} + +uint64 +pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) +{ + svbool_t pred = svptrue_b64(); + svuint8_t vec8; + svuint64_t vec, accum = svdup_u64(0); + + uint32 i = 0, vec_len = svcntb(); + uint32 num_vals_vec = svlen_u64(vec); + uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec; + uint64 popcnt, mask64 = ~UINT64CONST(0) / 0xFF * mask; + + /* Process full vectors */ + for (; i < num_vals_full_vec; i += num_vals_vec) + { + vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */ + accum = svadd_x(pred, accum, svcnt_x(pred, vec)); + buf += vec_len; + } + + popcnt = svaddv(pred, accum); + + /* Process the last incomplete vector */ + if(i < bytes) + { + pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */ + vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */ + popcnt += svaddv(pred, svcnt_z(pred, vec8)); + } + + return popcnt; +} + +#endif /* USE_SVE_POPCNT_WITH_RUNTIME_CHECK */ diff --git a/src/port/pg_popcount_sve_choose.c b/src/port/pg_popcount_sve_choose.c index aad59755f9..a6f9e0c3dc 100644 --- a/src/port/pg_popcount_sve_choose.c +++ b/src/port/pg_popcount_sve_choose.c @@ -1 +1,32 @@ -// DUMMY \ No newline at end of file +/*------------------------------------------------------------------------- + * + * pg_popcount_sve_choose.c + * Test whether we can use the SVE pg_popcount() implementation. + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/port/pg_popcount_sve_choose.c + * + *------------------------------------------------------------------------- + */ +#include "c.h" +#include "port/pg_bitutils.h" + +#include +#include + +#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK + +/* + * Returns true if the CPU supports the instructions required for the SVE + * pg_popcount() implementation. + */ +bool +pg_popcount_sve_available(void) +{ + unsigned long hwcap = getauxval(AT_HWCAP); /* get the HWCAP flags */ + return (hwcap & HWCAP_SVE) != 0; /* return true if SVE is supported */ +} + +#endif /* USE_SVE_POPCNT_WITH_RUNTIME_CHECK */ -- 2.34.1 From b0acc73b3ecebc50c8992a8354674a3b946e23a3 Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Tue, 26 Nov 2024 15:30:18 +0530 Subject: [PATCH v1 4/6] whitespaces and debug printing removed --- config/c-compiler.m4 | 3 - configure | 192 ++++++++++++++++++------------------- configure.ac | 28 +----- src/port/pg_popcount_sve.c | 31 +++--- 4 files changed, 116 insertions(+), 138 deletions(-) diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 500c4e6662..fcfee6b2e2 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -753,7 +753,6 @@ fi undefine([Ac_cachevar])dnl ])# PGAC_AVX512_POPCNT_INTRINSICS - # PGAC_ARM_SVE_POPCNT_INTRINSICS # ------------------------------ # Check if the compiler supports the ARM SVE popcount instructions using the @@ -794,5 +793,3 @@ AC_DEFUN([PGAC_ARM_SVE_POPCNT_INTRINSICS], pgac_arm_sve_popcnt_intrinsics=yes fi ]) - - diff --git a/configure b/configure index 0cd8f2879f..6297c46dd6 100755 --- a/configure +++ b/configure @@ -2789,12 +2789,10 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu -# Set architecture flags for ARM SVE -if [ "$target" = "aarch64" ]; then - CFLAGS="$CFLAGS -march=armv8-a+sve" -elif [ "$target" = "arm" ]; then - CFLAGS="$CFLAGS -march=armv8-a+sve" -fi + + + + ac_aux_dir= for ac_dir in config "$srcdir"/config; do @@ -17586,97 +17584,6 @@ $as_echo "#define HAVE_XSAVE_INTRINSICS 1" >>confdefs.h fi -# Check for ARM SVE popcount intrinsics -CFLAGS_POPCNT_ARM="" -PG_POPCNT_OBJS_ARM="" - -if test x"$host_cpu" = x"aarch64"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5 -$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; } -if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then : - $as_echo_n "(cached) " >&6 -else - pgac_save_CFLAGS=$CFLAGS - CFLAGS="$pgac_save_CFLAGS " - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ - svbool_t predicate = svptrue_b64(); - svuint64_t segment, accum = svdup_u64(0); - uint64_t numVals = svlen_u64(segment); // 64-bit count check - - svuint64_t counts = svcnt_u64_z(predicate, segment); - accum = svadd_u64_m(predicate, accum, counts); - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_arm_sve_popcnt_intrinsics_=yes -else - pgac_cv_arm_sve_popcnt_intrinsics_=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5 -$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; } -if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then - CFLAGS_POPCNT_ARM="" - pgac_arm_sve_popcnt_intrinsics=yes -fi - -if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5 -$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; } -if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then : - $as_echo_n "(cached) " >&6 -else - pgac_save_CFLAGS=$CFLAGS - CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ - svbool_t predicate = svptrue_b64(); - svuint64_t segment, accum = svdup_u64(0); - uint64_t numVals = svlen_u64(segment); // 64-bit count check - - svuint64_t counts = svcnt_u64_z(predicate, segment); - accum = svadd_u64_m(predicate, accum, counts); - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes -else - pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -CFLAGS="$pgac_save_CFLAGS" -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5 -$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; } -if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then - CFLAGS_POPCNT_ARM="-march=armv8-a+sve" - pgac_arm_sve_popcnt_intrinsics=yes -fi - -fi -if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then - PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" - - $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h - -fi -fi - # Check for AVX-512 popcount intrinsics # @@ -17778,6 +17685,97 @@ $as_echo "#define USE_AVX512_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h fi +# Check for ARM SVE popcount intrinsics +CFLAGS_POPCNT_ARM="" +PG_POPCNT_OBJS_ARM="" + +if test x"$host_cpu" = x"aarch64"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5 +$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; } +if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_save_CFLAGS=$CFLAGS + CFLAGS="$pgac_save_CFLAGS " + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ + svbool_t predicate = svptrue_b64(); + svuint64_t segment, accum = svdup_u64(0); + uint64_t numVals = svlen_u64(segment); // 64-bit count check + + svuint64_t counts = svcnt_u64_z(predicate, segment); + accum = svadd_u64_m(predicate, accum, counts); + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_arm_sve_popcnt_intrinsics_=yes +else + pgac_cv_arm_sve_popcnt_intrinsics_=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +CFLAGS="$pgac_save_CFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5 +$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; } +if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then + CFLAGS_POPCNT_ARM="" + pgac_arm_sve_popcnt_intrinsics=yes +fi + +if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5 +$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; } +if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_save_CFLAGS=$CFLAGS + CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ + svbool_t predicate = svptrue_b64(); + svuint64_t segment, accum = svdup_u64(0); + uint64_t numVals = svlen_u64(segment); // 64-bit count check + + svuint64_t counts = svcnt_u64_z(predicate, segment); + accum = svadd_u64_m(predicate, accum, counts); + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes +else + pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +CFLAGS="$pgac_save_CFLAGS" +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5 +$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; } +if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then + CFLAGS_POPCNT_ARM="-march=armv8-a+sve" + pgac_arm_sve_popcnt_intrinsics=yes +fi + +fi +if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then + PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" + + $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h + +fi +fi + # Check for Intel SSE 4.2 intrinsics to do CRC calculations. # diff --git a/configure.ac b/configure.ac index 2b67cd2753..5a5ae8f7ca 100644 --- a/configure.ac +++ b/configure.ac @@ -2092,35 +2092,16 @@ AC_SUBST(PG_POPCNT_OBJS) # Check for ARM popcount intrinsics CFLAGS_POPCNT_ARM="" PG_POPCNT_OBJS_ARM="" - -# Debug: Print the host CPU type -{ $as_echo "Host CPU: $host_cpu" >&5; } - if test x"$host_cpu" = x"aarch64"; then - #PGAC_ARM_SVE_POPCNT_INTRINSICS([]) - - # Debug: Check if SVE intrinsics were found - { $as_echo "pgac_arm_sve_popcnt_intrinsics: $pgac_arm_sve_popcnt_intrinsics" >&5; } - - #if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then - PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve]) - - # Debug: Check if SVE intrinsics were found after trying with -march flag - { $as_echo "pgac_arm_sve_popcnt_intrinsics after -march flag: $pgac_arm_sve_popcnt_intrinsics" >&5; } - - #fi + PGAC_ARM_SVE_POPCNT_INTRINSICS([]) + if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then + PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve]) + fi if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.]) - - # Debug: Print the objects being used - { $as_echo "Using ARM popcount objects: $PG_POPCNT_OBJS_ARM" >&5; } - fi fi -# Debug: Print final values of flags -{ $as_echo "CFLAGS_POPCNT_ARM: $CFLAGS_POPCNT_ARM" >&5; } -{ $as_echo "PG_POPCNT_OBJS_ARM: $PG_POPCNT_OBJS_ARM" >&5; } AC_SUBST(CFLAGS_POPCNT_ARM) AC_SUBST(PG_POPCNT_OBJS_ARM) @@ -2150,7 +2131,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ PGAC_ARMV8_CRC32C_INTRINSICS([]) if test x"$pgac_armv8_crc32c_intrinsics" != x"yes"; then PGAC_ARMV8_CRC32C_INTRINSICS([-march=armv8-a+crc]) - { $as_echo "pgac_armv8_crc32_intrinsics after -march flag: $pgac_armv8_crc32_intrinsics" >&5; } fi # Check for LoongArch CRC intrinsics to do CRC calculations. diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c index 94e97eaff9..564a20bf3d 100644 --- a/src/port/pg_popcount_sve.c +++ b/src/port/pg_popcount_sve.c @@ -24,12 +24,13 @@ uint64 pg_popcount_sve(const char *buf, int bytes) { - svbool_t pred = svptrue_b64(); - svuint64_t vec, accum = svdup_u64(0); - - uint32 i = 0, vec_len = svcntb(); - uint32 num_vals_vec = svlen_u64(vec); - uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec; + svbool_t pred = svptrue_b64(); + svuint64_t vec, + accum = svdup_u64(0); + uint32 i = 0, + vec_len = svcntb(), + num_vals_vec = svlen_u64(vec), + num_vals_full_vec = (bytes / vec_len) * num_vals_vec; uint64 popcnt; /* Process full vectors */ @@ -55,14 +56,16 @@ pg_popcount_sve(const char *buf, int bytes) uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) { - svbool_t pred = svptrue_b64(); - svuint8_t vec8; - svuint64_t vec, accum = svdup_u64(0); - - uint32 i = 0, vec_len = svcntb(); - uint32 num_vals_vec = svlen_u64(vec); - uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec; - uint64 popcnt, mask64 = ~UINT64CONST(0) / 0xFF * mask; + svbool_t pred = svptrue_b64(); + svuint8_t vec8; + svuint64_t vec, + accum = svdup_u64(0); + uint32 i = 0, + vec_len = svcntb(), + num_vals_vec = svlen_u64(vec), + num_vals_full_vec = (bytes / vec_len) * num_vals_vec; + uint64 popcnt, + mask64 = ~UINT64CONST(0) / 0xFF * mask; /* Process full vectors */ for (; i < num_vals_full_vec; i += num_vals_vec) -- 2.34.1 From 58e90123c88ecc10602ff84e8e061237c6dff830 Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Tue, 26 Nov 2024 15:48:57 +0530 Subject: [PATCH v1 5/6] updated the macros used and comments added --- configure.ac | 2 +- src/include/port/pg_bitutils.h | 8 +++----- src/port/pg_bitutils.c | 5 ++--- src/port/pg_popcount_sve.c | 12 ++++++++---- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 5a5ae8f7ca..a45a01c20a 100644 --- a/configure.ac +++ b/configure.ac @@ -2098,7 +2098,7 @@ if test x"$host_cpu" = x"aarch64"; then PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve]) fi if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then - PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation + PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.]) fi fi diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 811b1cafe5..8e09f78862 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -299,14 +299,12 @@ pg_ceil_log2_64(uint64 num) #endif /* - * On ARM builds, try SVE popcount instructions. + * On AArch64 builds, try using SVE popcount instructions, but only if + * we can verify that the CPU supports it via a runtime check. */ -#if defined(__aarch64__) || defined(__arm__) -#define ARM_BUILD 1 -#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK) +#if defined(__aarch64__) #define TRY_POPCNT_FAST 1 #endif -#endif #ifdef TRY_POPCNT_FAST /* Attempt to use the POPCNT instruction, but perform a runtime check first */ diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c index 15b3ea2e05..9bd5985330 100644 --- a/src/port/pg_bitutils.c +++ b/src/port/pg_bitutils.c @@ -129,15 +129,14 @@ uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask) /* * Return true if CPUID indicates that the POPCNT instruction is available. - * Return false for ARM builds. */ static bool pg_popcount_available(void) { unsigned int exx[4] = {0, 0, 0, 0}; -#if defined(ARM_BUILD) - return false; +#if defined(__aarch64__) + return false; /* cpuid not available in __aarch64__ */ #elif defined(HAVE__GET_CPUID) __get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); #elif defined(HAVE__CPUID) diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c index 564a20bf3d..140bf37a2d 100644 --- a/src/port/pg_popcount_sve.c +++ b/src/port/pg_popcount_sve.c @@ -46,13 +46,17 @@ pg_popcount_sve(const char *buf, int bytes) /* Process the last incomplete vector */ if(i < bytes) { - pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */ + pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */ popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf))); } return popcnt; } +/* + * pg_popcount_masked_sve + * Returns the number of 1-bits in buf after applying the mask to each byte + */ uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) { @@ -70,7 +74,7 @@ pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) /* Process full vectors */ for (; i < num_vals_full_vec; i += num_vals_vec) { - vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */ + vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */ accum = svadd_x(pred, accum, svcnt_x(pred, vec)); buf += vec_len; } @@ -80,8 +84,8 @@ pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) /* Process the last incomplete vector */ if(i < bytes) { - pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */ - vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */ + pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */ + vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */ popcnt += svaddv(pred, svcnt_z(pred, vec8)); } -- 2.34.1 From a4413f476ce0e9c58914150e5874eec340bd136c Mon Sep 17 00:00:00 2001 From: Chiranmoy Bhattacharya Date: Wed, 4 Dec 2024 12:55:36 +0530 Subject: [PATCH v1 6/6] buffer alignment for popcount sve impl --- src/include/port/pg_bitutils.h | 2 +- src/port/pg_popcount_sve.c | 75 +++++++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h index 8e09f78862..a690a46a88 100644 --- a/src/include/port/pg_bitutils.h +++ b/src/include/port/pg_bitutils.h @@ -302,7 +302,7 @@ pg_ceil_log2_64(uint64 num) * On AArch64 builds, try using SVE popcount instructions, but only if * we can verify that the CPU supports it via a runtime check. */ -#if defined(__aarch64__) +#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK) #define TRY_POPCNT_FAST 1 #endif diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c index 140bf37a2d..392441a9b1 100644 --- a/src/port/pg_popcount_sve.c +++ b/src/port/pg_popcount_sve.c @@ -24,16 +24,33 @@ uint64 pg_popcount_sve(const char *buf, int bytes) { - svbool_t pred = svptrue_b64(); - svuint64_t vec, + svbool_t pred; + svuint64_t vec, accum = svdup_u64(0); uint32 i = 0, - vec_len = svcntb(), - num_vals_vec = svlen_u64(vec), - num_vals_full_vec = (bytes / vec_len) * num_vals_vec; - uint64 popcnt; + num_vals_full_vec, + pre_align, + vec_len = svcntb(), + num_vals_vec = svlen_u64(vec); + uint64 popcnt = 0; + + /* + * For smaller inputs, aligning the buffer degrades performance. + * Therefore, we align the buffers only when the input size is sufficiently large. + */ + if (bytes > 4 * vec_len) + { + pre_align = (const char *) TYPEALIGN_DOWN(sizeof(uint64_t), buf) + sizeof(uint64_t) - buf; + pred = svwhilelt_b8(0U, pre_align); + popcnt = svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf))); + buf += pre_align; + bytes -= pre_align; + } + + pred = svptrue_b64(); + num_vals_full_vec = (bytes / vec_len) * num_vals_vec; - /* Process full vectors */ + /* Process complete vectors */ for (; i < num_vals_full_vec; i += num_vals_vec) { vec = svld1(pred, (const uint64 *) buf); @@ -41,12 +58,12 @@ pg_popcount_sve(const char *buf, int bytes) buf += vec_len; } - popcnt = svaddv(pred, accum); + popcnt += svaddv(pred, accum); /* Process the last incomplete vector */ if(i < bytes) { - pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */ + pred = svwhilelt_b8(i * 8, (uint32) bytes); popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf))); } @@ -60,32 +77,50 @@ pg_popcount_sve(const char *buf, int bytes) uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask) { - svbool_t pred = svptrue_b64(); + svbool_t pred; svuint8_t vec8; svuint64_t vec, accum = svdup_u64(0); uint32 i = 0, - vec_len = svcntb(), - num_vals_vec = svlen_u64(vec), - num_vals_full_vec = (bytes / vec_len) * num_vals_vec; - uint64 popcnt, - mask64 = ~UINT64CONST(0) / 0xFF * mask; + num_vals_full_vec, + pre_align, + vec_len = svcntb(), + num_vals_vec = svlen_u64(vec); + uint64 popcnt = 0, + mask64 = ~UINT64CONST(0) / 0xFF * mask; + + /* + * For smaller inputs, aligning the buffer degrades the performance. + * Therefore, we align the buffers only when the input size is sufficiently large. + */ + if (bytes > 4 * vec_len) + { + pre_align = (const char *) TYPEALIGN_DOWN(sizeof(uint64_t), buf) + sizeof(uint64_t) - buf; + pred = svwhilelt_b8(0U, pre_align); + vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *) buf), mask); /* load and mask */ + popcnt = svaddv(pred, svcnt_z(pred, vec8)); + buf += pre_align; + bytes -= pre_align; + } + + pred = svptrue_b64(); + num_vals_full_vec = (bytes / vec_len) * num_vals_vec; - /* Process full vectors */ + /* Process complete vectors */ for (; i < num_vals_full_vec; i += num_vals_vec) { - vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */ + vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *) buf), mask64); /* load and mask */ accum = svadd_x(pred, accum, svcnt_x(pred, vec)); buf += vec_len; } - popcnt = svaddv(pred, accum); + popcnt += svaddv(pred, accum); /* Process the last incomplete vector */ if(i < bytes) { - pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */ - vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */ + pred = svwhilelt_b8(i * 8, (uint32) bytes); + vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *) buf), mask); /* load and mask */ popcnt += svaddv(pred, svcnt_z(pred, vec8)); } -- 2.34.1