SVE_support_for_popcount.patch
application/octet-stream
Filename: SVE_support_for_popcount.patch
Type: application/octet-stream
Part: 0
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: format-patch
Series: patch v1-0001
Subject: Added new File of popcount SVE Implementation
| File | + | − |
|---|---|---|
| config/c-compiler.m4 | 44 | 3 |
| configure | 194 | 101 |
| configure.ac | 41 | 25 |
| meson.build | 31 | 0 |
| src/include/pg_config.h.in | 3 | 0 |
| src/include/port/pg_bitutils.h | 20 | 6 |
| src/Makefile.global.in | 2 | 0 |
| src/makefiles/meson.build | 2 | 1 |
| src/port/Makefile | 11 | 0 |
| src/port/meson.build | 3 | 1 |
| src/port/pg_bitutils.c | 12 | 4 |
| src/port/pg_popcount_sve.c | 169 | 39 |
| src/port/pg_popcount_sve_choose.c | 33 | 1 |
| src/port/popcount_sve.c | 44 | 44 |
From 04d2100e41f9e282c3f1cee7867f8c18841414d1 Mon Sep 17 00:00:00 2001
From: rajatma <rajat.ma@fujitsu.com>
Date: Wed, 6 Nov 2024 15:50:04 +0530
Subject: [PATCH v1 1/6] Added new File of popcount SVE Implementation
---
src/port/popcount_sve.c | 44 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
create mode 100644 src/port/popcount_sve.c
diff --git a/src/port/popcount_sve.c b/src/port/popcount_sve.c
new file mode 100644
index 0000000000..22185dda3b
--- /dev/null
+++ b/src/port/popcount_sve.c
@@ -0,0 +1,44 @@
+#include <stdio.h>
+#include <string.h>
+#include <arm_sve.h>
+
+typedef uint8_t uint8;
+typedef uint32_t uint32;
+typedef uint64_t uint64;
+
+uint64 pg_popcount_sve(const char *buf, uint32 bytes)
+{
+ // Version 2.3: 64bit value size, no loop predicate
+ svbool_t predicate = svptrue_b64();
+ svuint64_t segment, accum = svdup_u64(0);
+
+ uint64 popcnt;
+ uint32 i = 0, segment_len = svcntb();
+ uint32 num_vals_segment = svlen_u64(segment); // no. of 64bit values in a segment
+ uint32 num_vals_full_segment = (bytes / segment_len) * num_vals_segment; // no. of 64bit values in all full segments
+
+ // Process full segments, predicate is not needed here
+ for (; i < num_vals_full_segment; i += num_vals_segment)
+ {
+ segment = svld1(predicate, (const uint64 *) buf);
+ accum = svadd_x(predicate, accum, svcnt_x(predicate, segment));
+ buf += segment_len;
+ printf("i: %u, buf: %lu, accum: %lu\n", i, (uint64) buf, svaddv(predicate, accum));
+ }
+
+ popcnt = svaddv(predicate, accum); // horizontal reduce
+
+ // process the last segment, no. of bytes in the last segment may be odd, so we use 8bit values
+ predicate = svwhilelt_b8(i * 8, bytes); // i * 8 because, each value was of size 64bits in the loop
+ popcnt += svaddv(predicate, svcnt_z(predicate, svld1(predicate, (const uint8 *) buf)));
+
+ return popcnt;
+}
+
+int main() {
+ char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount should be 258
+ // char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount - 86
+ printf("\nTotal Bit Count: %lu\n", pg_popcount_sve(buf, strlen(buf)));
+ return 0;
+}
+
--
2.34.1
From 3e6229e8a85eb4a365aa9436b30c4162a8a8688b Mon Sep 17 00:00:00 2001
From: Chiranmoy Bhattacharya <chiranmoy.bhattacharya@fujitsu.com>
Date: Mon, 11 Nov 2024 15:55:12 +0530
Subject: [PATCH v1 2/6] Files added for popcount sve impl
---
src/port/pg_popcount_sve.c | 1 +
src/port/pg_popcount_sve_choose.c | 1 +
src/port/popcount_sve.c | 44 -------------------------------
3 files changed, 2 insertions(+), 44 deletions(-)
create mode 100644 src/port/pg_popcount_sve.c
create mode 100644 src/port/pg_popcount_sve_choose.c
delete mode 100644 src/port/popcount_sve.c
diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c
new file mode 100644
index 0000000000..aad59755f9
--- /dev/null
+++ b/src/port/pg_popcount_sve.c
@@ -0,0 +1 @@
+// DUMMY
\ No newline at end of file
diff --git a/src/port/pg_popcount_sve_choose.c b/src/port/pg_popcount_sve_choose.c
new file mode 100644
index 0000000000..aad59755f9
--- /dev/null
+++ b/src/port/pg_popcount_sve_choose.c
@@ -0,0 +1 @@
+// DUMMY
\ No newline at end of file
diff --git a/src/port/popcount_sve.c b/src/port/popcount_sve.c
deleted file mode 100644
index 22185dda3b..0000000000
--- a/src/port/popcount_sve.c
+++ /dev/null
@@ -1,44 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <arm_sve.h>
-
-typedef uint8_t uint8;
-typedef uint32_t uint32;
-typedef uint64_t uint64;
-
-uint64 pg_popcount_sve(const char *buf, uint32 bytes)
-{
- // Version 2.3: 64bit value size, no loop predicate
- svbool_t predicate = svptrue_b64();
- svuint64_t segment, accum = svdup_u64(0);
-
- uint64 popcnt;
- uint32 i = 0, segment_len = svcntb();
- uint32 num_vals_segment = svlen_u64(segment); // no. of 64bit values in a segment
- uint32 num_vals_full_segment = (bytes / segment_len) * num_vals_segment; // no. of 64bit values in all full segments
-
- // Process full segments, predicate is not needed here
- for (; i < num_vals_full_segment; i += num_vals_segment)
- {
- segment = svld1(predicate, (const uint64 *) buf);
- accum = svadd_x(predicate, accum, svcnt_x(predicate, segment));
- buf += segment_len;
- printf("i: %u, buf: %lu, accum: %lu\n", i, (uint64) buf, svaddv(predicate, accum));
- }
-
- popcnt = svaddv(predicate, accum); // horizontal reduce
-
- // process the last segment, no. of bytes in the last segment may be odd, so we use 8bit values
- predicate = svwhilelt_b8(i * 8, bytes); // i * 8 because, each value was of size 64bits in the loop
- popcnt += svaddv(predicate, svcnt_z(predicate, svld1(predicate, (const uint8 *) buf)));
-
- return popcnt;
-}
-
-int main() {
- char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount should be 258
- // char buf[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // bitcount - 86
- printf("\nTotal Bit Count: %lu\n", pg_popcount_sve(buf, strlen(buf)));
- return 0;
-}
-
--
2.34.1
From c1df7d044f7068285e6e5b7acf4e33afeb710107 Mon Sep 17 00:00:00 2001
From: Chiranmoy Bhattacharya <chiranmoy.bhattacharya@fujitsu.com>
Date: Wed, 13 Nov 2024 12:34:55 +0530
Subject: [PATCH v1 3/6] Optimizing pg_popcount() with SVE intrinsics.
---
config/c-compiler.m4 | 44 +++++++++++++
configure | 103 ++++++++++++++++++++++++++++--
configure.ac | 36 +++++++++++
meson.build | 31 +++++++++
src/Makefile.global.in | 2 +
src/include/pg_config.h.in | 3 +
src/include/port/pg_bitutils.h | 16 +++++
src/makefiles/meson.build | 3 +-
src/port/Makefile | 11 ++++
src/port/meson.build | 4 +-
src/port/pg_bitutils.c | 11 +++-
src/port/pg_popcount_sve.c | 89 +++++++++++++++++++++++++-
src/port/pg_popcount_sve_choose.c | 33 +++++++++-
13 files changed, 377 insertions(+), 9 deletions(-)
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 10f8c7bd0a..500c4e6662 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -752,3 +752,47 @@ if test x"$Ac_cachevar" = x"yes"; then
fi
undefine([Ac_cachevar])dnl
])# PGAC_AVX512_POPCNT_INTRINSICS
+
+
+# PGAC_ARM_SVE_POPCNT_INTRINSICS
+# ------------------------------
+# Check if the compiler supports the ARM SVE popcount instructions using the
+# svdup_u64, svptrue_b64, svcnt_z, svcnt_x, svadd_x, svaddv, and svwhilelt_b8
+# intrinsic functions.
+#
+# Optional compiler flags can be passed as arguments (e.g., -march=armv8-a+sve).
+AC_DEFUN([PGAC_ARM_SVE_POPCNT_INTRINSICS],
+[
+ AC_CACHE_CHECK([for svdup_u64 and other intrinsics with CFLAGS=$1],
+ [pgac_cv_arm_sve_popcnt_intrinsics],
+ [
+ pgac_save_CFLAGS=$CFLAGS
+ CFLAGS="$pgac_save_CFLAGS $1"
+
+ AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <arm_sve.h>],
+ [svbool_t predicate = svptrue_b64();
+ svuint64_t segment = svdup_u64(0), accum = svdup_u64(0);
+ const char *buf = NULL; // Simulating a buffer pointer
+ uint32_t num_vals_segment = svlen_u64(segment); // 64-bit values per segment
+
+ // Using other intrinsics as per the updated code
+ predicate = svwhilelt_b8(0, 128); // Simulating a conditional predicate
+ segment = svld1(predicate, (const uint64_t *)buf);
+ accum = svadd_x(predicate, accum, svcnt_x(predicate, segment));
+ uint64_t popcnt = svaddv(predicate, accum);
+
+ /* Return computed value, to prevent the above being optimized away */
+ return popcnt;])],
+ [pgac_cv_arm_sve_popcnt_intrinsics=yes],
+ [pgac_cv_arm_sve_popcnt_intrinsics=no])
+
+ CFLAGS="$pgac_save_CFLAGS"
+ ])
+
+ if test x"$pgac_cv_arm_sve_popcnt_intrinsics" = x"yes"; then
+ CFLAGS_POPCNT_ARM="$1"
+ pgac_arm_sve_popcnt_intrinsics=yes
+ fi
+])
+
+
diff --git a/configure b/configure
index 53c8a1f2ba..0cd8f2879f 100755
--- a/configure
+++ b/configure
@@ -648,7 +648,9 @@ MSGFMT
PG_CRC32C_OBJS
CFLAGS_CRC
PG_POPCNT_OBJS
+PG_POPCNT_OBJS_ARM
CFLAGS_POPCNT
+CFLAGS_POPCNT_ARM
CFLAGS_XSAVE
LIBOBJS
OPENSSL
@@ -2787,10 +2789,12 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
-
-
-
-
+# Set architecture flags for ARM SVE
+if [ "$target" = "aarch64" ]; then
+ CFLAGS="$CFLAGS -march=armv8-a+sve"
+elif [ "$target" = "arm" ]; then
+ CFLAGS="$CFLAGS -march=armv8-a+sve"
+fi
ac_aux_dir=
for ac_dir in config "$srcdir"/config; do
@@ -17582,6 +17586,97 @@ $as_echo "#define HAVE_XSAVE_INTRINSICS 1" >>confdefs.h
fi
+# Check for ARM SVE popcount intrinsics
+CFLAGS_POPCNT_ARM=""
+PG_POPCNT_OBJS_ARM=""
+
+if test x"$host_cpu" = x"aarch64"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5
+$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; }
+if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+ CFLAGS="$pgac_save_CFLAGS "
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <arm_sve.h>
+int
+main ()
+{
+ svbool_t predicate = svptrue_b64();
+ svuint64_t segment, accum = svdup_u64(0);
+ uint64_t numVals = svlen_u64(segment); // 64-bit count check
+
+ svuint64_t counts = svcnt_u64_z(predicate, segment);
+ accum = svadd_u64_m(predicate, accum, counts);
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_arm_sve_popcnt_intrinsics_=yes
+else
+ pgac_cv_arm_sve_popcnt_intrinsics_=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5
+$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; }
+if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then
+ CFLAGS_POPCNT_ARM=""
+ pgac_arm_sve_popcnt_intrinsics=yes
+fi
+
+if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5
+$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; }
+if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+ CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <arm_sve.h>
+int
+main ()
+{
+ svbool_t predicate = svptrue_b64();
+ svuint64_t segment, accum = svdup_u64(0);
+ uint64_t numVals = svlen_u64(segment); // 64-bit count check
+
+ svuint64_t counts = svcnt_u64_z(predicate, segment);
+ accum = svadd_u64_m(predicate, accum, counts);
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes
+else
+ pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5
+$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; }
+if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then
+ CFLAGS_POPCNT_ARM="-march=armv8-a+sve"
+ pgac_arm_sve_popcnt_intrinsics=yes
+fi
+
+fi
+if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
+ PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o"
+
+ $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h
+
+fi
+fi
+
# Check for AVX-512 popcount intrinsics
#
diff --git a/configure.ac b/configure.ac
index 6a35b2880b..2b67cd2753 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2089,6 +2089,41 @@ fi
AC_SUBST(CFLAGS_POPCNT)
AC_SUBST(PG_POPCNT_OBJS)
+# Check for ARM popcount intrinsics
+CFLAGS_POPCNT_ARM=""
+PG_POPCNT_OBJS_ARM=""
+
+# Debug: Print the host CPU type
+{ $as_echo "Host CPU: $host_cpu" >&5; }
+
+if test x"$host_cpu" = x"aarch64"; then
+ #PGAC_ARM_SVE_POPCNT_INTRINSICS([])
+
+ # Debug: Check if SVE intrinsics were found
+ { $as_echo "pgac_arm_sve_popcnt_intrinsics: $pgac_arm_sve_popcnt_intrinsics" >&5; }
+
+ #if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
+ PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve])
+
+ # Debug: Check if SVE intrinsics were found after trying with -march flag
+ { $as_echo "pgac_arm_sve_popcnt_intrinsics after -march flag: $pgac_arm_sve_popcnt_intrinsics" >&5; }
+
+ #fi
+ if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
+ PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation
+ AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.])
+
+ # Debug: Print the objects being used
+ { $as_echo "Using ARM popcount objects: $PG_POPCNT_OBJS_ARM" >&5; }
+
+ fi
+fi
+# Debug: Print final values of flags
+{ $as_echo "CFLAGS_POPCNT_ARM: $CFLAGS_POPCNT_ARM" >&5; }
+{ $as_echo "PG_POPCNT_OBJS_ARM: $PG_POPCNT_OBJS_ARM" >&5; }
+AC_SUBST(CFLAGS_POPCNT_ARM)
+AC_SUBST(PG_POPCNT_OBJS_ARM)
+
# Check for Intel SSE 4.2 intrinsics to do CRC calculations.
#
# First check if the _mm_crc32_u8 and _mm_crc32_u64 intrinsics can be used
@@ -2115,6 +2150,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
PGAC_ARMV8_CRC32C_INTRINSICS([])
if test x"$pgac_armv8_crc32c_intrinsics" != x"yes"; then
PGAC_ARMV8_CRC32C_INTRINSICS([-march=armv8-a+crc])
+ { $as_echo "pgac_armv8_crc32_intrinsics after -march flag: $pgac_armv8_crc32_intrinsics" >&5; }
fi
# Check for LoongArch CRC intrinsics to do CRC calculations.
diff --git a/meson.build b/meson.build
index 7150f85e0f..ba3bc4cd2b 100644
--- a/meson.build
+++ b/meson.build
@@ -2217,6 +2217,37 @@ int main(void)
endif
+###############################################################
+# Check for the availability of ARM SVE popcount intrinsics.
+###############################################################
+
+cflags_popcnt_arm = []
+if host_cpu == 'aarch64'
+
+ prog = '''
+#include <arm_sve.h>
+
+int main(void)
+{
+ const svuint64_t val = svdup_u64(0xFFFFFFFFFFFFFFFF); // Example value
+ svuint64_t popcnt = svcntb(val); // Count the number of 1 bits
+ /* return computed value, to prevent the above being optimized away */
+ return popcnt == 0; // Ensure to return a valid value
+}
+'''
+
+ if cc.links(prog, name: 'ARM SVE popcount without -march=armv8-a+sve',
+ args: test_c_args + ['-DSVINT64=@0@'.format(cdata.get('SV_INT64_TYPE'))])
+ cdata.set('USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 1)
+ elif cc.links(prog, name: 'ARM SVE popcount with -march=armv8-a+sve',
+ args: test_c_args + ['-DSVINT64=@0@'.format(cdata.get('SV_INT64_TYPE'))] + ['-march=armv8-a+sve'])
+ cdata.set('USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 1)
+ cflags_popcnt_arm += ['-march=armv8-a+sve']
+ endif
+
+endif
+
+
###############################################################
# Select CRC-32C implementation.
#
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 42f50b4976..2e79a24a2d 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -263,6 +263,7 @@ CXXFLAGS_SL_MODULE = @CXXFLAGS_SL_MODULE@
CFLAGS_UNROLL_LOOPS = @CFLAGS_UNROLL_LOOPS@
CFLAGS_VECTORIZE = @CFLAGS_VECTORIZE@
CFLAGS_POPCNT = @CFLAGS_POPCNT@
+CFLAGS_POPCNT_ARM = @CFLAGS_POPCNT_ARM@
CFLAGS_CRC = @CFLAGS_CRC@
CFLAGS_XSAVE = @CFLAGS_XSAVE@
PERMIT_DECLARATION_AFTER_STATEMENT = @PERMIT_DECLARATION_AFTER_STATEMENT@
@@ -764,6 +765,7 @@ PG_CRC32C_OBJS = @PG_CRC32C_OBJS@
# files needed for the chosen popcount implementation
PG_POPCNT_OBJS = @PG_POPCNT_OBJS@
+PG_POPCNT_OBJS_ARM = @PG_POPCNT_OBJS_ARM@
LIBS := -lpgcommon -lpgport $(LIBS)
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 38006367a4..f2573ad4d1 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -678,6 +678,9 @@
/* Define to 1 to use AVX-512 popcount instructions with a runtime check. */
#undef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK
+/* Define to 1 to use SVE popcount instructions with a runtime check. */
+#undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
+
/* Define to 1 to build with Bonjour support. (--with-bonjour) */
#undef USE_BONJOUR
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 4d88478c9c..811b1cafe5 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -298,6 +298,16 @@ pg_ceil_log2_64(uint64 num)
#endif
#endif
+/*
+ * On ARM builds, try SVE popcount instructions.
+ */
+#if defined(__aarch64__) || defined(__arm__)
+#define ARM_BUILD 1
+#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
+#define TRY_POPCNT_FAST 1
+#endif
+#endif
+
#ifdef TRY_POPCNT_FAST
/* Attempt to use the POPCNT instruction, but perform a runtime check first */
extern PGDLLIMPORT int (*pg_popcount32) (uint32 word);
@@ -317,6 +327,12 @@ extern uint64 pg_popcount_avx512(const char *buf, int bytes);
extern uint64 pg_popcount_masked_avx512(const char *buf, int bytes, bits8 mask);
#endif
+#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
+extern bool pg_popcount_sve_available(void);
+extern uint64 pg_popcount_sve(const char *buf, int bytes);
+extern uint64 pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask);
+#endif
+
#else
/* Use a portable implementation -- no need for a function pointer. */
extern int pg_popcount32(uint32 word);
diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build
index 850e927584..0ff4e05993 100644
--- a/src/makefiles/meson.build
+++ b/src/makefiles/meson.build
@@ -103,6 +103,7 @@ pgxs_kv = {
'CFLAGS_CRC': ' '.join(cflags_crc),
'CFLAGS_POPCNT': ' '.join(cflags_popcnt),
+ 'CFLAGS_POPCNT_ARM': ' '.join(cflags_popcnt_arm)
'CFLAGS_UNROLL_LOOPS': ' '.join(unroll_loops_cflags),
'CFLAGS_VECTORIZE': ' '.join(vectorize_cflags),
'CFLAGS_XSAVE': ' '.join(cflags_xsave),
@@ -181,7 +182,7 @@ pgxs_empty = [
'WANTED_LANGUAGES',
# Not needed because we don't build the server / PLs with the generated makefile
- 'LIBOBJS', 'PG_CRC32C_OBJS', 'PG_POPCNT_OBJS', 'TAS',
+ 'LIBOBJS', 'PG_CRC32C_OBJS', 'PG_POPCNT_OBJS', 'PG_POPCNT_OBJS_ARM', 'TAS',
'DTRACEFLAGS', # only server has dtrace probes
'perl_archlibexp', 'perl_embed_ccflags', 'perl_embed_ldflags', 'perl_includespec', 'perl_privlibexp',
diff --git a/src/port/Makefile b/src/port/Makefile
index 9324ec2d9f..73ec5847d7 100644
--- a/src/port/Makefile
+++ b/src/port/Makefile
@@ -39,6 +39,7 @@ OBJS = \
$(LIBOBJS) \
$(PG_CRC32C_OBJS) \
$(PG_POPCNT_OBJS) \
+ $(PG_POPCNT_OBJS_ARM) \
bsearch_arg.o \
chklocale.o \
inet_net_ntop.o \
@@ -102,6 +103,16 @@ pg_popcount_avx512.o: CFLAGS+=$(CFLAGS_POPCNT)
pg_popcount_avx512_shlib.o: CFLAGS+=$(CFLAGS_POPCNT)
pg_popcount_avx512_srv.o: CFLAGS+=$(CFLAGS_POPCNT)
+# all version of pg_popcount_sve.o need CFLAGS_POPCNT_ARM
+pg_popcount_sve.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+pg_popcount_sve_shlib.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+pg_popcount_sve_srv.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+
+# all versions of pg_popcount_sve_choose.o need CFLAGS_POPCNT_ARM
+pg_popcount_sve_choose.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+pg_popcount_sve_choose_shlib.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+pg_popcount_sve_choose_srv.o: CFLAGS+=$(CFLAGS_POPCNT_ARM)
+
#
# Shared library versions of object files
#
diff --git a/src/port/meson.build b/src/port/meson.build
index 1150966ab7..5308898bb4 100644
--- a/src/port/meson.build
+++ b/src/port/meson.build
@@ -91,6 +91,8 @@ replace_funcs_pos = [
['pg_crc32c_armv8', 'USE_ARMV8_CRC32C'],
['pg_crc32c_armv8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 'crc'],
['pg_crc32c_armv8_choose', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'],
+ ['pg_popcount_sve', 'USE_SVE_POPCNT_WITH_RUNTIME_CHECK', 'popcnt'],
+ ['pg_popcount_sve_choose', 'USE_SVE_POPCNT_WITH_RUNTIME_CHECK'],
['pg_crc32c_sb8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'],
# loongarch
@@ -100,7 +102,7 @@ replace_funcs_pos = [
['pg_crc32c_sb8', 'USE_SLICING_BY_8_CRC32C'],
]
-pgport_cflags = {'crc': cflags_crc, 'popcnt': cflags_popcnt, 'xsave': cflags_xsave}
+pgport_cflags = {'crc': cflags_crc, 'popcnt': cflags_popcnt + cflags_popcnt_arm, 'xsave': cflags_xsave}
pgport_sources_cflags = {'crc': [], 'popcnt': [], 'xsave': []}
foreach f : replace_funcs_neg
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index 87f56e82b8..15b3ea2e05 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -129,13 +129,16 @@ uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask)
/*
* Return true if CPUID indicates that the POPCNT instruction is available.
+ * Return false for ARM builds.
*/
static bool
pg_popcount_available(void)
{
unsigned int exx[4] = {0, 0, 0, 0};
-#if defined(HAVE__GET_CPUID)
+#if defined(ARM_BUILD)
+ return false;
+#elif defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#elif defined(HAVE__CPUID)
__cpuid(exx, 1);
@@ -176,6 +179,12 @@ choose_popcount_functions(void)
pg_popcount_optimized = pg_popcount_avx512;
pg_popcount_masked_optimized = pg_popcount_masked_avx512;
}
+#elif USE_SVE_POPCNT_WITH_RUNTIME_CHECK
+ if (pg_popcount_sve_available())
+ {
+ pg_popcount_optimized = pg_popcount_sve;
+ pg_popcount_masked_optimized = pg_popcount_masked_sve;
+ }
#endif
}
diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c
index aad59755f9..94e97eaff9 100644
--- a/src/port/pg_popcount_sve.c
+++ b/src/port/pg_popcount_sve.c
@@ -1 +1,88 @@
-// DUMMY
\ No newline at end of file
+/*-------------------------------------------------------------------------
+ *
+ * pg_popcount_sve.c
+ * Holds the SVE pg_popcount() implementation.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/port/pg_popcount_sve.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "c.h"
+#include "port/pg_bitutils.h"
+
+#include <arm_sve.h>
+
+#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
+
+/*
+ * pg_popcount_sve
+ * Returns the number of 1-bits in buf
+ */
+uint64
+pg_popcount_sve(const char *buf, int bytes)
+{
+ svbool_t pred = svptrue_b64();
+ svuint64_t vec, accum = svdup_u64(0);
+
+ uint32 i = 0, vec_len = svcntb();
+ uint32 num_vals_vec = svlen_u64(vec);
+ uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
+ uint64 popcnt;
+
+ /* Process full vectors */
+ for (; i < num_vals_full_vec; i += num_vals_vec)
+ {
+ vec = svld1(pred, (const uint64 *) buf);
+ accum = svadd_x(pred, accum, svcnt_x(pred, vec));
+ buf += vec_len;
+ }
+
+ popcnt = svaddv(pred, accum);
+
+ /* Process the last incomplete vector */
+ if(i < bytes)
+ {
+ pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */
+ popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf)));
+ }
+
+ return popcnt;
+}
+
+uint64
+pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
+{
+ svbool_t pred = svptrue_b64();
+ svuint8_t vec8;
+ svuint64_t vec, accum = svdup_u64(0);
+
+ uint32 i = 0, vec_len = svcntb();
+ uint32 num_vals_vec = svlen_u64(vec);
+ uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
+ uint64 popcnt, mask64 = ~UINT64CONST(0) / 0xFF * mask;
+
+ /* Process full vectors */
+ for (; i < num_vals_full_vec; i += num_vals_vec)
+ {
+ vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */
+ accum = svadd_x(pred, accum, svcnt_x(pred, vec));
+ buf += vec_len;
+ }
+
+ popcnt = svaddv(pred, accum);
+
+ /* Process the last incomplete vector */
+ if(i < bytes)
+ {
+ pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */
+ vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */
+ popcnt += svaddv(pred, svcnt_z(pred, vec8));
+ }
+
+ return popcnt;
+}
+
+#endif /* USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
diff --git a/src/port/pg_popcount_sve_choose.c b/src/port/pg_popcount_sve_choose.c
index aad59755f9..a6f9e0c3dc 100644
--- a/src/port/pg_popcount_sve_choose.c
+++ b/src/port/pg_popcount_sve_choose.c
@@ -1 +1,32 @@
-// DUMMY
\ No newline at end of file
+/*-------------------------------------------------------------------------
+ *
+ * pg_popcount_sve_choose.c
+ * Test whether we can use the SVE pg_popcount() implementation.
+ *
+ * Copyright (c) 2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ * src/port/pg_popcount_sve_choose.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "c.h"
+#include "port/pg_bitutils.h"
+
+#include <asm/hwcap.h>
+#include <sys/auxv.h>
+
+#ifdef USE_SVE_POPCNT_WITH_RUNTIME_CHECK
+
+/*
+ * Returns true if the CPU supports the instructions required for the SVE
+ * pg_popcount() implementation.
+ */
+bool
+pg_popcount_sve_available(void)
+{
+ unsigned long hwcap = getauxval(AT_HWCAP); /* get the HWCAP flags */
+ return (hwcap & HWCAP_SVE) != 0; /* return true if SVE is supported */
+}
+
+#endif /* USE_SVE_POPCNT_WITH_RUNTIME_CHECK */
--
2.34.1
From b0acc73b3ecebc50c8992a8354674a3b946e23a3 Mon Sep 17 00:00:00 2001
From: Chiranmoy Bhattacharya <chiranmoy.bhattacharya@fujitsu.com>
Date: Tue, 26 Nov 2024 15:30:18 +0530
Subject: [PATCH v1 4/6] whitespaces and debug printing removed
---
config/c-compiler.m4 | 3 -
configure | 192 ++++++++++++++++++-------------------
configure.ac | 28 +-----
src/port/pg_popcount_sve.c | 31 +++---
4 files changed, 116 insertions(+), 138 deletions(-)
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 500c4e6662..fcfee6b2e2 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -753,7 +753,6 @@ fi
undefine([Ac_cachevar])dnl
])# PGAC_AVX512_POPCNT_INTRINSICS
-
# PGAC_ARM_SVE_POPCNT_INTRINSICS
# ------------------------------
# Check if the compiler supports the ARM SVE popcount instructions using the
@@ -794,5 +793,3 @@ AC_DEFUN([PGAC_ARM_SVE_POPCNT_INTRINSICS],
pgac_arm_sve_popcnt_intrinsics=yes
fi
])
-
-
diff --git a/configure b/configure
index 0cd8f2879f..6297c46dd6 100755
--- a/configure
+++ b/configure
@@ -2789,12 +2789,10 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'
ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
ac_compiler_gnu=$ac_cv_c_compiler_gnu
-# Set architecture flags for ARM SVE
-if [ "$target" = "aarch64" ]; then
- CFLAGS="$CFLAGS -march=armv8-a+sve"
-elif [ "$target" = "arm" ]; then
- CFLAGS="$CFLAGS -march=armv8-a+sve"
-fi
+
+
+
+
ac_aux_dir=
for ac_dir in config "$srcdir"/config; do
@@ -17586,97 +17584,6 @@ $as_echo "#define HAVE_XSAVE_INTRINSICS 1" >>confdefs.h
fi
-# Check for ARM SVE popcount intrinsics
-CFLAGS_POPCNT_ARM=""
-PG_POPCNT_OBJS_ARM=""
-
-if test x"$host_cpu" = x"aarch64"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5
-$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; }
-if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- pgac_save_CFLAGS=$CFLAGS
- CFLAGS="$pgac_save_CFLAGS "
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <arm_sve.h>
-int
-main ()
-{
- svbool_t predicate = svptrue_b64();
- svuint64_t segment, accum = svdup_u64(0);
- uint64_t numVals = svlen_u64(segment); // 64-bit count check
-
- svuint64_t counts = svcnt_u64_z(predicate, segment);
- accum = svadd_u64_m(predicate, accum, counts);
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv_arm_sve_popcnt_intrinsics_=yes
-else
- pgac_cv_arm_sve_popcnt_intrinsics_=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-CFLAGS="$pgac_save_CFLAGS"
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5
-$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; }
-if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then
- CFLAGS_POPCNT_ARM=""
- pgac_arm_sve_popcnt_intrinsics=yes
-fi
-
-if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5
-$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; }
-if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then :
- $as_echo_n "(cached) " >&6
-else
- pgac_save_CFLAGS=$CFLAGS
- CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve"
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-#include <arm_sve.h>
-int
-main ()
-{
- svbool_t predicate = svptrue_b64();
- svuint64_t segment, accum = svdup_u64(0);
- uint64_t numVals = svlen_u64(segment); // 64-bit count check
-
- svuint64_t counts = svcnt_u64_z(predicate, segment);
- accum = svadd_u64_m(predicate, accum, counts);
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
- pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes
-else
- pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
- conftest$ac_exeext conftest.$ac_ext
-CFLAGS="$pgac_save_CFLAGS"
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5
-$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; }
-if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then
- CFLAGS_POPCNT_ARM="-march=armv8-a+sve"
- pgac_arm_sve_popcnt_intrinsics=yes
-fi
-
-fi
-if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
- PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o"
-
- $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h
-
-fi
-fi
-
# Check for AVX-512 popcount intrinsics
#
@@ -17778,6 +17685,97 @@ $as_echo "#define USE_AVX512_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h
fi
+# Check for ARM SVE popcount intrinsics
+CFLAGS_POPCNT_ARM=""
+PG_POPCNT_OBJS_ARM=""
+
+if test x"$host_cpu" = x"aarch64"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=" >&5
+$as_echo_n "checking for svcnt_u64 with CFLAGS=... " >&6; }
+if ${pgac_cv_arm_sve_popcnt_intrinsics_+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+ CFLAGS="$pgac_save_CFLAGS "
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <arm_sve.h>
+int
+main ()
+{
+ svbool_t predicate = svptrue_b64();
+ svuint64_t segment, accum = svdup_u64(0);
+ uint64_t numVals = svlen_u64(segment); // 64-bit count check
+
+ svuint64_t counts = svcnt_u64_z(predicate, segment);
+ accum = svadd_u64_m(predicate, accum, counts);
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_arm_sve_popcnt_intrinsics_=yes
+else
+ pgac_cv_arm_sve_popcnt_intrinsics_=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics_" >&5
+$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics_" >&6; }
+if test x"$pgac_cv_arm_sve_popcnt_intrinsics_" = x"yes"; then
+ CFLAGS_POPCNT_ARM=""
+ pgac_arm_sve_popcnt_intrinsics=yes
+fi
+
+if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve" >&5
+$as_echo_n "checking for svcnt_u64 with CFLAGS=-march=armv8-a+sve... " >&6; }
+if ${pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+ CFLAGS="$pgac_save_CFLAGS -march=armv8-a+sve"
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <arm_sve.h>
+int
+main ()
+{
+ svbool_t predicate = svptrue_b64();
+ svuint64_t segment, accum = svdup_u64(0);
+ uint64_t numVals = svlen_u64(segment); // 64-bit count check
+
+ svuint64_t counts = svcnt_u64_z(predicate, segment);
+ accum = svadd_u64_m(predicate, accum, counts);
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=yes
+else
+ pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&5
+$as_echo "$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" >&6; }
+if test x"$pgac_cv_arm_sve_popcnt_intrinsics__march_armv8_a_sve" = x"yes"; then
+ CFLAGS_POPCNT_ARM="-march=armv8-a+sve"
+ pgac_arm_sve_popcnt_intrinsics=yes
+fi
+
+fi
+if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
+ PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o"
+
+ $as_echo "#define USE_SVE_POPCNT_WITH_RUNTIME_CHECK 1" >>confdefs.h
+
+fi
+fi
+
# Check for Intel SSE 4.2 intrinsics to do CRC calculations.
#
diff --git a/configure.ac b/configure.ac
index 2b67cd2753..5a5ae8f7ca 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2092,35 +2092,16 @@ AC_SUBST(PG_POPCNT_OBJS)
# Check for ARM popcount intrinsics
CFLAGS_POPCNT_ARM=""
PG_POPCNT_OBJS_ARM=""
-
-# Debug: Print the host CPU type
-{ $as_echo "Host CPU: $host_cpu" >&5; }
-
if test x"$host_cpu" = x"aarch64"; then
- #PGAC_ARM_SVE_POPCNT_INTRINSICS([])
-
- # Debug: Check if SVE intrinsics were found
- { $as_echo "pgac_arm_sve_popcnt_intrinsics: $pgac_arm_sve_popcnt_intrinsics" >&5; }
-
- #if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
- PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve])
-
- # Debug: Check if SVE intrinsics were found after trying with -march flag
- { $as_echo "pgac_arm_sve_popcnt_intrinsics after -march flag: $pgac_arm_sve_popcnt_intrinsics" >&5; }
-
- #fi
+ PGAC_ARM_SVE_POPCNT_INTRINSICS([])
+ if test x"$pgac_arm_sve_popcnt_intrinsics" != x"yes"; then
+ PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve])
+ fi
if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation
AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.])
-
- # Debug: Print the objects being used
- { $as_echo "Using ARM popcount objects: $PG_POPCNT_OBJS_ARM" >&5; }
-
fi
fi
-# Debug: Print final values of flags
-{ $as_echo "CFLAGS_POPCNT_ARM: $CFLAGS_POPCNT_ARM" >&5; }
-{ $as_echo "PG_POPCNT_OBJS_ARM: $PG_POPCNT_OBJS_ARM" >&5; }
AC_SUBST(CFLAGS_POPCNT_ARM)
AC_SUBST(PG_POPCNT_OBJS_ARM)
@@ -2150,7 +2131,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
PGAC_ARMV8_CRC32C_INTRINSICS([])
if test x"$pgac_armv8_crc32c_intrinsics" != x"yes"; then
PGAC_ARMV8_CRC32C_INTRINSICS([-march=armv8-a+crc])
- { $as_echo "pgac_armv8_crc32_intrinsics after -march flag: $pgac_armv8_crc32_intrinsics" >&5; }
fi
# Check for LoongArch CRC intrinsics to do CRC calculations.
diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c
index 94e97eaff9..564a20bf3d 100644
--- a/src/port/pg_popcount_sve.c
+++ b/src/port/pg_popcount_sve.c
@@ -24,12 +24,13 @@
uint64
pg_popcount_sve(const char *buf, int bytes)
{
- svbool_t pred = svptrue_b64();
- svuint64_t vec, accum = svdup_u64(0);
-
- uint32 i = 0, vec_len = svcntb();
- uint32 num_vals_vec = svlen_u64(vec);
- uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
+ svbool_t pred = svptrue_b64();
+ svuint64_t vec,
+ accum = svdup_u64(0);
+ uint32 i = 0,
+ vec_len = svcntb(),
+ num_vals_vec = svlen_u64(vec),
+ num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
uint64 popcnt;
/* Process full vectors */
@@ -55,14 +56,16 @@ pg_popcount_sve(const char *buf, int bytes)
uint64
pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
{
- svbool_t pred = svptrue_b64();
- svuint8_t vec8;
- svuint64_t vec, accum = svdup_u64(0);
-
- uint32 i = 0, vec_len = svcntb();
- uint32 num_vals_vec = svlen_u64(vec);
- uint32 num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
- uint64 popcnt, mask64 = ~UINT64CONST(0) / 0xFF * mask;
+ svbool_t pred = svptrue_b64();
+ svuint8_t vec8;
+ svuint64_t vec,
+ accum = svdup_u64(0);
+ uint32 i = 0,
+ vec_len = svcntb(),
+ num_vals_vec = svlen_u64(vec),
+ num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
+ uint64 popcnt,
+ mask64 = ~UINT64CONST(0) / 0xFF * mask;
/* Process full vectors */
for (; i < num_vals_full_vec; i += num_vals_vec)
--
2.34.1
From 58e90123c88ecc10602ff84e8e061237c6dff830 Mon Sep 17 00:00:00 2001
From: Chiranmoy Bhattacharya <chiranmoy.bhattacharya@fujitsu.com>
Date: Tue, 26 Nov 2024 15:48:57 +0530
Subject: [PATCH v1 5/6] updated the macros used and comments added
---
configure.ac | 2 +-
src/include/port/pg_bitutils.h | 8 +++-----
src/port/pg_bitutils.c | 5 ++---
src/port/pg_popcount_sve.c | 12 ++++++++----
4 files changed, 14 insertions(+), 13 deletions(-)
diff --git a/configure.ac b/configure.ac
index 5a5ae8f7ca..a45a01c20a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2098,7 +2098,7 @@ if test x"$host_cpu" = x"aarch64"; then
PGAC_ARM_SVE_POPCNT_INTRINSICS([-march=armv8-a+sve])
fi
if test x"$pgac_arm_sve_popcnt_intrinsics" = x"yes"; then
- PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o" # Adjust this as necessary to your implementation
+ PG_POPCNT_OBJS_ARM="pg_popcount_sve.o pg_popcount_sve_choose.o"
AC_DEFINE(USE_SVE_POPCNT_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARM popcount instructions.])
fi
fi
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 811b1cafe5..8e09f78862 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -299,14 +299,12 @@ pg_ceil_log2_64(uint64 num)
#endif
/*
- * On ARM builds, try SVE popcount instructions.
+ * On AArch64 builds, try using SVE popcount instructions, but only if
+ * we can verify that the CPU supports it via a runtime check.
*/
-#if defined(__aarch64__) || defined(__arm__)
-#define ARM_BUILD 1
-#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
+#if defined(__aarch64__)
#define TRY_POPCNT_FAST 1
#endif
-#endif
#ifdef TRY_POPCNT_FAST
/* Attempt to use the POPCNT instruction, but perform a runtime check first */
diff --git a/src/port/pg_bitutils.c b/src/port/pg_bitutils.c
index 15b3ea2e05..9bd5985330 100644
--- a/src/port/pg_bitutils.c
+++ b/src/port/pg_bitutils.c
@@ -129,15 +129,14 @@ uint64 (*pg_popcount_masked_optimized) (const char *buf, int bytes, bits8 mask)
/*
* Return true if CPUID indicates that the POPCNT instruction is available.
- * Return false for ARM builds.
*/
static bool
pg_popcount_available(void)
{
unsigned int exx[4] = {0, 0, 0, 0};
-#if defined(ARM_BUILD)
- return false;
+#if defined(__aarch64__)
+ return false; /* cpuid not available in __aarch64__ */
#elif defined(HAVE__GET_CPUID)
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]);
#elif defined(HAVE__CPUID)
diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c
index 564a20bf3d..140bf37a2d 100644
--- a/src/port/pg_popcount_sve.c
+++ b/src/port/pg_popcount_sve.c
@@ -46,13 +46,17 @@ pg_popcount_sve(const char *buf, int bytes)
/* Process the last incomplete vector */
if(i < bytes)
{
- pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */
+ pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */
popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf)));
}
return popcnt;
}
+/*
+ * pg_popcount_masked_sve
+ * Returns the number of 1-bits in buf after applying the mask to each byte
+ */
uint64
pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
{
@@ -70,7 +74,7 @@ pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
/* Process full vectors */
for (; i < num_vals_full_vec; i += num_vals_vec)
{
- vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */
+ vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */
accum = svadd_x(pred, accum, svcnt_x(pred, vec));
buf += vec_len;
}
@@ -80,8 +84,8 @@ pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
/* Process the last incomplete vector */
if(i < bytes)
{
- pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 because value size is 64bits */
- vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */
+ pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */
+ vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */
popcnt += svaddv(pred, svcnt_z(pred, vec8));
}
--
2.34.1
From a4413f476ce0e9c58914150e5874eec340bd136c Mon Sep 17 00:00:00 2001
From: Chiranmoy Bhattacharya <chiranmoy.bhattacharya@fujitsu.com>
Date: Wed, 4 Dec 2024 12:55:36 +0530
Subject: [PATCH v1 6/6] buffer alignment for popcount sve impl
---
src/include/port/pg_bitutils.h | 2 +-
src/port/pg_popcount_sve.c | 75 +++++++++++++++++++++++++---------
2 files changed, 56 insertions(+), 21 deletions(-)
diff --git a/src/include/port/pg_bitutils.h b/src/include/port/pg_bitutils.h
index 8e09f78862..a690a46a88 100644
--- a/src/include/port/pg_bitutils.h
+++ b/src/include/port/pg_bitutils.h
@@ -302,7 +302,7 @@ pg_ceil_log2_64(uint64 num)
* On AArch64 builds, try using SVE popcount instructions, but only if
* we can verify that the CPU supports it via a runtime check.
*/
-#if defined(__aarch64__)
+#if defined(USE_SVE_POPCNT_WITH_RUNTIME_CHECK)
#define TRY_POPCNT_FAST 1
#endif
diff --git a/src/port/pg_popcount_sve.c b/src/port/pg_popcount_sve.c
index 140bf37a2d..392441a9b1 100644
--- a/src/port/pg_popcount_sve.c
+++ b/src/port/pg_popcount_sve.c
@@ -24,16 +24,33 @@
uint64
pg_popcount_sve(const char *buf, int bytes)
{
- svbool_t pred = svptrue_b64();
- svuint64_t vec,
+ svbool_t pred;
+ svuint64_t vec,
accum = svdup_u64(0);
uint32 i = 0,
- vec_len = svcntb(),
- num_vals_vec = svlen_u64(vec),
- num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
- uint64 popcnt;
+ num_vals_full_vec,
+ pre_align,
+ vec_len = svcntb(),
+ num_vals_vec = svlen_u64(vec);
+ uint64 popcnt = 0;
+
+ /*
+ * For smaller inputs, aligning the buffer degrades performance.
+ * Therefore, we align the buffers only when the input size is sufficiently large.
+ */
+ if (bytes > 4 * vec_len)
+ {
+ pre_align = (const char *) TYPEALIGN_DOWN(sizeof(uint64_t), buf) + sizeof(uint64_t) - buf;
+ pred = svwhilelt_b8(0U, pre_align);
+ popcnt = svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf)));
+ buf += pre_align;
+ bytes -= pre_align;
+ }
+
+ pred = svptrue_b64();
+ num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
- /* Process full vectors */
+ /* Process complete vectors */
for (; i < num_vals_full_vec; i += num_vals_vec)
{
vec = svld1(pred, (const uint64 *) buf);
@@ -41,12 +58,12 @@ pg_popcount_sve(const char *buf, int bytes)
buf += vec_len;
}
- popcnt = svaddv(pred, accum);
+ popcnt += svaddv(pred, accum);
/* Process the last incomplete vector */
if(i < bytes)
{
- pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */
+ pred = svwhilelt_b8(i * 8, (uint32) bytes);
popcnt += svaddv(pred, svcnt_z(pred, svld1(pred, (const uint8 *) buf)));
}
@@ -60,32 +77,50 @@ pg_popcount_sve(const char *buf, int bytes)
uint64
pg_popcount_masked_sve(const char *buf, int bytes, bits8 mask)
{
- svbool_t pred = svptrue_b64();
+ svbool_t pred;
svuint8_t vec8;
svuint64_t vec,
accum = svdup_u64(0);
uint32 i = 0,
- vec_len = svcntb(),
- num_vals_vec = svlen_u64(vec),
- num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
- uint64 popcnt,
- mask64 = ~UINT64CONST(0) / 0xFF * mask;
+ num_vals_full_vec,
+ pre_align,
+ vec_len = svcntb(),
+ num_vals_vec = svlen_u64(vec);
+ uint64 popcnt = 0,
+ mask64 = ~UINT64CONST(0) / 0xFF * mask;
+
+ /*
+ * For smaller inputs, aligning the buffer degrades the performance.
+ * Therefore, we align the buffers only when the input size is sufficiently large.
+ */
+ if (bytes > 4 * vec_len)
+ {
+ pre_align = (const char *) TYPEALIGN_DOWN(sizeof(uint64_t), buf) + sizeof(uint64_t) - buf;
+ pred = svwhilelt_b8(0U, pre_align);
+ vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *) buf), mask); /* load and mask */
+ popcnt = svaddv(pred, svcnt_z(pred, vec8));
+ buf += pre_align;
+ bytes -= pre_align;
+ }
+
+ pred = svptrue_b64();
+ num_vals_full_vec = (bytes / vec_len) * num_vals_vec;
- /* Process full vectors */
+ /* Process complete vectors */
for (; i < num_vals_full_vec; i += num_vals_vec)
{
- vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *)buf), mask64); /* load and mask */
+ vec = svand_n_u64_x(pred, svld1(pred, (const uint64 *) buf), mask64); /* load and mask */
accum = svadd_x(pred, accum, svcnt_x(pred, vec));
buf += vec_len;
}
- popcnt = svaddv(pred, accum);
+ popcnt += svaddv(pred, accum);
/* Process the last incomplete vector */
if(i < bytes)
{
- pred = svwhilelt_b8(i * 8, (uint32) bytes); /* i * 8 bytes are processed in loop */
- vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *)buf), mask); /* load and mask */
+ pred = svwhilelt_b8(i * 8, (uint32) bytes);
+ vec8 = svand_n_u8_m(pred, svld1(pred, (const uint8 *) buf), mask); /* load and mask */
popcnt += svaddv(pred, svcnt_z(pred, vec8));
}
--
2.34.1