v1-0002-Compute-CRC32C-on-ARM-using-the-Crypto-Extension-.patch

From d447d42e5b0a89e4c2d0e6c40e38980df558b899 Mon Sep 17 00:00:00 2001
From: John Naylor <john.naylor@postgresql.org>
Date: Fri, 9 May 2025 19:48:26 +0700
Subject: [PATCH v1 2/3] Compute CRC32C on ARM using the Crypto Extension where
 available

---
 meson.build                       |  32 ++++++++
 src/include/port/pg_crc32c.h      |  24 ++++--
 src/port/meson.build              |   1 +
 src/port/pg_crc32c_armv8.c        | 125 ++++++++++++++++++++++++++++++
 src/port/pg_crc32c_armv8_choose.c |  36 +++++++++
 5 files changed, 213 insertions(+), 5 deletions(-)

diff --git a/meson.build b/meson.build
index 12de5e80c31..5f7a10f4c8d 100644
--- a/meson.build
+++ b/meson.build
@@ -2523,6 +2523,38 @@ int main(void)
     have_optimized_crc = true
   endif
 
+    # Check if the compiler supports ARMv8 CRYPTO carryless multiplication
+    # and three-way exclusive-or instructions used for computing CRC.
+    # We test with cflags_crc since we also need __crc32cd.
+    prog = '''
+#include <arm_acle.h>
+#include <arm_neon.h>
+uint64x2_t	a;
+uint64x2_t	b;
+uint64x2_t	c;
+
+int main(void)
+{
+    uint64x2_t	r;
+    uint64x2_t	r2;
+
+__asm("pmull %0.1q, %2.1d, %3.1d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
+__asm("pmull2 %0.1q, %2.2d, %3.2d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r2), "+w"(c):"w"(a), "w"(b));
+
+    /* return computed value, to prevent the above being optimized away */
+    r = veorq_u64(r, r2);
+    return __crc32cd(0, vgetq_lane_u64(r, 0));
+}
+'''
+
+  if cc.links(prog,
+      name: 'PMULL CRC32C',
+      args: test_c_args + ['-march=armv8-a+crc+simd+crypto'])
+    # Use ARM CRYPTO Extension, with runtime check
+    cflags_crc += '-march=armv8-a+crc+simd+crypto'
+    cdata.set('USE_PMULL_CRC32C_WITH_RUNTIME_CHECK', 1)
+  endif
+
 elif host_cpu == 'loongarch64'
 
   prog = '''
diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h
index 9f021db83b2..def3de5dfb9 100644
--- a/src/include/port/pg_crc32c.h
+++ b/src/include/port/pg_crc32c.h
@@ -111,13 +111,22 @@ extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t l
 #endif
 
 #elif defined(USE_ARMV8_CRC32C)
-/* Use ARMv8 CRC Extension instructions. */
+/*
+ * Use either ARMv8 CRC Extension or CRYPTO Extension (PMULL) instructions.
+ * We don't need a runtime check for CRC, so we can inline those in some cases.
+ */
+
+#include <arm_acle.h>
 
 #define COMP_CRC32C(crc, data, len)							\
 	((crc) = pg_comp_crc32c_dispatch((crc), (data), (len)))
 #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF)
 
+extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len);
 extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len);
+#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
+extern pg_crc32c pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len);
+#endif
 
 static inline
 pg_crc32c
@@ -132,13 +141,14 @@ pg_comp_crc32c_dispatch(pg_crc32c crc, const void *data, size_t len)
 
 		/*
 		 * For small constant inputs, inline the computation to avoid a
-		 * function call and allow the compiler to unroll loops.
+		 * function call.
 		 */
 #if 1
 
 		/*
-		 * WIP: is it better to avoid branching by unrolling the loop and
-		 * processing only 4-bytes per iteration?
+		 * WIP: Unlike x86, using 8-byte variants requires 8-byte alignment,
+		 * so the compiler cannot unroll the loop unless we restrict ourselves
+		 * to the 4-byte variant. Needs testing to choose the best one.
 		 */
 		if (!PointerIsAligned(p, uint64) && len > 4)
 		{
@@ -158,7 +168,8 @@ pg_comp_crc32c_dispatch(pg_crc32c crc, const void *data, size_t len)
 		return crc;
 	}
 	else
-		return pg_comp_crc32c_armv8(crc, data, len);
+		/* Otherwise, use a runtime check for PMULL instructions. */
+		return pg_comp_crc32c(crc, data, len);
 }
 
 #elif defined(USE_LOONGARCH_CRC32C)
@@ -183,6 +194,9 @@ extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_
 extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len);
 extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len);
 extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len);
+#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
+extern pg_crc32c pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len);
+#endif
 
 #else
 /*
diff --git a/src/port/meson.build b/src/port/meson.build
index fc7b059fee5..8b0ac6b931a 100644
--- a/src/port/meson.build
+++ b/src/port/meson.build
@@ -94,6 +94,7 @@ replace_funcs_pos = [
   # arm / aarch64
   ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C'],
   ['pg_crc32c_armv8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK', 'crc'],
+  ['pg_crc32c_armv8_choose', 'USE_ARMV8_CRC32C'],
   ['pg_crc32c_armv8_choose', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'],
   ['pg_crc32c_sb8', 'USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK'],
 
diff --git a/src/port/pg_crc32c_armv8.c b/src/port/pg_crc32c_armv8.c
index 5ba070bb99d..f67de2016b4 100644
--- a/src/port/pg_crc32c_armv8.c
+++ b/src/port/pg_crc32c_armv8.c
@@ -15,6 +15,9 @@
 #include "c.h"
 
 #include <arm_acle.h>
+#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
+#include <arm_neon.h>
+#endif
 
 #include "port/pg_crc32c.h"
 
@@ -73,3 +76,125 @@ pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len)
 
 	return crc;
 }
+
+#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
+
+/*
+ * Note: There is no copyright notice in the following generated code.
+ *
+ * We have modified the output to
+ *   - match our function declaration and alignment macro
+ *   - match whitespace to our project style
+ */
+
+/* Generated by https://github.com/corsix/fast-crc32/ using: */
+/* ./generate -i neon -p crc32c -a v4e */
+/* MIT licensed */
+
+static inline
+uint64x2_t
+clmul_lo_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
+{
+	uint64x2_t	r;
+
+__asm("pmull %0.1q, %2.1d, %3.1d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
+	return r;
+}
+
+static inline
+uint64x2_t
+clmul_hi_e(uint64x2_t a, uint64x2_t b, uint64x2_t c)
+{
+	uint64x2_t	r;
+
+__asm("pmull2 %0.1q, %2.2d, %3.2d\neor %0.16b, %0.16b, %1.16b\n":"=w"(r), "+w"(c):"w"(a), "w"(b));
+	return r;
+}
+
+pg_crc32c
+pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len)
+{
+	/* adjust names to match generated code */
+	pg_crc32c	crc0 = crc;
+	const char *buf = data;
+
+	/* Align to 16 bytes to prevent straddling cacheline boundaries. */
+	for (; len && ((uintptr_t) buf & 7); --len)
+	{
+		crc0 = __crc32cb(crc0, *buf++);
+	}
+	if (((uintptr_t) buf & 8) && len >= 8)
+	{
+		crc0 = __crc32cd(crc0, *(const uint64_t *) buf);
+		buf += 8;
+		len -= 8;
+	}
+
+	if (len >= 64)
+	{
+		const char *end = buf + len;
+		const char *limit = buf + len - 64;
+
+		/* First vector chunk. */
+		uint64x2_t	x0 = vld1q_u64((const uint64_t *) buf),
+					y0;
+		uint64x2_t	x1 = vld1q_u64((const uint64_t *) (buf + 16)),
+					y1;
+		uint64x2_t	x2 = vld1q_u64((const uint64_t *) (buf + 32)),
+					y2;
+		uint64x2_t	x3 = vld1q_u64((const uint64_t *) (buf + 48)),
+					y3;
+		uint64x2_t	k;
+
+		{
+			static const uint64_t pg_attribute_aligned(16) k_[] = {0x740eef02, 0x9e4addf8};
+
+			k = vld1q_u64(k_);
+		}
+
+
+		/* pgindent doesn't like this: */
+		x0 = veorq_u64((uint64x2_t) {crc0, 0}, x0);
+		/*
+		 * a possible alternative?
+		 *
+		 * x0 = veorq_u64((uint64x2_t) vsetq_lane_u32(crc0, vdupq_n_u32(0), 0), x0);
+		 */
+
+		buf += 64;
+
+		/* Main loop. */
+		while (buf <= limit)
+		{
+			y0 = clmul_lo_e(x0, k, vld1q_u64((const uint64_t *) buf)), x0 = clmul_hi_e(x0, k, y0);
+			y1 = clmul_lo_e(x1, k, vld1q_u64((const uint64_t *) (buf + 16))), x1 = clmul_hi_e(x1, k, y1);
+			y2 = clmul_lo_e(x2, k, vld1q_u64((const uint64_t *) (buf + 32))), x2 = clmul_hi_e(x2, k, y2);
+			y3 = clmul_lo_e(x3, k, vld1q_u64((const uint64_t *) (buf + 48))), x3 = clmul_hi_e(x3, k, y3);
+			buf += 64;
+		}
+
+		/* Reduce x0 ... x3 to just x0. */
+		{
+			static const uint64_t pg_attribute_aligned(16) k_[] = {0xf20c0dfe, 0x493c7d27};
+
+			k = vld1q_u64(k_);
+		}
+		y0 = clmul_lo_e(x0, k, x1), x0 = clmul_hi_e(x0, k, y0);
+		y2 = clmul_lo_e(x2, k, x3), x2 = clmul_hi_e(x2, k, y2);
+		{
+			static const uint64_t pg_attribute_aligned(16) k_[] = {0x3da6d0cb, 0xba4fc28e};
+
+			k = vld1q_u64(k_);
+		}
+		y0 = clmul_lo_e(x0, k, x2), x0 = clmul_hi_e(x0, k, y0);
+
+		/* Reduce 128 bits to 32 bits, and multiply by x^32. */
+		crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0));
+		crc0 = __crc32cd(crc0, vgetq_lane_u64(x0, 1));
+		len = end - buf;
+	}
+
+	return pg_comp_crc32c_armv8(crc0, buf, len);
+}
+
+#endif
diff --git a/src/port/pg_crc32c_armv8_choose.c b/src/port/pg_crc32c_armv8_choose.c
index ec12be1bbc3..9a1656a2685 100644
--- a/src/port/pg_crc32c_armv8_choose.c
+++ b/src/port/pg_crc32c_armv8_choose.c
@@ -107,6 +107,27 @@ pg_crc32c_armv8_available(void)
 #endif
 }
 
+static inline bool
+pg_pmull_available(void)
+{
+#ifdef __aarch64__
+
+#ifdef HAVE_ELF_AUX_INFO
+	unsigned long value;
+
+	return elf_aux_info(AT_HWCAP, &value, sizeof(value)) == 0 &&
+		(value & HWCAP_PMULL) != 0;
+#elif defined(HAVE_GETAUXVAL)
+	return (getauxval(AT_HWCAP) & HWCAP_PMULL) != 0;
+#else
+	return false;
+#endif
+
+#else
+	return false;
+#endif
+}
+
 /*
  * This gets called on the first call. It replaces the function pointer
  * so that subsequent calls are routed directly to the chosen implementation.
@@ -114,11 +135,26 @@ pg_crc32c_armv8_available(void)
 static pg_crc32c
 pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len)
 {
+#if defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK)
 	if (pg_crc32c_armv8_available())
 		pg_comp_crc32c = pg_comp_crc32c_armv8;
 	else
 		pg_comp_crc32c = pg_comp_crc32c_sb8;
 
+#elif defined(USE_ARMV8_CRC32C)
+	/*
+	 * On MacOS, compilers may emit CRC instructions without extra CFLAGS, but
+	 * we can't use Linux-isms to detect CPU features, so we just set it here
+	 * as a fallback for PMULL.
+	 */
+	pg_comp_crc32c = pg_comp_crc32c_armv8;
+#endif
+
+#ifdef USE_PMULL_CRC32C_WITH_RUNTIME_CHECK
+	if (pg_pmull_available())
+		pg_comp_crc32c = pg_comp_crc32c_pmull;
+#endif
+
 	return pg_comp_crc32c(crc, data, len);
 }
 
-- 
2.49.0