0007-TAP-12-crashes-at-injection-points.patch

From 57a5e353dbfcaaac1fa7cd25dc38f3e1807ccafa Mon Sep 17 00:00:00 2001
From: Tomas Vondra <tomas@vondra.me>
Date: Wed, 15 Apr 2026 23:06:14 +0200
Subject: [PATCH 07/13] TAP 12: crashes at injection points

---
 .../t/012_concurrent_checkpoint_crash.pl      | 282 ++++++++++++++++++
 1 file changed, 282 insertions(+)
 create mode 100644 src/test/modules/test_checksums/t/012_concurrent_checkpoint_crash.pl

diff --git a/src/test/modules/test_checksums/t/012_concurrent_checkpoint_crash.pl b/src/test/modules/test_checksums/t/012_concurrent_checkpoint_crash.pl
new file mode 100644
index 00000000000..de169f47034
--- /dev/null
+++ b/src/test/modules/test_checksums/t/012_concurrent_checkpoint_crash.pl
@@ -0,0 +1,282 @@
+
+# Copyright (c) 2026, PostgreSQL Global Development Group
+
+# Test suite for testing enabling data checksums in an online cluster with
+# injection point tests injecting failures into the processing
+
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+use FindBin;
+use lib $FindBin::RealBin;
+
+use DataChecksums::Utils;
+
+# This test suite is expensive, or very expensive, to execute.  There are two
+# PG_TEST_EXTRA options for running it, "checksum" for a pared-down test suite
+# an "checksum_extended" for the full suite.  The full suite can run for hours
+# on slow or constrained systems.
+my $extended = undef;
+if ($ENV{PG_TEST_EXTRA})
+{
+	$extended = 1 if ($ENV{PG_TEST_EXTRA} =~ /\bchecksum_extended\b/);
+	plan skip_all => 'Expensive data checksums test disabled'
+	  unless ($ENV{PG_TEST_EXTRA} =~ /\bchecksum(_extended)?\b/);
+}
+
+if ($ENV{enable_injection_points} ne 'yes')
+{
+	plan skip_all => 'Injection points not supported by this build';
+}
+
+# ---------------------------------------------------------------------------
+# Test cluster setup
+#
+
+# Initiate testcluster
+my $node = PostgreSQL::Test::Cluster->new('injection_node');
+$node->init(no_data_checksums => 1);
+$node->start;
+
+# Set up test environment
+$node->safe_psql('postgres', 'CREATE EXTENSION test_checksums;');
+$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;');
+
+my $pgbench = undef;
+my $scalefactor = ($extended ? 10 : 1);
+my $node_loglocation = 0;
+
+$node->command_ok(
+	[
+		'pgbench', '-p', $node->port, '-i',
+		'-s', $scalefactor, '-q', 'postgres'
+	]);
+
+# Start a pgbench run in the background against the server specified via the
+# port passed as parameter.
+sub background_rw_pgbench
+{
+	my $port = shift;
+
+	# If a previous pgbench is still running, start by shutting it down.
+	$pgbench->finish if $pgbench;
+
+	my $clients = 1;
+	my $runtime = 2;
+
+	if ($extended)
+	{
+		# Randomize the number of pgbench clients a bit (range 1-16)
+		$clients = 1 + int(rand(15));
+		$runtime = 600;
+	}
+	my @cmd = ('pgbench', '-p', $port, '-T', $runtime, '-c', $clients);
+
+	# Randomize whether we spawn connections or not
+	push(@cmd, '-C') if ($extended && cointoss);
+	# Finally add the database name to use
+	push(@cmd, 'postgres');
+
+	$pgbench = IPC::Run::start(
+		\@cmd,
+		'<' => '/dev/null',
+		'>' => '/dev/null',
+		'2>' => '/dev/null',
+		IPC::Run::timer($PostgreSQL::Test::Utils::timeout_default));
+}
+
+sub attach_injection_point
+{
+	my ($node, $point) = @_;
+	note('attaching injection point: ' . $point);
+	$node->safe_psql('postgres',
+		"SELECT injection_points_attach('" . $point . "','wait');"
+	);
+}
+
+sub wait_injection_point
+{
+	my ($node, $point) = @_;
+
+	note("waiting for the injection point to be hit");
+	$node->poll_query_until(
+		'postgres',
+		"SELECT COUNT(*) FROM pg_catalog.pg_stat_activity WHERE wait_event = '" . $point . "'",
+		'1');
+}
+
+sub wakeup_injection_point
+{
+	my ($node, $point) = @_;
+
+	note('waking the init injection point: ' . $point);
+	$node->safe_psql('postgres',
+		"SELECT injection_points_wakeup('" . $point . "');"
+	);
+
+	note("detaching the init injection point");
+	$node->safe_psql('postgres',
+		"SELECT injection_points_detach('" . $point . "');");
+}
+
+# Test behavior with a checksum transitions and a concurrent checkpoint,
+# followed by a crash.
+#
+# The test puts the instance into the initial checksum state $start,
+# triggers a checksum change that pauses on a first injection point. Then
+# a checkpoint is performed, and the checksum change proceeds either to
+# a second injection point or finishes.
+#
+# Then the instance get restarted in immediate mode to simulate failure,
+# and the final checksum state (after recovery) is validated against the
+# expected value. The server log is checked for checksum failures.
+#
+# While the checksum change is happening, there's a r/w pgbench running in
+# the background, to generate writes.
+#
+# arguments:
+#
+# - start  - start checksum state (enabled/disabled)
+# - change - checksum change to initiate
+# - point1 - injection point before checkpoint
+# - point2 - injection point after checkpoint
+# - final  - expected checksum state at the end
+#
+# XXX Some of the injection points are in a critical section, which does
+# not allow memory allocations etc. INJECTION_POINT_LOAD/_CACHED handles
+# just private memory allocation, but 'wait' requires a shmem allocation.
+# To deal with that, we setup an injection point $init outside a critical
+# section to initialize the shmem stuff needed by 'wait'.
+#
+# XXX We could also validate the checksums using pg_checksums, if the
+# state is 'on'.
+sub test_checksum_transition
+{
+	my ($start, $change, $init, $point1, $point2, $final) = @_;
+
+	my $initstr = 'undef';
+	my $point2str = 'undef';
+
+	# print the current test instructions, both into TAP output and into
+	# the server log, to make correlation easier
+
+	$initstr = $init if defined($init);
+	$point2str = $point2 if defined($point2);
+
+	$node->safe_psql('postgres',
+		"SELECT '========== " . $start . " / " . $change . " / " . $initstr . " / " . $point1 . " / " . $point2str . " / " . $final . " =========='");
+
+	note($start . " / " . $change . " / " . $initstr . " / " . $point1 . " / " . $point2str . " / " . $final);
+
+	# Start the test suite with pgbench running.
+	background_rw_pgbench($node->port);
+
+	# put the cluster into the initial checksum state, synchronously
+	note('changing checksums into initial state: ' . $start);
+	enable_data_checksums($node, wait => 'on') if ($start eq 'enabled');
+	disable_data_checksums($node, wait => 'off') if ($start eq 'disabled');
+
+	 # Wait on an injection point outside a critical section, to initialize
+	 # the shmem (which can't be done in critical section).
+	attach_injection_point($node, $init) if defined($init);
+
+	# Wait on the two injection points, to pause the checksum change.
+	attach_injection_point($node, $point1);
+
+	# The second injection point is optional, so only attach it then.
+	attach_injection_point($node, $point2) if defined($point2);
+
+	# Trigger the checksum change, asynchronously
+	note("triggering checksum change: " . $change);
+	enable_data_checksums($node) if ($change eq 'enable');
+	disable_data_checksums($node) if ($change eq 'disable');
+
+	# Handle the initial injection point - wait, wakeup and detatch. This
+	# initializes the shmem for the 'wait' action.
+	wait_injection_point($node, $init) if defined($init);
+	wakeup_injection_point($node, $init) if defined($init);
+
+	# Wait for the first injection point to be hit by the state change.
+	wait_injection_point($node, $point1);
+
+	# The checksum state change is paused on the first injection point.
+	# Perform the checkpoint (synchronously).
+	note('checkpoint');
+	$node->safe_psql('postgres', "CHECKPOINT");
+
+	# Wake the injection point, so that the first change can proceed.
+	wakeup_injection_point($node, $point1);
+
+	# Either wait for the second injection point - if defined, or for the
+	# checksum change to complete.
+
+	if (defined($point2))
+	{
+		wait_injection_point($node, $point2);
+	}
+	else
+	{
+		# Wait until there are no ongoing checksum changes, which we determine
+		# by looking for a checksum launcher process.
+		note('wait for the checksum launcher to exit');
+		$node->poll_query_until('postgres',
+				"SELECT count(*) = 0 "
+			  . "FROM pg_catalog.pg_stat_activity "
+			  . "WHERE backend_type = 'datachecksum launcher';");
+		note('checksum launcher exited');
+	}
+
+	# restart the cluster, in immediate mode, to simulate a crash
+	$node->stop('immediate');
+	$node->start;
+
+	# Does the final checksum state match the expected state?
+	test_checksum_state($node, $final);
+
+	# Since the log isn't being written to now, parse the log and check
+	# for instances of checksum verification failures.
+	my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile,
+		$node_loglocation);
+	unlike(
+		$log,
+		qr/page verification failed,.+\d$/,
+		"no checksum validation errors in primary log (during WAL recovery)"
+	);
+	$node_loglocation = -s $node->logfile;
+}
+
+# concurrent enable + checkpoint, different injection points in the "enable" process
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-inprogress-checksums-start', 'datachecksums-enable-inprogress-checksums-after-xlog', 'off');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-inprogress-checksums-start', 'datachecksums-enable-inprogress-checksums-after-xlog', 'datachecksums-enable-inprogress-checksums-after-xlogctl', 'off');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-inprogress-checksums-start', 'datachecksums-enable-inprogress-checksums-after-xlogctl', 'datachecksums-enable-inprogress-checksums-after-controlfile', 'off');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-inprogress-checksums-start', 'datachecksums-enable-inprogress-checksums-after-controlfile', 'datachecksums-enable-inprogress-checksums-before-barrier-wait', 'off');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-inprogress-checksums-before-barrier-wait', 'datachecksums-enable-inprogress-checksums-end', 'off');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-inprogress-checksums-end', 'datachecksums-enable-checksums-start', 'off');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-checksums-start', 'datachecksums-enable-checksums-after-xlog', 'on');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-checksums-start', 'datachecksums-enable-checksums-after-xlog', 'datachecksums-enable-checksums-after-xlogctl', 'off');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-checksums-start', 'datachecksums-enable-checksums-after-xlogctl', 'datachecksums-enable-checksums-after-controlfile', 'on');
+test_checksum_transition('disabled', 'enable', 'datachecksums-enable-checksums-start', 'datachecksums-enable-checksums-after-controlfile', 'datachecksums-enable-checksums-before-checkpoint', 'on');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-checksums-before-checkpoint', 'datachecksums-enable-checksums-before-barrier-wait', 'on');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-checksums-before-barrier-wait', 'datachecksums-enable-checksums-end', 'on');
+test_checksum_transition('disabled', 'enable', undef, 'datachecksums-enable-checksums-end', undef, 'on');
+
+# concurrent disable + checkpoint, different injection points in the "disable" process
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-inprogress-checksums-start', 'datachecksums-disable-inprogress-checksums-after-xlog', 'off');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-inprogress-checksums-start', 'datachecksums-disable-inprogress-checksums-after-xlog', 'datachecksums-disable-inprogress-checksums-after-xlogctl', 'on');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-inprogress-checksums-start', 'datachecksums-disable-inprogress-checksums-after-xlogctl', 'datachecksums-disable-inprogress-checksums-after-controlfile', 'off');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-inprogress-checksums-start', 'datachecksums-disable-inprogress-checksums-after-controlfile', 'datachecksums-disable-inprogress-checksums-before-barrier-wait', 'off');
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-inprogress-checksums-before-barrier-wait', 'datachecksums-disable-checksums-start', 'off');
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-checksums-start', 'datachecksums-disable-checksums-after-xlog', 'off');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-checksums-start', 'datachecksums-disable-checksums-after-xlog', 'datachecksums-disable-checksums-after-xlogctl', 'off');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-checksums-start', 'datachecksums-disable-checksums-after-xlogctl', 'datachecksums-disable-checksums-after-controlfile', 'off');
+test_checksum_transition('enabled', 'disable', 'datachecksums-disable-checksums-start', 'datachecksums-disable-checksums-after-controlfile', 'datachecksums-disable-checksums-before-checkpoint', 'off');
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-checksums-before-checkpoint', 'datachecksums-disable-checksums-before-barrier-wait', 'off');
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-checksums-before-barrier-wait', 'datachecksums-disable-checksums-end', 'off');
+test_checksum_transition('enabled', 'disable', undef, 'datachecksums-disable-checksums-end', undef, 'off');
+
+$node->stop;
+done_testing();
-- 
2.54.0