0016-Tests-for-dynamic-shared_buffers-resizing-20250918.patch

From 359800f4e40a4ac97cb5aec1198bd0b8584ce53e Mon Sep 17 00:00:00 2001
From: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>
Date: Wed, 3 Sep 2025 10:59:20 +0530
Subject: [PATCH 16/16] Tests for dynamic shared_buffers resizing

The commit adds two tests:

1. TAP test to stress test buffer pool resizing under concurrent load.

2. SQL test to test sanity of shared memory allocations and mappings
   after buffer pool resizing operation.

Author: Palak Chaturvedi <chaturvedipalak1911@gmail.com>
Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>
---
 src/test/Makefile                             |   2 +-
 src/test/README                               |   3 +
 src/test/buffermgr/Makefile                   |  27 ++
 src/test/buffermgr/README                     |  26 ++
 src/test/buffermgr/expected/buffer_resize.out | 237 ++++++++++++++++++
 src/test/buffermgr/meson.build                |  17 ++
 src/test/buffermgr/sql/buffer_resize.sql      |  73 ++++++
 src/test/buffermgr/t/001_resize_buffer.pl     | 126 ++++++++++
 src/test/meson.build                          |   1 +
 9 files changed, 511 insertions(+), 1 deletion(-)
 create mode 100644 src/test/buffermgr/Makefile
 create mode 100644 src/test/buffermgr/README
 create mode 100644 src/test/buffermgr/expected/buffer_resize.out
 create mode 100644 src/test/buffermgr/meson.build
 create mode 100644 src/test/buffermgr/sql/buffer_resize.sql
 create mode 100644 src/test/buffermgr/t/001_resize_buffer.pl

diff --git a/src/test/Makefile b/src/test/Makefile
index 511a72e6238..95f8858a818 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -12,7 +12,7 @@ subdir = src/test
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription
+SUBDIRS = perl postmaster regress isolation modules authentication recovery subscription buffermgr
 
 ifeq ($(with_icu),yes)
 SUBDIRS += icu
diff --git a/src/test/README b/src/test/README
index afdc7676519..77f11607ff7 100644
--- a/src/test/README
+++ b/src/test/README
@@ -15,6 +15,9 @@ examples/
   Demonstration programs for libpq that double as regression tests via
   "make check"
 
+buffermgr/
+  Tests for resizing buffer pool without restarting the server
+
 isolation/
   Tests for concurrent behavior at the SQL level
 
diff --git a/src/test/buffermgr/Makefile b/src/test/buffermgr/Makefile
new file mode 100644
index 00000000000..97c3da9e20a
--- /dev/null
+++ b/src/test/buffermgr/Makefile
@@ -0,0 +1,27 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/test/buffermgr
+#
+# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/test/buffermgr/Makefile
+#
+#-------------------------------------------------------------------------
+
+EXTRA_INSTALL = contrib/pg_buffercache
+
+REGRESS = buffer_resize
+
+subdir = src/test/buffermgr
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+check:
+	$(prove_check)
+
+installcheck:
+	$(prove_installcheck)
+
+clean distclean:
+	rm -rf tmp_check
diff --git a/src/test/buffermgr/README b/src/test/buffermgr/README
new file mode 100644
index 00000000000..c375ad80989
--- /dev/null
+++ b/src/test/buffermgr/README
@@ -0,0 +1,26 @@
+src/test/buffermgr/README
+
+Regression tests for buffer manager
+===================================
+
+This directory contains a test suite for resizing buffer manager without restarting the server.
+
+
+Running the tests
+=================
+
+NOTE: You must have given the --enable-tap-tests argument to configure.
+
+Run
+    make check
+or
+    make installcheck
+You can use "make installcheck" if you previously did "make install".
+In that case, the code in the installation tree is tested.  With
+"make check", a temporary installation tree is built from the current
+sources and then tested.
+
+Either way, this test initializes, starts, and stops a test Postgres
+cluster.
+
+See src/test/perl/README for more info about running these tests.
diff --git a/src/test/buffermgr/expected/buffer_resize.out b/src/test/buffermgr/expected/buffer_resize.out
new file mode 100644
index 00000000000..a986be9a5da
--- /dev/null
+++ b/src/test/buffermgr/expected/buffer_resize.out
@@ -0,0 +1,237 @@
+-- Test buffer pool resizing and shared memory allocation tracking
+-- This test resizes the buffer pool multiple times and monitors
+-- shared memory allocations related to buffer management
+-- Create a separate schema for this test
+CREATE SCHEMA buffer_resize_test;
+SET search_path TO buffer_resize_test, public;
+-- Create a view for buffer-related shared memory allocations
+CREATE VIEW buffer_allocations AS
+SELECT name, segment, size, allocated_size 
+FROM pg_shmem_allocations 
+WHERE name IN ('Buffer Blocks', 'Buffer Descriptors', 'Buffer IO Condition Variables', 
+               'Checkpoint BufferIds')
+ORDER BY name;
+-- Note: We exclude the 'main' segment even if it contains the shared buffer
+-- lookup table because it contains other shared structures whose total sizes
+-- may vary as the code changes.
+CREATE VIEW buffer_segments AS
+SELECT name, size, mapping_size, mapping_reserved_size
+FROM pg_shmem_segments
+WHERE name <> 'main'
+ORDER BY name;
+-- Enable pg_buffercache for buffer count verification
+CREATE EXTENSION IF NOT EXISTS pg_buffercache;
+-- Test 1: Default shared_buffers 
+SHOW shared_buffers;
+ shared_buffers 
+----------------
+ 128MB
+(1 row)
+
+SELECT * FROM buffer_allocations;
+             name              |   segment   |   size    | allocated_size 
+-------------------------------+-------------+-----------+----------------
+ Buffer Blocks                 | buffers     | 134221824 |      134221824
+ Buffer Descriptors            | descriptors |   1048576 |        1048576
+ Buffer IO Condition Variables | iocv        |    262144 |         262144
+ Checkpoint BufferIds          | checkpoint  |    327680 |         327680
+(4 rows)
+
+SELECT * FROM buffer_segments;
+    name     |   size    | mapping_size | mapping_reserved_size 
+-------------+-----------+--------------+-----------------------
+ buffers     | 134225920 |    134225920 |            2576982016
+ checkpoint  |    335872 |       335872 |             214753280
+ descriptors |   1056768 |      1056768 |             429498368
+ iocv        |    270336 |       270336 |             429498368
+(4 rows)
+
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+ buffer_count 
+--------------
+        16384
+(1 row)
+
+-- Test 2: Set to 64MB  
+ALTER SYSTEM SET shared_buffers = '64MB';
+SELECT pg_reload_conf();
+ pg_reload_conf 
+----------------
+ t
+(1 row)
+
+SELECT pg_sleep(1);
+ pg_sleep 
+----------
+ 
+(1 row)
+
+SHOW shared_buffers;
+ shared_buffers 
+----------------
+ 64MB
+(1 row)
+
+SELECT * FROM buffer_allocations;
+             name              |   segment   |   size   | allocated_size 
+-------------------------------+-------------+----------+----------------
+ Buffer Blocks                 | buffers     | 67112960 |       67112960
+ Buffer Descriptors            | descriptors |   524288 |         524288
+ Buffer IO Condition Variables | iocv        |   131072 |         131072
+ Checkpoint BufferIds          | checkpoint  |   163840 |         163840
+(4 rows)
+
+SELECT * FROM buffer_segments;
+    name     |   size   | mapping_size | mapping_reserved_size 
+-------------+----------+--------------+-----------------------
+ buffers     | 67117056 |     67117056 |            2576982016
+ checkpoint  |   172032 |       172032 |             214753280
+ descriptors |   532480 |       532480 |             429498368
+ iocv        |   139264 |       139264 |             429498368
+(4 rows)
+
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+ buffer_count 
+--------------
+         8192
+(1 row)
+
+-- Test 3: Set to 256MB
+ALTER SYSTEM SET shared_buffers = '256MB';
+SELECT pg_reload_conf();
+ pg_reload_conf 
+----------------
+ t
+(1 row)
+
+SELECT pg_sleep(1);
+ pg_sleep 
+----------
+ 
+(1 row)
+
+SHOW shared_buffers;
+ shared_buffers 
+----------------
+ 256MB
+(1 row)
+
+SELECT * FROM buffer_allocations;
+             name              |   segment   |   size    | allocated_size 
+-------------------------------+-------------+-----------+----------------
+ Buffer Blocks                 | buffers     | 268439552 |      268439552
+ Buffer Descriptors            | descriptors |   2097152 |        2097152
+ Buffer IO Condition Variables | iocv        |    524288 |         524288
+ Checkpoint BufferIds          | checkpoint  |    655360 |         655360
+(4 rows)
+
+SELECT * FROM buffer_segments;
+    name     |   size    | mapping_size | mapping_reserved_size 
+-------------+-----------+--------------+-----------------------
+ buffers     | 268443648 |    268443648 |            2576982016
+ checkpoint  |    663552 |       663552 |             214753280
+ descriptors |   2105344 |      2105344 |             429498368
+ iocv        |    532480 |       532480 |             429498368
+(4 rows)
+
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+ buffer_count 
+--------------
+        32768
+(1 row)
+
+-- Test 4: Set to 100MB (non-power-of-two)
+ALTER SYSTEM SET shared_buffers = '100MB';
+SELECT pg_reload_conf();
+ pg_reload_conf 
+----------------
+ t
+(1 row)
+
+SELECT pg_sleep(1);
+ pg_sleep 
+----------
+ 
+(1 row)
+
+SHOW shared_buffers;
+ shared_buffers 
+----------------
+ 100MB
+(1 row)
+
+SELECT * FROM buffer_allocations;
+             name              |   segment   |   size    | allocated_size 
+-------------------------------+-------------+-----------+----------------
+ Buffer Blocks                 | buffers     | 104861696 |      104861696
+ Buffer Descriptors            | descriptors |    819200 |         819200
+ Buffer IO Condition Variables | iocv        |    204800 |         204800
+ Checkpoint BufferIds          | checkpoint  |    256000 |         256000
+(4 rows)
+
+SELECT * FROM buffer_segments;
+    name     |   size    | mapping_size | mapping_reserved_size 
+-------------+-----------+--------------+-----------------------
+ buffers     | 104865792 |    104865792 |            2576982016
+ checkpoint  |    262144 |       262144 |             214753280
+ descriptors |    827392 |       827392 |             429498368
+ iocv        |    212992 |       212992 |             429498368
+(4 rows)
+
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+ buffer_count 
+--------------
+        12800
+(1 row)
+
+-- Test 5: Set to minimum 128kB
+ALTER SYSTEM SET shared_buffers = '128kB';
+SELECT pg_reload_conf();
+ pg_reload_conf 
+----------------
+ t
+(1 row)
+
+SELECT pg_sleep(1);
+ pg_sleep 
+----------
+ 
+(1 row)
+
+SHOW shared_buffers;
+ shared_buffers 
+----------------
+ 128kB
+(1 row)
+
+SELECT * FROM buffer_allocations;
+             name              |   segment   |  size  | allocated_size 
+-------------------------------+-------------+--------+----------------
+ Buffer Blocks                 | buffers     | 135168 |         135168
+ Buffer Descriptors            | descriptors |   1024 |           1024
+ Buffer IO Condition Variables | iocv        |    256 |            256
+ Checkpoint BufferIds          | checkpoint  |    320 |            320
+(4 rows)
+
+SELECT * FROM buffer_segments;
+    name     |  size  | mapping_size | mapping_reserved_size 
+-------------+--------+--------------+-----------------------
+ buffers     | 139264 |       139264 |            2576982016
+ checkpoint  |   8192 |         8192 |             214753280
+ descriptors |   8192 |         8192 |             429498368
+ iocv        |   8192 |         8192 |             429498368
+(4 rows)
+
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+ buffer_count 
+--------------
+           16
+(1 row)
+
+-- Clean up the schema and all its objects
+RESET search_path;
+DROP SCHEMA buffer_resize_test CASCADE;
+NOTICE:  drop cascades to 3 other objects
+DETAIL:  drop cascades to view buffer_resize_test.buffer_allocations
+drop cascades to view buffer_resize_test.buffer_segments
+drop cascades to extension pg_buffercache
diff --git a/src/test/buffermgr/meson.build b/src/test/buffermgr/meson.build
new file mode 100644
index 00000000000..e71dcdea685
--- /dev/null
+++ b/src/test/buffermgr/meson.build
@@ -0,0 +1,17 @@
+# Copyright (c) 2022-2025, PostgreSQL Global Development Group
+
+tests += {
+  'name': 'buffermgr',
+  'sd': meson.current_source_dir(),
+  'bd': meson.current_build_dir(),
+  'regress': {
+    'sql': [
+      'buffer_resize',
+    ],
+  },
+  'tap': {
+    'tests': [
+      't/001_resize_buffer.pl',
+    ],
+  },
+}
diff --git a/src/test/buffermgr/sql/buffer_resize.sql b/src/test/buffermgr/sql/buffer_resize.sql
new file mode 100644
index 00000000000..45f5bb6d78b
--- /dev/null
+++ b/src/test/buffermgr/sql/buffer_resize.sql
@@ -0,0 +1,73 @@
+-- Test buffer pool resizing and shared memory allocation tracking
+-- This test resizes the buffer pool multiple times and monitors
+-- shared memory allocations related to buffer management
+
+-- Create a separate schema for this test
+CREATE SCHEMA buffer_resize_test;
+SET search_path TO buffer_resize_test, public;
+
+-- Create a view for buffer-related shared memory allocations
+CREATE VIEW buffer_allocations AS
+SELECT name, segment, size, allocated_size 
+FROM pg_shmem_allocations 
+WHERE name IN ('Buffer Blocks', 'Buffer Descriptors', 'Buffer IO Condition Variables', 
+               'Checkpoint BufferIds')
+ORDER BY name;
+
+-- Note: We exclude the 'main' segment even if it contains the shared buffer
+-- lookup table because it contains other shared structures whose total sizes
+-- may vary as the code changes.
+CREATE VIEW buffer_segments AS
+SELECT name, size, mapping_size, mapping_reserved_size
+FROM pg_shmem_segments
+WHERE name <> 'main'
+ORDER BY name;
+
+-- Enable pg_buffercache for buffer count verification
+CREATE EXTENSION IF NOT EXISTS pg_buffercache;
+
+-- Test 1: Default shared_buffers 
+SHOW shared_buffers;
+SELECT * FROM buffer_allocations;
+SELECT * FROM buffer_segments;
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+
+-- Test 2: Set to 64MB  
+ALTER SYSTEM SET shared_buffers = '64MB';
+SELECT pg_reload_conf();
+SELECT pg_sleep(1);
+SHOW shared_buffers;
+SELECT * FROM buffer_allocations;
+SELECT * FROM buffer_segments;
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+
+-- Test 3: Set to 256MB
+ALTER SYSTEM SET shared_buffers = '256MB';
+SELECT pg_reload_conf();
+SELECT pg_sleep(1);
+SHOW shared_buffers;
+SELECT * FROM buffer_allocations;
+SELECT * FROM buffer_segments;
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+
+-- Test 4: Set to 100MB (non-power-of-two)
+ALTER SYSTEM SET shared_buffers = '100MB';
+SELECT pg_reload_conf();
+SELECT pg_sleep(1);
+SHOW shared_buffers;
+SELECT * FROM buffer_allocations;
+SELECT * FROM buffer_segments;
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+
+-- Test 5: Set to minimum 128kB
+ALTER SYSTEM SET shared_buffers = '128kB';
+SELECT pg_reload_conf();
+SELECT pg_sleep(1);
+SHOW shared_buffers;
+SELECT * FROM buffer_allocations;
+SELECT * FROM buffer_segments;
+SELECT COUNT(*) AS buffer_count FROM pg_buffercache;
+
+-- Clean up the schema and all its objects
+RESET search_path;
+DROP SCHEMA buffer_resize_test CASCADE;
diff --git a/src/test/buffermgr/t/001_resize_buffer.pl b/src/test/buffermgr/t/001_resize_buffer.pl
new file mode 100644
index 00000000000..8cf9e4539ab
--- /dev/null
+++ b/src/test/buffermgr/t/001_resize_buffer.pl
@@ -0,0 +1,126 @@
+# Copyright (c) 2025-2025, PostgreSQL Global Development Group
+#
+# Minimal test testing shared_buffer resizing under load
+
+use strict;
+use warnings;
+use IPC::Run;
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+use Test::More;
+
+# Function to resize buffer pool and verify the change.
+sub apply_and_verify_buffer_change
+{
+	my ($node, $new_size) = @_;
+	
+	# Use a single background_psql session for consistency
+	my $psql_session = $node->background_psql('postgres');
+	$psql_session->query_safe("ALTER SYSTEM SET shared_buffers = '$new_size'");
+	$psql_session->query_safe("SELECT pg_reload_conf()");
+	
+	# Wait till the resizing finishes using the same session
+	# 
+	# TODO: Right now there is no way to know when the resize has finished and
+	# all the backends are using new value of shared_buffers. Hence we poll
+	# manually until we get the expected value in the same session.
+	my $current_size;
+	my $attempts = 0;
+	my $max_attempts = 60; # 60 seconds timeout
+	do {
+		$current_size = $psql_session->query_safe("SHOW shared_buffers");
+		$attempts++;
+		
+		# Only sleep if we didn't get the expected result and haven't timed out yet
+		if ($current_size ne $new_size && $attempts < $max_attempts) {
+			sleep(1);
+		}
+	} while ($current_size ne $new_size && $attempts < $max_attempts);
+	
+	$psql_session->quit;
+	
+	# Check if we succeeded or timed out
+	if ($current_size ne $new_size) {
+		die "Timeout waiting for shared_buffers to change to $new_size (got $current_size after ${attempts}s)";
+	}
+}
+
+# Initialize a cluster and start pgbench in the background for concurrent load.
+my $node = PostgreSQL::Test::Cluster->new('main');
+$node->init;
+$node->start;
+$node->safe_psql('postgres', "CREATE EXTENSION pg_buffercache");
+my $pgb_scale = 10;
+my $pgb_duration = 120;
+my $pgb_num_clients = 10;
+$node->pgbench(
+	"--initialize --init-steps=dtpvg --scale=$pgb_scale --quiet",
+	0,
+	[qr{^$}],
+	[   # stderr patterns to verify initialization stages
+		qr{dropping old tables},
+		qr{creating tables},
+		qr{done in \d+\.\d\d s }
+	],
+	"pgbench initialization (scale=$pgb_scale)"
+);
+my ($pgbench_stdin, $pgbench_stdout, $pgbench_stderr) = ('', '', '');
+my $pgbench_process = IPC::Run::start(
+	[
+		'pgbench',
+		'-p', $node->port,
+		'-T', $pgb_duration,
+		'-c', $pgb_num_clients,
+		'postgres'
+	],
+	'<'  => \$pgbench_stdin,
+	'>'  => \$pgbench_stdout,
+	'2>' => \$pgbench_stderr
+);
+
+ok($pgbench_process, "pgbench started successfully");
+
+# Allow pgbench to establish connections and start generating load.
+# 
+# TODO: When creating new backends is known to work well with buffer pool
+# resizing, this wait should be removed.
+sleep(1);
+
+# Resize buffer pool to various sizes while pgbench is running in the
+# background.
+# 
+# TODO: These are pseudo-randomly picked sizes, but we can do better.
+my $tests_completed = 0;
+my @buffer_sizes = ('900MB', '500MB', '250MB', '400MB', '120MB', '600MB');
+for my $target_size (@buffer_sizes)
+{
+	# Verify workload generator is still running
+	if (!$pgbench_process->pumpable) {
+		ok(0, "pgbench is still running");
+		last;
+	}
+	
+	apply_and_verify_buffer_change($node, $target_size);
+	$tests_completed++;
+	
+	# Wait for the resized buffer pool to stabilize. If the resized buffer pool
+	# is utilized fully, it might hit any wrongly initialized areas of shared
+	# memory.
+	sleep(2);
+}
+is($tests_completed, scalar(@buffer_sizes), "All buffer sizes were tested");
+
+# Make sure that pgbench can end normally.
+$pgbench_process->signal('TERM');
+IPC::Run::finish $pgbench_process;
+ok(grep { $pgbench_process->result == $_ } (0, 15),  "pgbench exited gracefully");
+
+# Log any error output from pgbench for debugging
+diag("pgbench stderr:\n$pgbench_stderr");
+diag("pgbench stdout:\n$pgbench_stdout");
+
+# Ensure database is still functional after all the buffer changes
+$node->connect_ok("dbname=postgres", 
+	"Database remains accessible after $tests_completed buffer resize operations");
+
+done_testing();
\ No newline at end of file
diff --git a/src/test/meson.build b/src/test/meson.build
index ccc31d6a86a..2a5ba1dec39 100644
--- a/src/test/meson.build
+++ b/src/test/meson.build
@@ -4,6 +4,7 @@ subdir('regress')
 subdir('isolation')
 
 subdir('authentication')
+subdir('buffermgr')
 subdir('postmaster')
 subdir('recovery')
 subdir('subscription')
-- 
2.34.1