BUG #19520: PANIC when concurrently manipulating stored procedures with pg_stat_statements and track_functions =

The following bug has been logged on the website:

Bug reference:      19520
Logged by:          zhanglihui
Email address:      zlh21343@163.com
PostgreSQL version: 19beta1
Operating system:   Ubuntu 25.04
Description:        

=== Configuration ===
Enabled extensions: pg_stat_statements
postgresql.conf:
shared_preload_libraries = 'pg_stat_statements'
track_functions = 'all'

=== Problem Description ===
PostgreSQL server throws PANIC under high concurrent create, CALL and DROP
of stored procedures.
This issue **only reproduces when pg_stat_statements is enabled and
track_functions = all**.
It cannot be triggered if pg_stat_statements is disabled or track_functions
is set to none/pl.

=== Run Steps ===
javac -cp postgresql-42.7.5.jar -d out     src/ConcurrentSqlTest.java
src/ProcCrashReprodure.java
# Terminal 1 — Mixed DDL/DML load test (40 threads, 10000 iterations each)
java -cp out:postgresql-42.7.5.jar ConcurrentSqlTest

# Terminal 2 — Pure CALL load test (20 threads, infinite loop)
java -cp out:postgresql-42.7.5.jar ProcCrashReprodure
# Note: If no PANIC log or core dump is generated after the execution of
Terminal 1, please re-run the command repeatedly until the issue occurs.

PANIC log：
postgresql-2026-06-14_235441.log:2026-06-14 23:54:41.949 CST [691761] PANIC:
XX000: cannot abort transaction 4166281, it was already committed
postgresql-2026-06-14_235931.log:2026-06-14 23:59:31.556 CST [696980] PANIC:
XX000: cannot abort transaction 4641943, it was already committed

(gdb) bt
#0  __pthread_kill_implementation (threadid=<optimized out>, signo=6,
no_tid=0) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (threadid=<optimized out>, signo=6) at
./nptl/pthread_kill.c:89
#2  __GI___pthread_kill (threadid=<optimized out>, signo=signo@entry=6) at
./nptl/pthread_kill.c:100
#3  0x000073e62ec4579e in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#4  0x000073e62ec288cd in __GI_abort () at ./stdlib/abort.c:73
#5  0x000056aeb574138c in errfinish (filename=0x56aeb57ff545 "xact.c",
lineno=1835, funcname=0x56aeb5800d00 <__func__.26> "RecordTransactionAbort")
at elog.c:621
#6  0x000056aeb4faae67 in RecordTransactionAbort (isSubXact=false) at
xact.c:1835
#7  0x000056aeb4fac19f in AbortTransaction () at xact.c:2982
#8  0x000056aeb4facc22 in AbortCurrentTransactionInternal () at xact.c:3553
#9  0x000056aeb4facb93 in AbortCurrentTransaction () at xact.c:3507
#10 0x000056aeb5525552 in PostgresMain (dbname=0x56aedecf30d0 "postgres",
username=0x56aedecf30b0 "zlh_user") at postgres.c:4539
#11 0x000056aeb551b59c in BackendMain (startup_data=0x7fffb9501000,
startup_data_len=24) at backend_startup.c:124
#12 0x000056aeb5405686 in postmaster_child_launch (child_type=B_BACKEND,
child_slot=24, startup_data=0x7fffb9501000, startup_data_len=24,
client_sock=0x7fffb9501060) at launch_backend.c:268
#13 0x000056aeb540c11b in BackendStartup (client_sock=0x7fffb9501060) at
postmaster.c:3627
#14 0x000056aeb540969f in ServerLoop () at postmaster.c:1728
#15 0x000056aeb5408f7c in PostmasterMain (argc=1, argv=0x56aedeca1430) at
postmaster.c:1415
#16 0x000056aeb528ce51 in main (argc=1, argv=0x56aedeca1430) at main.c:231


---- File 1: ConcurrentSqlTest.java (Mixed DDL/DML test, 40 threads × 10000
iterations) ----
import java.sql.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * PostgreSQL concurrent DDL/DML stress test — reproduces backend crash
 * (signal 5 / SIGTRAP) when DROP/CREATE PROCEDURE and CALL PROCEDURE
 * execute concurrently at high concurrency.
 *
 * Compile:  javac -cp postgresql-42.7.5.jar ConcurrentSqlTest.java
 * Run:      java -cp .:postgresql-42.7.5.jar ConcurrentSqlTest
 *
 * Optional env vars: PG_HOST, PG_PORT, PG_DATABASE, PG_USER, PG_PASSWORD
 */
public final class ConcurrentSqlTest {

    // ── connection defaults ──
    private static final String HOST     = envOr("PG_HOST",
"192.168.239.128");
    private static final String PORT     = envOr("PG_PORT",     "5432");
    private static final String DATABASE = envOr("PG_DATABASE", "postgres");
    private static final String USER     = envOr("PG_USER",     "zlh_user");
    private static final String PASSWORD = envOr("PG_PASSWORD",
"Gauss@123");
    private static final String JDBC_URL =
            "jdbc:postgresql://" + HOST + ":" + PORT + "/" + DATABASE;

    // ── test parameters ──
    private static final int THREADS    = 40;
    private static final int ITERATIONS = 10_000;

    // ── SQL batches ──
    private static final String[] SQL = {
        "DROP PROCEDURE IF EXISTS proc_test",
        "CREATE OR REPLACE PROCEDURE proc_test()\n"
                + "LANGUAGE plpgsql\n"
                + "AS $$\n"
                + "BEGIN\n"
                + "END;\n"
                + "$$",
        "CALL proc_test()",
    };

    // ── counters ──
    private static final AtomicInteger totalOk   = new AtomicInteger(0);
    private static final AtomicInteger totalFail = new AtomicInteger(0);

    static {
        try { Class.forName("org.postgresql.Driver"); }
        catch (ClassNotFoundException e) {
            System.err.println("PostgreSQL JDBC driver not found on
classpath");
            System.exit(1);
        }
    }

    public static void main(String[] args) throws InterruptedException {
        System.out.println("=== PostgreSQL concurrent DROP/CREATE/CALL
PROCEDURE ===");
        System.out.println("URL: " + JDBC_URL);
        System.out.println("Threads: " + THREADS + " | Iterations/thread: "
+ ITERATIONS);
        System.out.println("Watch for: backend terminated by signal 5
(SIGTRAP)");
System.out.println("========================================================");

        long t0 = System.currentTimeMillis();
        ExecutorService pool = Executors.newFixedThreadPool(THREADS);
        List<Future<?>> futures = new ArrayList<>();

        for (int i = 0; i < THREADS; i++) {
            final int tid = i;
            futures.add(pool.submit(() -> runWorker(tid)));
        }

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            System.out.println("\nShutting down...");
            pool.shutdownNow();
        }));

        // drain
        for (Future<?> f : futures) {
            try { f.get(); } catch (ExecutionException e) {
                System.err.println("Thread crashed: " +
e.getCause().getMessage());
            } catch (CancellationException ignored) { }
        }
        pool.shutdown();

        long elapsed = System.currentTimeMillis() - t0;
System.out.println("========================================================");
        System.out.printf("Done.  OK: %d  FAIL: %d  Time: %.1fs%n",
                totalOk.get(), totalFail.get(), elapsed / 1000.0);
        System.exit(totalFail.get() > 0 ? 1 : 0);
    }

    // ── single worker (reuses one connection) ──
    private static void runWorker(int tid) {
        int ok = 0, fail = 0;
        Connection c = openConnection();
        for (int i = 0; i < ITERATIONS; i++) {
            if (Thread.currentThread().isInterrupted()) break;
            try {
                for (String sql : SQL) {
                    try (Statement s = c.createStatement()) {
                        s.execute(sql);
                    }
                }
                ok++;
            } catch (SQLException e) {
                fail++;
                if (totalFail.incrementAndGet() <= 10) {
                    System.err.printf("Thread %d loop %d: [%s] %s%n",
                            tid, i, e.getSQLState(),
e.getMessage().replace('\n', ' '));
                }
                // reconnect if server closed the connection (e.g. after
crash)
                try {
                    if (c.isClosed()) c = openConnection();
                } catch (SQLException ignored) { }
            }
        }
        totalOk.addAndGet(ok);
        close(c);
        System.out.printf("Thread %2d: %d ok / %d fail%n", tid, ok, fail);
    }

    private static Connection openConnection() {
        try {
            Connection c = DriverManager.getConnection(JDBC_URL, USER,
PASSWORD);
            c.setAutoCommit(true);
            return c;
        } catch (SQLException e) {
            throw new RuntimeException("Failed to connect: " +
e.getMessage(), e);
        }
    }

    private static void close(Connection c) {
        try { if (c != null) c.close(); } catch (SQLException ignored) { }
    }

    private static String envOr(String key, String def) {
        String v = System.getenv(key);
        return (v != null && !v.trim().isEmpty()) ? v : def;
    }
}


---- File 2: ProcCrashReprodure.java (Pure CALL test, 20 threads, infinite
loop) ----
import java.sql.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * PostgreSQL concurrent CALL stress test — run this together with
 * ConcurrentSqlTest to reproduce backend crash (signal 5 / SIGTRAP).
 *
 * This program does ONLY repeated CALL proc_test() across 20 threads
 * (infinite loop).  ConcurrentSqlTest does DROP → CREATE → CALL in
 * a loop.  Run both simultaneously.
 *
 * Compile:  javac -cp postgresql-42.7.5.jar ProcCrashReprodure.java
 * Run:      java -cp .:postgresql-42.7.5.jar ProcCrashReprodure
 *
 * Optional env vars: PG_HOST, PG_PORT, PG_DATABASE, PG_USER, PG_PASSWORD
 */
public final class ProcCrashReprodure {

    private static final String HOST     = envOr("PG_HOST",
"192.168.239.128");
    private static final String PORT     = envOr("PG_PORT",     "5432");
    private static final String DATABASE = envOr("PG_DATABASE", "postgres");
    private static final String USER     = envOr("PG_USER",     "zlh_user");
    private static final String PASSWORD = envOr("PG_PASSWORD",
"Gauss@123");
    private static final String JDBC_URL =
            "jdbc:postgresql://" + HOST + ":" + PORT + "/" + DATABASE;

    private static final int THREADS = 20;

    private static final AtomicBoolean stop = new AtomicBoolean(false);

    static {
        try { Class.forName("org.postgresql.Driver"); }
        catch (ClassNotFoundException e) {
            System.err.println("PostgreSQL JDBC driver not found on
classpath");
            System.exit(1);
        }
    }

    public static void main(String[] args) {
        System.out.println("=== PostgreSQL CALL stress test (run
ConcurrentSqlTest too) ===");
        System.out.println("URL: " + JDBC_URL);
        System.out.println("Threads: " + THREADS + "  |  Loop: infinite
(Ctrl+C to stop)");
        System.out.println("Watch for: backend terminated by signal 5
(SIGTRAP)");
System.out.println("================================================================");

        ExecutorService pool = Executors.newFixedThreadPool(THREADS);
        CountDownLatch latch = new CountDownLatch(THREADS);

        for (int i = 0; i < THREADS; i++) {
            final int tid = i;
            pool.submit(() -> {
                try {
                    runWorker(tid);
                } finally {
                    latch.countDown();
                }
            });
        }

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            System.out.println("\nShutting down...");
            stop.set(true);
            pool.shutdownNow();
        }));

        try { latch.await(); }
        catch (InterruptedException e) { Thread.currentThread().interrupt();
}

        pool.shutdownNow();
        System.out.println("Done.");
    }

    private static void runWorker(int tid) {
        while (!stop.get()) {
            try (Connection c = DriverManager.getConnection(JDBC_URL, USER,
PASSWORD)) {
                c.setAutoCommit(true);
                while (!stop.get()) {
                    try (CallableStatement cs = c.prepareCall("{ CALL
proc_test() }")) {
                        cs.execute();
                    } catch (SQLException e) {
                        // expected when proc is being dropped/recreated
                        if (c.isClosed()) break;  // reconnect
                    }
                }
            } catch (SQLException e) {
                // connection failure — pause then reconnect
                try { Thread.sleep(100); }
                catch (InterruptedException ie) { break; }
            }
        }
    }

    private static String envOr(String key, String def) {
        String v = System.getenv(key);
        return (v != null && !v.trim().isEmpty()) ? v : def;
    }
}