v8-master-0001-Fix-exponential-memory-allocation-issue-in.patch

From 81db514f0703c8752b8ce3e7cbe95cc7be5530ce Mon Sep 17 00:00:00 2001
From: Vignesh <vignesh21@gmail.com>
Date: Fri, 30 May 2025 12:06:07 +0530
Subject: [PATCH v8-master] Fix exponential memory allocation issue in logical
 decoding due to invalidation message distribution

Commit 4909b38af0 added logic to distribute invalidation messages from
one transaction to all concurrent in-progress transactions. However,
since the source of each invalidation message was not tracked, transactions
could redundantly re-distribute already propagated messages. This led to an
exponential increase in invalidation messages and, ultimately, memory
allocation failures.

To fix this, a new field is introduced to track distributed invalidation
messages separately. This ensures that only transactions in the pre-existing
list receive the invalidations, preventing redundant propagation.

The accumulated distributed invalidations are now consumed at the end of
transaction processing in `ReorderBufferProcessTXN()`.
---
 .../replication/logical/reorderbuffer.c       | 175 ++++++++++++++----
 src/backend/replication/logical/snapbuild.c   |   5 +-
 src/include/replication/reorderbuffer.h       |  18 ++
 3 files changed, 159 insertions(+), 39 deletions(-)

diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 67655111875..e2e02315639 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -109,6 +109,7 @@
 #include "storage/procarray.h"
 #include "storage/sinval.h"
 #include "utils/builtins.h"
+#include "utils/inval.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
 #include "utils/relfilenumbermap.h"
@@ -472,6 +473,12 @@ ReorderBufferFreeTXN(ReorderBuffer *rb, ReorderBufferTXN *txn)
 		txn->invalidations = NULL;
 	}
 
+	if (txn->distributed_invalidations)
+	{
+		pfree(txn->distributed_invalidations);
+		txn->distributed_invalidations = NULL;
+	}
+
 	/* Reset the toast hash */
 	ReorderBufferToastReset(rb, txn);
 
@@ -2661,7 +2668,14 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn,
 		AbortCurrentTransaction();
 
 		/* make sure there's no cache pollution */
-		ReorderBufferExecuteInvalidations(txn->ninvalidations, txn->invalidations);
+		if (rbtxn_inval_all_cache(txn))
+			InvalidateSystemCaches();
+		else
+		{
+			ReorderBufferExecuteInvalidations(txn->ninvalidations, txn->invalidations);
+			ReorderBufferExecuteInvalidations(txn->ninvalidations_distr,
+											  txn->distributed_invalidations);
+		}
 
 		if (using_subtxn)
 			RollbackAndReleaseCurrentSubTransaction();
@@ -2747,6 +2761,20 @@ ReorderBufferProcessTXN(ReorderBuffer *rb, ReorderBufferTXN *txn,
 			if (stream_started)
 				ReorderBufferMaybeMarkTXNStreamed(rb, txn);
 
+			/*
+			 * Make sure there's no cache pollution. Unlike the PG_TRY section,
+			 * this must be done unconditionally, since processing may fail
+			 * before we reach the invalidation messages. This isn't needed
+			 * in non-rollback cases, as the system cache is cleared either
+			 * by the API's error handling or when the walsender exits.
+			 */
+			if (rbtxn_inval_all_cache(txn))
+				InvalidateSystemCaches();
+			else
+				ReorderBufferExecuteInvalidations(txn->ninvalidations_distr,
+												  txn->distributed_invalidations);
+
+
 			/* Reset the TXN so that it is allowed to stream remaining data. */
 			ReorderBufferResetTXN(rb, txn, snapshot_now,
 								  command_id, prev_lsn,
@@ -3422,58 +3450,57 @@ ReorderBufferAddNewTupleCids(ReorderBuffer *rb, TransactionId xid,
 }
 
 /*
- * Accumulate the invalidations for executing them later.
+ * Helper function to store invalidation messages and queue them as a
+ * change in the reorder buffer. Handles both regular and distributed
+ * invalidations.
  *
- * This needs to be called for each XLOG_XACT_INVALIDATIONS message and
- * accumulates all the invalidation messages in the toplevel transaction, if
- * available, otherwise in the current transaction, as well as in the form of
- * change in reorder buffer.  We require to record it in form of the change
+ * This accumulates all the invalidation messages in the toplevel transaction,
+ * if available, otherwise in the current transaction, as well as in the form
+ * of change in reorder buffer.  We require to record it in form of the change
  * so that we can execute only the required invalidations instead of executing
  * all the invalidations on each CommandId increment.  We also need to
  * accumulate these in the txn buffer because in some cases where we skip
  * processing the transaction (see ReorderBufferForget), we need to execute
  * all the invalidations together.
  */
-void
-ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid,
-							  XLogRecPtr lsn, Size nmsgs,
-							  SharedInvalidationMessage *msgs)
+static void
+ReorderBufferAddInvalidationsCommon(ReorderBuffer *rb, TransactionId xid,
+									XLogRecPtr lsn, Size nmsgs,
+									SharedInvalidationMessage *msgs,
+									ReorderBufferTXN *txn,
+									bool for_inval)
 {
-	ReorderBufferTXN *txn;
 	MemoryContext oldcontext;
 	ReorderBufferChange *change;
 
-	txn = ReorderBufferTXNByXid(rb, xid, true, NULL, lsn, true);
-
 	oldcontext = MemoryContextSwitchTo(rb->context);
 
-	/*
-	 * Collect all the invalidations under the top transaction, if available,
-	 * so that we can execute them all together.  See comments atop this
-	 * function.
-	 */
-	txn = rbtxn_get_toptxn(txn);
+	if (!for_inval || (for_inval && !rbtxn_inval_all_cache(txn)))
+	{
+		uint32	   *ninvalidations;
+		SharedInvalidationMessage **invalidations;
 
-	Assert(nmsgs > 0);
+		ninvalidations = for_inval ? &txn->ninvalidations_distr : &txn->ninvalidations;
+		invalidations = for_inval ? &txn->distributed_invalidations : &txn->invalidations;
 
-	/* Accumulate invalidations. */
-	if (txn->ninvalidations == 0)
-	{
-		txn->ninvalidations = nmsgs;
-		txn->invalidations = (SharedInvalidationMessage *)
-			palloc(sizeof(SharedInvalidationMessage) * nmsgs);
-		memcpy(txn->invalidations, msgs,
-			   sizeof(SharedInvalidationMessage) * nmsgs);
-	}
-	else
-	{
-		txn->invalidations = (SharedInvalidationMessage *)
-			repalloc(txn->invalidations, sizeof(SharedInvalidationMessage) *
-					 (txn->ninvalidations + nmsgs));
+		/* Accumulate invalidations. */
+		if (*ninvalidations == 0)
+		{
+			*ninvalidations = nmsgs;
+			*invalidations = (SharedInvalidationMessage *)
+				palloc(sizeof(SharedInvalidationMessage) * nmsgs);
+			memcpy(*invalidations, msgs, sizeof(SharedInvalidationMessage) * nmsgs);
+		}
+		else
+		{
+			*invalidations = (SharedInvalidationMessage *)
+				repalloc(*invalidations, sizeof(SharedInvalidationMessage) *
+						 (*ninvalidations + nmsgs));
 
-		memcpy(txn->invalidations + txn->ninvalidations, msgs,
-			   nmsgs * sizeof(SharedInvalidationMessage));
-		txn->ninvalidations += nmsgs;
+			memcpy(*invalidations + *ninvalidations, msgs,
+				   nmsgs * sizeof(SharedInvalidationMessage));
+			*ninvalidations += nmsgs;
+		}
 	}
 
 	change = ReorderBufferAllocChange(rb);
@@ -3489,6 +3516,80 @@ ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid,
 	MemoryContextSwitchTo(oldcontext);
 }
 
+/*
+ * Accumulate the invalidations for executing them later.
+ *
+ * This needs to be called for each XLOG_XACT_INVALIDATIONS message.
+ */
+void
+ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid,
+							  XLogRecPtr lsn, Size nmsgs,
+							  SharedInvalidationMessage *msgs)
+{
+	ReorderBufferTXN *txn;
+
+	txn = ReorderBufferTXNByXid(rb, xid, true, NULL, lsn, true);
+
+	/*
+	 * Collect all the invalidations under the top transaction, if available,
+	 * so that we can execute them all together.  See comments atop
+	 * ReorderBufferAddInvalidationsCommon function.
+	 */
+	txn = rbtxn_get_toptxn(txn);
+
+	Assert(nmsgs > 0);
+
+	ReorderBufferAddInvalidationsCommon(rb, xid, lsn, nmsgs, msgs, txn, false);
+}
+
+/*
+ * Accumulate the invalidations sent by committed transactions for executing
+ * them later.
+ *
+ * This needs to be called by committed transactions to distribute the
+ * invalidations to the in-progress transactions.
+ */
+void
+ReorderBufferAddDistributedInvalidtions(ReorderBuffer *rb, TransactionId xid,
+										XLogRecPtr lsn, Size nmsgs,
+										SharedInvalidationMessage *msgs)
+{
+	ReorderBufferTXN *txn;
+	Size		req_mem_size;
+
+	txn = ReorderBufferTXNByXid(rb, xid, true, NULL, lsn, true);
+
+	/*
+	 * Collect all the invalidations under the top transaction, if available,
+	 * so that we can execute them all together.  See comments
+	 * ReorderBufferAddInvalidationsCommon ReorderBufferAddInvalidations.
+	 */
+	txn = rbtxn_get_toptxn(txn);
+
+	Assert(nmsgs > 0);
+
+	req_mem_size = sizeof(SharedInvalidationMessage) * (txn->ninvalidations_distr + nmsgs);
+
+	/*
+	 * If the number of invalidation messages is larger than 8MB, it's more
+	 * efficient to invalidate the entire cache rather than processing each
+	 * message individually.
+	 */
+	if (req_mem_size > (8 * 1024 * 1024) || rbtxn_inval_all_cache(txn))
+	{
+		txn->txn_flags |= RBTXN_INVAL_ALL_CACHE;
+
+		if (txn->distributed_invalidations)
+		{
+			pfree(txn->distributed_invalidations);
+			txn->distributed_invalidations = NULL;
+			txn->ninvalidations_distr = 0;
+		}
+	}
+
+	ReorderBufferAddInvalidationsCommon(rb, xid, lsn, nmsgs, msgs, txn, true);
+}
+
 /*
  * Apply all invalidations we know. Possibly we only need parts at this point
  * in the changestream but we don't know which those are.
diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c
index 0d7bddbe4ed..a2d93f35a2f 100644
--- a/src/backend/replication/logical/snapbuild.c
+++ b/src/backend/replication/logical/snapbuild.c
@@ -807,8 +807,9 @@ SnapBuildDistributeSnapshotAndInval(SnapBuild *builder, XLogRecPtr lsn, Transact
 			{
 				Assert(msgs != NULL);
 
-				ReorderBufferAddInvalidations(builder->reorder, txn->xid, lsn,
-											  ninvalidations, msgs);
+				ReorderBufferAddDistributedInvalidtions(builder->reorder,
+														txn->xid, lsn,
+														ninvalidations, msgs);
 			}
 		}
 	}
diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index 24e88c409ba..a56c3e25d5a 100644
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -176,6 +176,7 @@ typedef struct ReorderBufferChange
 #define RBTXN_SENT_PREPARE			0x0200
 #define RBTXN_IS_COMMITTED			0x0400
 #define RBTXN_IS_ABORTED			0x0800
+#define RBTXN_INVAL_ALL_CACHE		0x1000
 
 #define RBTXN_PREPARE_STATUS_MASK	(RBTXN_IS_PREPARED | RBTXN_SKIPPED_PREPARE | RBTXN_SENT_PREPARE)
 
@@ -215,6 +216,12 @@ typedef struct ReorderBufferChange
 	((txn)->txn_flags & RBTXN_HAS_STREAMABLE_CHANGE) != 0 \
 )
 
+/* Should the complete cache be invalidated? */
+#define rbtxn_inval_all_cache(txn) \
+( \
+	((txn)->txn_flags & RBTXN_INVAL_ALL_CACHE) != 0 \
+)
+
 /*
  * Has this transaction been streamed to downstream?
  *
@@ -422,6 +429,14 @@ typedef struct ReorderBufferTXN
 	uint32		ninvalidations;
 	SharedInvalidationMessage *invalidations;
 
+	/*
+	 * Stores the cache invalidation messages distributed by the committing
+	 * transaction.
+	 */
+	uint32		ninvalidations_distr;
+	SharedInvalidationMessage *distributed_invalidations;
+
+
 	/* ---
 	 * Position in one of two lists:
 	 * * list of subtransactions if we are *known* to be subxact
@@ -738,6 +753,9 @@ extern void ReorderBufferAddNewTupleCids(ReorderBuffer *rb, TransactionId xid,
 										 CommandId cmin, CommandId cmax, CommandId combocid);
 extern void ReorderBufferAddInvalidations(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn,
 										  Size nmsgs, SharedInvalidationMessage *msgs);
+extern void ReorderBufferAddDistributedInvalidtions(ReorderBuffer *rb, TransactionId xid,
+													XLogRecPtr lsn, Size nmsgs,
+													SharedInvalidationMessage *msgs);
 extern void ReorderBufferImmediateInvalidation(ReorderBuffer *rb, uint32 ninvalidations,
 											   SharedInvalidationMessage *invalidations);
 extern void ReorderBufferProcessXid(ReorderBuffer *rb, TransactionId xid, XLogRecPtr lsn);
-- 
2.43.0