vacuum_mem_estimate.v1.patch

diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c
index b5fb325..478ec8c 100644
--- a/src/backend/commands/vacuumlazy.c
+++ b/src/backend/commands/vacuumlazy.c
@@ -88,9 +88,13 @@
 /*
  * Guesstimation of number of dead tuples per page.  This is used to
  * provide an upper limit to memory allocated when vacuuming small
- * tables.
+ * tables. The guesstimation becomes ineffective as table size grows
+ * so we use a heuristic to say that beyond LAZY_ESTIMATION_THRESHOLD
+ * we need better estimates and will use more precise pg_stats
+ * data after that point, noting that it is still only an estimate.
  */
 #define LAZY_ALLOC_TUPLES		MaxHeapTuplesPerPage
+#define LAZY_ESTIMATION_THRESHOLD	1000
 
 /*
  * Before we consider skipping a page that's marked as clean in
@@ -117,8 +121,8 @@ typedef struct LVRelStats
 	BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */
 	/* List of TIDs of tuples we intend to delete */
 	/* NB: this list is ordered by TID address */
-	int			num_dead_tuples;	/* current # of entries */
-	int			max_dead_tuples;	/* # slots allocated in array */
+	long			num_dead_tuples;	/* current # of entries */
+	long			max_dead_tuples;	/* # slots allocated in array */
 	ItemPointer dead_tuples;	/* array of ItemPointerData */
 	int			num_index_scans;
 	TransactionId latestRemovedXid;
@@ -154,7 +158,7 @@ static bool should_attempt_truncation(LVRelStats *vacrelstats);
 static void lazy_truncate_heap(Relation onerel, LVRelStats *vacrelstats);
 static BlockNumber count_nondeletable_pages(Relation onerel,
 						 LVRelStats *vacrelstats);
-static void lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks);
+static void lazy_space_alloc(Relation onerel, LVRelStats *vacrelstats, BlockNumber relblocks);
 static void lazy_record_dead_tuple(LVRelStats *vacrelstats,
 					   ItemPointer itemptr);
 static bool lazy_tid_reaped(ItemPointer itemptr, void *state);
@@ -492,7 +496,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats,
 	vacrelstats->nonempty_pages = 0;
 	vacrelstats->latestRemovedXid = InvalidTransactionId;
 
-	lazy_space_alloc(vacrelstats, nblocks);
+	lazy_space_alloc(onerel, vacrelstats, nblocks);
 	frozen = palloc(sizeof(xl_heap_freeze_tuple) * MaxHeapTuplesPerPage);
 
 	/* Report that we're scanning the heap, advertising total # of blocks */
@@ -1597,7 +1601,7 @@ lazy_vacuum_index(Relation indrel,
 							   lazy_tid_reaped, (void *) vacrelstats);
 
 	ereport(elevel,
-			(errmsg("scanned index \"%s\" to remove %d row versions",
+			(errmsg("scanned index \"%s\" to remove %ld row versions",
 					RelationGetRelationName(indrel),
 					vacrelstats->num_dead_tuples),
 			 errdetail("%s.", pg_rusage_show(&ru0))));
@@ -1941,7 +1945,7 @@ count_nondeletable_pages(Relation onerel, LVRelStats *vacrelstats)
  * See the comments at the head of this file for rationale.
  */
 static void
-lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks)
+lazy_space_alloc(Relation onerel, LVRelStats *vacrelstats, BlockNumber relblocks)
 {
 	long		maxtuples;
 	int			vac_work_mem = IsAutoVacuumWorkerProcess() &&
@@ -1950,13 +1954,52 @@ lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks)
 
 	if (vacrelstats->hasindex)
 	{
+		long	tuple_estimate = 0;
+
+		/*
+		 * We need to allocate an array to track dead tuples, so apply initial
+		 * criteria based upon user preferences and limits.
+		 */
 		maxtuples = (vac_work_mem * 1024L) / sizeof(ItemPointerData);
-		maxtuples = Min(maxtuples, INT_MAX);
-		maxtuples = Min(maxtuples, MaxAllocSize / sizeof(ItemPointerData));
+		maxtuples = Min(maxtuples, LONG_MAX);
+		maxtuples = Min(maxtuples, MaxAllocHugeSize / sizeof(ItemPointerData));
+
+		/*
+		 * The amount of memory used for tracking dead tuples will be
+		 * limited by the number of blocks. For large tables this limit
+		 * becomes ineffective in many cases, so we need to more
+		 * accurately estimate the number of rows likely to be found
+		 * during the VACUUM, if we are scanning indexes. We could
+		 * use this in all cases, but the stats lookup is slow and
+		 * we don't gain much from it for smaller tables.
+		 */
+		if (relblocks > LAZY_ESTIMATION_THRESHOLD)
+		{
+			PgStat_StatTabEntry *tabentry = NULL;
+
+			tabentry = pgstat_fetch_stat_tabentry(RelationGetRelid(onerel));
+			if (PointerIsValid(tabentry))
+				tuple_estimate = tabentry->n_dead_tuples;
+		}
 
-		/* curious coding here to ensure the multiplication can't overflow */
-		if ((BlockNumber) (maxtuples / LAZY_ALLOC_TUPLES) > relblocks)
-			maxtuples = relblocks * LAZY_ALLOC_TUPLES;
+		/*
+		 * Apply the information we have about the table to further reduce
+		 * the size of the allocation so we do not overallocate memory.
+		 */
+		if (tuple_estimate > 0)
+		{
+			/*
+			 * We have a tuple estimate, but allow 10% slop for timing and
+			 * other issues affecting the accuracy.
+			 */
+			maxtuples = Min(maxtuples, (long) tuple_estimate * 1.1);
+		}
+		else
+		{
+			/* curious coding here to ensure the multiplication can't overflow */
+			if ((BlockNumber) (maxtuples / LAZY_ALLOC_TUPLES) > relblocks)
+				maxtuples = relblocks * LAZY_ALLOC_TUPLES;
+		}
 
 		/* stay sane if small maintenance_work_mem */
 		maxtuples = Max(maxtuples, MaxHeapTuplesPerPage);
@@ -1967,9 +2010,10 @@ lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks)
 	}
 
 	vacrelstats->num_dead_tuples = 0;
-	vacrelstats->max_dead_tuples = (int) maxtuples;
+	vacrelstats->max_dead_tuples = (long) maxtuples;
 	vacrelstats->dead_tuples = (ItemPointer)
-		palloc(maxtuples * sizeof(ItemPointerData));
+		MemoryContextAllocHuge(CurrentMemoryContext,
+							maxtuples * sizeof(ItemPointerData));
 }
 
 /*