sortmem_grow-v5.patch

diff src/backend/utils/sort/tuplesort.c
index ce27e40..04a4cbf
*** a/src/backend/utils/sort/tuplesort.c
--- b/src/backend/utils/sort/tuplesort.c
*************** struct Tuplesortstate
*** 276,281 ****
--- 276,282 ----
  	SortTuple  *memtuples;		/* array of SortTuple structs */
  	int			memtupcount;	/* number of tuples currently present */
  	int			memtupsize;		/* allocated length of memtuples array */
+ 	bool		growmemtuples;	/* memtuples' growth still underway */
  
  	/*
  	 * While building initial runs, this is the current output run number
*************** tuplesort_begin_common(int workMem, bool
*** 570,575 ****
--- 571,577 ----
  
  	state->memtupcount = 0;
  	state->memtupsize = 1024;	/* initial guess */
+ 	state->growmemtuples = true;
  	state->memtuples = (SortTuple *) palloc(state->memtupsize * sizeof(SortTuple));
  
  	USEMEM(state, GetMemoryChunkSpace(state->memtuples));
*************** tuplesort_end(Tuplesortstate *state)
*** 957,991 ****
   * Grow the memtuples[] array, if possible within our memory constraint.
   * Return TRUE if able to enlarge the array, FALSE if not.
   *
!  * At each increment we double the size of the array.  When we are short
!  * on memory we could consider smaller increases, but because availMem
!  * moves around with tuple addition/removal, this might result in thrashing.
!  * Small increases in the array size are likely to be pretty inefficient.
   */
  static bool
  grow_memtuples(Tuplesortstate *state)
  {
  	/*
! 	 * We need to be sure that we do not cause LACKMEM to become true, else
! 	 * the space management algorithm will go nuts.  We assume here that the
! 	 * memory chunk overhead associated with the memtuples array is constant
! 	 * and so there will be no unexpected addition to what we ask for.	(The
! 	 * minimum array size established in tuplesort_begin_common is large
! 	 * enough to force palloc to treat it as a separate chunk, so this
! 	 * assumption should be good.  But let's check it.)
  	 */
  	if (state->availMem <= (long) (state->memtupsize * sizeof(SortTuple)))
! 		return false;
  
  	/*
  	 * On a 64-bit machine, allowedMem could be high enough to get us into
  	 * trouble with MaxAllocSize, too.
  	 */
! 	if ((Size) (state->memtupsize * 2) >= MaxAllocSize / sizeof(SortTuple))
! 		return false;
  
  	FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
! 	state->memtupsize *= 2;
  	state->memtuples = (SortTuple *)
  		repalloc(state->memtuples,
  				 state->memtupsize * sizeof(SortTuple));
--- 959,1050 ----
   * Grow the memtuples[] array, if possible within our memory constraint.
   * Return TRUE if able to enlarge the array, FALSE if not.
   *
!  * At each increment we double the size of the array.  When we are short on
!  * memory we do attempt one last, smaller increase.  This only happens at most
!  * once, since availMem moves around with tuple addition/removal. To do othewise
!  * might result in thrashing.  This is nothing more than a last-ditch effort to
!  * avoid exceeding allowedMem, an undesirable outcome if avoidable.
   */
  static bool
  grow_memtuples(Tuplesortstate *state)
  {
+ 	int			newmemtupsize;
+ 	int			memtupsize = state->memtupsize;
+ 	long		memNowUsed = state->allowedMem - state->availMem;
+ 
+ 	/* This may not be our first time through */
+ 	if (!state->growmemtuples)
+ 		return false;
+ 
  	/*
! 	 * We need to be sure that we do not cause LACKMEM to become true, else the
! 	 * space management algorithm will go nuts.
! 	 */
! 	if (memNowUsed <= state->availMem)
! 	{
! 		newmemtupsize = memtupsize * 2;
! 	}
! 	else
! 	{
! 		uint64		allowedMem = state->allowedMem;
! 
! 		/*
! 		 * For this last increment, abandon doubling strategy.
! 		 *
! 		 * To make sure LACKMEM doesn't become true, we can't increase
! 		 * memtupsize by more than state->availMem / sizeof(SortTuple)
! 		 * elements.  In practice, we want to increase it by considerably less,
! 		 * because we need to leave some space for the tuples to which the new
! 		 * array slots will refer.  We assume the new tuples will be about the
! 		 * same size as the tuples we've already seen, and thus use the known
! 		 * size (in bytes) of the tuples seen so far to estimate an appropriate
! 		 * new size for the memtuples array.  The optimal value might be higher
! 		 * or lower than we estimate, but it's hard to know that in advance.
! 		 *
! 		 * In any case, we're definitely safe against enlarging the array so
! 		 * much that LACKMEM becomes true, because the memory currently used
! 		 * includes the present array; thus, there would be enough allowedMem
! 		 * for the new array elements even if no other memory were currently
! 		 * used.
! 		 *
! 		 * allowedMem has been converted to uint64 to prevent overflow on
! 		 * platforms where long is only 32 bits wide.  Even still, the first
! 		 * argument to Max() below could overflow.
! 		 *
! 		 * XXX: This approach could prevent us from allocating a very large
! 		 * amount of memory that is still within allowedMem in some cases.
! 		 * However, the MaxAllocSize limitation would prevent such an
! 		 * allocation in that situation.
! 		 */
! 		state->growmemtuples = false;
! 		newmemtupsize = Min(Max(memtupsize * allowedMem / memNowUsed,
! 								memtupsize),
! 							memtupsize * 2);
! 	}
! 
! 	/*
! 	 * We assume here that the memory chunk overhead associated with the
! 	 * memtuples array is constant and so there will be no unexpected addition
! 	 * to what we ask for.	(The minimum array size established in
! 	 * tuplesort_begin_common is large enough to force palloc to treat it as a
! 	 * separate chunk, so this assumption should be good.  But let's check it.)
  	 */
  	if (state->availMem <= (long) (state->memtupsize * sizeof(SortTuple)))
! 		goto noalloc;
  
  	/*
  	 * On a 64-bit machine, allowedMem could be high enough to get us into
  	 * trouble with MaxAllocSize, too.
  	 */
! 	if ((Size) (newmemtupsize) >= MaxAllocSize / sizeof(SortTuple))
! 		goto noalloc;
! 
! 	/* Avoid redundant repalloc */
! 	if (newmemtupsize <= memtupsize)
! 		goto noalloc;
  
  	FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
! 	state->memtupsize = newmemtupsize;
  	state->memtuples = (SortTuple *)
  		repalloc(state->memtuples,
  				 state->memtupsize * sizeof(SortTuple));
*************** grow_memtuples(Tuplesortstate *state)
*** 993,998 ****
--- 1052,1061 ----
  	if (LACKMEM(state))
  		elog(ERROR, "unexpected out-of-memory situation during sort");
  	return true;
+ 
+ noalloc:
+ 	state->growmemtuples = false;
+ 	return false;
  }
  
  /*
diff src/backend/utils/sort/tuplestore.c
index 1b1cf35..743e578
*** a/src/backend/utils/sort/tuplestore.c
--- b/src/backend/utils/sort/tuplestore.c
*************** tuplestore_puttuple_common(Tuplestoresta
*** 632,639 ****
  			if (state->memtupcount >= state->memtupsize - 1)
  			{
  				/*
! 				 * See grow_memtuples() in tuplesort.c for the rationale
! 				 * behind these two tests.
  				 */
  				if (state->availMem > (long) (state->memtupsize * sizeof(void *)) &&
  					(Size) (state->memtupsize * 2) < MaxAllocSize / sizeof(void *))
--- 632,640 ----
  			if (state->memtupcount >= state->memtupsize - 1)
  			{
  				/*
! 				 * See grow_memtuples() in tuplesort.c for the rationale behind
! 				 * these two tests.  Note that that module uses a slightly more
! 				 * sophisticated strategy for sizing its memtuples array.
  				 */
  				if (state->availMem > (long) (state->memtupsize * sizeof(void *)) &&
  					(Size) (state->memtupsize * 2) < MaxAllocSize / sizeof(void *))