huge-stringinfo-v3.patch

diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c
index 6d0f3f3..ed9cabd 100644
--- a/src/backend/access/common/heaptuple.c
+++ b/src/backend/access/common/heaptuple.c
@@ -741,7 +741,9 @@ heap_form_tuple(TupleDesc tupleDescriptor,
 	 * Allocate and zero the space needed.  Note that the tuple body and
 	 * HeapTupleData management structure are allocated in one chunk.
 	 */
-	tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
+	tuple = MemoryContextAllocExtended(CurrentMemoryContext,
+									   HEAPTUPLESIZE + len,
+									   MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
 	tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
 
 	/*
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 432b0ca..7419362 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -397,7 +397,7 @@ ReceiveCopyBegin(CopyState cstate)
 			pq_sendint(&buf, format, 2);		/* per-column formats */
 		pq_endmessage(&buf);
 		cstate->copy_dest = COPY_NEW_FE;
-		cstate->fe_msgbuf = makeStringInfo();
+		cstate->fe_msgbuf = makeLongStringInfo();
 	}
 	else if (PG_PROTOCOL_MAJOR(FrontendProtocol) >= 2)
 	{
@@ -1892,7 +1892,7 @@ CopyTo(CopyState cstate)
 	cstate->null_print_client = cstate->null_print;		/* default */
 
 	/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
-	cstate->fe_msgbuf = makeStringInfo();
+	cstate->fe_msgbuf = makeLongStringInfo();
 
 	/* Get info about the columns we need to process. */
 	cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
@@ -2707,8 +2707,8 @@ BeginCopyFrom(ParseState *pstate,
 	cstate->cur_attval = NULL;
 
 	/* Set up variables to avoid per-attribute overhead. */
-	initStringInfo(&cstate->attribute_buf);
-	initStringInfo(&cstate->line_buf);
+	initLongStringInfo(&cstate->attribute_buf);
+	initLongStringInfo(&cstate->line_buf);
 	cstate->line_buf_converted = false;
 	cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
 	cstate->raw_buf_index = cstate->raw_buf_len = 0;
diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c
index 7382e08..0125047 100644
--- a/src/backend/lib/stringinfo.c
+++ b/src/backend/lib/stringinfo.c
@@ -37,6 +37,24 @@ makeStringInfo(void)
 }
 
 /*
+ * makeLongStringInfo
+ *
+ * Same as makeStringInfo for larger strings.
+ */
+StringInfo
+makeLongStringInfo(void)
+{
+	StringInfo	res;
+
+	res = (StringInfo) palloc(sizeof(StringInfoData));
+
+	initLongStringInfo(res);
+
+	return res;
+}
+
+
+/*
  * initStringInfo
  *
  * Initialize a StringInfoData struct (with previously undefined contents)
@@ -47,12 +65,25 @@ initStringInfo(StringInfo str)
 {
 	int			size = 1024;	/* initial default buffer size */
 
-	str->data = (char *) palloc(size);
+	str->data = (char *) palloc(size);	/* no need for "huge" at this point */
 	str->maxlen = size;
+	str->allow_long = false;
 	resetStringInfo(str);
 }
 
 /*
+ * initLongStringInfo
+ * Same as initLongStringInfo for larger strings.
+ */
+void
+initLongStringInfo(StringInfo str)
+{
+	initStringInfo(str);
+	str->allow_long = true;
+}
+
+
+/*
  * resetStringInfo
  *
  * Reset the StringInfo: the data buffer remains valid, but its
@@ -245,6 +276,14 @@ void
 enlargeStringInfo(StringInfo str, int needed)
 {
 	int			newlen;
+	int			limit;
+
+	/*
+	 * Maximum size for strings with allow_long=true.
+	 * It must not exceed INT_MAX, as the StringInfo routines
+	 * expect offsets into the buffer to fit into an int.
+	 */
+	const int max_alloc_long = 0x7fffffff;
 
 	/*
 	 * Guard against out-of-range "needed" values.  Without this, we can get
@@ -252,7 +291,10 @@ enlargeStringInfo(StringInfo str, int needed)
 	 */
 	if (needed < 0)				/* should not happen */
 		elog(ERROR, "invalid string enlargement request size: %d", needed);
-	if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
+
+	/* choose the proper limit and verify this allocation wouldn't exceed it */
+	limit = str->allow_long ? max_alloc_long : MaxAllocSize;
+	if (((Size) needed) >= (limit - (Size) str->len))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("out of memory"),
@@ -261,7 +303,7 @@ enlargeStringInfo(StringInfo str, int needed)
 
 	needed += str->len + 1;		/* total space required now */
 
-	/* Because of the above test, we now have needed <= MaxAllocSize */
+	/* Because of the above test, we now have needed <= limit */
 
 	if (needed <= str->maxlen)
 		return;					/* got enough space already */
@@ -271,19 +313,18 @@ enlargeStringInfo(StringInfo str, int needed)
 	 * for efficiency, double the buffer size each time it overflows.
 	 * Actually, we might need to more than double it if 'needed' is big...
 	 */
-	newlen = 2 * str->maxlen;
-	while (needed > newlen)
-		newlen = 2 * newlen;
 
-	/*
-	 * Clamp to MaxAllocSize in case we went past it.  Note we are assuming
-	 * here that MaxAllocSize <= INT_MAX/2, else the above loop could
-	 * overflow.  We will still have newlen >= needed.
-	 */
-	if (newlen > (int) MaxAllocSize)
-		newlen = (int) MaxAllocSize;
+	newlen = str->maxlen;
+
+	do
+	{
+		if (newlen < limit / 2)		/* prevent integer overflow */
+			newlen = 2 * newlen;
+		else
+			newlen = limit;
+	} while (newlen < needed);		/* can not loop forever, because needed <= limit */
 
-	str->data = (char *) repalloc(str->data, newlen);
+	str->data = (char *) repalloc_huge(str->data, (Size)newlen);
 
 	str->maxlen = newlen;
 }
diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h
index f644067..3d8621f 100644
--- a/src/include/lib/stringinfo.h
+++ b/src/include/lib/stringinfo.h
@@ -30,6 +30,8 @@
  *		cursor	is initialized to zero by makeStringInfo or initStringInfo,
  *				but is not otherwise touched by the stringinfo.c routines.
  *				Some routines use it to scan through a StringInfo.
+ *		allow_long boolean flag indicating whether this StringInfo can allocate
+ *				more than MaxAllocSize bytes.
  *-------------------------
  */
 typedef struct StringInfoData
@@ -38,6 +40,7 @@ typedef struct StringInfoData
 	int			len;
 	int			maxlen;
 	int			cursor;
+	bool		allow_long;
 } StringInfoData;
 
 typedef StringInfoData *StringInfo;
@@ -70,6 +73,7 @@ typedef StringInfoData *StringInfo;
  * Create an empty 'StringInfoData' & return a pointer to it.
  */
 extern StringInfo makeStringInfo(void);
+extern StringInfo makeLongStringInfo(void);
 
 /*------------------------
  * initStringInfo
@@ -79,6 +83,12 @@ extern StringInfo makeStringInfo(void);
 extern void initStringInfo(StringInfo str);
 
 /*------------------------
+ * initLongStringInfo
+ * Same as initStringInfo but for larger strings (up to 2GB)
+ */
+extern void initLongStringInfo(StringInfo str);
+
+/*------------------------
  * resetStringInfo
  * Clears the current content of the StringInfo, if any. The
  * StringInfo remains valid.