0002-addendum-to-detoast-v5.patch

diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index fb52bfba3e..f0f84699c5 100644
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -375,18 +375,14 @@ DetoastIterator create_detoast_iterator(struct varlena *attr) {
 			/* If it's compressed, prepare buffer for raw data */
 			iterator->buf = (ToastBuffer *) palloc0(sizeof(ToastBuffer));
 			init_toast_buffer(iterator->buf, toast_pointer.va_rawsize, false);
-			iterator->source = NULL;
 			iterator->ctrlc = 0;
 			iterator->compressed = true;
-			iterator->done = false;
 		}
 		else
 		{
 			iterator->buf = iterator->fetch_datum_iterator->buf;
-			iterator->source = NULL;
 			iterator->ctrlc = 0;
 			iterator->compressed = false;
-			iterator->done = false;
 		}
 	}
 	else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
@@ -410,21 +406,9 @@ DetoastIterator create_detoast_iterator(struct varlena *attr) {
 	{
 		/*
 		 * This is a compressed value inside of the main tuple
+		 * Skip the iterator and just decompress the whole thing.
 		 */
-		iterator = (DetoastIterator) palloc0(sizeof(DetoastIteratorData));
-		iterator->fetch_datum_iterator = NULL;
-		iterator->source = palloc0(sizeof(ToastBuffer));
-		iterator->source->buf = (const char*) attr;
-		iterator->source->position = TOAST_COMPRESS_RAWDATA(attr);
-		iterator->source->limit = (char *)attr + VARSIZE(attr);
-		iterator->source->capacity = iterator->source->limit;
-
-		iterator->buf = palloc0(sizeof(ToastBuffer));
-		init_toast_buffer(iterator->buf, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ, false);
-
-		iterator->ctrlc = 0;
-		iterator->compressed = true;
-		iterator->done = false;
+		return NULL;
 	}
 
 	return iterator;
@@ -447,7 +431,6 @@ bool free_detoast_iterator(DetoastIterator iter) {
 		free_toast_buffer(iter->buf);
 	}
 	free_fetch_datum_iterator(iter->fetch_datum_iterator);
-	free_toast_buffer(iter->source);
 	pfree(iter);
 	return true;
 }
@@ -463,34 +446,26 @@ bool free_detoast_iterator(DetoastIterator iter) {
  * Return -1 when no more data.
  * ----------
  */
-extern int32 detoast_iterate(int32 length, DetoastIterator iter)
+extern int32 detoast_iterate(DetoastIterator iter)
 {
 	if (iter == NULL)
 	{
 		elog(ERROR, "detoast_iterate shouln't be called for NULL iterator");
 	}
 
-	if (iter->buf->limit - iter->buf->position >= length || iter->done)
-	{
-		return iter->buf->limit - iter->buf->position;
-	}
-
-	if (iter->fetch_datum_iterator != NULL)
-	{
-		ToastBuffer *buf = iter->fetch_datum_iterator->buf;
-		FetchDatumIterator fetch_iter = iter->fetch_datum_iterator;
-		while(iter->buf->limit - iter->buf->position < length && !fetch_iter->done) {
-			fetch_datum_iterate(fetch_iter);
-			if (iter->compressed)
-			{
-				pglz_decompress_iterate(buf, iter->buf, iter, -1);
-			}
-		}
+	FetchDatumIterator fetch_iter = iter->fetch_datum_iterator;
+	Assert(fetch_iter != NULL);
 
+	fetch_datum_iterate(fetch_iter);
+	if (fetch_iter->done)
+		/* XXX should this an error? */
 		return iter->buf->limit - iter->buf->position;
-	}
 
-	return pglz_decompress_iterate(iter->source, iter->buf, iter, length);
+	if (iter->compressed)
+		pglz_decompress_iterate(fetch_iter->buf, iter->buf, iter, -1);
+
+	/* XXX Can the caller do any error checking with this? If not, just return void */
+	return iter->buf->limit - iter->buf->position;
 }
 
 
@@ -2665,6 +2640,7 @@ fetch_datum_iterate(FetchDatumIterator iter) {
 		elog(ERROR, "fetch_datum_iterate shouln't be called for NULL iterator");
 	}
 
+	/* XXX maybe should be an assert? */
 	if (iter->done)
 	{
 		return -1;
@@ -2763,6 +2739,8 @@ fetch_datum_iterate(FetchDatumIterator iter) {
 	iter->buf->limit += chunksize;
 
 	iter->nextidx++;
+
+	/* XXX Can the caller do error checking with this? */
 	return chunksize;
 }
 
@@ -2821,7 +2799,8 @@ pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest, DetoastIterator
 	dp = (unsigned char *) dest->limit;
 	destend = (unsigned char *) dest->capacity;
 
-	decompress_all = need_len < 0 ? true : false;
+	/* TODO: remove need_len parameter */
+	decompress_all = true;
 
 	while (sp + 1 < srcend && dp < destend &&
 	      (decompress_all || ((char *)dp - dest->position) < need_len))
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 72e10a3664..ff2a01b2b1 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -1368,9 +1368,9 @@ text_position_next_internal(char *start_ptr, TextPositionState *state, DetoastIt
 		hptr = start_ptr;
 		while (hptr < haystack_end)
 		{
-			if (iter != NULL) {
-				detoast_iterate(hptr - iter->buf->position + 1, iter);
-			}
+			if (iter != NULL && hptr >= iter->buf->limit)
+				detoast_iterate(iter);
+
 			if (*hptr == nchar)
 				return (char *) hptr;
 			hptr++;
@@ -1388,8 +1388,14 @@ text_position_next_internal(char *start_ptr, TextPositionState *state, DetoastIt
 			const char *nptr;
 			const char *p;
 
-			if (iter != NULL) {
-				detoast_iterate(hptr - iter->buf->position + 1, iter);
+			if (iter != NULL)
+			{
+				/*
+				 * The needle could be long enough that we need
+				 * to detoast more than one chunk each time.
+				 */
+				while (hptr >= iter->buf->limit)
+					detoast_iterate(iter);
 			}
 
 			nptr = needle_last;
diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index e60815d2fd..3dbddd6c70 100644
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -69,11 +69,9 @@ typedef struct DetoastIteratorData
 {
 	ToastBuffer 		*buf;
 	FetchDatumIterator	fetch_datum_iterator;
-	ToastBuffer			*source;
 	unsigned char		ctrl;
 	int					ctrlc;
 	bool				compressed;		/* toast value is compressed? */
-	bool				done;			/* iterator exhausted? */
 }			DetoastIteratorData;
 
 typedef struct DetoastIteratorData *DetoastIterator;
@@ -104,7 +102,7 @@ extern bool free_detoast_iterator(DetoastIterator iter);
  * Return -1 when no more data.
  * ----------
  */
-extern int32 detoast_iterate(int32 length, DetoastIterator iter);
+extern int32 detoast_iterate(DetoastIterator iter);
 
 #endif