v2-0002-Create-an-API-for-inserting-and-deleting-rows-in-.patch

From 975b88d0018949805a4309d6af8820ab43620377 Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Tue, 14 May 2019 17:11:28 -0400
Subject: [PATCH v2 2/3] Create an API for inserting and deleting rows in TOAST
 tables.

This moves much of the non-heap-specific logic from toast_delete and
toast_insert_or_update into a helper functions accessible via a new
header, toast_helper.h.  Using the functions in this module, a table
AM can implement creation and deletion of TOAST table rows with
much less code duplication than was possible heretofore.  Some
table AMs won't want to use the TOAST logic at all, but for those
that do this will make that easier.
---
 src/backend/access/heap/heaptoast.c     | 400 +++---------------------
 src/backend/access/table/Makefile       |   2 +-
 src/backend/access/table/toast_helper.c | 331 ++++++++++++++++++++
 src/include/access/toast_helper.h       | 114 +++++++
 4 files changed, 490 insertions(+), 357 deletions(-)
 create mode 100644 src/backend/access/table/toast_helper.c
 create mode 100644 src/include/access/toast_helper.h

diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index 5d105e3517..3a2118e1d8 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -27,6 +27,7 @@
 #include "access/detoast.h"
 #include "access/heapam.h"
 #include "access/heaptoast.h"
+#include "access/toast_helper.h"
 #include "access/toast_internals.h"
 
 
@@ -40,8 +41,6 @@ void
 toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative)
 {
 	TupleDesc	tupleDesc;
-	int			numAttrs;
-	int			i;
 	Datum		toast_values[MaxHeapAttributeNumber];
 	bool		toast_isnull[MaxHeapAttributeNumber];
 
@@ -64,27 +63,12 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative)
 	 * least one varlena column, by the way.)
 	 */
 	tupleDesc = rel->rd_att;
-	numAttrs = tupleDesc->natts;
 
-	Assert(numAttrs <= MaxHeapAttributeNumber);
+	Assert(tupleDesc->natts <= MaxHeapAttributeNumber);
 	heap_deform_tuple(oldtup, tupleDesc, toast_values, toast_isnull);
 
-	/*
-	 * Check for external stored attributes and delete them from the secondary
-	 * relation.
-	 */
-	for (i = 0; i < numAttrs; i++)
-	{
-		if (TupleDescAttr(tupleDesc, i)->attlen == -1)
-		{
-			Datum		value = toast_values[i];
-
-			if (toast_isnull[i])
-				continue;
-			else if (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value)))
-				toast_delete_datum(rel, value, is_speculative);
-		}
-	}
+	/* Do the real work. */
+	toast_delete_external(rel, toast_values, toast_isnull, is_speculative);
 }
 
 
@@ -113,25 +97,16 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	HeapTuple	result_tuple;
 	TupleDesc	tupleDesc;
 	int			numAttrs;
-	int			i;
-
-	bool		need_change = false;
-	bool		need_free = false;
-	bool		need_delold = false;
-	bool		has_nulls = false;
 
 	Size		maxDataLen;
 	Size		hoff;
 
-	char		toast_action[MaxHeapAttributeNumber];
 	bool		toast_isnull[MaxHeapAttributeNumber];
 	bool		toast_oldisnull[MaxHeapAttributeNumber];
 	Datum		toast_values[MaxHeapAttributeNumber];
 	Datum		toast_oldvalues[MaxHeapAttributeNumber];
-	struct varlena *toast_oldexternal[MaxHeapAttributeNumber];
-	int32		toast_sizes[MaxHeapAttributeNumber];
-	bool		toast_free[MaxHeapAttributeNumber];
-	bool		toast_delold[MaxHeapAttributeNumber];
+	ToastAttrInfo	toast_attr[MaxHeapAttributeNumber];
+	ToastTupleContext	ttc;
 
 	/*
 	 * Ignore the INSERT_SPECULATIVE option. Speculative insertions/super
@@ -160,129 +135,24 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		heap_deform_tuple(oldtup, tupleDesc, toast_oldvalues, toast_oldisnull);
 
 	/* ----------
-	 * Then collect information about the values given
-	 *
-	 * NOTE: toast_action[i] can have these values:
-	 *		' '		default handling
-	 *		'p'		already processed --- don't touch it
-	 *		'x'		incompressible, but OK to move off
-	 *
-	 * NOTE: toast_sizes[i] is only made valid for varlena attributes with
-	 *		toast_action[i] different from 'p'.
+	 * Prepare for toasting
 	 * ----------
 	 */
-	memset(toast_action, ' ', numAttrs * sizeof(char));
-	memset(toast_oldexternal, 0, numAttrs * sizeof(struct varlena *));
-	memset(toast_free, 0, numAttrs * sizeof(bool));
-	memset(toast_delold, 0, numAttrs * sizeof(bool));
-
-	for (i = 0; i < numAttrs; i++)
+	ttc.ttc_rel = rel;
+	ttc.ttc_values = toast_values;
+	ttc.ttc_isnull = toast_isnull;
+	if (oldtup == NULL)
 	{
-		Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
-		struct varlena *old_value;
-		struct varlena *new_value;
-
-		if (oldtup != NULL)
-		{
-			/*
-			 * For UPDATE get the old and new values of this attribute
-			 */
-			old_value = (struct varlena *) DatumGetPointer(toast_oldvalues[i]);
-			new_value = (struct varlena *) DatumGetPointer(toast_values[i]);
-
-			/*
-			 * If the old value is stored on disk, check if it has changed so
-			 * we have to delete it later.
-			 */
-			if (att->attlen == -1 && !toast_oldisnull[i] &&
-				VARATT_IS_EXTERNAL_ONDISK(old_value))
-			{
-				if (toast_isnull[i] || !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
-					memcmp((char *) old_value, (char *) new_value,
-						   VARSIZE_EXTERNAL(old_value)) != 0)
-				{
-					/*
-					 * The old external stored value isn't needed any more
-					 * after the update
-					 */
-					toast_delold[i] = true;
-					need_delold = true;
-				}
-				else
-				{
-					/*
-					 * This attribute isn't changed by this update so we reuse
-					 * the original reference to the old value in the new
-					 * tuple.
-					 */
-					toast_action[i] = 'p';
-					continue;
-				}
-			}
-		}
-		else
-		{
-			/*
-			 * For INSERT simply get the new value
-			 */
-			new_value = (struct varlena *) DatumGetPointer(toast_values[i]);
-		}
-
-		/*
-		 * Handle NULL attributes
-		 */
-		if (toast_isnull[i])
-		{
-			toast_action[i] = 'p';
-			has_nulls = true;
-			continue;
-		}
-
-		/*
-		 * Now look at varlena attributes
-		 */
-		if (att->attlen == -1)
-		{
-			/*
-			 * If the table's attribute says PLAIN always, force it so.
-			 */
-			if (att->attstorage == 'p')
-				toast_action[i] = 'p';
-
-			/*
-			 * We took care of UPDATE above, so any external value we find
-			 * still in the tuple must be someone else's that we cannot reuse
-			 * (this includes the case of an out-of-line in-memory datum).
-			 * Fetch it back (without decompression, unless we are forcing
-			 * PLAIN storage).  If necessary, we'll push it out as a new
-			 * external value below.
-			 */
-			if (VARATT_IS_EXTERNAL(new_value))
-			{
-				toast_oldexternal[i] = new_value;
-				if (att->attstorage == 'p')
-					new_value = heap_tuple_untoast_attr(new_value);
-				else
-					new_value = heap_tuple_fetch_attr(new_value);
-				toast_values[i] = PointerGetDatum(new_value);
-				toast_free[i] = true;
-				need_change = true;
-				need_free = true;
-			}
-
-			/*
-			 * Remember the size of this attribute
-			 */
-			toast_sizes[i] = VARSIZE_ANY(new_value);
-		}
-		else
-		{
-			/*
-			 * Not a varlena attribute, plain storage always
-			 */
-			toast_action[i] = 'p';
-		}
+		ttc.ttc_oldvalues = NULL;
+		ttc.ttc_oldisnull = NULL;
 	}
+	else
+	{
+		ttc.ttc_oldvalues = toast_oldvalues;
+		ttc.ttc_oldisnull = toast_oldisnull;
+	}
+	ttc.ttc_attr = toast_attr;
+	toast_tuple_init(&ttc);
 
 	/* ----------
 	 * Compress and/or save external until data fits into target length
@@ -297,7 +167,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 
 	/* compute header overhead --- this should match heap_form_tuple() */
 	hoff = SizeofHeapTupleHeader;
-	if (has_nulls)
+	if ((ttc.ttc_flags & TOAST_HAS_NULLS) != 0)
 		hoff += BITMAPLEN(numAttrs);
 	hoff = MAXALIGN(hoff);
 	/* now convert to a limit on the tuple data size */
@@ -310,66 +180,21 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	while (heap_compute_data_size(tupleDesc,
 								  toast_values, toast_isnull) > maxDataLen)
 	{
-		int			biggest_attno = -1;
-		int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
-		Datum		old_value;
-		Datum		new_value;
-
-		/*
-		 * Search for the biggest yet unprocessed internal attribute
-		 */
-		for (i = 0; i < numAttrs; i++)
-		{
-			Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
-
-			if (toast_action[i] != ' ')
-				continue;
-			if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i])))
-				continue;		/* can't happen, toast_action would be 'p' */
-			if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i])))
-				continue;
-			if (att->attstorage != 'x' && att->attstorage != 'e')
-				continue;
-			if (toast_sizes[i] > biggest_size)
-			{
-				biggest_attno = i;
-				biggest_size = toast_sizes[i];
-			}
-		}
+		int			biggest_attno;
 
+		biggest_attno = toast_tuple_find_biggest_attribute(&ttc, true, false);
 		if (biggest_attno < 0)
 			break;
 
 		/*
 		 * Attempt to compress it inline, if it has attstorage 'x'
 		 */
-		i = biggest_attno;
-		if (TupleDescAttr(tupleDesc, i)->attstorage == 'x')
-		{
-			old_value = toast_values[i];
-			new_value = toast_compress_datum(old_value);
-
-			if (DatumGetPointer(new_value) != NULL)
-			{
-				/* successful compression */
-				if (toast_free[i])
-					pfree(DatumGetPointer(old_value));
-				toast_values[i] = new_value;
-				toast_free[i] = true;
-				toast_sizes[i] = VARSIZE(DatumGetPointer(toast_values[i]));
-				need_change = true;
-				need_free = true;
-			}
-			else
-			{
-				/* incompressible, ignore on subsequent compression passes */
-				toast_action[i] = 'x';
-			}
-		}
+		if (TupleDescAttr(tupleDesc, biggest_attno)->attstorage == 'x')
+			toast_tuple_try_compression(&ttc, biggest_attno);
 		else
 		{
 			/* has attstorage 'e', ignore on subsequent compression passes */
-			toast_action[i] = 'x';
+			toast_attr[biggest_attno].tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
 		}
 
 		/*
@@ -380,72 +205,26 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		 *
 		 * XXX maybe the threshold should be less than maxDataLen?
 		 */
-		if (toast_sizes[i] > maxDataLen &&
+		if (toast_attr[biggest_attno].tai_size > maxDataLen &&
 			rel->rd_rel->reltoastrelid != InvalidOid)
-		{
-			old_value = toast_values[i];
-			toast_action[i] = 'p';
-			toast_values[i] = toast_save_datum(rel, toast_values[i],
-											   toast_oldexternal[i], options);
-			if (toast_free[i])
-				pfree(DatumGetPointer(old_value));
-			toast_free[i] = true;
-			need_change = true;
-			need_free = true;
-		}
+			toast_tuple_externalize(&ttc, biggest_attno, options);
 	}
 
 	/*
 	 * Second we look for attributes of attstorage 'x' or 'e' that are still
-	 * inline.  But skip this if there's no toast table to push them to.
+	 * inline, and make them external.  But skip this if there's no toast table
+	 * to push them to.
 	 */
 	while (heap_compute_data_size(tupleDesc,
 								  toast_values, toast_isnull) > maxDataLen &&
 		   rel->rd_rel->reltoastrelid != InvalidOid)
 	{
-		int			biggest_attno = -1;
-		int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
-		Datum		old_value;
-
-		/*------
-		 * Search for the biggest yet inlined attribute with
-		 * attstorage equals 'x' or 'e'
-		 *------
-		 */
-		for (i = 0; i < numAttrs; i++)
-		{
-			Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
-
-			if (toast_action[i] == 'p')
-				continue;
-			if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i])))
-				continue;		/* can't happen, toast_action would be 'p' */
-			if (att->attstorage != 'x' && att->attstorage != 'e')
-				continue;
-			if (toast_sizes[i] > biggest_size)
-			{
-				biggest_attno = i;
-				biggest_size = toast_sizes[i];
-			}
-		}
+		int			biggest_attno;
 
+		biggest_attno = toast_tuple_find_biggest_attribute(&ttc, false, false);
 		if (biggest_attno < 0)
 			break;
-
-		/*
-		 * Store this external
-		 */
-		i = biggest_attno;
-		old_value = toast_values[i];
-		toast_action[i] = 'p';
-		toast_values[i] = toast_save_datum(rel, toast_values[i],
-										   toast_oldexternal[i], options);
-		if (toast_free[i])
-			pfree(DatumGetPointer(old_value));
-		toast_free[i] = true;
-
-		need_change = true;
-		need_free = true;
+		toast_tuple_externalize(&ttc, biggest_attno, options);
 	}
 
 	/*
@@ -455,57 +234,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	while (heap_compute_data_size(tupleDesc,
 								  toast_values, toast_isnull) > maxDataLen)
 	{
-		int			biggest_attno = -1;
-		int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
-		Datum		old_value;
-		Datum		new_value;
-
-		/*
-		 * Search for the biggest yet uncompressed internal attribute
-		 */
-		for (i = 0; i < numAttrs; i++)
-		{
-			if (toast_action[i] != ' ')
-				continue;
-			if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i])))
-				continue;		/* can't happen, toast_action would be 'p' */
-			if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i])))
-				continue;
-			if (TupleDescAttr(tupleDesc, i)->attstorage != 'm')
-				continue;
-			if (toast_sizes[i] > biggest_size)
-			{
-				biggest_attno = i;
-				biggest_size = toast_sizes[i];
-			}
-		}
+		int			biggest_attno;
 
+		biggest_attno = toast_tuple_find_biggest_attribute(&ttc, true, true);
 		if (biggest_attno < 0)
 			break;
 
-		/*
-		 * Attempt to compress it inline
-		 */
-		i = biggest_attno;
-		old_value = toast_values[i];
-		new_value = toast_compress_datum(old_value);
-
-		if (DatumGetPointer(new_value) != NULL)
-		{
-			/* successful compression */
-			if (toast_free[i])
-				pfree(DatumGetPointer(old_value));
-			toast_values[i] = new_value;
-			toast_free[i] = true;
-			toast_sizes[i] = VARSIZE(DatumGetPointer(toast_values[i]));
-			need_change = true;
-			need_free = true;
-		}
-		else
-		{
-			/* incompressible, ignore on subsequent compression passes */
-			toast_action[i] = 'x';
-		}
+		toast_tuple_try_compression(&ttc, biggest_attno);
 	}
 
 	/*
@@ -519,54 +254,20 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 								  toast_values, toast_isnull) > maxDataLen &&
 		   rel->rd_rel->reltoastrelid != InvalidOid)
 	{
-		int			biggest_attno = -1;
-		int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
-		Datum		old_value;
-
-		/*--------
-		 * Search for the biggest yet inlined attribute with
-		 * attstorage = 'm'
-		 *--------
-		 */
-		for (i = 0; i < numAttrs; i++)
-		{
-			if (toast_action[i] == 'p')
-				continue;
-			if (VARATT_IS_EXTERNAL(DatumGetPointer(toast_values[i])))
-				continue;		/* can't happen, toast_action would be 'p' */
-			if (TupleDescAttr(tupleDesc, i)->attstorage != 'm')
-				continue;
-			if (toast_sizes[i] > biggest_size)
-			{
-				biggest_attno = i;
-				biggest_size = toast_sizes[i];
-			}
-		}
+		int			biggest_attno;
 
+		biggest_attno = toast_tuple_find_biggest_attribute(&ttc, false, true);
 		if (biggest_attno < 0)
 			break;
 
-		/*
-		 * Store this external
-		 */
-		i = biggest_attno;
-		old_value = toast_values[i];
-		toast_action[i] = 'p';
-		toast_values[i] = toast_save_datum(rel, toast_values[i],
-										   toast_oldexternal[i], options);
-		if (toast_free[i])
-			pfree(DatumGetPointer(old_value));
-		toast_free[i] = true;
-
-		need_change = true;
-		need_free = true;
+		toast_tuple_externalize(&ttc, biggest_attno, options);
 	}
 
 	/*
 	 * In the case we toasted any values, we need to build a new heap tuple
 	 * with the changed values.
 	 */
-	if (need_change)
+	if ((ttc.ttc_flags & TOAST_NEEDS_CHANGE) != 0)
 	{
 		HeapTupleHeader olddata = newtup->t_data;
 		HeapTupleHeader new_data;
@@ -585,7 +286,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		 * whether there needs to be one at all.
 		 */
 		new_header_len = SizeofHeapTupleHeader;
-		if (has_nulls)
+		if ((ttc.ttc_flags & TOAST_HAS_NULLS) != 0)
 			new_header_len += BITMAPLEN(numAttrs);
 		new_header_len = MAXALIGN(new_header_len);
 		new_data_len = heap_compute_data_size(tupleDesc,
@@ -616,26 +317,13 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 						(char *) new_data + new_header_len,
 						new_data_len,
 						&(new_data->t_infomask),
-						has_nulls ? new_data->t_bits : NULL);
+						((ttc.ttc_flags & TOAST_HAS_NULLS) != 0) ?
+						new_data->t_bits : NULL);
 	}
 	else
 		result_tuple = newtup;
 
-	/*
-	 * Free allocated temp values
-	 */
-	if (need_free)
-		for (i = 0; i < numAttrs; i++)
-			if (toast_free[i])
-				pfree(DatumGetPointer(toast_values[i]));
-
-	/*
-	 * Delete external values from the old tuple
-	 */
-	if (need_delold)
-		for (i = 0; i < numAttrs; i++)
-			if (toast_delold[i])
-				toast_delete_datum(rel, toast_oldvalues[i], false);
+	toast_tuple_cleanup(&ttc);
 
 	return result_tuple;
 }
diff --git a/src/backend/access/table/Makefile b/src/backend/access/table/Makefile
index 55a0e5efad..b29df3f333 100644
--- a/src/backend/access/table/Makefile
+++ b/src/backend/access/table/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/access/table
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = table.o tableam.o tableamapi.o
+OBJS = table.o tableam.o tableamapi.o toast_helper.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
new file mode 100644
index 0000000000..ee119cea2d
--- /dev/null
+++ b/src/backend/access/table/toast_helper.c
@@ -0,0 +1,331 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_helper.c
+ *	  Helper functions for table AMs implementing compressed or
+ *    out-of-line storage of varlena attributes.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/common/toast_helper.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/detoast.h"
+#include "access/table.h"
+#include "access/toast_helper.h"
+#include "access/toast_internals.h"
+
+/*
+ * Prepare to TOAST a tuple.
+ *
+ * tupleDesc, toast_values, and toast_isnull are required parameters; they
+ * provide the necessary details about the tuple to be toasted.
+ *
+ * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
+ * tuple; for an update, they should describe the existing tuple.
+ *
+ * All of these arrays should have a length equal to tupleDesc->natts.
+ *
+ * On return, toast_flags and toast_attr will have been initialized.
+ * toast_flags is just a single uint8, but toast_attr is an caller-provided
+ * array with a length equal to tupleDesc->natts.  The caller need not
+ * perform any initialization of the array before calling this function.
+ */
+void
+toast_tuple_init(ToastTupleContext *ttc)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int		i;
+
+	ttc->ttc_flags = 0;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
+		struct varlena *old_value;
+		struct varlena *new_value;
+
+		ttc->ttc_attr[i].tai_colflags = 0;
+		ttc->ttc_attr[i].tai_oldexternal = NULL;
+
+		if (ttc->ttc_oldvalues != NULL)
+		{
+			/*
+			 * For UPDATE get the old and new values of this attribute
+			 */
+			old_value =
+				(struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
+			new_value =
+				(struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
+
+			/*
+			 * If the old value is stored on disk, check if it has changed so
+			 * we have to delete it later.
+			 */
+			if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
+				VARATT_IS_EXTERNAL_ONDISK(old_value))
+			{
+				if (ttc->ttc_isnull[i] ||
+					!VARATT_IS_EXTERNAL_ONDISK(new_value) ||
+					memcmp((char *) old_value, (char *) new_value,
+						   VARSIZE_EXTERNAL(old_value)) != 0)
+				{
+					/*
+					 * The old external stored value isn't needed any more
+					 * after the update
+					 */
+					ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_DELETE_OLD;
+					ttc->ttc_flags |= TOAST_NEEDS_DELETE_OLD;
+				}
+				else
+				{
+					/*
+					 * This attribute isn't changed by this update so we reuse
+					 * the original reference to the old value in the new
+					 * tuple.
+					 */
+					ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+					continue;
+				}
+			}
+		}
+		else
+		{
+			/*
+			 * For INSERT simply get the new value
+			 */
+			new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
+		}
+
+		/*
+		 * Handle NULL attributes
+		 */
+		if (ttc->ttc_isnull[i])
+		{
+			ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+			ttc->ttc_flags |= TOAST_HAS_NULLS;
+			continue;
+		}
+
+		/*
+		 * Now look at varlena attributes
+		 */
+		if (att->attlen == -1)
+		{
+			/*
+			 * If the table's attribute says PLAIN always, force it so.
+			 */
+			if (att->attstorage == 'p')
+				ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+
+			/*
+			 * We took care of UPDATE above, so any external value we find
+			 * still in the tuple must be someone else's that we cannot reuse
+			 * (this includes the case of an out-of-line in-memory datum).
+			 * Fetch it back (without decompression, unless we are forcing
+			 * PLAIN storage).  If necessary, we'll push it out as a new
+			 * external value below.
+			 */
+			if (VARATT_IS_EXTERNAL(new_value))
+			{
+				ttc->ttc_attr[i].tai_oldexternal = new_value;
+				if (att->attstorage == 'p')
+					new_value = heap_tuple_untoast_attr(new_value);
+				else
+					new_value = heap_tuple_fetch_attr(new_value);
+				ttc->ttc_values[i] = PointerGetDatum(new_value);
+				ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
+				ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+			}
+
+			/*
+			 * Remember the size of this attribute
+			 */
+			ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
+		}
+		else
+		{
+			/*
+			 * Not a varlena attribute, plain storage always
+			 */
+			ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+		}
+	}
+}
+
+/*
+ * Find the largest varlena attribute that satisfies certain criteria.
+ *
+ * The relevant column must not be marked TOASTCOL_IGNORE, and if the
+ * for_compression flag is passed as true, it must also not be marked
+ * TOASTCOL_INCOMPRESSIBLE.
+ *
+ * The column must have attstorage 'e' or 'x' if check_main is false, and
+ * must have attstorage 'm' if check_main is true.
+ *
+ * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
+ * if not, no benefit is to be expected by compressing it.
+ *
+ * The return value is the index of the biggest suitable column, or
+ * -1 if there is none.
+ */
+int
+toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
+								   bool for_compression, bool check_main)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int			biggest_attno = -1;
+	int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
+	int32		skip_colflags = TOASTCOL_IGNORE;
+	int			i;
+
+	if (for_compression)
+		skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
+
+		if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
+			continue;
+		if (VARATT_IS_EXTERNAL(DatumGetPointer(ttc->ttc_values[i])))
+			continue;		/* can't happen, toast_action would be 'p' */
+		if (for_compression &&
+			VARATT_IS_COMPRESSED(DatumGetPointer(ttc->ttc_values[i])))
+			continue;
+		if (check_main && att->attstorage != 'm')
+			continue;
+		if (!check_main && att->attstorage != 'x' && att->attstorage != 'e')
+			continue;
+
+		if (ttc->ttc_attr[i].tai_size > biggest_size)
+		{
+			biggest_attno = i;
+			biggest_size = ttc->ttc_attr[i].tai_size;
+		}
+	}
+
+	return biggest_attno;
+}
+
+/*
+ * Try compression for an attribute.
+ *
+ * If we find that the attribute is not compressible, mark it so.
+ */
+void
+toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
+{
+	Datum  *value = &ttc->ttc_values[attribute];
+	Datum	new_value = toast_compress_datum(*value);
+	ToastAttrInfo  *attr = &ttc->ttc_attr[attribute];
+
+	if (DatumGetPointer(new_value) != NULL)
+	{
+		/* successful compression */
+		if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+			pfree(DatumGetPointer(*value));
+		*value = new_value;
+		attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
+		attr->tai_size = VARSIZE(DatumGetPointer(*value));
+		ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+	}
+	else
+	{
+		/* incompressible, ignore on subsequent compression passes */
+		attr->tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
+	}
+}
+
+/*
+ * Move an attribute to external storage.
+ */
+void
+toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
+{
+	Datum  *value = &ttc->ttc_values[attribute];
+	Datum	old_value = *value;
+	ToastAttrInfo  *attr = &ttc->ttc_attr[attribute];
+
+	attr->tai_colflags |= TOASTCOL_IGNORE;
+	*value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
+							  options);
+	if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+		pfree(DatumGetPointer(old_value));
+	attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
+	ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+}
+
+/*
+ * Perform appropriate cleanup after one tuple has been subjected to TOAST.
+ */
+void
+toast_tuple_cleanup(ToastTupleContext *ttc)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+
+	/*
+	 * Free allocated temp values
+	 */
+	if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
+	{
+		int		i;
+
+		for (i = 0; i < numAttrs; i++)
+		{
+			ToastAttrInfo  *attr = &ttc->ttc_attr[i];
+
+			if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+				pfree(DatumGetPointer(ttc->ttc_values[i]));
+		}
+	}
+
+	/*
+	 * Delete external values from the old tuple
+	 */
+	if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
+	{
+		int		i;
+
+		for (i = 0; i < numAttrs; i++)
+		{
+			ToastAttrInfo  *attr = &ttc->ttc_attr[i];
+
+			if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
+				toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
+		}
+	}
+}
+
+/*
+ * Check for external stored attributes and delete them from the secondary
+ * relation.
+ */
+void
+toast_delete_external(Relation rel, Datum *values, bool *isnull,
+					  bool is_speculative)
+{
+	TupleDesc	tupleDesc = rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int			i;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		if (TupleDescAttr(tupleDesc, i)->attlen == -1)
+		{
+			Datum		value = values[i];
+
+			if (isnull[i])
+				continue;
+			else if (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value)))
+				toast_delete_datum(rel, value, is_speculative);
+		}
+	}
+}
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
new file mode 100644
index 0000000000..0fac6cc772
--- /dev/null
+++ b/src/include/access/toast_helper.h
@@ -0,0 +1,114 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_helper.h
+ *	  Helper functions for table AMs implementing compressed or
+ *    out-of-line storage of varlena attributes.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/toast_helper.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TOAST_HELPER_H
+#define TOAST_HELPER_H
+
+#include "utils/rel.h"
+
+/*
+ * Information about one column of a tuple being toasted.
+ *
+ * NOTE: toast_action[i] can have these values:
+ *		' '		default handling
+ *		'p'		already processed --- don't touch it
+ *		'x'		incompressible, but OK to move off
+ *
+ * NOTE: toast_attr[i].tai_size is only made valid for varlena attributes with
+ * toast_action[i] different from 'p'.
+ */
+typedef struct
+{
+	struct varlena *tai_oldexternal;
+	int32		tai_size;
+	uint8		tai_colflags;
+} ToastAttrInfo;
+
+/*
+ * Information about one tuple being toasted.
+ */
+typedef struct
+{
+	/*
+	 * Before calling toast_tuple_init, the caller must initialize the
+	 * following fields.  Each array must have a length equal to
+	 * ttc_rel->rd_att->natts.  The tts_oldvalues and tts_oldisnull fields
+	 * should be NULL in the case of an insert.
+	 */
+	Relation	ttc_rel;			/* the relation that contains the tuple */
+	Datum	   *ttc_values;			/* values from the tuple columns */
+	bool	   *ttc_isnull;			/* null flags for the tuple columns */
+	Datum	   *ttc_oldvalues;		/* values from previous tuple */
+	bool	   *ttc_oldisnull;		/* null flags from previous tuple */
+
+	/*
+	 * Before calling toast_tuple_init, the caller should set tts_attr to
+	 * point to an array of ToastAttrInfo structures of a length equal to
+	 * tts_rel->rd_att->natts.  The contents of the array need not be
+	 * initialized.  ttc_flags also does not need to be initialized.
+	 */
+	uint8		ttc_flags;
+	ToastAttrInfo	*ttc_attr;
+} ToastTupleContext;
+
+/*
+ * Flags indicating the overall state of a TOAST operation.
+ *
+ * TOAST_NEEDS_DELETE_OLD indicates that one or more old TOAST datums need
+ * to be deleted.
+ *
+ * TOAST_NEEDS_FREE indicates that one or more TOAST values need to be freed.
+ *
+ * TOAST_HAS_NULLS indicates that nulls were found in the tuple being toasted.
+ *
+ * TOAST_NEEDS_CHANGE indicates that a new tuple needs to built; in other
+ * words, the toaster did something.
+ */
+#define TOAST_NEEDS_DELETE_OLD				0x0001
+#define TOAST_NEEDS_FREE					0x0002
+#define TOAST_HAS_NULLS						0x0004
+#define TOAST_NEEDS_CHANGE					0x0008
+
+/*
+ * Flags indicating the status of a TOAST operation with respect to a
+ * particular column.
+ *
+ * TOASTCOL_NEEDS_DELETE_OLD indicates that the old TOAST datums for this
+ * column need to be deleted.
+ *
+ * TOASTCOL_NEEDS_FREE indicates that the value for this column needs to
+ * be freed.
+ *
+ * TOASTCOL_IGNORE indicates that the toaster should not further process
+ * this column.
+ *
+ * TOASTCOL_INCOMPRESSIBLE indicates that this column has been found to
+ * be incompressible, but could be moved out-of-line.
+ */
+#define TOASTCOL_NEEDS_DELETE_OLD			TOAST_NEEDS_DELETE_OLD
+#define TOASTCOL_NEEDS_FREE					TOAST_NEEDS_FREE
+#define TOASTCOL_IGNORE						0x0010
+#define TOASTCOL_INCOMPRESSIBLE				0x0020
+
+extern void toast_tuple_init(ToastTupleContext *ttc);
+extern int toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
+								   bool for_compression, bool check_main);
+extern void toast_tuple_try_compression(ToastTupleContext *ttc, int attribute);
+extern void toast_tuple_externalize(ToastTupleContext *ttc, int attribute,
+						int options);
+extern void toast_tuple_cleanup(ToastTupleContext *ttc);
+
+extern void toast_delete_external(Relation rel, Datum *values, bool *isnull,
+					  bool is_speculative);
+
+#endif
-- 
2.17.2 (Apple Git-113)