0001-Refactor-partitonwise-aggregate-signalling.patch

From 71bb618f69872623366ca85ff4799c99c7ca9e1f Mon Sep 17 00:00:00 2001
From: Robert Haas <rhaas@postgresql.org>
Date: Wed, 21 Mar 2018 17:34:17 -0400
Subject: [PATCH] Refactor partitonwise aggregate signalling.

---
 src/backend/optimizer/plan/planner.c | 215 ++++++++++++++---------------------
 src/include/nodes/relation.h         |  26 ++++-
 2 files changed, 105 insertions(+), 136 deletions(-)

diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index f2b1a8bf39..e3d4dcaae7 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -107,14 +107,10 @@ typedef struct
  * GROUPING_CAN_PARTIAL_AGG should be set if the aggregation is of a type
  * for which we support partial aggregation (not, for example, grouping sets).
  * It says nothing about parallel-safety or the availability of suitable paths.
- *
- * GROUPING_CAN_PARTITIONWISE_AGG should be set if it's possible to perform
- * partitionwise grouping and/or aggregation.
  */
 #define GROUPING_CAN_USE_SORT       0x0001
 #define GROUPING_CAN_USE_HASH       0x0002
 #define GROUPING_CAN_PARTIAL_AGG	0x0004
-#define GROUPING_CAN_PARTITIONWISE_AGG	0x0008
 
 /*
  * Data specific to grouping sets
@@ -237,7 +233,8 @@ static RelOptInfo *create_partial_grouping_paths(PlannerInfo *root,
 							  RelOptInfo *grouped_rel,
 							  RelOptInfo *input_rel,
 							  grouping_sets_data *gd,
-							  GroupPathExtraData *extra);
+							  GroupPathExtraData *extra,
+							  bool force_rel_creation);
 static void gather_grouping_paths(PlannerInfo *root, RelOptInfo *rel);
 static bool can_partial_agg(PlannerInfo *root,
 				const AggClauseCosts *agg_costs);
@@ -246,18 +243,13 @@ static void apply_scanjoin_target_to_paths(PlannerInfo *root,
 							   PathTarget *scanjoin_target,
 							   bool scanjoin_target_parallel_safe,
 							   bool modify_in_place);
-static bool can_partitionwise_grouping(PlannerInfo *root,
-						   RelOptInfo *input_rel,
-						   RelOptInfo *grouped_rel,
-						   GroupPathExtraData *extra,
-						   grouping_sets_data *gd,
-						   bool *perform_partial_partitionwise_aggregation);
 static void create_partitionwise_grouping_paths(PlannerInfo *root,
 									RelOptInfo *input_rel,
 									RelOptInfo *grouped_rel,
 									RelOptInfo *partially_grouped_rel,
 									const AggClauseCosts *agg_costs,
 									grouping_sets_data *gd,
+									PartitionwiseAggregateType patype,
 									GroupPathExtraData *extra);
 static bool group_by_has_partkey(RelOptInfo *input_rel,
 					 RelOptInfo *grouped_rel,
@@ -3742,16 +3734,17 @@ create_grouping_paths(PlannerInfo *root,
 		extra.havingQual = parse->havingQual;
 		extra.targetList = parse->targetList;
 		extra.partial_costs_set = false;
-		extra.is_partial_aggregation = false;
 
 		/*
-		 * Check whether we can perform partitionwise grouping and/or
-		 * aggregation.
+		 * Determine whether partitionwise aggregation is in theory possible.
+		 * It can be disabled by the user, and for now, we don't try to
+		 * support grouping sets.  create_ordinary_grouping_paths() will check
+		 * additional conditions, such as whether input_rel is partitioned.
 		 */
-		if (can_partitionwise_grouping(root, input_rel, grouped_rel, &extra,
-									   gd,
-									   &extra.perform_partial_partitionwise_aggregation))
-			extra.flags |= GROUPING_CAN_PARTITIONWISE_AGG;
+		if (enable_partitionwise_aggregate && !parse->groupingSets)
+			extra.patype = PARTITIONWISE_AGGREGATE_FULL;
+		else
+			extra.patype = PARTITIONWISE_AGGREGATE_NONE;
 
 		create_ordinary_grouping_paths(root, input_rel, grouped_rel,
 									   agg_costs, gd, &extra,
@@ -3923,6 +3916,38 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel,
 	Path	   *cheapest_path = input_rel->cheapest_total_path;
 	RelOptInfo *partially_grouped_rel = NULL;
 	double		dNumGroups;
+	PartitionwiseAggregateType patype = PARTITIONWISE_AGGREGATE_NONE;
+
+	/*
+	 * If this is the topmost grouping relation or if the parent relation is
+	 * doing some form of partitionwise aggregation, then we may be able to do
+	 * it at this level also.  However, if the input relation is not
+	 * partitioned, partition-wise aggregate is impossible, and if it is dummy,
+	 * partition-wise aggregate is pointless.
+	 */
+	if (extra->patype != PARTITIONWISE_AGGREGATE_NONE &&
+		input_rel->part_scheme && input_rel->part_rels &&
+		!IS_DUMMY_REL(input_rel))
+	{
+		/*
+		 * If this is the topmost relation or if the parent relation is doing
+		 * full partitionwise aggregation, then we can do full partitionwise
+		 * aggregation provided that the GROUP BY clause contains all of the
+		 * partitioning columns at this level.  Otherwise, we can do at most
+		 * partial partitionwise aggregation.  But if partial aggregation is
+		 * not supported in general then we can't use it for partitionwise
+		 * aggregation either.
+		 */
+		if (extra->patype == PARTITIONWISE_AGGREGATE_FULL &&
+			group_by_has_partkey(input_rel, grouped_rel,
+								 extra->targetList,
+								 root->parse->groupClause))
+			patype = PARTITIONWISE_AGGREGATE_FULL;
+		else if ((extra->flags & GROUPING_CAN_PARTIAL_AGG) != 0)
+			patype = PARTITIONWISE_AGGREGATE_PARTIAL;
+		else
+			patype = PARTITIONWISE_AGGREGATE_NONE;
+	}
 
 	/*
 	 * Before generating paths for grouped_rel, we first generate any possible
@@ -3930,12 +3955,24 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel,
 	 * parallel and non-parallel approaches to grouping.
 	 */
 	if ((extra->flags & GROUPING_CAN_PARTIAL_AGG) != 0)
+	{
+		bool	force_rel_creation;
+
+		/*
+		 * If we're doing partition-wise aggregation at this level, force
+		 * creation of a partially_grouped_rel so we can add partition-wise
+		 * paths to it.
+		 */
+		force_rel_creation = (patype == PARTITIONWISE_AGGREGATE_PARTIAL);
+
 		partially_grouped_rel =
 			create_partial_grouping_paths(root,
 										  grouped_rel,
 										  input_rel,
 										  gd,
-										  extra);
+										  extra,
+										  force_rel_creation);
+	}
 
 	/*
 	 * Set partially_grouped_rel_p so that the caller get the newly created
@@ -3944,13 +3981,13 @@ create_ordinary_grouping_paths(PlannerInfo *root, RelOptInfo *input_rel,
 	*partially_grouped_rel_p = partially_grouped_rel;
 
 	/* Apply partitionwise aggregation technique, if possible. */
-	if ((extra->flags & GROUPING_CAN_PARTITIONWISE_AGG) != 0)
+	if (patype != PARTITIONWISE_AGGREGATE_NONE)
 		create_partitionwise_grouping_paths(root, input_rel, grouped_rel,
 											partially_grouped_rel, agg_costs,
-											gd, extra);
+											gd, patype, extra);
 
 	/* If we are doing partial aggregation only, return. */
-	if (extra->is_partial_aggregation)
+	if (extra->patype == PARTITIONWISE_AGGREGATE_PARTIAL)
 	{
 		Assert(partially_grouped_rel);
 
@@ -6393,7 +6430,8 @@ create_partial_grouping_paths(PlannerInfo *root,
 							  RelOptInfo *grouped_rel,
 							  RelOptInfo *input_rel,
 							  grouping_sets_data *gd,
-							  GroupPathExtraData *extra)
+							  GroupPathExtraData *extra,
+							  bool force_rel_creation)
 {
 	Query	   *parse = root->parse;
 	RelOptInfo *partially_grouped_rel;
@@ -6409,11 +6447,13 @@ create_partial_grouping_paths(PlannerInfo *root,
 
 	/*
 	 * Consider whether we should generate partially aggregated non-partial
-	 * paths.  We can only do this if we have a non-partial path, and in
-	 * addition the caller must have requested it by setting
-	 * extra->is_partial_aggregation.
+	 * paths.  We can only do this if we have a non-partial path, and only if
+	 * the parent of the input rel is performing partial partitionwise
+	 * aggregation.  (Note that extra->patype is the type of partitionwise
+	 * aggregation being used at the parent level, not this level.)
 	 */
-	if (input_rel->pathlist != NIL && extra->is_partial_aggregation)
+	if (input_rel->pathlist != NIL &&
+		extra->patype == PARTITIONWISE_AGGREGATE_PARTIAL)
 		cheapest_total_path = input_rel->cheapest_total_path;
 
 	/*
@@ -6426,16 +6466,12 @@ create_partial_grouping_paths(PlannerInfo *root,
 
 	/*
 	 * If we can't partially aggregate partial paths, and we can't partially
-	 * aggregate non-partial paths, then there may not be any point to
-	 * creating a new RelOptInfo after all.  However, if partitionwise
-	 * aggregate is a possibility and going to perform a partial aggregation,
-	 * then we need to create the RelOptInfo anyway, because the caller may
-	 * want to add Append paths to it.
+	 * aggregate non-partial paths, then don't bother creating the new
+	 * RelOptInfo at all, unless the caller specified force_rel_creation.
 	 */
 	if (cheapest_total_path == NULL &&
 		cheapest_partial_path == NULL &&
-		((extra->flags & GROUPING_CAN_PARTITIONWISE_AGG) == 0 ||
-		 !extra->perform_partial_partitionwise_aggregation))
+		!force_rel_creation)
 		return NULL;
 
 	/*
@@ -6855,77 +6891,6 @@ apply_scanjoin_target_to_paths(PlannerInfo *root,
 	}
 }
 
-/*
- * can_partitionwise_grouping
- *
- * Can we use partitionwise grouping technique?
- */
-static bool
-can_partitionwise_grouping(PlannerInfo *root,
-						   RelOptInfo *input_rel,
-						   RelOptInfo *grouped_rel,
-						   GroupPathExtraData *extra,
-						   grouping_sets_data *gd,
-						   bool *perform_partial_partitionwise_aggregation)
-{
-	Query	   *parse = root->parse;
-
-	*perform_partial_partitionwise_aggregation = false;
-
-	/* No, if user disabled partitionwise aggregation. */
-	if (!enable_partitionwise_aggregate)
-		return false;
-
-	/*
-	 * Currently, grouping sets plan does not work with an inheritance subtree
-	 * (see notes in create_groupingsets_plan). Moreover, grouping sets
-	 * implies multiple group by clauses, each of which may not have all
-	 * partition keys. Those sets which have all partition keys will be
-	 * computed completely for each partition, but others will require partial
-	 * aggregation. We will need to apply partitionwise aggregation at each
-	 * derived group by clause and not as a whole-sale strategy.  Due to this
-	 * we won't be able to compute "whole" grouping sets here and thus bail
-	 * out.
-	 */
-	if (parse->groupingSets || gd)
-		return false;
-
-	/*
-	 * Nothing to do, if the input relation is not partitioned or it has no
-	 * partitioned relations.
-	 */
-	if (!input_rel->part_scheme || !input_rel->part_rels)
-		return false;
-
-	/* Nothing to do, if the input relation itself is dummy. */
-	if (IS_DUMMY_REL(input_rel))
-		return false;
-
-	/*
-	 * If partition keys are part of group by clauses, then we can do full
-	 * partitionwise aggregation.  Otherwise need to calculate partial
-	 * aggregates for each partition and combine them.
-	 *
-	 * However, if caller forces to perform partial aggregation, then do that
-	 * unconditionally.
-	 */
-	*perform_partial_partitionwise_aggregation = (extra->is_partial_aggregation ||
-												  !group_by_has_partkey(input_rel,
-																		grouped_rel,
-																		extra->targetList,
-																		parse->groupClause));
-
-	/*
-	 * If we need to perform partial aggregation for every child but cannot
-	 * compute partial aggregates, no partitionwise grouping is possible.
-	 */
-	if (*perform_partial_partitionwise_aggregation &&
-		(extra->flags & GROUPING_CAN_PARTIAL_AGG) == 0)
-		return false;
-
-	return true;
-}
-
 /*
  * create_partitionwise_grouping_paths
  *
@@ -6951,25 +6916,22 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 									RelOptInfo *partially_grouped_rel,
 									const AggClauseCosts *agg_costs,
 									grouping_sets_data *gd,
+									PartitionwiseAggregateType patype,
 									GroupPathExtraData *extra)
 {
-	int			nparts;
+	int			nparts = input_rel->nparts;
 	int			cnt_parts;
 	RelOptInfo **part_rels;
 	List	   *grouped_live_children = NIL;
 	List	   *partially_grouped_live_children = NIL;
 	PathTarget *target = extra->target;
 
-	nparts = input_rel->nparts;
-	part_rels = (RelOptInfo **) palloc(nparts * sizeof(RelOptInfo *));
-
-	/*
-	 * If partial partitionwise aggregation needs to be performed, then we
-	 * must have created a partially_grouped_rel already.
-	 */
-	Assert(!extra->perform_partial_partitionwise_aggregation ||
+	Assert(patype != PARTITIONWISE_AGGREGATE_NONE);
+	Assert(patype != PARTITIONWISE_AGGREGATE_PARTIAL ||
 		   partially_grouped_rel != NULL);
 
+	part_rels = (RelOptInfo **) palloc(nparts * sizeof(RelOptInfo *));
+
 	/*
 	 * For full aggregation or grouping, each partition produces a disjoint
 	 * groups which can simply be appended and thus we can say that the child
@@ -6977,7 +6939,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 	 * the partitioning details for this grouped rel. In case of a partial
 	 * aggregation, this is not true.
 	 */
-	if (!extra->perform_partial_partitionwise_aggregation)
+	if (patype == PARTITIONWISE_AGGREGATE_FULL)
 	{
 		grouped_rel->part_scheme = input_rel->part_scheme;
 		grouped_rel->nparts = nparts;
@@ -7026,8 +6988,12 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 																 nappinfos,
 																 appinfos);
 
-		/* Is partially aggregated result expected from every child? */
-		child_extra.is_partial_aggregation = extra->perform_partial_partitionwise_aggregation;
+		/*
+		 * extra->patype was the value computed for our parent rel; patype
+		 * is the value for this relation.  For the child, our value is it's
+		 * parent rel's value.
+		 */
+		child_extra.patype = patype;
 
 		/*
 		 * Create grouping relation to hold fully aggregated grouping and/or
@@ -7046,17 +7012,6 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 			continue;
 		}
 
-		/*
-		 * Check whether we can perform partitionwise grouping and/or
-		 * aggregation on this child grouped rel.
-		 */
-		if (can_partitionwise_grouping(root, child_input_rel,
-									   child_grouped_rel, &child_extra, gd,
-									   &child_extra.perform_partial_partitionwise_aggregation))
-			child_extra.flags |= GROUPING_CAN_PARTITIONWISE_AGG;
-		else
-			child_extra.flags &= ~GROUPING_CAN_PARTITIONWISE_AGG;
-
 		/*
 		 * Copy pathtarget from underneath scan/join as we are modifying it
 		 * and translate its Vars with respect to this appendrel.  We use
@@ -7090,7 +7045,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 													  child_partially_grouped_rel);
 		}
 
-		if (!extra->perform_partial_partitionwise_aggregation)
+		if (patype == PARTITIONWISE_AGGREGATE_FULL)
 		{
 			part_rels[cnt_parts] = child_grouped_rel;
 			grouped_live_children = lappend(grouped_live_children,
@@ -7134,7 +7089,7 @@ create_partitionwise_grouping_paths(PlannerInfo *root,
 	 * Now, create append rel for all grouped children and stick them into the
 	 * grouped_rel.
 	 */
-	if (!extra->perform_partial_partitionwise_aggregation)
+	if (patype == PARTITIONWISE_AGGREGATE_FULL)
 		add_paths_to_append_rel(root, grouped_rel, grouped_live_children);
 }
 
diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h
index a920b30def..f2883de1f2 100644
--- a/src/include/nodes/relation.h
+++ b/src/include/nodes/relation.h
@@ -2295,6 +2295,24 @@ typedef struct JoinPathExtraData
 	Relids		param_source_rels;
 } JoinPathExtraData;
 
+/*
+ * What kind of partitionwise aggregation is in use?
+ *
+ * PARTITIONWISE_AGGREGATE_NONE: Not used.
+ *
+ * PARTITIONWISE_AGGREGATE_FULL: Aggregate each partition separately, and
+ * append the results.
+ *
+ * PARTITIONWISE_AGGREGATE_PARTIAL: Partially aggregate each partition
+ * separately, append the results, and then finalize aggregation.
+ */
+typedef enum
+{
+	PARTITIONWISE_AGGREGATE_NONE,
+	PARTITIONWISE_AGGREGATE_FULL,
+	PARTITIONWISE_AGGREGATE_PARTIAL
+} PartitionwiseAggregateType;
+
 /*
  * Struct for extra information passed to subroutines of create_grouping_paths
  *
@@ -2307,9 +2325,7 @@ typedef struct JoinPathExtraData
  * target_parallel_safe is true if target is parallel safe.
  * havingQual gives list of quals to be applied post aggregation.
  * targetList gives list of columns to be projected.
- * is_partial_aggregation is true if doing partial aggregation.
- * perform_partial_partitionwise_aggregation is true if child needs to perform
- * 		partial partitionwise aggregation.
+ * patype is the type of partitionwise aggregation that is being performed.
  */
 typedef struct
 {
@@ -2327,9 +2343,7 @@ typedef struct
 	bool		target_parallel_safe;
 	Node	   *havingQual;
 	List	   *targetList;
-
-	bool		is_partial_aggregation;
-	bool		perform_partial_partitionwise_aggregation;
+	PartitionwiseAggregateType patype;
 } GroupPathExtraData;
 
 /*
-- 
2.14.3 (Apple Git-98)