0002-headline-cosmetic-improvements.patch

diff --git a/src/backend/tsearch/wparser_def.c b/src/backend/tsearch/wparser_def.c
index 898466f..3e9d1f8 100644
--- a/src/backend/tsearch/wparser_def.c
+++ b/src/backend/tsearch/wparser_def.c
@@ -2064,6 +2064,20 @@ mark_fragment(HeadlineParsedText *prs, int highlight, int startpos, int endpos)
 	}
 }
 
+/*
+ * Macros useful in headline selection.  These rely on availability of
+ * "HeadlineParsedText *prs" describing some text, and "int shortword"
+ * describing the "short word" length parameter.
+ */
+
+/* Interesting words are non-repeated search terms */
+#define INTERESTINGWORD(j) \
+	(prs->words[j].item && !prs->words[j].repeated)
+
+/* Don't want to end at a non-word or a short word */
+#define BADENDPOINT(j) \
+	(NOENDTOKEN(prs->words[j].type) || prs->words[j].len <= shortword)
+
 typedef struct
 {
 	int32		startpos;
@@ -2091,7 +2105,7 @@ get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos,
 	for (i = *startpos; i <= *endpos; i++)
 	{
 		*startpos = i;
-		if (prs->words[i].item && !prs->words[i].repeated)
+		if (INTERESTINGWORD(i))
 			break;
 	}
 	/* cut endpos to have only max_words */
@@ -2101,7 +2115,7 @@ get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos,
 	{
 		if (!NONWORDTOKEN(prs->words[i].type))
 			*curlen += 1;
-		if (prs->words[i].item && !prs->words[i].repeated)
+		if (INTERESTINGWORD(i))
 			*poslen += 1;
 	}
 	/* if the cover was cut then move back endpos to a query item */
@@ -2111,7 +2125,7 @@ get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos,
 		for (i = *endpos; i >= *startpos; i--)
 		{
 			*endpos = i;
-			if (prs->words[i].item && !prs->words[i].repeated)
+			if (INTERESTINGWORD(i))
 				break;
 			if (!NONWORDTOKEN(prs->words[i].type))
 				*curlen -= 1;
@@ -2197,8 +2211,9 @@ mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, int highlight,
 		for (i = 0; i < numcovers; i++)
 		{
 			if (!covers[i].in && !covers[i].excluded &&
-				(maxitems < covers[i].poslen || (maxitems == covers[i].poslen
-												 && minwords > covers[i].curlen)))
+				(maxitems < covers[i].poslen ||
+				 (maxitems == covers[i].poslen &&
+				  minwords > covers[i].curlen)))
 			{
 				maxitems = covers[i].poslen;
 				minwords = covers[i].curlen;
@@ -2235,8 +2250,8 @@ mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, int highlight,
 					}
 					posmarker = i;
 				}
-				/* cut back startpos till we find a non short token */
-				for (i = posmarker; i < startpos && (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword); i++)
+				/* cut back startpos till we find a good endpoint */
+				for (i = posmarker; i < startpos && BADENDPOINT(i); i++)
 				{
 					if (!NONWORDTOKEN(prs->words[i].type))
 						curlen--;
@@ -2250,8 +2265,8 @@ mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, int highlight,
 						curlen++;
 					posmarker = i;
 				}
-				/* cut back endpos till we find a non-short token */
-				for (i = posmarker; i > endpos && (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword); i--)
+				/* cut back endpos till we find a good endpoint */
+				for (i = posmarker; i > endpos && BADENDPOINT(i); i--)
 				{
 					if (!NONWORDTOKEN(prs->words[i].type))
 						curlen--;
@@ -2267,7 +2282,11 @@ mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, int highlight,
 			/* exclude overlapping covers */
 			for (i = 0; i < numcovers; i++)
 			{
-				if (i != minI && ((covers[i].startpos >= covers[minI].startpos && covers[i].startpos <= covers[minI].endpos) || (covers[i].endpos >= covers[minI].startpos && covers[i].endpos <= covers[minI].endpos)))
+				if (i != minI &&
+					((covers[i].startpos >= covers[minI].startpos &&
+					  covers[i].startpos <= covers[minI].endpos) ||
+					 (covers[i].endpos >= covers[minI].startpos &&
+					  covers[i].endpos <= covers[minI].endpos)))
 					covers[i].excluded = 1;
 			}
 		}
@@ -2275,7 +2294,7 @@ mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, int highlight,
 			break;
 	}
 
-	/* show at least min_words we have not marked anything */
+	/* show at least min_words if we have not marked anything */
 	if (num_f <= 0)
 	{
 		startpos = endpos = curlen = 0;
@@ -2299,7 +2318,7 @@ mark_hl_words(HeadlineParsedText *prs, TSQuery query, int highlight,
 	int			bestb = -1,
 				beste = -1;
 	int			bestlen = -1;
-	int			pose = 0,
+	int			pose,
 				posb,
 				poslen,
 				curlen;
@@ -2310,55 +2329,69 @@ mark_hl_words(HeadlineParsedText *prs, TSQuery query, int highlight,
 	{
 		while (hlCover(prs, query, &p, &q))
 		{
-			/* find cover len in words */
+			/*
+			 * Count words (curlen) and interesting words (poslen) within
+			 * cover, but stop once we reach max_words.  This step doesn't
+			 * consider whether that's a good stopping point.  posb and pose
+			 * are set to the start and end indexes of the possible headline.
+			 */
 			curlen = 0;
 			poslen = 0;
+			posb = pose = p;
 			for (i = p; i <= q && curlen < max_words; i++)
 			{
 				if (!NONWORDTOKEN(prs->words[i].type))
 					curlen++;
-				if (prs->words[i].item && !prs->words[i].repeated)
+				if (INTERESTINGWORD(i))
 					poslen++;
 				pose = i;
 			}
 
-			if (poslen < bestlen && !(NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword))
+			/* XXX this optimization seems unnecessary and wrong */
+			if (poslen < bestlen && !BADENDPOINT(beste))
 			{
-				/* best already found, so try one more cover */
+				/* better cover already found, so try next cover */
 				p++;
 				continue;
 			}
 
-			posb = p;
 			if (curlen < max_words)
-			{					/* find good end */
+			{
+				/*
+				 * We have room to lengthen the headline, so search forward
+				 * until it's full or we find a good stopping point.  We'll
+				 * reconsider the word at "q", then move forward.
+				 */
 				for (i = i - 1; i < prs->curwords && curlen < max_words; i++)
 				{
-					if (i != q)
+					if (i > q)
 					{
 						if (!NONWORDTOKEN(prs->words[i].type))
 							curlen++;
-						if (prs->words[i].item && !prs->words[i].repeated)
+						if (INTERESTINGWORD(i))
 							poslen++;
 					}
 					pose = i;
-					if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
+					if (BADENDPOINT(i))
 						continue;
 					if (curlen >= min_words)
 						break;
 				}
-				if (curlen < min_words && i >= prs->curwords)
-				{				/* got end of text and our cover is shorter
-								 * than min_words */
+				if (curlen < min_words)
+				{
+					/*
+					 * Reached end of text and our headline is still shorter
+					 * than min_words, so try to extend it to the left.
+					 */
 					for (i = p - 1; i >= 0; i--)
 					{
 						if (!NONWORDTOKEN(prs->words[i].type))
 							curlen++;
-						if (prs->words[i].item && !prs->words[i].repeated)
+						if (INTERESTINGWORD(i))
 							poslen++;
 						if (curlen >= max_words)
 							break;
-						if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
+						if (BADENDPOINT(i))
 							continue;
 						if (curlen >= min_words)
 							break;
@@ -2367,25 +2400,34 @@ mark_hl_words(HeadlineParsedText *prs, TSQuery query, int highlight,
 				}
 			}
 			else
-			{					/* shorter cover :((( */
+			{
+				/*
+				 * Can't make headline longer, so consider making it shorter
+				 * if needed to avoid a bad endpoint.
+				 */
 				if (i > q)
 					i = q;
 				for (; curlen > min_words; i--)
 				{
 					if (!NONWORDTOKEN(prs->words[i].type))
 						curlen--;
-					if (prs->words[i].item && !prs->words[i].repeated)
+					if (INTERESTINGWORD(i))
 						poslen--;
 					pose = i;
-					if (NOENDTOKEN(prs->words[i].type) || prs->words[i].len <= shortword)
-						continue;
-					break;
+					if (!BADENDPOINT(i))
+						break;
 				}
 			}
 
-			if (bestlen < 0 || (poslen > bestlen && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword)) ||
-				(bestlen >= 0 && !(NOENDTOKEN(prs->words[pose].type) || prs->words[pose].len <= shortword) &&
-				 (NOENDTOKEN(prs->words[beste].type) || prs->words[beste].len <= shortword)))
+			/*
+			 * Adopt this headline if it's the first, or if it has more
+			 * interesting words and isn't ending at a bad endpoint, or if it
+			 * replaces a bad endpoint with a good one (XXX even if it has
+			 * fewer interesting words?  Really?)
+			 */
+			if (bestlen < 0 ||
+				(poslen > bestlen && !BADENDPOINT(pose)) ||
+				(!BADENDPOINT(pose) && BADENDPOINT(beste)))
 			{
 				bestb = posb;
 				beste = pose;
@@ -2395,6 +2437,10 @@ mark_hl_words(HeadlineParsedText *prs, TSQuery query, int highlight,
 			p++;
 		}
 
+		/*
+		 * If we found nothing acceptable, select min_words words starting at
+		 * the beginning.
+		 */
 		if (bestlen < 0)
 		{
 			curlen = 0;
@@ -2433,7 +2479,6 @@ mark_hl_words(HeadlineParsedText *prs, TSQuery query, int highlight,
 
 		prs->words[i].in = (prs->words[i].repeated) ? 0 : 1;
 	}
-
 }
 
 Datum