v2-0001-pg_regress-TAP-output-format.patch

From f31d962c1d1101b7d90ab1956e18b4c94eda63b8 Mon Sep 17 00:00:00 2001
From: Daniel Gustafsson <daniel@yesql.se>
Date: Fri, 18 Feb 2022 15:13:54 +0100
Subject: [PATCH v2] pg_regress TAP output format

---
 src/test/regress/pg_regress.c | 396 +++++++++++++++++++++++++++-------
 1 file changed, 317 insertions(+), 79 deletions(-)

diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index e6f71c7582..81f3bb7bab 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -95,6 +95,8 @@ static char *dlpath = PKGLIBDIR;
 static char *user = NULL;
 static _stringlist *extraroles = NULL;
 static char *config_auth_datadir = NULL;
+static char *format = NULL;
+static char *psql_formatting = NULL;
 
 /* internal variables */
 static const char *progname;
@@ -120,11 +122,73 @@ static int	fail_ignore_count = 0;
 static bool directory_exists(const char *dir);
 static void make_directory(const char *dir);
 
-static void header(const char *fmt,...) pg_attribute_printf(1, 2);
+struct output_func
+{
+	void (*header)(const char *line);
+	void (*footer)(const char *difffilename, const char *logfilename);
+	void (*comment)(const char *comment);
+
+	void (*test_status_preamble)(const char *testname);
+
+	void (*test_status_ok)(const char *testname);
+	void (*test_status_failed)(const char *testname);
+	void (*test_status_ignored)(const char *testname);
+
+	void (*test_runtime)(const char *testname, double runtime);
+};
+
+
+void (*test_runtime)(const char *testname, double runtime);
+/* Text output format */
+static void header_text(const char *line);
+static void footer_text(const char *difffilename, const char *logfilename);
+static void comment_text(const char *comment);
+static void test_status_preamble_text(const char *testname);
+static void test_status_ok_text(const char *testname);
+static void test_status_failed_text(const char *testname);
+static void test_runtime_text(const char *testname, double runtime);
+
+struct output_func output_func_text =
+{
+	header_text,
+	footer_text,
+	comment_text,
+	test_status_preamble_text,
+	test_status_ok_text,
+	test_status_failed_text,
+	NULL,
+	test_runtime_text
+};
+
+/* TAP output format */
+static void header_tap(const char *line);
+static void footer_tap(const char *difffilename, const char *logfilename);
+static void comment_tap(const char *comment);
+static void test_status_ok_tap(const char *testname);
+static void test_status_failed_tap(const char *testname);
+static void test_status_ignored_tap(const char *testname);
+
+struct output_func output_func_tap =
+{
+	header_tap,
+	footer_tap,
+	comment_tap,
+	NULL,
+	test_status_ok_tap,
+	test_status_failed_tap,
+	test_status_ignored_tap,
+	NULL
+};
+
+struct output_func *output = &output_func_text;
+
+static void test_status_ok(const char *testname);
+
 static void status(const char *fmt,...) pg_attribute_printf(1, 2);
 static StringInfo psql_start_command(void);
 static void psql_add_command(StringInfo buf, const char *query,...) pg_attribute_printf(2, 3);
 static void psql_end_command(StringInfo buf, const char *database);
+static void status_end(void);
 
 /*
  * allow core files if possible.
@@ -208,18 +272,214 @@ split_to_stringlist(const char *s, const char *delim, _stringlist **listhead)
 /*
  * Print a progress banner on stdout.
  */
+static void
+header_text(const char *line)
+{
+	fprintf(stdout, "============== %-38s ==============\n", line);
+	fflush(stdout);
+}
+
+static void
+header_tap(const char *line)
+{
+	fprintf(stdout, "# %s\n", line);
+	fflush(stdout);
+}
+
 static void
 header(const char *fmt,...)
 {
 	char		tmp[64];
 	va_list		ap;
 
+	if (!output->header)
+		return;
+
 	va_start(ap, fmt);
 	vsnprintf(tmp, sizeof(tmp), fmt, ap);
 	va_end(ap);
 
-	fprintf(stdout, "============== %-38s ==============\n", tmp);
-	fflush(stdout);
+	output->header(tmp);
+}
+
+static void
+footer_tap(const char *difffilename, const char *logfilename)
+{
+	status("1..%i\n", (fail_count + fail_ignore_count + success_count));
+	status_end();
+}
+
+static void
+footer(const char *difffilename, const char *logfilename)
+{
+	if (output->footer)
+		output->footer(difffilename, logfilename);
+}
+
+static void
+comment_text(const char *comment)
+{
+	status("%s", comment);
+}
+
+static void
+comment_tap(const char *comment)
+{
+	status("# %s", comment);
+}
+
+static void
+comment(const char *fmt,...)
+{
+	char		tmp[256];
+	va_list		ap;
+
+	if (!output->comment)
+		return;
+
+	va_start(ap, fmt);
+	vsnprintf(tmp, sizeof(tmp), fmt, ap);
+	va_end(ap);
+
+	output->comment(tmp);
+}
+
+static void
+test_status_preamble_text(const char *testname)
+{
+	status(_("test %-28s ... "), testname);
+}
+
+static void
+test_status_preamble(const char *testname)
+{
+	if (output->test_status_preamble)
+		output->test_status_preamble(testname);
+}
+
+static void
+test_status_ok_tap(const char *testname)
+{
+	/* There is no NLS translation here as "ok" is a protocol message */
+	status("ok %i - %s",
+		   (fail_count + fail_ignore_count + success_count),
+		   testname);
+}
+
+static void
+test_status_ok_text(const char *testname)
+{
+	(void) testname; /* unused */
+	status(_("ok    "));	/* align with FAILED */
+}
+
+static void
+test_status_ok(const char *testname)
+{
+	success_count++;
+	if (output->test_status_ok)
+		output->test_status_ok(testname);
+}
+
+static void
+test_status_failed_tap(const char *testname)
+{
+	status("not ok %i - %s",
+		   (fail_count + fail_ignore_count + success_count),
+		   testname);
+}
+
+static void
+test_status_failed_text(const char *testname)
+{
+	status(_("FAILED"));
+}
+
+static void
+test_status_failed(const char *testname)
+{
+	fail_count++;
+	if (output->test_status_failed)
+		output->test_status_failed(testname);
+}
+
+static void
+test_status_ignored(const char *testname)
+{
+	fail_ignore_count++;
+	if (output->test_status_ignored)
+		output->test_status_ignored(testname);
+}
+
+static void
+test_status_ignored_tap(const char *testname)
+{
+	status("ok %i - %s # SKIP (ignored)",
+		   (fail_count + fail_ignore_count + success_count),
+		   testname);
+}
+
+static void
+test_runtime_text(const char *testname, double runtime)
+{
+	(void)testname;
+	status(_(" %8.0f ms"), runtime);
+}
+
+static void
+runtime(const char *testname, double runtime)
+{
+	if (output->test_runtime)
+		output->test_runtime(testname, runtime);
+}
+
+static void
+footer_text(const char *difffilename, const char *logfilename)
+{
+	char buf[256];
+
+	/*
+	 * Emit nice-looking summary message
+	 */
+	if (fail_count == 0 && fail_ignore_count == 0)
+		snprintf(buf, sizeof(buf),
+				 _(" All %d tests passed. "),
+				 success_count);
+	else if (fail_count == 0)	/* fail_count=0, fail_ignore_count>0 */
+		snprintf(buf, sizeof(buf),
+				 _(" %d of %d tests passed, %d failed test(s) ignored. "),
+				 success_count,
+				 success_count + fail_ignore_count,
+				 fail_ignore_count);
+	else if (fail_ignore_count == 0)	/* fail_count>0 && fail_ignore_count=0 */
+		snprintf(buf, sizeof(buf),
+				 _(" %d of %d tests failed. "),
+				 fail_count,
+				 success_count + fail_count);
+	else
+		/* fail_count>0 && fail_ignore_count>0 */
+		snprintf(buf, sizeof(buf),
+				 _(" %d of %d tests failed, %d of these failures ignored. "),
+				 fail_count + fail_ignore_count,
+				 success_count + fail_count + fail_ignore_count,
+				 fail_ignore_count);
+
+	putchar('\n');
+	for (int i = strlen(buf); i > 0; i--)
+		putchar('=');
+	printf("\n%s\n", buf);
+	for (int i = strlen(buf); i > 0; i--)
+		putchar('=');
+	putchar('\n');
+	putchar('\n');
+
+	if (difffilename && logfilename)
+	{
+		printf(_("The differences that caused some tests to fail can be viewed in the\n"
+				 "file \"%s\".  A copy of the test summary that you see\n"
+				 "above is saved in the file \"%s\".\n\n"),
+			   difffilename, logfilename);
+	}
 }
 
 /*
@@ -752,13 +1012,13 @@ initialize_environment(void)
 #endif
 
 		if (pghost && pgport)
-			printf(_("(using postmaster on %s, port %s)\n"), pghost, pgport);
+			comment(_("(using postmaster on %s, port %s)\n"), pghost, pgport);
 		if (pghost && !pgport)
-			printf(_("(using postmaster on %s, default port)\n"), pghost);
+			comment(_("(using postmaster on %s, default port)\n"), pghost);
 		if (!pghost && pgport)
-			printf(_("(using postmaster on Unix socket, port %s)\n"), pgport);
+			comment(_("(using postmaster on Unix socket, port %s)\n"), pgport);
 		if (!pghost && !pgport)
-			printf(_("(using postmaster on Unix socket, default port)\n"));
+			comment(_("(using postmaster on Unix socket, default port)\n"));
 	}
 
 	load_resultmap();
@@ -984,9 +1244,10 @@ psql_start_command(void)
 	StringInfo	buf = makeStringInfo();
 
 	appendStringInfo(buf,
-					 "\"%s%spsql\" -X",
+					 "\"%s%spsql\" -X %s",
 					 bindir ? bindir : "",
-					 bindir ? "/" : "");
+					 bindir ? "/" : "",
+					 psql_formatting ? psql_formatting : "");
 	return buf;
 }
 
@@ -1585,6 +1846,9 @@ run_schedule(const char *schedule, test_start_function startfunc,
 				c++;
 			add_stringlist_item(&ignorelist, c);
 
+			test_status_ignored(c);
+			status_end();
+
 			/*
 			 * Note: ignore: lines do not run the test, they just say that
 			 * failure of this test when run later on is to be ignored. A bit
@@ -1643,7 +1907,7 @@ run_schedule(const char *schedule, test_start_function startfunc,
 
 		if (num_tests == 1)
 		{
-			status(_("test %-28s ... "), tests[0]);
+			test_status_preamble(tests[0]);
 			pids[0] = (startfunc) (tests[0], &resultfiles[0], &expectfiles[0], &tags[0]);
 			INSTR_TIME_SET_CURRENT(starttimes[0]);
 			wait_for_tests(pids, statuses, stoptimes, NULL, 1);
@@ -1659,8 +1923,8 @@ run_schedule(const char *schedule, test_start_function startfunc,
 		{
 			int			oldest = 0;
 
-			status(_("parallel group (%d tests, in groups of %d): "),
-				   num_tests, max_connections);
+			comment(_("parallel group (%d tests, in groups of %d): "),
+					  num_tests, max_connections);
 			for (i = 0; i < num_tests; i++)
 			{
 				if (i - oldest >= max_connections)
@@ -1680,7 +1944,7 @@ run_schedule(const char *schedule, test_start_function startfunc,
 		}
 		else
 		{
-			status(_("parallel group (%d tests): "), num_tests);
+			comment(_("parallel group (%d tests): "), num_tests);
 			for (i = 0; i < num_tests; i++)
 			{
 				pids[i] = (startfunc) (tests[i], &resultfiles[i], &expectfiles[i], &tags[i]);
@@ -1699,7 +1963,7 @@ run_schedule(const char *schedule, test_start_function startfunc,
 			bool		differ = false;
 
 			if (num_tests > 1)
-				status(_("     %-28s ... "), tests[i]);
+				test_status_preamble(tests[i]);
 
 			/*
 			 * Advance over all three lists simultaneously.
@@ -1739,27 +2003,18 @@ run_schedule(const char *schedule, test_start_function startfunc,
 					}
 				}
 				if (ignore)
-				{
-					status(_("failed (ignored)"));
-					fail_ignore_count++;
-				}
+					test_status_ignored(tests[i]);
 				else
-				{
-					status(_("FAILED"));
-					fail_count++;
-				}
+					test_status_failed(tests[i]);
 			}
 			else
-			{
-				status(_("ok    "));	/* align with FAILED */
-				success_count++;
-			}
+				test_status_ok(tests[i]);
 
 			if (statuses[i] != 0)
 				log_child_failure(statuses[i]);
 
 			INSTR_TIME_SUBTRACT(stoptimes[i], starttimes[i]);
-			status(_(" %8.0f ms"), INSTR_TIME_GET_MILLISEC(stoptimes[i]));
+			runtime(tests[i], INSTR_TIME_GET_MILLISEC(stoptimes[i]));
 
 			status_end();
 		}
@@ -1798,7 +2053,7 @@ run_single_test(const char *test, test_start_function startfunc,
 			   *tl;
 	bool		differ = false;
 
-	status(_("test %-28s ... "), test);
+	test_status_preamble(test);
 	pid = (startfunc) (test, &resultfiles, &expectfiles, &tags);
 	INSTR_TIME_SET_CURRENT(starttime);
 	wait_for_tests(&pid, &exit_status, &stoptime, NULL, 1);
@@ -1828,15 +2083,9 @@ run_single_test(const char *test, test_start_function startfunc,
 	}
 
 	if (differ)
-	{
-		status(_("FAILED"));
-		fail_count++;
-	}
+		test_status_failed(test);
 	else
-	{
-		status(_("ok    "));	/* align with FAILED */
-		success_count++;
-	}
+		test_status_ok(test);
 
 	if (exit_status != 0)
 		log_child_failure(exit_status);
@@ -1983,6 +2232,7 @@ help(void)
 	printf(_("      --debug                   turn on debug mode in programs that are run\n"));
 	printf(_("      --dlpath=DIR              look for dynamic libraries in DIR\n"));
 	printf(_("      --encoding=ENCODING       use ENCODING as the encoding\n"));
+	printf(_("      --format=(regress|tap)    output format to use\n"));
 	printf(_("  -h, --help                    show this help, then exit\n"));
 	printf(_("      --inputdir=DIR            take input files from DIR (default \".\")\n"));
 	printf(_("      --launcher=CMD            use CMD as launcher of psql\n"));
@@ -2046,6 +2296,7 @@ regression_main(int argc, char *argv[],
 		{"load-extension", required_argument, NULL, 22},
 		{"config-auth", required_argument, NULL, 24},
 		{"max-concurrent-tests", required_argument, NULL, 25},
+ 		{"format", required_argument, NULL, 26},
 		{NULL, 0, NULL, 0}
 	};
 
@@ -2175,6 +2426,9 @@ regression_main(int argc, char *argv[],
 			case 25:
 				max_concurrent_tests = atoi(optarg);
 				break;
+			case 26:
+				format = pg_strdup(optarg);
+				break;
 			default:
 				/* getopt_long already emitted a complaint */
 				fprintf(stderr, _("\nTry \"%s -h\" for more information.\n"),
@@ -2201,6 +2455,24 @@ regression_main(int argc, char *argv[],
 		exit(0);
 	}
 
+	/*
+	 * text (regress) is the default so we don't need to set any variables in
+	 * that case.
+	 */
+	if (format)
+	{
+		if (strcmp(format, "tap") == 0)
+		{
+			output = &output_func_tap;
+			psql_formatting = pg_strdup("-q");
+		}
+		else if (strcmp(format, "regress") != 0)
+		{
+			fprintf(stderr, _("\n%s: invalid format specified: \"%s\". Supported formats are \"tap\" and \"regress\"\n"), progname, format);
+			exit(2);
+		}
+	}
+
 	if (temp_instance && !port_specified_by_user)
 
 		/*
@@ -2523,54 +2795,20 @@ regression_main(int argc, char *argv[],
 
 	fclose(logfile);
 
-	/*
-	 * Emit nice-looking summary message
-	 */
-	if (fail_count == 0 && fail_ignore_count == 0)
-		snprintf(buf, sizeof(buf),
-				 _(" All %d tests passed. "),
-				 success_count);
-	else if (fail_count == 0)	/* fail_count=0, fail_ignore_count>0 */
-		snprintf(buf, sizeof(buf),
-				 _(" %d of %d tests passed, %d failed test(s) ignored. "),
-				 success_count,
-				 success_count + fail_ignore_count,
-				 fail_ignore_count);
-	else if (fail_ignore_count == 0)	/* fail_count>0 && fail_ignore_count=0 */
-		snprintf(buf, sizeof(buf),
-				 _(" %d of %d tests failed. "),
-				 fail_count,
-				 success_count + fail_count);
-	else
-		/* fail_count>0 && fail_ignore_count>0 */
-		snprintf(buf, sizeof(buf),
-				 _(" %d of %d tests failed, %d of these failures ignored. "),
-				 fail_count + fail_ignore_count,
-				 success_count + fail_count + fail_ignore_count,
-				 fail_ignore_count);
-
-	putchar('\n');
-	for (i = strlen(buf); i > 0; i--)
-		putchar('=');
-	printf("\n%s\n", buf);
-	for (i = strlen(buf); i > 0; i--)
-		putchar('=');
-	putchar('\n');
-	putchar('\n');
-
-	if (file_size(difffilename) > 0)
-	{
-		printf(_("The differences that caused some tests to fail can be viewed in the\n"
-				 "file \"%s\".  A copy of the test summary that you see\n"
-				 "above is saved in the file \"%s\".\n\n"),
-			   difffilename, logfilename);
-	}
-	else
+	if (file_size(difffilename) <= 0)
 	{
 		unlink(difffilename);
 		unlink(logfilename);
+
+		free(difffilename);
+		difffilename = NULL;
+		free(logfilename);
+		logfilename = NULL;
 	}
 
+	footer(difffilename, logfilename);
+	status_end();
+
 	if (fail_count != 0)
 		exit(1);
 
-- 
2.24.3 (Apple Git-128)