v21-0001-common-jsonapi-support-libpq-as-a-client.patch

From 557370eabb9e7c8004267dd617ac8936f499b2cf Mon Sep 17 00:00:00 2001
From: Jacob Champion <pchampion@vmware.com>
Date: Mon, 3 May 2021 15:38:26 -0700
Subject: [PATCH v21 1/7] common/jsonapi: support libpq as a client

Based on a patch by Michael Paquier.

For frontend code, use PQExpBuffer instead of StringInfo. This requires
us to track allocation failures so that we can return JSON_OUT_OF_MEMORY
as needed rather than exit()ing.
---
 src/bin/pg_combinebackup/Makefile    |   4 +-
 src/bin/pg_combinebackup/meson.build |   2 +-
 src/bin/pg_verifybackup/Makefile     |   2 +-
 src/common/Makefile                  |   2 +-
 src/common/jsonapi.c                 | 173 ++++++++++++++++++++-------
 src/common/meson.build               |   8 +-
 src/include/common/jsonapi.h         |  19 ++-
 7 files changed, 158 insertions(+), 52 deletions(-)

diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile
index c3729755ba..2f7dc1ed87 100644
--- a/src/bin/pg_combinebackup/Makefile
+++ b/src/bin/pg_combinebackup/Makefile
@@ -18,6 +18,8 @@ include $(top_builddir)/src/Makefile.global
 
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils
+# TODO: fix this properly
+LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
@@ -30,7 +32,7 @@ OBJS = \
 
 all: pg_combinebackup
 
-pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils
+pg_combinebackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
 	$(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
 
 install: all installdirs
diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build
index 1d4b9c218f..cab677b574 100644
--- a/src/bin/pg_combinebackup/meson.build
+++ b/src/bin/pg_combinebackup/meson.build
@@ -17,7 +17,7 @@ endif
 
 pg_combinebackup = executable('pg_combinebackup',
   pg_combinebackup_sources,
-  dependencies: [frontend_code],
+  dependencies: [frontend_code, libpq],
   kwargs: default_bin_args,
 )
 bin_targets += pg_combinebackup
diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile
index 7c045f142e..3372fada01 100644
--- a/src/bin/pg_verifybackup/Makefile
+++ b/src/bin/pg_verifybackup/Makefile
@@ -17,7 +17,7 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 # We need libpq only because fe_utils does.
-LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpgcommon $(libpq_pgport)
 
 OBJS = \
 	$(WIN32RES) \
diff --git a/src/common/Makefile b/src/common/Makefile
index 3d83299432..c4f4448e2f 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -40,7 +40,7 @@ override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\""
 override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\""
 override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\""
 
-override CPPFLAGS := -DFRONTEND -I. -I$(top_srcdir)/src/common $(CPPFLAGS)
+override CPPFLAGS := -DFRONTEND -I. -I$(top_srcdir)/src/common -I$(libpq_srcdir) $(CPPFLAGS)
 LIBS += $(PTHREAD_LIBS)
 
 OBJS_COMMON = \
diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c
index 98d6e66a21..4aeedc0bc5 100644
--- a/src/common/jsonapi.c
+++ b/src/common/jsonapi.c
@@ -21,10 +21,43 @@
 #include "mb/pg_wchar.h"
 #include "port/pg_lfind.h"
 
-#ifndef FRONTEND
+#ifdef FRONTEND
+#include "pqexpbuffer.h"
+#else
+#include "lib/stringinfo.h"
 #include "miscadmin.h"
 #endif
 
+/*
+ * In backend, we will use palloc/pfree along with StringInfo.  In frontend, use
+ * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory.
+ */
+#ifdef FRONTEND
+
+#define STRDUP(s) strdup(s)
+#define ALLOC(size) malloc(size)
+
+#define appendStrVal		appendPQExpBuffer
+#define appendBinaryStrVal  appendBinaryPQExpBuffer
+#define appendStrValChar	appendPQExpBufferChar
+#define createStrVal		createPQExpBuffer
+#define resetStrVal			resetPQExpBuffer
+#define destroyStrVal		destroyPQExpBuffer
+
+#else							/* !FRONTEND */
+
+#define STRDUP(s) pstrdup(s)
+#define ALLOC(size) palloc(size)
+
+#define appendStrVal		appendStringInfo
+#define appendBinaryStrVal  appendBinaryStringInfo
+#define appendStrValChar	appendStringInfoChar
+#define createStrVal		makeStringInfo
+#define resetStrVal			resetStringInfo
+#define destroyStrVal		destroyStringInfo
+
+#endif
+
 /*
  * The context of the parser is maintained by the recursive descent
  * mechanism, but is passed explicitly to the error reporting routine
@@ -168,9 +201,16 @@ makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
 	lex->input_encoding = encoding;
 	if (need_escapes)
 	{
-		lex->strval = makeStringInfo();
+		/*
+		 * This call can fail in FRONTEND code. We defer error handling to
+		 * time of use (json_lex_string()) since we might not need to parse
+		 * any strings anyway.
+		 */
+		lex->strval = createStrVal();
 		lex->flags |= JSONLEX_FREE_STRVAL;
+		lex->parse_strval = true;
 	}
+	lex->errormsg = NULL;
 
 	return lex;
 }
@@ -185,14 +225,18 @@ makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
 void
 freeJsonLexContext(JsonLexContext *lex)
 {
+	static const JsonLexContext empty = {0};
+
 	if (lex->flags & JSONLEX_FREE_STRVAL)
-		destroyStringInfo(lex->strval);
+		destroyStrVal(lex->strval);
 
 	if (lex->errormsg)
-		destroyStringInfo(lex->errormsg);
+		destroyStrVal(lex->errormsg);
 
 	if (lex->flags & JSONLEX_FREE_STRUCT)
 		pfree(lex);
+	else
+		*lex = empty;
 }
 
 /*
@@ -258,7 +302,7 @@ json_count_array_elements(JsonLexContext *lex, int *elements)
 	 * etc, so doing this with a copy makes that safe.
 	 */
 	memcpy(&copylex, lex, sizeof(JsonLexContext));
-	copylex.strval = NULL;		/* not interested in values here */
+	copylex.parse_strval = false;	/* not interested in values here */
 	copylex.lex_level++;
 
 	count = 0;
@@ -320,14 +364,21 @@ parse_scalar(JsonLexContext *lex, JsonSemAction *sem)
 	/* extract the de-escaped string value, or the raw lexeme */
 	if (lex_peek(lex) == JSON_TOKEN_STRING)
 	{
-		if (lex->strval != NULL)
-			val = pstrdup(lex->strval->data);
+		if (lex->parse_strval)
+		{
+			val = STRDUP(lex->strval->data);
+			if (val == NULL)
+				return JSON_OUT_OF_MEMORY;
+		}
 	}
 	else
 	{
 		int			len = (lex->token_terminator - lex->token_start);
 
-		val = palloc(len + 1);
+		val = ALLOC(len + 1);
+		if (val == NULL)
+			return JSON_OUT_OF_MEMORY;
+
 		memcpy(val, lex->token_start, len);
 		val[len] = '\0';
 	}
@@ -361,8 +412,12 @@ parse_object_field(JsonLexContext *lex, JsonSemAction *sem)
 
 	if (lex_peek(lex) != JSON_TOKEN_STRING)
 		return report_parse_error(JSON_PARSE_STRING, lex);
-	if ((ostart != NULL || oend != NULL) && lex->strval != NULL)
-		fname = pstrdup(lex->strval->data);
+	if ((ostart != NULL || oend != NULL) && lex->parse_strval)
+	{
+		fname = STRDUP(lex->strval->data);
+		if (fname == NULL)
+			return JSON_OUT_OF_MEMORY;
+	}
 	result = json_lex(lex);
 	if (result != JSON_SUCCESS)
 		return result;
@@ -418,6 +473,11 @@ parse_object(JsonLexContext *lex, JsonSemAction *sem)
 	JsonParseErrorType result;
 
 #ifndef FRONTEND
+
+	/*
+	 * TODO: clients need some way to put a bound on stack growth. Parse level
+	 * limits maybe?
+	 */
 	check_stack_depth();
 #endif
 
@@ -766,8 +826,15 @@ json_lex_string(JsonLexContext *lex)
 		return code; \
 	} while (0)
 
-	if (lex->strval != NULL)
-		resetStringInfo(lex->strval);
+	if (lex->parse_strval)
+	{
+#ifdef FRONTEND
+		/* make sure initialization succeeded */
+		if (lex->strval == NULL)
+			return JSON_OUT_OF_MEMORY;
+#endif
+		resetStrVal(lex->strval);
+	}
 
 	Assert(lex->input_length > 0);
 	s = lex->token_start;
@@ -804,7 +871,7 @@ json_lex_string(JsonLexContext *lex)
 					else
 						FAIL_AT_CHAR_END(JSON_UNICODE_ESCAPE_FORMAT);
 				}
-				if (lex->strval != NULL)
+				if (lex->parse_strval)
 				{
 					/*
 					 * Combine surrogate pairs.
@@ -861,19 +928,19 @@ json_lex_string(JsonLexContext *lex)
 
 						unicode_to_utf8(ch, (unsigned char *) utf8str);
 						utf8len = pg_utf_mblen((unsigned char *) utf8str);
-						appendBinaryStringInfo(lex->strval, utf8str, utf8len);
+						appendBinaryPQExpBuffer(lex->strval, utf8str, utf8len);
 					}
 					else if (ch <= 0x007f)
 					{
 						/* The ASCII range is the same in all encodings */
-						appendStringInfoChar(lex->strval, (char) ch);
+						appendPQExpBufferChar(lex->strval, (char) ch);
 					}
 					else
 						FAIL_AT_CHAR_END(JSON_UNICODE_HIGH_ESCAPE);
 #endif							/* FRONTEND */
 				}
 			}
-			else if (lex->strval != NULL)
+			else if (lex->parse_strval)
 			{
 				if (hi_surrogate != -1)
 					FAIL_AT_CHAR_END(JSON_UNICODE_LOW_SURROGATE);
@@ -883,22 +950,22 @@ json_lex_string(JsonLexContext *lex)
 					case '"':
 					case '\\':
 					case '/':
-						appendStringInfoChar(lex->strval, *s);
+						appendStrValChar(lex->strval, *s);
 						break;
 					case 'b':
-						appendStringInfoChar(lex->strval, '\b');
+						appendStrValChar(lex->strval, '\b');
 						break;
 					case 'f':
-						appendStringInfoChar(lex->strval, '\f');
+						appendStrValChar(lex->strval, '\f');
 						break;
 					case 'n':
-						appendStringInfoChar(lex->strval, '\n');
+						appendStrValChar(lex->strval, '\n');
 						break;
 					case 'r':
-						appendStringInfoChar(lex->strval, '\r');
+						appendStrValChar(lex->strval, '\r');
 						break;
 					case 't':
-						appendStringInfoChar(lex->strval, '\t');
+						appendStrValChar(lex->strval, '\t');
 						break;
 					default:
 
@@ -933,7 +1000,7 @@ json_lex_string(JsonLexContext *lex)
 
 			/*
 			 * Skip to the first byte that requires special handling, so we
-			 * can batch calls to appendBinaryStringInfo.
+			 * can batch calls to appendBinaryStrVal.
 			 */
 			while (p < end - sizeof(Vector8) &&
 				   !pg_lfind8('\\', (uint8 *) p, sizeof(Vector8)) &&
@@ -957,8 +1024,8 @@ json_lex_string(JsonLexContext *lex)
 				}
 			}
 
-			if (lex->strval != NULL)
-				appendBinaryStringInfo(lex->strval, s, p - s);
+			if (lex->parse_strval)
+				appendBinaryStrVal(lex->strval, s, p - s);
 
 			/*
 			 * s will be incremented at the top of the loop, so set it to just
@@ -974,6 +1041,11 @@ json_lex_string(JsonLexContext *lex)
 		return JSON_UNICODE_LOW_SURROGATE;
 	}
 
+#ifdef FRONTEND
+	if (lex->parse_strval && PQExpBufferBroken(lex->strval))
+		return JSON_OUT_OF_MEMORY;
+#endif
+
 	/* Hooray, we found the end of the string! */
 	lex->prev_token_terminator = lex->token_terminator;
 	lex->token_terminator = s + 1;
@@ -1158,19 +1230,25 @@ report_parse_error(JsonParseContext ctx, JsonLexContext *lex)
 char *
 json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
 {
+	if (error == JSON_OUT_OF_MEMORY)
+	{
+		/* Short circuit. Allocating anything for this case is unhelpful. */
+		return _("out of memory");
+	}
+
 	if (lex->errormsg)
-		resetStringInfo(lex->errormsg);
+		resetStrVal(lex->errormsg);
 	else
-		lex->errormsg = makeStringInfo();
+		lex->errormsg = createStrVal();
 
 	/*
 	 * A helper for error messages that should print the current token. The
 	 * format must contain exactly one %.*s specifier.
 	 */
 #define token_error(lex, format) \
-	appendStringInfo((lex)->errormsg, _(format), \
-					 (int) ((lex)->token_terminator - (lex)->token_start), \
-					 (lex)->token_start);
+	appendStrVal((lex)->errormsg, _(format), \
+				 (int) ((lex)->token_terminator - (lex)->token_start), \
+				 (lex)->token_start);
 
 	switch (error)
 	{
@@ -1181,9 +1259,9 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
 			token_error(lex, "Escape sequence \"\\%.*s\" is invalid.");
 			break;
 		case JSON_ESCAPING_REQUIRED:
-			appendStringInfo(lex->errormsg,
-							 _("Character with value 0x%02x must be escaped."),
-							 (unsigned char) *(lex->token_terminator));
+			appendStrVal(lex->errormsg,
+						 _("Character with value 0x%02x must be escaped."),
+						 (unsigned char) *(lex->token_terminator));
 			break;
 		case JSON_EXPECTED_END:
 			token_error(lex, "Expected end of input, but found \"%.*s\".");
@@ -1214,6 +1292,9 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
 		case JSON_INVALID_TOKEN:
 			token_error(lex, "Token \"%.*s\" is invalid.");
 			break;
+		case JSON_OUT_OF_MEMORY:
+			/* should have been handled above; use the error path */
+			break;
 		case JSON_UNICODE_CODE_POINT_ZERO:
 			return _("\\u0000 cannot be converted to text.");
 		case JSON_UNICODE_ESCAPE_FORMAT:
@@ -1245,15 +1326,23 @@ json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
 	}
 #undef token_error
 
-	/*
-	 * We don't use a default: case, so that the compiler will warn about
-	 * unhandled enum values.  But this needs to be here anyway to cover the
-	 * possibility of an incorrect input.
-	 */
-	if (lex->errormsg->len == 0)
-		appendStringInfo(lex->errormsg,
-						 _("unexpected json parse error type: %d"),
-						 (int) error);
+	/* Note that lex->errormsg can be NULL in FRONTEND code. */
+	if (lex->errormsg && lex->errormsg->len == 0)
+	{
+		/*
+		 * We don't use a default: case, so that the compiler will warn about
+		 * unhandled enum values.  But this needs to be here anyway to cover
+		 * the possibility of an incorrect input.
+		 */
+		appendStrVal(lex->errormsg,
+					 _("unexpected json parse error type: %d"),
+					 (int) error);
+	}
+
+#ifdef FRONTEND
+	if (PQExpBufferBroken(lex->errormsg))
+		return _("out of memory while constructing error description");
+#endif
 
 	return lex->errormsg->data;
 }
diff --git a/src/common/meson.build b/src/common/meson.build
index de68e408fa..379b228d86 100644
--- a/src/common/meson.build
+++ b/src/common/meson.build
@@ -125,13 +125,18 @@ common_sources_frontend_static += files(
 # least cryptohash_openssl.c, hmac_openssl.c depend on it.
 # controldata_utils.c depends on wait_event_types_h. That's arguably a
 # layering violation, but ...
+#
+# XXX Frontend builds need libpq's pqexpbuffer.h, so adjust the include paths
+# appropriately. This seems completely broken.
 pgcommon = {}
 pgcommon_variants = {
   '_srv': internal_lib_args + {
+    'include_directories': include_directories('.'),
     'sources': common_sources + [lwlocknames_h] + [wait_event_types_h],
     'dependencies': [backend_common_code],
   },
   '': default_lib_args + {
+    'include_directories': include_directories('../interfaces/libpq', '.'),
     'sources': common_sources_frontend_static,
     'dependencies': [frontend_common_code],
     # Files in libpgcommon.a should use/export the "xxx_private" versions
@@ -140,6 +145,7 @@ pgcommon_variants = {
   },
   '_shlib': default_lib_args + {
     'pic': true,
+    'include_directories': include_directories('../interfaces/libpq', '.'),
     'sources': common_sources_frontend_shlib,
     'dependencies': [frontend_common_code],
   },
@@ -157,7 +163,6 @@ foreach name, opts : pgcommon_variants
     c_args = opts.get('c_args', []) + common_cflags[cflagname]
     cflag_libs += static_library('libpgcommon@0@_@1@'.format(name, cflagname),
       c_pch: pch_c_h,
-      include_directories: include_directories('.'),
       kwargs: opts + {
         'sources': sources,
         'c_args': c_args,
@@ -170,7 +175,6 @@ foreach name, opts : pgcommon_variants
   lib = static_library('libpgcommon@0@'.format(name),
       link_with: cflag_libs,
       c_pch: pch_c_h,
-      include_directories: include_directories('.'),
       kwargs: opts + {
         'dependencies': opts['dependencies'] + [ssl],
       }
diff --git a/src/include/common/jsonapi.h b/src/include/common/jsonapi.h
index 86a0fc2d00..75d444c17a 100644
--- a/src/include/common/jsonapi.h
+++ b/src/include/common/jsonapi.h
@@ -14,8 +14,6 @@
 #ifndef JSONAPI_H
 #define JSONAPI_H
 
-#include "lib/stringinfo.h"
-
 typedef enum JsonTokenType
 {
 	JSON_TOKEN_INVALID,
@@ -48,6 +46,7 @@ typedef enum JsonParseErrorType
 	JSON_EXPECTED_OBJECT_NEXT,
 	JSON_EXPECTED_STRING,
 	JSON_INVALID_TOKEN,
+	JSON_OUT_OF_MEMORY,
 	JSON_UNICODE_CODE_POINT_ZERO,
 	JSON_UNICODE_ESCAPE_FORMAT,
 	JSON_UNICODE_HIGH_ESCAPE,
@@ -57,6 +56,17 @@ typedef enum JsonParseErrorType
 	JSON_SEM_ACTION_FAILED,		/* error should already be reported */
 } JsonParseErrorType;
 
+/*
+ * Don't depend on the internal type header for strval; if callers need access
+ * then they can include the appropriate header themselves.
+ */
+#ifdef FRONTEND
+#define StrValType PQExpBufferData
+#else
+#define StrValType StringInfoData
+#endif
+
+typedef struct StrValType StrValType;
 
 /*
  * All the fields in this structure should be treated as read-only.
@@ -88,8 +98,9 @@ typedef struct JsonLexContext
 	bits32		flags;
 	int			line_number;	/* line number, starting from 1 */
 	char	   *line_start;		/* where that line starts within input */
-	StringInfo	strval;
-	StringInfo	errormsg;
+	bool		parse_strval;
+	StrValType *strval;			/* only used if parse_strval == true */
+	StrValType *errormsg;
 } JsonLexContext;
 
 typedef JsonParseErrorType (*json_struct_action) (void *state);
-- 
2.34.1