since-v20.diff.txt
text/plain
Filename: since-v20.diff.txt
Type: text/plain
Part: 0
1: 231c6fb165 ! 1: 557370eabb common/jsonapi: support FRONTEND clients
@@ Metadata
Author: Jacob Champion <pchampion@vmware.com>
## Commit message ##
- common/jsonapi: support FRONTEND clients
+ common/jsonapi: support libpq as a client
Based on a patch by Michael Paquier.
For frontend code, use PQExpBuffer instead of StringInfo. This requires
us to track allocation failures so that we can return JSON_OUT_OF_MEMORY
- as needed. json_errdetail() now allocates its error message inside
- memory owned by the JsonLexContext, so clients don't need to worry about
- freeing it.
+ as needed rather than exit()ing.
- We can now partially revert b44669b2ca, now that json_errdetail() works
- correctly.
+ ## src/bin/pg_combinebackup/Makefile ##
+@@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global
- Co-authored-by: Daniel Gustafsson <daniel@yesql.se>
+ override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+ LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils
++# TODO: fix this properly
++LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport)
- ## src/bin/pg_verifybackup/t/005_bad_manifest.pl ##
-@@ src/bin/pg_verifybackup/t/005_bad_manifest.pl: use Test::More;
- my $tempdir = PostgreSQL::Test::Utils::tempdir;
+ OBJS = \
+ $(WIN32RES) \
+@@ src/bin/pg_combinebackup/Makefile: OBJS = \
- test_bad_manifest('input string ended unexpectedly',
-- qr/could not parse backup manifest: parsing failed/, <<EOM);
-+ qr/could not parse backup manifest: The input string ended unexpectedly/,
-+ <<EOM);
- {
- EOM
+ all: pg_combinebackup
+
+-pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils
++pg_combinebackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+ install: all installdirs
+
+ ## src/bin/pg_combinebackup/meson.build ##
+@@ src/bin/pg_combinebackup/meson.build: endif
+
+ pg_combinebackup = executable('pg_combinebackup',
+ pg_combinebackup_sources,
+- dependencies: [frontend_code],
++ dependencies: [frontend_code, libpq],
+ kwargs: default_bin_args,
+ )
+ bin_targets += pg_combinebackup
+ ## src/bin/pg_verifybackup/Makefile ##
+@@ src/bin/pg_verifybackup/Makefile: top_builddir = ../../..
+ include $(top_builddir)/src/Makefile.global
+
+ # We need libpq only because fe_utils does.
+-LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
++LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpgcommon $(libpq_pgport)
+
+ OBJS = \
+ $(WIN32RES) \
## src/common/Makefile ##
@@ src/common/Makefile: override CPPFLAGS += -DVAL_LDFLAGS_EX="\"$(LDFLAGS_EX)\""
@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js
+ static const JsonLexContext empty = {0};
+
if (lex->flags & JSONLEX_FREE_STRVAL)
-- {
-- pfree(lex->strval->data);
-- pfree(lex->strval);
-- }
+- destroyStringInfo(lex->strval);
+ destroyStrVal(lex->strval);
-+
-+ if (lex->errormsg)
+
+ if (lex->errormsg)
+- destroyStringInfo(lex->errormsg);
+ destroyStrVal(lex->errormsg);
-+
+
if (lex->flags & JSONLEX_FREE_STRUCT)
pfree(lex);
+ else
@@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex)
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
@@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *lex)
- return JSON_SUCCESS; /* silence stupider compilers */
- }
-
--
--#ifndef FRONTEND
--/*
-- * Extract the current token from a lexing context, for error reporting.
-- */
--static char *
--extract_token(JsonLexContext *lex)
--{
-- int toklen = lex->token_terminator - lex->token_start;
-- char *token = palloc(toklen + 1);
--
-- memcpy(token, lex->token_start, toklen);
-- token[toklen] = '\0';
-- return token;
--}
--
- /*
- * Construct an (already translated) detail message for a JSON error.
- *
-- * Note that the error message generated by this routine may not be
-- * palloc'd, making it unsafe for frontend code as there is no way to
-- * know if this can be safely pfree'd or not.
-+ * The returned allocation is either static or owned by the JsonLexContext and
-+ * should not be freed.
- */
char *
json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
{
-+ int toklen = lex->token_terminator - lex->token_start;
-+
+ if (error == JSON_OUT_OF_MEMORY)
+ {
+ /* Short circuit. Allocating anything for this case is unhelpful. */
+ return _("out of memory");
+ }
+
-+ if (lex->errormsg)
+ if (lex->errormsg)
+- resetStringInfo(lex->errormsg);
+ resetStrVal(lex->errormsg);
-+ else
+ else
+- lex->errormsg = makeStringInfo();
+ lex->errormsg = createStrVal();
-+
+
+ /*
+ * A helper for error messages that should print the current token. The
+ * format must contain exactly one %.*s specifier.
+ */
+ #define token_error(lex, format) \
+- appendStringInfo((lex)->errormsg, _(format), \
+- (int) ((lex)->token_terminator - (lex)->token_start), \
+- (lex)->token_start);
++ appendStrVal((lex)->errormsg, _(format), \
++ (int) ((lex)->token_terminator - (lex)->token_start), \
++ (lex)->token_start);
+
switch (error)
{
- case JSON_SUCCESS:
- /* fall through to the error code after switch */
+@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
+ token_error(lex, "Escape sequence \"\\%.*s\" is invalid.");
break;
- case JSON_ESCAPING_INVALID:
-- return psprintf(_("Escape sequence \"\\%s\" is invalid."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Escape sequence \"\\%.*s\" is invalid."),
-+ toklen, lex->token_start);
-+ break;
case JSON_ESCAPING_REQUIRED:
-- return psprintf(_("Character with value 0x%02x must be escaped."),
+- appendStringInfo(lex->errormsg,
+- _("Character with value 0x%02x must be escaped."),
- (unsigned char) *(lex->token_terminator));
+ appendStrVal(lex->errormsg,
+ _("Character with value 0x%02x must be escaped."),
+ (unsigned char) *(lex->token_terminator));
-+ break;
+ break;
case JSON_EXPECTED_END:
-- return psprintf(_("Expected end of input, but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected end of input, but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_ARRAY_FIRST:
-- return psprintf(_("Expected array element or \"]\", but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected array element or \"]\", but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_ARRAY_NEXT:
-- return psprintf(_("Expected \",\" or \"]\", but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected \",\" or \"]\", but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_COLON:
-- return psprintf(_("Expected \":\", but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected \":\", but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_JSON:
-- return psprintf(_("Expected JSON value, but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected JSON value, but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_MORE:
- return _("The input string ended unexpectedly.");
- case JSON_EXPECTED_OBJECT_FIRST:
-- return psprintf(_("Expected string or \"}\", but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected string or \"}\", but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_OBJECT_NEXT:
-- return psprintf(_("Expected \",\" or \"}\", but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected \",\" or \"}\", but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
- case JSON_EXPECTED_STRING:
-- return psprintf(_("Expected string, but found \"%s\"."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Expected string, but found \"%.*s\"."),
-+ toklen, lex->token_start);
-+ break;
+ token_error(lex, "Expected end of input, but found \"%.*s\".");
+@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
case JSON_INVALID_TOKEN:
-- return psprintf(_("Token \"%s\" is invalid."),
-- extract_token(lex));
-+ appendStrVal(lex->errormsg,
-+ _("Token \"%.*s\" is invalid."),
-+ toklen, lex->token_start);
-+ break;
+ token_error(lex, "Token \"%.*s\" is invalid.");
+ break;
+ case JSON_OUT_OF_MEMORY:
+ /* should have been handled above; use the error path */
+ break;
@@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l
return _("\\u0000 cannot be converted to text.");
case JSON_UNICODE_ESCAPE_FORMAT:
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
- /* note: this case is only reachable in frontend not backend */
- return _("Unicode escape values cannot be used for code point values above 007F when the encoding is not UTF8.");
- case JSON_UNICODE_UNTRANSLATABLE:
-- /* note: this case is only reachable in backend not frontend */
-+
-+ /*
-+ * note: this case is only reachable in backend not frontend.
-+ * #ifdef it away so the frontend doesn't try to link against
-+ * backend functionality.
-+ */
-+#ifndef FRONTEND
- return psprintf(_("Unicode escape value could not be translated to the server's encoding %s."),
- GetDatabaseEncodingName());
-+#else
-+ Assert(false);
-+ break;
-+#endif
- case JSON_UNICODE_HIGH_SURROGATE:
- return _("Unicode high surrogate must not follow a high surrogate.");
- case JSON_UNICODE_LOW_SURROGATE:
-@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
- break;
}
+ #undef token_error
- /*
- * We don't use a default: case, so that the compiler will warn about
- * unhandled enum values. But this needs to be here anyway to cover the
- * possibility of an incorrect input.
- */
-- elog(ERROR, "unexpected json parse error type: %d", (int) error);
-- return NULL;
--}
+- if (lex->errormsg->len == 0)
+- appendStringInfo(lex->errormsg,
+- _("unexpected json parse error type: %d"),
+- (int) error);
+ /* Note that lex->errormsg can be NULL in FRONTEND code. */
-+ if (lex->errormsg && !lex->errormsg->data[0])
++ if (lex->errormsg && lex->errormsg->len == 0)
+ {
+ /*
+ * We don't use a default: case, so that the compiler will warn about
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l
+ * the possibility of an incorrect input.
+ */
+ appendStrVal(lex->errormsg,
-+ "unexpected json parse error type: %d", (int) error);
++ _("unexpected json parse error type: %d"),
++ (int) error);
+ }
+
+#ifdef FRONTEND
+ if (PQExpBufferBroken(lex->errormsg))
+ return _("out of memory while constructing error description");
- #endif
-+
-+ return lex->errormsg->data;
-+}
++#endif
+
+ return lex->errormsg->data;
+ }
## src/common/meson.build ##
@@ src/common/meson.build: common_sources_frontend_static += files(
@@ src/common/meson.build: foreach name, opts : pgcommon_variants
'dependencies': opts['dependencies'] + [ssl],
}
- ## src/common/parse_manifest.c ##
-@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, char *buffer,
- /* Run the actual JSON parser. */
- json_error = pg_parse_json(lex, &sem);
- if (json_error != JSON_SUCCESS)
-- json_manifest_parse_failure(context, "parsing failed");
-+ json_manifest_parse_failure(context, json_errdetail(json_error, lex));
- if (parse.state != JM_EXPECT_EOF)
- json_manifest_parse_failure(context, "manifest ended unexpectedly");
-
-
- ## src/common/stringinfo.c ##
-@@ src/common/stringinfo.c: enlargeStringInfo(StringInfo str, int needed)
-
- str->maxlen = newlen;
- }
-+
-+void
-+destroyStringInfo(StringInfo str)
-+{
-+ pfree(str->data);
-+ pfree(str);
-+}
-
## src/include/common/jsonapi.h ##
@@
#ifndef JSONAPI_H
@@ src/include/common/jsonapi.h: typedef struct JsonLexContext
int line_number; /* line number, starting from 1 */
char *line_start; /* where that line starts within input */
- StringInfo strval;
+- StringInfo errormsg;
+ bool parse_strval;
+ StrValType *strval; /* only used if parse_strval == true */
+ StrValType *errormsg;
} JsonLexContext;
typedef JsonParseErrorType (*json_struct_action) (void *state);
-
- ## src/include/lib/stringinfo.h ##
-@@ src/include/lib/stringinfo.h: extern void appendBinaryStringInfoNT(StringInfo str,
- */
- extern void enlargeStringInfo(StringInfo str, int needed);
-
-+
-+extern void destroyStringInfo(StringInfo str);
- #endif /* STRINGINFO_H */
2: f78c79ea68 < -: ---------- Refactor SASL exchange to return tri-state status
3: 10b6d2a6b9 < -: ---------- Explicitly require password for SCRAM exchange
4: 2a55d9c806 ! 2: b754a2f8fb libpq: add OAUTHBEARER SASL mechanism
@@ src/interfaces/libpq/Makefile: endif
endif
## src/interfaces/libpq/exports.txt ##
-@@ src/interfaces/libpq/exports.txt: PQsendClosePrepared 190
- PQsendClosePortal 191
- PQchangePassword 192
- PQsendPipelineSync 193
-+PQsetAuthDataHook 194
-+PQgetAuthDataHook 195
-+PQdefaultAuthDataHook 196
+@@ src/interfaces/libpq/exports.txt: PQcancelSocket 199
+ PQcancelErrorMessage 200
+ PQcancelReset 201
+ PQcancelFinish 202
++PQsetAuthDataHook 203
++PQgetAuthDataHook 204
++PQdefaultAuthDataHook 205
## src/interfaces/libpq/fe-auth-oauth-curl.c (new) ##
@@
@@ src/interfaces/libpq/fe-connect.c: static const internalPQconninfoOption PQconni
{NULL, NULL, NULL, NULL,
NULL, NULL, 0}
@@ src/interfaces/libpq/fe-connect.c: pqDropServerData(PGconn *conn)
+ conn->write_failed = false;
+ free(conn->write_err_msg);
conn->write_err_msg = NULL;
- conn->be_pid = 0;
- conn->be_key = 0;
+ /* conn->oauth_want_retry = false; TODO */
- }
-
+ /*
+ * Cancel connections need to retain their be_pid and be_key across
@@ src/interfaces/libpq/fe-connect.c: PQconnectPoll(PGconn *conn)
case CONNECTION_NEEDED:
case CONNECTION_GSS_STARTUP:
@@ src/interfaces/libpq/libpq-fe.h: extern "C"
/*
* Option flags for PQcopyResult
@@ src/interfaces/libpq/libpq-fe.h: typedef enum
- CONNECTION_CONSUME, /* Consuming any extra messages. */
- CONNECTION_GSS_STARTUP, /* Negotiating GSSAPI. */
- CONNECTION_CHECK_TARGET, /* Checking target server properties. */
-- CONNECTION_CHECK_STANDBY /* Checking if server is in standby mode. */
-+ CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */
+ CONNECTION_CHECK_TARGET, /* Internal state: checking target server
+ * properties. */
+ CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */
+- CONNECTION_ALLOCATED /* Waiting for connection attempt to be
++ CONNECTION_ALLOCATED, /* Waiting for connection attempt to be
+ * started. */
+ CONNECTION_AUTHENTICATING, /* Authentication is in progress with some
+ * external system. */
} ConnStatusType;
@@ src/interfaces/libpq/libpq-int.h: typedef struct pg_conn_host
* PGconn stores all the state data associated with a single connection
* to a backend.
@@ src/interfaces/libpq/libpq-int.h: struct pg_conn
- char *require_auth; /* name of the expected auth method */
- char *load_balance_hosts; /* load balance over hosts */
+ * cancel request, instead of being a normal
+ * connection that's used for queries */
+ /* OAuth v2 */
+ char *oauth_issuer; /* token issuer URL */
5: 5488ac25f5 ! 3: 27251b63c1 backend: add OAUTHBEARER SASL mechanism
@@ src/backend/utils/misc/guc_tables.c
#include "nodes/queryjumble.h"
#include "optimizer/cost.h"
@@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[] =
- check_debug_io_direct, assign_debug_io_direct, NULL
+ check_standby_slot_names, assign_standby_slot_names, NULL
},
+ {
6: fdbad1976a ! 4: 78617fa9ba Introduce OAuth validator libraries
@@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[]
NULL, NULL, NULL
},
- ## src/bin/pg_combinebackup/Makefile ##
-@@ src/bin/pg_combinebackup/Makefile: OBJS = \
- all: pg_combinebackup
-
- pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils
-- $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(libpq_pgport) $(LIBS) -o $@$(X)
-
- install: all installdirs
- $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)'
-
## src/common/Makefile ##
@@ src/common/Makefile: override CPPFLAGS += -DVAL_LDFLAGS_SL="\"$(LDFLAGS_SL)\""
override CPPFLAGS += -DVAL_LIBS="\"$(LIBS)\""
7: e1da97fb50 ! 5: 1b871542f5 Add pytest suite for OAuth
@@ src/test/python/client/test_oauth.py (new)
+ discovery_uri = issuer + "/.well-known/openid-configuration"
+ access_token = secrets.token_urlsafe()
+
-+ sock, _ = accept(
++ sock, client = accept(
+ oauth_issuer=issuer,
+ oauth_client_id="some-id",
+ oauth_scope=scope,
@@ src/test/python/client/test_oauth.py (new)
+ assert call.openid_configuration.decode() == discovery_uri
+ assert call.scope == (None if scope is None else scope.encode())
+
-+ # Make sure we cleaned up after ourselves.
++ # Make sure we clean up after ourselves when the connection is finished.
++ client.check_completed()
+ assert cleanup_calls == 1
+
+
8: 185f9902fd ! 6: e6c5d94682 XXX temporary patches to build and test
@@ Metadata
## Commit message ##
XXX temporary patches to build and test
- - the new pg_combinebackup utility uses JSON in the frontend without
- 0001; has something changed?
- construct 2.10.70 has some incompatibilities with the current tests
- temporarily skip the exit check (from Daniel Gustafsson); this needs
to be turned into an exception for curl rather than a plain exit call
- ## src/bin/pg_combinebackup/Makefile ##
-@@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global
-
- override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
- LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils
-+# TODO: fix this properly
-+LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport)
-
- OBJS = \
- $(WIN32RES) \
-@@ src/bin/pg_combinebackup/Makefile: OBJS = \
-
- all: pg_combinebackup
-
--pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils
-- $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(libpq_pgport) $(LIBS) -o $@$(X)
-+pg_combinebackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
-+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
-
- install: all installdirs
- $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)'
-
- ## src/bin/pg_combinebackup/meson.build ##
-@@ src/bin/pg_combinebackup/meson.build: endif
-
- pg_combinebackup = executable('pg_combinebackup',
- pg_combinebackup_sources,
-- dependencies: [frontend_code],
-+ # XXX linking against libpq isn't good, but how was JSON working?
-+ dependencies: [frontend_code, libpq],
- kwargs: default_bin_args,
- )
- bin_targets += pg_combinebackup
-
- ## src/bin/pg_verifybackup/Makefile ##
-@@ src/bin/pg_verifybackup/Makefile: top_builddir = ../../..
- include $(top_builddir)/src/Makefile.global
-
- # We need libpq only because fe_utils does.
--LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
-+LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpgcommon $(libpq_pgport)
-
- OBJS = \
- $(WIN32RES) \
-
## src/interfaces/libpq/Makefile ##
@@ src/interfaces/libpq/Makefile: libpq-refs-stamp: $(shlib)
ifneq ($(enable_coverage), yes)
9: c4d850a7c4 = 7: decc90579a WIP: Python OAuth provider implementation