since-v22.diff.txt
text/plain
Filename: since-v22.diff.txt
Type: text/plain
Part: 0
-: ---------- > 1: aa553b7700 Revert ECPG's use of pnstrdup()
-: ---------- > 2: d6cae9157e Remove fe_memutils from libpgcommon_shlib
1: 9efb0d64e5 ! 3: 5543539169 common/jsonapi: support libpq as a client
@@ Commit message
us to track allocation failures so that we can return JSON_OUT_OF_MEMORY
as needed rather than exit()ing.
+ Co-authored-by: Michael Paquier <michael@paquier.xyz>
+ Co-authored-by: Daniel Gustafsson <daniel@yesql.se>
+
## src/bin/pg_combinebackup/Makefile ##
@@ src/bin/pg_combinebackup/Makefile: include $(top_builddir)/src/Makefile.global
@@ src/common/jsonapi.c
#endif
+/*
-+ * In backend, we will use palloc/pfree along with StringInfo. In frontend, use
-+ * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory.
++ * In backend, we will use palloc/pfree along with StringInfo. In frontend,
++ * use malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory.
+ */
+#ifdef FRONTEND
+
+#define STRDUP(s) strdup(s)
+#define ALLOC(size) malloc(size)
++#define ALLOC0(size) calloc(1, size)
++#define REALLOC realloc
++#define FREE(s) free(s)
+
-+#define appendStrVal appendPQExpBuffer
-+#define appendBinaryStrVal appendBinaryPQExpBuffer
-+#define appendStrValChar appendPQExpBufferChar
-+#define createStrVal createPQExpBuffer
-+#define resetStrVal resetPQExpBuffer
-+#define destroyStrVal destroyPQExpBuffer
++#define appendStrVal appendPQExpBuffer
++#define appendBinaryStrVal appendBinaryPQExpBuffer
++#define appendStrValChar appendPQExpBufferChar
++/* XXX should we add a macro version to PQExpBuffer? */
++#define appendStrValCharMacro appendPQExpBufferChar
++#define createStrVal createPQExpBuffer
++#define initStrVal initPQExpBuffer
++#define resetStrVal resetPQExpBuffer
++#define termStrVal termPQExpBuffer
++#define destroyStrVal destroyPQExpBuffer
+
+#else /* !FRONTEND */
+
+#define STRDUP(s) pstrdup(s)
+#define ALLOC(size) palloc(size)
++#define ALLOC0(size) palloc0(size)
++#define REALLOC repalloc
+
-+#define appendStrVal appendStringInfo
-+#define appendBinaryStrVal appendBinaryStringInfo
-+#define appendStrValChar appendStringInfoChar
-+#define createStrVal makeStringInfo
-+#define resetStrVal resetStringInfo
-+#define destroyStrVal destroyStringInfo
++/*
++ * Backend pfree() doesn't handle NULL pointers like the frontend's does; smooth
++ * that over to reduce mental gymnastics. Avoid multiple evaluation of the macro
++ * argument to avoid future hair-pulling.
++ */
++#define FREE(s) do { \
++ void *__v = (s); \
++ if (__v) \
++ pfree(__v); \
++} while (0)
++
++#define appendStrVal appendStringInfo
++#define appendBinaryStrVal appendBinaryStringInfo
++#define appendStrValChar appendStringInfoChar
++#define appendStrValCharMacro appendStringInfoCharMacro
++#define createStrVal makeStringInfo
++#define initStrVal initStringInfo
++#define resetStrVal resetStringInfo
++#define termStrVal(s) pfree((s)->data)
++#define destroyStrVal destroyStringInfo
+
+#endif
+
/*
* The context of the parser is maintained by the recursive descent
* mechanism, but is passed explicitly to the error reporting routine
-@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
+@@ src/common/jsonapi.c: struct JsonIncrementalState
+ {
+ bool is_last_chunk;
+ bool partial_completed;
+- StringInfoData partial_token;
++ StrValType partial_token;
+ };
+
+ /*
+@@ src/common/jsonapi.c: static JsonParseErrorType parse_object(JsonLexContext *lex, JsonSemAction *sem);
+ static JsonParseErrorType parse_array_element(JsonLexContext *lex, JsonSemAction *sem);
+ static JsonParseErrorType parse_array(JsonLexContext *lex, JsonSemAction *sem);
+ static JsonParseErrorType report_parse_error(JsonParseContext ctx, JsonLexContext *lex);
++static bool allocate_incremental_state(JsonLexContext *lex);
+
+ /* the null action object used for pure validation */
+ JsonSemAction nullSemAction =
+@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, size_t len)
+ {
+ bool numeric_error;
+ size_t total_len;
+- JsonLexContext dummy_lex;
++ JsonLexContext dummy_lex = {0};
+
+ if (len <= 0)
+ return false;
+
+- dummy_lex.incremental = false;
+- dummy_lex.inc_state = NULL;
+- dummy_lex.pstack = NULL;
+-
+ /*
+ * json_lex_number expects a leading '-' to have been eaten already.
+ *
+@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, size_t len)
+ * responsible for freeing the returned struct, either by calling
+ * freeJsonLexContext() or (in backend environment) via memory context
+ * cleanup.
++ *
++ * In frontend code this can return NULL on OOM, so callers must inspect the
++ * returned pointer.
+ */
+ JsonLexContext *
+ makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json,
+@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json,
+ {
+ if (lex == NULL)
+ {
+- lex = palloc0(sizeof(JsonLexContext));
++ lex = ALLOC0(sizeof(JsonLexContext));
++ if (!lex)
++ return NULL;
+ lex->flags |= JSONLEX_FREE_STRUCT;
+ }
+ else
+@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json,
lex->input_encoding = encoding;
if (need_escapes)
{
@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js
lex->flags |= JSONLEX_FREE_STRVAL;
+ lex->parse_strval = true;
}
-+ lex->errormsg = NULL;
return lex;
}
-@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
+
++/*
++ * Allocates the internal bookkeeping structures for incremental parsing. This
++ * can only fail in-band with FRONTEND code.
++ */
++#define JS_STACK_CHUNK_SIZE 64
++#define JS_MAX_PROD_LEN 10 /* more than we need */
++#define JSON_TD_MAX_STACK 6400 /* hard coded for now - this is a REALLY high
++ * number */
++static bool
++allocate_incremental_state(JsonLexContext *lex)
++{
++ void *pstack,
++ *prediction,
++ *fnames,
++ *fnull;
++
++ lex->inc_state = ALLOC0(sizeof(JsonIncrementalState));
++ pstack = ALLOC(sizeof(JsonParserStack));
++ prediction = ALLOC(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN);
++ fnames = ALLOC(JS_STACK_CHUNK_SIZE * sizeof(char *));
++ fnull = ALLOC(JS_STACK_CHUNK_SIZE * sizeof(bool));
++
++#ifdef FRONTEND
++ if (!lex->inc_state
++ || !pstack
++ || !prediction
++ || !fnames
++ || !fnull)
++ {
++ FREE(lex->inc_state);
++ FREE(pstack);
++ FREE(prediction);
++ FREE(fnames);
++ FREE(fnull);
++
++ return false;
++ }
++#endif
++
++ initStrVal(&(lex->inc_state->partial_token));
++ lex->pstack = pstack;
++ lex->pstack->stack_size = JS_STACK_CHUNK_SIZE;
++ lex->pstack->prediction = prediction;
++ lex->pstack->pred_index = 0;
++ lex->pstack->fnames = fnames;
++ lex->pstack->fnull = fnull;
++
++ lex->incremental = true;
++ return true;
++}
++
+
+ /*
+ * makeJsonLexContextIncremental
+@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, const char *json,
+ * we don't need the input, that will be handed in bit by bit to the
+ * parse routine. We also need an accumulator for partial tokens in case
+ * the boundary between chunks happens to fall in the middle of a token.
++ *
++ * In frontend code this can return NULL on OOM, so callers must inspect the
++ * returned pointer.
+ */
+-#define JS_STACK_CHUNK_SIZE 64
+-#define JS_MAX_PROD_LEN 10 /* more than we need */
+-#define JSON_TD_MAX_STACK 6400 /* hard coded for now - this is a REALLY high
+- * number */
+-
+ JsonLexContext *
+ makeJsonLexContextIncremental(JsonLexContext *lex, int encoding,
+ bool need_escapes)
+ {
+ if (lex == NULL)
+ {
+- lex = palloc0(sizeof(JsonLexContext));
++ lex = ALLOC0(sizeof(JsonLexContext));
++ if (!lex)
++ return NULL;
++
+ lex->flags |= JSONLEX_FREE_STRUCT;
+ }
+ else
+@@ src/common/jsonapi.c: makeJsonLexContextIncremental(JsonLexContext *lex, int encoding,
+
+ lex->line_number = 1;
+ lex->input_encoding = encoding;
+- lex->incremental = true;
+- lex->inc_state = palloc0(sizeof(JsonIncrementalState));
+- initStringInfo(&(lex->inc_state->partial_token));
+- lex->pstack = palloc(sizeof(JsonParserStack));
+- lex->pstack->stack_size = JS_STACK_CHUNK_SIZE;
+- lex->pstack->prediction = palloc(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN);
+- lex->pstack->pred_index = 0;
+- lex->pstack->fnames = palloc(JS_STACK_CHUNK_SIZE * sizeof(char *));
+- lex->pstack->fnull = palloc(JS_STACK_CHUNK_SIZE * sizeof(bool));
++
++ if (!allocate_incremental_state(lex))
++ {
++ if (lex->flags & JSONLEX_FREE_STRUCT)
++ FREE(lex);
++ return NULL;
++ }
++
+ if (need_escapes)
+ {
+- lex->strval = makeStringInfo();
++ /*
++ * This call can fail in FRONTEND code. We defer error handling to
++ * time of use (json_lex_string()) since we might not need to parse
++ * any strings anyway.
++ */
++ lex->strval = createStrVal();
+ lex->flags |= JSONLEX_FREE_STRVAL;
++ lex->parse_strval = true;
+ }
++
+ return lex;
+ }
+
+-static inline void
++static inline bool
+ inc_lex_level(JsonLexContext *lex)
+ {
+- lex->lex_level += 1;
+-
+- if (lex->incremental && lex->lex_level >= lex->pstack->stack_size)
++ if (lex->incremental && (lex->lex_level + 1) >= lex->pstack->stack_size)
+ {
+- lex->pstack->stack_size += JS_STACK_CHUNK_SIZE;
+- lex->pstack->prediction =
+- repalloc(lex->pstack->prediction,
+- lex->pstack->stack_size * JS_MAX_PROD_LEN);
+- if (lex->pstack->fnames)
+- lex->pstack->fnames =
+- repalloc(lex->pstack->fnames,
+- lex->pstack->stack_size * sizeof(char *));
+- if (lex->pstack->fnull)
+- lex->pstack->fnull =
+- repalloc(lex->pstack->fnull, lex->pstack->stack_size * sizeof(bool));
++ size_t new_stack_size;
++ char *new_prediction;
++ char **new_fnames;
++ bool *new_fnull;
++
++ new_stack_size = lex->pstack->stack_size + JS_STACK_CHUNK_SIZE;
++
++ new_prediction = REALLOC(lex->pstack->prediction,
++ new_stack_size * JS_MAX_PROD_LEN);
++ new_fnames = REALLOC(lex->pstack->fnames,
++ new_stack_size * sizeof(char *));
++ new_fnull = REALLOC(lex->pstack->fnull, new_stack_size * sizeof(bool));
++
++#ifdef FRONTEND
++ if (!new_prediction || !new_fnames || !new_fnull)
++ return false;
++#endif
++
++ lex->pstack->stack_size = new_stack_size;
++ lex->pstack->prediction = new_prediction;
++ lex->pstack->fnames = new_fnames;
++ lex->pstack->fnull = new_fnull;
+ }
++
++ lex->lex_level += 1;
++ return true;
+ }
+
+ static inline void
+@@ src/common/jsonapi.c: get_fnull(JsonLexContext *lex)
void
freeJsonLexContext(JsonLexContext *lex)
{
+ static const JsonLexContext empty = {0};
++
++ if (!lex)
++ return;
+
if (lex->flags & JSONLEX_FREE_STRVAL)
- destroyStringInfo(lex->strval);
@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *js
- destroyStringInfo(lex->errormsg);
+ destroyStrVal(lex->errormsg);
+ if (lex->incremental)
+ {
+- pfree(lex->inc_state->partial_token.data);
+- pfree(lex->inc_state);
+- pfree(lex->pstack->prediction);
+- pfree(lex->pstack->fnames);
+- pfree(lex->pstack->fnull);
+- pfree(lex->pstack);
++ termStrVal(&lex->inc_state->partial_token);
++ FREE(lex->inc_state);
++ FREE(lex->pstack->prediction);
++ FREE(lex->pstack->fnames);
++ FREE(lex->pstack->fnull);
++ FREE(lex->pstack);
+ }
+
if (lex->flags & JSONLEX_FREE_STRUCT)
- pfree(lex);
+- pfree(lex);
++ FREE(lex);
+ else
+ *lex = empty;
}
/*
+@@ src/common/jsonapi.c: JsonParseErrorType
+ pg_parse_json(JsonLexContext *lex, JsonSemAction *sem)
+ {
+ #ifdef FORCE_JSON_PSTACK
+-
+- lex->incremental = true;
+- lex->inc_state = palloc0(sizeof(JsonIncrementalState));
+-
+ /*
+ * We don't need partial token processing, there is only one chunk. But we
+ * still need to init the partial token string so that freeJsonLexContext
+- * works.
++ * works, so perform the full incremental initialization.
+ */
+- initStringInfo(&(lex->inc_state->partial_token));
+- lex->pstack = palloc(sizeof(JsonParserStack));
+- lex->pstack->stack_size = JS_STACK_CHUNK_SIZE;
+- lex->pstack->prediction = palloc(JS_STACK_CHUNK_SIZE * JS_MAX_PROD_LEN);
+- lex->pstack->pred_index = 0;
+- lex->pstack->fnames = palloc(JS_STACK_CHUNK_SIZE * sizeof(char *));
+- lex->pstack->fnull = palloc(JS_STACK_CHUNK_SIZE * sizeof(bool));
++ if (!allocate_incremental_state(lex))
++ return JSON_OUT_OF_MEMORY;
+
+ return pg_parse_json_incremental(lex, sem, lex->input, lex->input_length, true);
+
@@ src/common/jsonapi.c: json_count_array_elements(JsonLexContext *lex, int *elements)
* etc, so doing this with a copy makes that safe.
*/
@@ src/common/jsonapi.c: json_count_array_elements(JsonLexContext *lex, int *elemen
copylex.lex_level++;
count = 0;
+@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex,
+ if (result != JSON_SUCCESS)
+ return result;
+ }
+- inc_lex_level(lex);
++
++ if (!inc_lex_level(lex))
++ return JSON_OUT_OF_MEMORY;
+ }
+ break;
+ case JSON_SEM_OEND:
+@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex,
+ if (result != JSON_SUCCESS)
+ return result;
+ }
+- inc_lex_level(lex);
++
++ if (!inc_lex_level(lex))
++ return JSON_OUT_OF_MEMORY;
+ }
+ break;
+ case JSON_SEM_AEND:
+@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex,
+ json_ofield_action ostart = sem->object_field_start;
+ json_ofield_action oend = sem->object_field_end;
+
+- if ((ostart != NULL || oend != NULL) && lex->strval != NULL)
++ if ((ostart != NULL || oend != NULL) && lex->parse_strval)
+ {
+- fname = pstrdup(lex->strval->data);
++ fname = STRDUP(lex->strval->data);
++ if (fname == NULL)
++ return JSON_OUT_OF_MEMORY;
+ }
+ set_fname(lex, fname);
+ }
+@@ src/common/jsonapi.c: pg_parse_json_incremental(JsonLexContext *lex,
+ */
+ if (tok == JSON_TOKEN_STRING)
+ {
+- if (lex->strval != NULL)
+- pstack->scalar_val = pstrdup(lex->strval->data);
++ if (lex->parse_strval)
++ {
++ pstack->scalar_val = STRDUP(lex->strval->data);
++ if (pstack->scalar_val == NULL)
++ return JSON_OUT_OF_MEMORY;
++ }
+ }
+ else
+ {
+ ptrdiff_t tlen = (lex->token_terminator - lex->token_start);
+
+- pstack->scalar_val = palloc(tlen + 1);
++ pstack->scalar_val = ALLOC(tlen + 1);
++ if (pstack->scalar_val == NULL)
++ return JSON_OUT_OF_MEMORY;
++
+ memcpy(pstack->scalar_val, lex->token_start, tlen);
+ pstack->scalar_val[tlen] = '\0';
+ }
@@ src/common/jsonapi.c: parse_scalar(JsonLexContext *lex, JsonSemAction *sem)
/* extract the de-escaped string value, or the raw lexeme */
if (lex_peek(lex) == JSON_TOKEN_STRING)
@@ src/common/jsonapi.c: parse_object(JsonLexContext *lex, JsonSemAction *sem)
check_stack_depth();
#endif
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ const char *const end = lex->input + lex->input_length;
+ JsonParseErrorType result;
+
+- if (lex->incremental && lex->inc_state->partial_completed)
++ if (lex->incremental)
+ {
+- /*
+- * We just lexed a completed partial token on the last call, so reset
+- * everything
+- */
+- resetStringInfo(&(lex->inc_state->partial_token));
+- lex->token_terminator = lex->input;
+- lex->inc_state->partial_completed = false;
++ if (lex->inc_state->partial_completed)
++ {
++ /*
++ * We just lexed a completed partial token on the last call, so
++ * reset everything
++ */
++ resetStrVal(&(lex->inc_state->partial_token));
++ lex->token_terminator = lex->input;
++ lex->inc_state->partial_completed = false;
++ }
++
++#ifdef FRONTEND
++ /* Make sure our partial token buffer is valid before using it below. */
++ if (PQExpBufferDataBroken(lex->inc_state->partial_token))
++ return JSON_OUT_OF_MEMORY;
++#endif
+ }
+
+ s = lex->token_terminator;
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ * We have a partial token. Extend it and if completed lex it by a
+ * recursive call
+ */
+- StringInfo ptok = &(lex->inc_state->partial_token);
++ StrValType *ptok = &(lex->inc_state->partial_token);
+ size_t added = 0;
+ bool tok_done = false;
+ JsonLexContext dummy_lex;
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ {
+ char c = lex->input[i];
+
+- appendStringInfoCharMacro(ptok, c);
++ appendStrValCharMacro(ptok, c);
+ added++;
+ if (c == '"' && escapes % 2 == 0)
+ {
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ case '8':
+ case '9':
+ {
+- appendStringInfoCharMacro(ptok, cc);
++ appendStrValCharMacro(ptok, cc);
+ added++;
+ }
+ break;
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+
+ if (JSON_ALPHANUMERIC_CHAR(cc))
+ {
+- appendStringInfoCharMacro(ptok, cc);
++ appendStrValCharMacro(ptok, cc);
+ added++;
+ }
+ else
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ dummy_lex.input_length = ptok->len;
+ dummy_lex.input_encoding = lex->input_encoding;
+ dummy_lex.incremental = false;
++ dummy_lex.parse_strval = lex->parse_strval;
+ dummy_lex.strval = lex->strval;
+
+ partial_result = json_lex(&dummy_lex);
+@@ src/common/jsonapi.c: json_lex(JsonLexContext *lex)
+ if (lex->incremental && !lex->inc_state->is_last_chunk &&
+ p == lex->input + lex->input_length)
+ {
+- appendBinaryStringInfo(
+- &(lex->inc_state->partial_token), s, end - s);
++ appendBinaryStrVal(
++ &(lex->inc_state->partial_token), s, end - s);
+ return JSON_INCOMPLETE;
+ }
+
+@@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex)
+ do { \
+ if (lex->incremental && !lex->inc_state->is_last_chunk) \
+ { \
+- appendBinaryStringInfo(&lex->inc_state->partial_token, \
+- lex->token_start, end - lex->token_start); \
++ appendBinaryStrVal(&lex->inc_state->partial_token, \
++ lex->token_start, end - lex->token_start); \
+ return JSON_INCOMPLETE; \
+ } \
+ lex->token_terminator = s; \
@@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex)
return code; \
} while (0)
@@ src/common/jsonapi.c: json_lex_string(JsonLexContext *lex)
/* Hooray, we found the end of the string! */
lex->prev_token_terminator = lex->token_terminator;
lex->token_terminator = s + 1;
+@@ src/common/jsonapi.c: json_lex_number(JsonLexContext *lex, const char *s,
+ if (lex->incremental && !lex->inc_state->is_last_chunk &&
+ len >= lex->input_length)
+ {
+- appendBinaryStringInfo(&lex->inc_state->partial_token,
+- lex->token_start, s - lex->token_start);
++ appendBinaryStrVal(&lex->inc_state->partial_token,
++ lex->token_start, s - lex->token_start);
+ if (num_err != NULL)
+ *num_err = error;
+
@@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *lex)
char *
json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
@@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l
* A helper for error messages that should print the current token. The
* format must contain exactly one %.*s specifier.
*/
- #define token_error(lex, format) \
+ #define json_token_error(lex, format) \
- appendStringInfo((lex)->errormsg, _(format), \
- (int) ((lex)->token_terminator - (lex)->token_start), \
- (lex)->token_start);
@@ src/common/jsonapi.c: report_parse_error(JsonParseContext ctx, JsonLexContext *l
switch (error)
{
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
- token_error(lex, "Escape sequence \"\\%.*s\" is invalid.");
+ json_token_error(lex, "Escape sequence \"\\%.*s\" is invalid.");
break;
case JSON_ESCAPING_REQUIRED:
- appendStringInfo(lex->errormsg,
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l
+ (unsigned char) *(lex->token_terminator));
break;
case JSON_EXPECTED_END:
- token_error(lex, "Expected end of input, but found \"%.*s\".");
+ json_token_error(lex, "Expected end of input, but found \"%.*s\".");
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
case JSON_INVALID_TOKEN:
- token_error(lex, "Token \"%.*s\" is invalid.");
+ json_token_error(lex, "Token \"%.*s\" is invalid.");
break;
+ case JSON_OUT_OF_MEMORY:
+ /* should have been handled above; use the error path */
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l
case JSON_UNICODE_ESCAPE_FORMAT:
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
}
- #undef token_error
+ #undef json_token_error
- /*
- * We don't use a default: case, so that the compiler will warn about
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l
- */
- if (lex->errormsg->len == 0)
- appendStringInfo(lex->errormsg,
-- _("unexpected json parse error type: %d"),
+- "unexpected json parse error type: %d",
- (int) error);
+ /* Note that lex->errormsg can be NULL in FRONTEND code. */
+ if (lex->errormsg && lex->errormsg->len == 0)
@@ src/common/jsonapi.c: json_errdetail(JsonParseErrorType error, JsonLexContext *l
+ * the possibility of an incorrect input.
+ */
+ appendStrVal(lex->errormsg,
-+ _("unexpected json parse error type: %d"),
++ "unexpected json parse error type: %d",
+ (int) error);
+ }
+
@@ src/common/meson.build: foreach name, opts : pgcommon_variants
'dependencies': opts['dependencies'] + [ssl],
}
+ ## src/common/parse_manifest.c ##
+@@ src/common/parse_manifest.c: json_parse_manifest_incremental_init(JsonManifestParseContext *context)
+ parse->state = JM_EXPECT_TOPLEVEL_START;
+ parse->saw_version_field = false;
+
+- makeJsonLexContextIncremental(&(incstate->lex), PG_UTF8, true);
++ if (!makeJsonLexContextIncremental(&(incstate->lex), PG_UTF8, true))
++ context->error_cb(context, "out of memory");
+
+ incstate->sem.semstate = parse;
+ incstate->sem.object_start = json_manifest_object_start;
+@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, const char *buffer,
+
+ /* Create a JSON lexing context. */
+ lex = makeJsonLexContextCstringLen(NULL, buffer, size, PG_UTF8, true);
++ if (!lex)
++ json_manifest_parse_failure(context, "out of memory");
+
+ /* Set up semantic actions. */
+ sem.semstate = &parse;
+
## src/include/common/jsonapi.h ##
@@
#ifndef JSONAPI_H
@@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType
JSON_UNICODE_ESCAPE_FORMAT,
JSON_UNICODE_HIGH_ESCAPE,
@@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType
- JSON_SEM_ACTION_FAILED, /* error should already be reported */
- } JsonParseErrorType;
+ typedef struct JsonParserStack JsonParserStack;
+ typedef struct JsonIncrementalState JsonIncrementalState;
+/*
+ * Don't depend on the internal type header for strval; if callers need access
@@ src/include/common/jsonapi.h: typedef enum JsonParseErrorType
+#endif
+
+typedef struct StrValType StrValType;
-
++
/*
* All the fields in this structure should be treated as read-only.
+ *
@@ src/include/common/jsonapi.h: typedef struct JsonLexContext
- bits32 flags;
- int line_number; /* line number, starting from 1 */
- char *line_start; /* where that line starts within input */
+ const char *line_start; /* where that line starts within input */
+ JsonParserStack *pstack;
+ JsonIncrementalState *inc_state;
- StringInfo strval;
- StringInfo errormsg;
+ bool parse_strval;
@@ src/include/common/jsonapi.h: typedef struct JsonLexContext
} JsonLexContext;
typedef JsonParseErrorType (*json_struct_action) (void *state);
+
+ ## src/test/modules/test_json_parser/Makefile ##
+@@ src/test/modules/test_json_parser/Makefile: include $(top_builddir)/src/Makefile.global
+ include $(top_srcdir)/contrib/contrib-global.mk
+ endif
+
++# TODO: fix this properly
++LDFLAGS_INTERNAL += -lpgcommon $(libpq_pgport)
++
+ all: test_json_parser_incremental$(X) test_json_parser_perf$(X)
+
+ %.o: $(top_srcdir)/$(subdir)/%.c
+
+ ## src/test/modules/test_json_parser/meson.build ##
+@@ src/test/modules/test_json_parser/meson.build: endif
+
+ test_json_parser_incremental = executable('test_json_parser_incremental',
+ test_json_parser_incremental_sources,
+- dependencies: [frontend_code],
++ dependencies: [frontend_code, libpq],
+ kwargs: default_bin_args + {
+ 'install': false,
+ },
+@@ src/test/modules/test_json_parser/meson.build: endif
+
+ test_json_parser_perf = executable('test_json_parser_perf',
+ test_json_parser_perf_sources,
+- dependencies: [frontend_code],
++ dependencies: [frontend_code, libpq],
+ kwargs: default_bin_args + {
+ 'install': false,
+ },
2: 31f2ffdf0b ! 4: 1639f7eb9a libpq: add OAUTHBEARER SASL mechanism
@@ src/interfaces/libpq/Makefile: endif
endif
## src/interfaces/libpq/exports.txt ##
-@@ src/interfaces/libpq/exports.txt: PQcancelSocket 199
- PQcancelErrorMessage 200
- PQcancelReset 201
- PQcancelFinish 202
-+PQsetAuthDataHook 203
-+PQgetAuthDataHook 204
-+PQdefaultAuthDataHook 205
+@@ src/interfaces/libpq/exports.txt: PQcancelFinish 202
+ PQsocketPoll 203
+ PQsetChunkedRowsMode 204
+ PQgetCurrentTimeUSec 205
++PQsetAuthDataHook 206
++PQgetAuthDataHook 207
++PQdefaultAuthDataHook 208
## src/interfaces/libpq/fe-auth-oauth-curl.c (new) ##
@@
@@ src/interfaces/libpq/fe-connect.c: keep_going: /* We will come back to here
+ }
+#endif
+
- #ifdef ENABLE_GSS
-
- /*
+ CONNECTION_FAILED();
+ }
+ else if (beresp == PqMsg_NegotiateProtocolVersion)
@@ src/interfaces/libpq/fe-connect.c: keep_going: /* We will come back to here until there is
* Note that conn->pghost must be non-NULL if we are going to
* avoid the Kerberos code doing a hostname look-up.
@@ src/interfaces/libpq/fe-connect.c: PQsocket(const PGconn *conn)
## src/interfaces/libpq/fe-misc.c ##
@@ src/interfaces/libpq/fe-misc.c: static int
- pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time)
+ pqSocketCheck(PGconn *conn, int forRead, int forWrite, pg_usec_time_t end_time)
{
int result;
+ pgsocket sock;
@@ src/interfaces/libpq/fe-misc.c: static int
{
libpq_append_conn_error(conn, "invalid socket");
return -1;
-@@ src/interfaces/libpq/fe-misc.c: pqSocketCheck(PGconn *conn, int forRead, int forWrite, time_t end_time)
+@@ src/interfaces/libpq/fe-misc.c: pqSocketCheck(PGconn *conn, int forRead, int forWrite, pg_usec_time_t end_time)
/* We will retry as long as we get EINTR */
do
-- result = pqSocketPoll(conn->sock, forRead, forWrite, end_time);
-+ result = pqSocketPoll(sock, forRead, forWrite, end_time);
+- result = PQsocketPoll(conn->sock, forRead, forWrite, end_time);
++ result = PQsocketPoll(sock, forRead, forWrite, end_time);
while (result < 0 && SOCK_ERRNO == EINTR);
if (result < 0)
@@ src/interfaces/libpq/libpq-fe.h: extern "C"
/*
* Option flags for PQcopyResult
@@ src/interfaces/libpq/libpq-fe.h: typedef enum
- CONNECTION_CHECK_TARGET, /* Internal state: checking target server
- * properties. */
CONNECTION_CHECK_STANDBY, /* Checking if server is in standby mode. */
-- CONNECTION_ALLOCATED /* Waiting for connection attempt to be
-+ CONNECTION_ALLOCATED, /* Waiting for connection attempt to be
+ CONNECTION_ALLOCATED, /* Waiting for connection attempt to be
* started. */
+ CONNECTION_AUTHENTICATING, /* Authentication is in progress with some
+ * external system. */
3: 96892c6249 ! 5: 044d8b08e9 backend: add OAUTHBEARER SASL mechanism
@@ Commit message
Co-authored-by: Daniel Gustafsson <daniel@yesql.se>
+ ## .cirrus.tasks.yml ##
+@@ .cirrus.tasks.yml: task:
+ chown root:postgres /tmp/cores
+ sysctl kern.corefile='/tmp/cores/%N.%P.core'
+ setup_additional_packages_script: |
+- #pkg install -y ...
++ pkg install -y curl
+
+ # NB: Intentionally build without -Dllvm. The freebsd image size is already
+ # large enough to make VM startup slow, and even without llvm freebsd
+@@ .cirrus.tasks.yml: task:
+ -Dcassert=true -Dinjection_points=true \
+ -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
+ -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
++ -Doauth=curl \
+ -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
+ build
+ EOF
+@@ .cirrus.tasks.yml: LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
+ --with-libxslt
+ --with-llvm
+ --with-lz4
++ --with-oauth=curl
+ --with-pam
+ --with-perl
+ --with-python
+@@ .cirrus.tasks.yml: LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
+
+ LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
+ -Dllvm=enabled
++ -Doauth=curl
+ -Duuid=e2fs
+
+
+@@ .cirrus.tasks.yml: task:
+ EOF
+
+ setup_additional_packages_script: |
+- #apt-get update
+- #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
++ apt-get update
++ DEBIAN_FRONTEND=noninteractive apt-get -y install \
++ libcurl4-openssl-dev \
++ libcurl4-openssl-dev:i386 \
+
+ matrix:
+ - name: Linux - Debian Bullseye - Autoconf
+@@ .cirrus.tasks.yml: task:
+ folder: $CCACHE_DIR
+
+ setup_additional_packages_script: |
+- #apt-get update
+- #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
++ apt-get update
++ DEBIAN_FRONTEND=noninteractive apt-get -y install libcurl4-openssl-dev
+
+ ###
+ # Test that code can be built with gcc/clang without warnings
+
## src/backend/libpq/Makefile ##
@@ src/backend/libpq/Makefile: include $(top_builddir)/src/Makefile.global
# be-fsstubs is here for historical reasons, probably belongs elsewhere
@@ src/backend/utils/misc/guc_tables.c
#include "nodes/queryjumble.h"
#include "optimizer/cost.h"
@@ src/backend/utils/misc/guc_tables.c: struct config_string ConfigureNamesString[] =
- check_standby_slot_names, assign_standby_slot_names, NULL
+ check_synchronized_standby_slots, assign_synchronized_standby_slots, NULL
},
+ {
@@ src/include/libpq/sasl.h: typedef struct pg_be_sasl_mech
/* Common implementation for auth.c */
+ ## src/test/modules/Makefile ##
+@@ src/test/modules/Makefile: SUBDIRS = \
+ dummy_index_am \
+ dummy_seclabel \
+ libpq_pipeline \
++ oauth_validator \
+ plsample \
+ spgist_name_ops \
+ test_bloomfilter \
+
## src/test/modules/meson.build ##
@@ src/test/modules/meson.build: subdir('gin')
subdir('injection_points')
@@ src/test/modules/oauth_validator/.gitignore (new)
## src/test/modules/oauth_validator/Makefile (new) ##
@@
+export PYTHON
++export with_oauth
+
+MODULES = validator
+PGFILEDESC = "validator - test OAuth validator module"
@@ src/test/modules/oauth_validator/meson.build (new)
+ ],
+ 'env': {
+ 'PYTHON': python.path(),
++ 'with_oauth': oauth_library,
+ },
+ },
+}
@@ src/test/modules/oauth_validator/t/001_server.pl (new)
+use PostgreSQL::Test::OAuthServer;
+use Test::More;
+
++if ($ENV{with_oauth} ne 'curl')
++{
++ plan skip_all => 'client-side OAuth not supported by this build';
++}
++
+my $node = PostgreSQL::Test::Cluster->new('primary');
+$node->init;
+$node->append_conf('postgresql.conf', "log_connections = on\n");
4: 5677e59152 ! 6: 84c6893325 Review comments
@@ Commit message
* Implement a version check for libcurl in autoconf, the
equivalent check for Meson is still a TODO.
* Address a few TODOs in the code
- * libpq JSON support memory management fixups
+ * libpq JSON support memory management fixups [ed: these have been moved
+ to an earlier commit]
## config/programs.m4 ##
@@ config/programs.m4: if test "$pgac_cv_ldap_safe" != yes; then
@@ src/backend/libpq/auth-oauth.c: validate(Port *port, const char *auth)
}
- ## src/common/jsonapi.c ##
-@@
- #endif
-
- /*
-- * In backend, we will use palloc/pfree along with StringInfo. In frontend, use
-- * malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory.
-+ * In backend, we will use palloc/pfree along with StringInfo. In frontend,
-+ * use malloc and PQExpBuffer, and return JSON_OUT_OF_MEMORY on out-of-memory.
- */
- #ifdef FRONTEND
-
- #define STRDUP(s) strdup(s)
- #define ALLOC(size) malloc(size)
-+#define ALLOC0(size) calloc(1, size)
-+#define FREE(s) free(s)
-
- #define appendStrVal appendPQExpBuffer
- #define appendBinaryStrVal appendBinaryPQExpBuffer
-@@
-
- #define STRDUP(s) pstrdup(s)
- #define ALLOC(size) palloc(size)
-+#define ALLOC0(size) palloc0(size)
-+#define FREE(s) pfree(s)
-
- #define appendStrVal appendStringInfo
- #define appendBinaryStrVal appendBinaryStringInfo
-@@ src/common/jsonapi.c: IsValidJsonNumber(const char *str, int len)
- * responsible for freeing the returned struct, either by calling
- * freeJsonLexContext() or (in backend environment) via memory context
- * cleanup.
-+ *
-+ * In frontend code this can return NULL on OOM, so callers must inspect the
-+ * returned pointer.
- */
- JsonLexContext *
- makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
-@@ src/common/jsonapi.c: makeJsonLexContextCstringLen(JsonLexContext *lex, char *json,
- {
- if (lex == NULL)
- {
-- lex = palloc0(sizeof(JsonLexContext));
-+ lex = ALLOC0(sizeof(JsonLexContext));
-+ if (!lex)
-+ return NULL;
- lex->flags |= JSONLEX_FREE_STRUCT;
- }
- else
-@@ src/common/jsonapi.c: freeJsonLexContext(JsonLexContext *lex)
- {
- static const JsonLexContext empty = {0};
-
-+ if (!lex)
-+ return;
-+
- if (lex->flags & JSONLEX_FREE_STRVAL)
- destroyStrVal(lex->strval);
-
-@@ src/common/jsonapi.c: freeJsonLexContext(JsonLexContext *lex)
- destroyStrVal(lex->errormsg);
-
- if (lex->flags & JSONLEX_FREE_STRUCT)
-- pfree(lex);
-+ FREE(lex);
- else
- *lex = empty;
- }
-
- ## src/common/parse_manifest.c ##
-@@ src/common/parse_manifest.c: json_parse_manifest(JsonManifestParseContext *context, char *buffer,
-
- /* Create a JSON lexing context. */
- lex = makeJsonLexContextCstringLen(NULL, buffer, size, PG_UTF8, true);
-+ if (!lex)
-+ json_manifest_parse_failure(context, "out of memory");
-
- /* Set up semantic actions. */
- sem.semstate = &parse;
-
## src/include/common/oauth-common.h ##
@@
* oauth-common.h