From 521ccf1daff979b9beb5bd08f395217175d5f948 Mon Sep 17 00:00:00 2001 From: Mark Dilger Date: Wed, 6 Jan 2021 14:37:44 -0800 Subject: [PATCH v32 8/9] Normalizing option_utils interface. The functions in option_utils were copied from pg_dump, mostly preserving the function signatures. But the signatures and corresponding functionality were originally written based solely on pg_dump's needs, not with the goal of creating a consistent interface. Fixing that. --- src/bin/pg_dump/pg_dump.c | 58 +++++++++++++++++++------ src/bin/pg_dump/pg_dumpall.c | 4 +- src/fe_utils/option_utils.c | 66 ++++++++++++++++++++--------- src/fe_utils/string_utils.c | 50 ++++++++++++++++++++++ src/include/fe_utils/option_utils.h | 29 +++++++++---- src/include/fe_utils/string_utils.h | 6 +++ 6 files changed, 169 insertions(+), 44 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index c334b9e829..5c446c0f24 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -125,6 +125,17 @@ static SimpleOidList tabledata_exclude_oids = {NULL, NULL}; static SimpleStringList foreign_servers_include_patterns = {NULL, NULL}; static SimpleOidList foreign_servers_include_oids = {NULL, NULL}; +/* + * Cstring list of relkinds which qualify as tables for our purposes when + * processing table inclusion or exclusion patterns. + */ +#define TABLE_RELKIND_LIST CppAsString2(RELKIND_RELATION) ", " \ + CppAsString2(RELKIND_SEQUENCE) ", " \ + CppAsString2(RELKIND_VIEW) ", " \ + CppAsString2(RELKIND_MATVIEW) ", " \ + CppAsString2(RELKIND_FOREIGN_TABLE) ", " \ + CppAsString2(RELKIND_PARTITIONED_TABLE) + static const CatalogId nilCatalogId = {0, 0}; /* override for standard extra_float_digits setting */ @@ -791,6 +802,7 @@ main(int argc, char **argv) { expand_schema_name_patterns(GetConnection(fout), &schema_include_patterns, + NULL, &schema_include_oids, strict_names); if (schema_include_oids.head == NULL) @@ -798,6 +810,7 @@ main(int argc, char **argv) } expand_schema_name_patterns(GetConnection(fout), &schema_exclude_patterns, + NULL, &schema_exclude_oids, false); /* non-matching exclusion patterns aren't an error */ @@ -805,26 +818,43 @@ main(int argc, char **argv) /* Expand table selection patterns into OID lists */ if (table_include_patterns.head != NULL) { - expand_table_name_patterns(GetConnection(fout), - &table_include_patterns, - &table_include_oids, - strict_names); + expand_rel_name_patterns(GetConnection(fout), + &table_include_patterns, + NULL, + NULL, + TABLE_RELKIND_LIST, + AMTYPE_TABLE, + &table_include_oids, + strict_names, + true); if (table_include_oids.head == NULL) fatal("no matching tables were found"); } - expand_table_name_patterns(GetConnection(fout), - &table_exclude_patterns, - &table_exclude_oids, - false); - - expand_table_name_patterns(GetConnection(fout), - &tabledata_exclude_patterns, - &tabledata_exclude_oids, - false); + expand_rel_name_patterns(GetConnection(fout), + &table_exclude_patterns, + NULL, + NULL, + TABLE_RELKIND_LIST, + AMTYPE_TABLE, + &table_exclude_oids, + false, + true); + + expand_rel_name_patterns(GetConnection(fout), + &tabledata_exclude_patterns, + NULL, + NULL, + TABLE_RELKIND_LIST, + AMTYPE_TABLE, + &tabledata_exclude_oids, + false, + true); expand_foreign_server_name_patterns(GetConnection(fout), &foreign_servers_include_patterns, - &foreign_servers_include_oids); + NULL, + &foreign_servers_include_oids, + true); /* non-matching exclusion patterns aren't an error */ diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 01db15dfda..2b3a4e3349 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -471,8 +471,8 @@ main(int argc, char *argv[]) /* * Get a list of database names that match the exclude patterns */ - expand_dbname_patterns(conn, &database_exclude_patterns, - &database_exclude_names); + expand_dbname_patterns(conn, &database_exclude_patterns, NULL, + &database_exclude_names, false); /* * Open the output file if required, otherwise use stdout diff --git a/src/fe_utils/option_utils.c b/src/fe_utils/option_utils.c index 7893df77aa..76ca456784 100644 --- a/src/fe_utils/option_utils.c +++ b/src/fe_utils/option_utils.c @@ -14,6 +14,7 @@ */ #include "postgres_fe.h" +#include "catalog/pg_am.h" #include "catalog/pg_class.h" #include "common/connect.h" #include "fe_utils/exit_utils.h" @@ -30,7 +31,8 @@ */ void expand_schema_name_patterns(PGconn *conn, - SimpleStringList *patterns, + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, SimpleOidList *oids, bool strict_names) { @@ -55,6 +57,7 @@ expand_schema_name_patterns(PGconn *conn, "SELECT oid FROM pg_catalog.pg_namespace n\n"); processSQLNamePattern(conn, query, cell->val, false, false, NULL, "n.nspname", NULL, NULL); + exclude_filter(query, "n.oid", exclude_oids); res = ExecuteSqlQuery(conn, query->data, PGRES_TUPLES_OK); if (strict_names && PQntuples(res) == 0) @@ -78,8 +81,10 @@ expand_schema_name_patterns(PGconn *conn, */ void expand_foreign_server_name_patterns(PGconn *conn, - SimpleStringList *patterns, - SimpleOidList *oids) + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, + SimpleOidList *oids, + bool strict_names) { PQExpBuffer query; PGresult *res; @@ -102,9 +107,10 @@ expand_foreign_server_name_patterns(PGconn *conn, "SELECT oid FROM pg_catalog.pg_foreign_server s\n"); processSQLNamePattern(conn, query, cell->val, false, false, NULL, "s.srvname", NULL, NULL); + exclude_filter(query, "s.oid", exclude_oids); res = ExecuteSqlQuery(conn, query->data, PGRES_TUPLES_OK); - if (PQntuples(res) == 0) + if (strict_names && PQntuples(res) == 0) fatal("no matching foreign servers were found for pattern \"%s\"", cell->val); for (i = 0; i < PQntuples(res); i++) @@ -118,18 +124,32 @@ expand_foreign_server_name_patterns(PGconn *conn, } /* - * Find the OIDs of all tables matching the given list of patterns, - * and append them to the given OID list. + * Find the OIDs of all relations matching the given list of patterns + * and restrictions, and append them to the given OID list. */ void -expand_table_name_patterns(PGconn *conn, - SimpleStringList *patterns, SimpleOidList *oids, - bool strict_names) +expand_rel_name_patterns(PGconn *conn, + const SimpleStringList *patterns, + const SimpleOidList *exclude_nsp_oids, + const SimpleOidList *exclude_oids, + const char *relkinds, + char amtype, + SimpleOidList *oids, + bool strict_names, + bool restrict_visible) { PQExpBuffer query; PGresult *res; SimpleStringListCell *cell; int i; + const char *visibility_rule; + + Assert(amtype == AMTYPE_TABLE || amtype == AMTYPE_INDEX); + + if (restrict_visible) + visibility_rule = "pg_catalog.pg_table_is_visible(c.oid)"; + else + visibility_rule = NULL; if (patterns->head == NULL) return; /* nothing to do */ @@ -154,20 +174,22 @@ expand_table_name_patterns(PGconn *conn, "\n LEFT JOIN pg_catalog.pg_namespace n" "\n ON n.oid OPERATOR(pg_catalog.=) c.relnamespace" "\nWHERE c.relkind OPERATOR(pg_catalog.=) ANY" - "\n (array['%c', '%c', '%c', '%c', '%c', '%c'])\n", - RELKIND_RELATION, RELKIND_SEQUENCE, RELKIND_VIEW, - RELKIND_MATVIEW, RELKIND_FOREIGN_TABLE, - RELKIND_PARTITIONED_TABLE); + "\n (array[%s])\n", relkinds); processSQLNamePattern(conn, query, cell->val, true, - false, "n.nspname", "c.relname", NULL, - "pg_catalog.pg_table_is_visible(c.oid)"); + false, "n.nspname", "c.relname", NULL, visibility_rule); + exclude_filter(query, "n.oid", exclude_nsp_oids); + exclude_filter(query, "c.oid", exclude_oids); ExecuteSqlStatement(conn, "RESET search_path"); res = ExecuteSqlQuery(conn, query->data, PGRES_TUPLES_OK); PQclear(ExecuteSqlQueryForSingleRow(conn, ALWAYS_SECURE_SEARCH_PATH_SQL)); if (strict_names && PQntuples(res) == 0) - fatal("no matching tables were found for pattern \"%s\"", cell->val); + { + if (amtype == AMTYPE_TABLE) + fatal("no matching tables were found for pattern \"%s\"", cell->val); + fatal("no matching indexes were found for pattern \"%s\"", cell->val); + } for (i = 0; i < PQntuples(res); i++) { @@ -186,8 +208,10 @@ expand_table_name_patterns(PGconn *conn, */ void expand_dbname_patterns(PGconn *conn, - SimpleStringList *patterns, - SimpleStringList *names) + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, + SimpleStringList *names, + bool strict_names) { PQExpBuffer query; PGresult *res; @@ -206,12 +230,16 @@ expand_dbname_patterns(PGconn *conn, for (SimpleStringListCell *cell = patterns->head; cell; cell = cell->next) { appendPQExpBufferStr(query, - "SELECT datname FROM pg_catalog.pg_database n\n"); + "SELECT datname FROM pg_catalog.pg_database d\n"); processSQLNamePattern(conn, query, cell->val, false, false, NULL, "datname", NULL, NULL); + exclude_filter(query, "d.oid", exclude_oids); pg_log_info("executing %s", query->data); res = ExecuteSqlQuery(conn, query->data, PGRES_TUPLES_OK); + if (strict_names && PQntuples(res) == 0) + fatal("no matching databases were found for pattern \"%s\"", cell->val); + for (int i = 0; i < PQntuples(res); i++) { simple_string_list_append(names, PQgetvalue(res, i, 0)); diff --git a/src/fe_utils/string_utils.c b/src/fe_utils/string_utils.c index a1a9d691d5..4e57a6f940 100644 --- a/src/fe_utils/string_utils.c +++ b/src/fe_utils/string_utils.c @@ -797,6 +797,56 @@ appendReloptionsArray(PQExpBuffer buffer, const char *reloptions, return true; } +/* + * Internal implementation of include_filter and exclude_filter. + */ +static void +apply_filter(PQExpBuffer querybuf, const char *lval, const SimpleOidList *oids, + bool include) +{ + const SimpleOidListCell *cell; + const char *comma; + + if (!oids || !oids->head) + return; + + if (include) + appendPQExpBuffer(querybuf, "\nAND %s OPERATOR(pg_catalog.=) ANY(array[", lval); + else + appendPQExpBuffer(querybuf, "\nAND %s OPERATOR(pg_catalog.!=) ALL(array[", lval); + + for (comma = "", cell = oids->head; cell; comma = ", ", cell = cell->next) + appendPQExpBuffer(querybuf, "%s%u", comma, cell->val); + appendPQExpBuffer(querybuf, "]::OID[])"); +} + +/* + * Conditionally add a restriction to a query such that lval must be an Oid in + * the given list of Oids, except that for a null or empty oids list argument, + * no filtering is done and we return without having modified the query buffer. + * + * The query argument must already have begun the WHERE clause and must be in a + * state where we can append an AND clause. No checking of this requirement is + * done here. + * + * On return, the query buffer will be extended with an AND clause that filters + * only those rows where the lval is an Oid present in the given list of oids. + */ +void +include_filter(PQExpBuffer querybuf, const char *lval, const SimpleOidList *oids) +{ + apply_filter(querybuf, lval, oids, true); +} + +/* + * Same as include_filter, above, except that for a non-null, non-empty oids + * list, the lval is restricted to not be any of the values in the list. + */ +void +exclude_filter(PQExpBuffer querybuf, const char *lval, const SimpleOidList *oids) +{ + apply_filter(querybuf, lval, oids, false); +} /* * processSQLNamePattern diff --git a/src/include/fe_utils/option_utils.h b/src/include/fe_utils/option_utils.h index d626a0bbc9..53da30754f 100644 --- a/src/include/fe_utils/option_utils.h +++ b/src/include/fe_utils/option_utils.h @@ -19,17 +19,28 @@ #include "libpq-fe.h" extern void expand_schema_name_patterns(PGconn *conn, - SimpleStringList *patterns, + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, SimpleOidList *oids, bool strict_names); extern void expand_foreign_server_name_patterns(PGconn *conn, - SimpleStringList *patterns, - SimpleOidList *oids); -extern void expand_table_name_patterns(PGconn *conn, - SimpleStringList *patterns, - SimpleOidList *oids, - bool strict_names); -extern void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, - SimpleStringList *names); + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, + SimpleOidList *oids, + bool strict_names); +extern void expand_rel_name_patterns(PGconn *conn, + const SimpleStringList *patterns, + const SimpleOidList *exclude_nsp_oids, + const SimpleOidList *exclude_oids, + const char *relkinds, + char amtype, + SimpleOidList *oids, + bool strict_names, + bool restrict_visible); +extern void expand_dbname_patterns(PGconn *conn, + const SimpleStringList *patterns, + const SimpleOidList *exclude_oids, + SimpleStringList *names, + bool strict_names); #endif /* OPTION_UTILS_H */ diff --git a/src/include/fe_utils/string_utils.h b/src/include/fe_utils/string_utils.h index c290c302f5..301a8eef4d 100644 --- a/src/include/fe_utils/string_utils.h +++ b/src/include/fe_utils/string_utils.h @@ -16,6 +16,7 @@ #ifndef STRING_UTILS_H #define STRING_UTILS_H +#include "fe_utils/simple_list.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -50,6 +51,11 @@ extern bool parsePGArray(const char *atext, char ***itemarray, int *nitems); extern bool appendReloptionsArray(PQExpBuffer buffer, const char *reloptions, const char *prefix, int encoding, bool std_strings); +extern void include_filter(PQExpBuffer querybuf, const char *lval, + const SimpleOidList *oids); +extern void exclude_filter(PQExpBuffer querybuf, const char *lval, + const SimpleOidList *oids); + extern bool processSQLNamePattern(PGconn *conn, PQExpBuffer buf, const char *pattern, bool have_where, bool force_escape, -- 2.21.1 (Apple Git-122.3)