From 607984bdcc91fa31fb7a12e9b24fb8704aa14975 Mon Sep 17 00:00:00 2001 From: Corey Huinker Date: Fri, 14 Mar 2025 01:06:19 -0400 Subject: [PATCH v12 1/3] Introduce CreateStmtPtr. CreateStmtPtr is a function pointer that can replace the createStmt/defn parameter. This is useful in situations where the amount of text generated for a definition is so large that it is undesirable to hold many such objects in memory at the same time. Using functions of this type, the text created is then immediately written out to the appropriate file for the given dump format. --- src/bin/pg_dump/pg_backup.h | 2 + src/bin/pg_dump/pg_backup_archiver.c | 22 ++- src/bin/pg_dump/pg_backup_archiver.h | 7 + src/bin/pg_dump/pg_dump.c | 229 +++++++++++++++------------ 4 files changed, 158 insertions(+), 102 deletions(-) diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index 658986de6f8..fdcccd64a70 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -289,6 +289,8 @@ typedef int (*DataDumperPtr) (Archive *AH, const void *userArg); typedef void (*SetupWorkerPtrType) (Archive *AH); +typedef char *(*CreateStmtPtr) (Archive *AH, const void *userArg); + /* * Main archiver interface. */ diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index 1d131e5a57d..1b4c62fd7d7 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -1265,6 +1265,9 @@ ArchiveEntry(Archive *AHX, CatalogId catalogId, DumpId dumpId, newToc->dataDumper = opts->dumpFn; newToc->dataDumperArg = opts->dumpArg; newToc->hadDumper = opts->dumpFn ? true : false; + newToc->createDumper = opts->createFn; + newToc->createDumperArg = opts->createArg; + newToc->hadCreateDumper = opts->createFn ? true : false; newToc->formatData = NULL; newToc->dataLength = 0; @@ -2621,7 +2624,17 @@ WriteToc(ArchiveHandle *AH) WriteStr(AH, te->tag); WriteStr(AH, te->desc); WriteInt(AH, te->section); - WriteStr(AH, te->defn); + + if (te->hadCreateDumper) + { + char *defn = te->createDumper((Archive *) AH, te->createDumperArg); + + WriteStr(AH, defn); + pg_free(defn); + } + else + WriteStr(AH, te->defn); + WriteStr(AH, te->dropStmt); WriteStr(AH, te->copyStmt); WriteStr(AH, te->namespace); @@ -3877,6 +3890,13 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, const char *pfx) { IssueACLPerBlob(AH, te); } + else if (te->hadCreateDumper) + { + char *ptr = te->createDumper((Archive *) AH, te->createDumperArg); + + ahwrite(ptr, 1, strlen(ptr), AH); + pg_free(ptr); + } else if (te->defn && strlen(te->defn) > 0) { ahprintf(AH, "%s\n\n", te->defn); diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h index a2064f471ed..e68db633995 100644 --- a/src/bin/pg_dump/pg_backup_archiver.h +++ b/src/bin/pg_dump/pg_backup_archiver.h @@ -368,6 +368,11 @@ struct _tocEntry const void *dataDumperArg; /* Arg for above routine */ void *formatData; /* TOC Entry data specific to file format */ + CreateStmtPtr createDumper; /* Routine for create statement creation */ + const void *createDumperArg; /* arg for the above routine */ + bool hadCreateDumper; /* Archiver was passed a create statement + * routine */ + /* working state while dumping/restoring */ pgoff_t dataLength; /* item's data size; 0 if none or unknown */ int reqs; /* do we need schema and/or data of object @@ -407,6 +412,8 @@ typedef struct _archiveOpts int nDeps; DataDumperPtr dumpFn; const void *dumpArg; + CreateStmtPtr createFn; + const void *createArg; } ArchiveOpts; #define ARCHIVE_OPTS(...) &(ArchiveOpts){__VA_ARGS__} /* Called to add a TOC entry */ diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 4ca34be230c..cc195d6cd9e 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -10560,42 +10560,44 @@ appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname, } /* - * dumpRelationStats -- + * printDumpRelationStats -- * - * Dump command to import stats into the relation on the new database. + * Generate the SQL statements needed to restore a relation's statistics. */ -static void -dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) +static char * +printRelationStats(Archive *fout, const void *userArg) { + const RelStatsInfo *rsinfo = (RelStatsInfo *) userArg; const DumpableObject *dobj = &rsinfo->dobj; + + PQExpBufferData query; + PQExpBufferData out; + PGresult *res; - PQExpBuffer query; - PQExpBuffer out; - int i_attname; - int i_inherited; - int i_null_frac; - int i_avg_width; - int i_n_distinct; - int i_most_common_vals; - int i_most_common_freqs; - int i_histogram_bounds; - int i_correlation; - int i_most_common_elems; - int i_most_common_elem_freqs; - int i_elem_count_histogram; - int i_range_length_histogram; - int i_range_empty_frac; - int i_range_bounds_histogram; - /* nothing to do if we are not dumping statistics */ - if (!fout->dopt->dumpStatistics) - return; + static bool first_query = true; + static int i_attname; + static int i_inherited; + static int i_null_frac; + static int i_avg_width; + static int i_n_distinct; + static int i_most_common_vals; + static int i_most_common_freqs; + static int i_histogram_bounds; + static int i_correlation; + static int i_most_common_elems; + static int i_most_common_elem_freqs; + static int i_elem_count_histogram; + static int i_range_length_histogram; + static int i_range_empty_frac; + static int i_range_bounds_histogram; - query = createPQExpBuffer(); - if (!fout->is_prepared[PREPQUERY_GETATTRIBUTESTATS]) + initPQExpBuffer(&query); + + if (first_query) { - appendPQExpBufferStr(query, - "PREPARE getAttributeStats(pg_catalog.name, pg_catalog.name) AS\n" + appendPQExpBufferStr(&query, + "PREPARE getAttributeStats(pg_catalog.text, pg_catalog.text) AS\n" "SELECT s.attname, s.inherited, " "s.null_frac, s.avg_width, s.n_distinct, " "s.most_common_vals, s.most_common_freqs, " @@ -10604,88 +10606,87 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) "s.elem_count_histogram, "); if (fout->remoteVersion >= 170000) - appendPQExpBufferStr(query, + appendPQExpBufferStr(&query, "s.range_length_histogram, " "s.range_empty_frac, " "s.range_bounds_histogram "); else - appendPQExpBufferStr(query, + appendPQExpBufferStr(&query, "NULL AS range_length_histogram," "NULL AS range_empty_frac," "NULL AS range_bounds_histogram "); - appendPQExpBufferStr(query, + appendPQExpBufferStr(&query, "FROM pg_catalog.pg_stats s " "WHERE s.schemaname = $1 " "AND s.tablename = $2 " "ORDER BY s.attname, s.inherited"); - ExecuteSqlStatement(fout, query->data); + ExecuteSqlStatement(fout, query.data); - fout->is_prepared[PREPQUERY_GETATTRIBUTESTATS] = true; - resetPQExpBuffer(query); + resetPQExpBuffer(&query); } - out = createPQExpBuffer(); + initPQExpBuffer(&out); /* restore relation stats */ - appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n"); - appendPQExpBuffer(out, "\t'version', '%u'::integer,\n", - fout->remoteVersion); - appendPQExpBufferStr(out, "\t'schemaname', "); - appendStringLiteralAH(out, rsinfo->dobj.namespace->dobj.name, fout); - appendPQExpBufferStr(out, ",\n"); - appendPQExpBufferStr(out, "\t'relname', "); - appendStringLiteralAH(out, rsinfo->dobj.name, fout); - appendPQExpBufferStr(out, ",\n"); - appendPQExpBuffer(out, "\t'relpages', '%d'::integer,\n", rsinfo->relpages); - appendPQExpBuffer(out, "\t'reltuples', '%s'::real,\n", rsinfo->reltuples); - appendPQExpBuffer(out, "\t'relallvisible', '%d'::integer", - rsinfo->relallvisible); + appendPQExpBufferStr(&out, "SELECT * FROM pg_catalog.pg_restore_relation_stats("); + appendPQExpBuffer(&out, "\n\t'version', '%u'::integer", fout->remoteVersion); + appendPQExpBufferStr(&out, ",\n\t'schemaname', "); + appendStringLiteralAH(&out, rsinfo->dobj.namespace->dobj.name, fout); + appendPQExpBufferStr(&out, ",\n\t'relname', "); + appendStringLiteralAH(&out, rsinfo->dobj.name, fout); + appendPQExpBuffer(&out, ",\n\t'relpages', '%d'::integer", rsinfo->relpages); + appendPQExpBuffer(&out, ",\n\t'reltuples', '%s'::real", rsinfo->reltuples); + appendPQExpBuffer(&out, ",\n\t'relallvisible', '%d'::integer", rsinfo->relallvisible); if (fout->remoteVersion >= 180000) - appendPQExpBuffer(out, ",\n\t'relallfrozen', '%d'::integer", rsinfo->relallfrozen); + appendPQExpBuffer(&out, ",\n\t'relallfrozen', '%d'::integer", rsinfo->relallfrozen); - appendPQExpBufferStr(out, "\n);\n"); + appendPQExpBufferStr(&out, "\n);\n"); /* fetch attribute stats */ - appendPQExpBufferStr(query, "EXECUTE getAttributeStats("); - appendStringLiteralAH(query, dobj->namespace->dobj.name, fout); - appendPQExpBufferStr(query, ", "); - appendStringLiteralAH(query, dobj->name, fout); - appendPQExpBufferStr(query, ");"); + appendPQExpBufferStr(&query, "EXECUTE getAttributeStats("); + appendStringLiteralAH(&query, dobj->namespace->dobj.name, fout); + appendPQExpBufferStr(&query, ", "); + appendStringLiteralAH(&query, dobj->name, fout); + appendPQExpBufferStr(&query, ")"); - res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK); + res = ExecuteSqlQuery(fout, query.data, PGRES_TUPLES_OK); - i_attname = PQfnumber(res, "attname"); - i_inherited = PQfnumber(res, "inherited"); - i_null_frac = PQfnumber(res, "null_frac"); - i_avg_width = PQfnumber(res, "avg_width"); - i_n_distinct = PQfnumber(res, "n_distinct"); - i_most_common_vals = PQfnumber(res, "most_common_vals"); - i_most_common_freqs = PQfnumber(res, "most_common_freqs"); - i_histogram_bounds = PQfnumber(res, "histogram_bounds"); - i_correlation = PQfnumber(res, "correlation"); - i_most_common_elems = PQfnumber(res, "most_common_elems"); - i_most_common_elem_freqs = PQfnumber(res, "most_common_elem_freqs"); - i_elem_count_histogram = PQfnumber(res, "elem_count_histogram"); - i_range_length_histogram = PQfnumber(res, "range_length_histogram"); - i_range_empty_frac = PQfnumber(res, "range_empty_frac"); - i_range_bounds_histogram = PQfnumber(res, "range_bounds_histogram"); + if (first_query) + { + i_attname = PQfnumber(res, "attname"); + i_inherited = PQfnumber(res, "inherited"); + i_null_frac = PQfnumber(res, "null_frac"); + i_avg_width = PQfnumber(res, "avg_width"); + i_n_distinct = PQfnumber(res, "n_distinct"); + i_most_common_vals = PQfnumber(res, "most_common_vals"); + i_most_common_freqs = PQfnumber(res, "most_common_freqs"); + i_histogram_bounds = PQfnumber(res, "histogram_bounds"); + i_correlation = PQfnumber(res, "correlation"); + i_most_common_elems = PQfnumber(res, "most_common_elems"); + i_most_common_elem_freqs = PQfnumber(res, "most_common_elem_freqs"); + i_elem_count_histogram = PQfnumber(res, "elem_count_histogram"); + i_range_length_histogram = PQfnumber(res, "range_length_histogram"); + i_range_empty_frac = PQfnumber(res, "range_empty_frac"); + i_range_bounds_histogram = PQfnumber(res, "range_bounds_histogram"); + first_query = false; + } /* restore attribute stats */ for (int rownum = 0; rownum < PQntuples(res); rownum++) { const char *attname; - appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n"); - appendPQExpBuffer(out, "\t'version', '%u'::integer,\n", + appendPQExpBufferStr(&out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n"); + appendPQExpBuffer(&out, "\t'version', '%u'::integer,\n", fout->remoteVersion); - appendPQExpBufferStr(out, "\t'schemaname', "); - appendStringLiteralAH(out, rsinfo->dobj.namespace->dobj.name, fout); - appendPQExpBufferStr(out, ",\n\t'relname', "); - appendStringLiteralAH(out, rsinfo->dobj.name, fout); + appendPQExpBufferStr(&out, "\t'schemaname', "); + appendStringLiteralAH(&out, rsinfo->dobj.namespace->dobj.name, fout); + appendPQExpBufferStr(&out, ",\n\t'relname', "); + appendStringLiteralAH(&out, rsinfo->dobj.name, fout); if (PQgetisnull(res, rownum, i_attname)) pg_fatal("attname cannot be NULL"); @@ -10698,8 +10699,8 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) */ if (rsinfo->nindAttNames == 0) { - appendPQExpBuffer(out, ",\n\t'attname', "); - appendStringLiteralAH(out, attname, fout); + appendPQExpBuffer(&out, ",\n\t'attname', "); + appendStringLiteralAH(&out, attname, fout); } else { @@ -10709,7 +10710,7 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) { if (strcmp(attname, rsinfo->indAttNames[i]) == 0) { - appendPQExpBuffer(out, ",\n\t'attnum', '%d'::smallint", + appendPQExpBuffer(&out, ",\n\t'attnum', '%d'::smallint", i + 1); found = true; break; @@ -10721,66 +10722,92 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) } if (!PQgetisnull(res, rownum, i_inherited)) - appendNamedArgument(out, fout, "inherited", "boolean", + appendNamedArgument(&out, fout, "inherited", "boolean", PQgetvalue(res, rownum, i_inherited)); if (!PQgetisnull(res, rownum, i_null_frac)) - appendNamedArgument(out, fout, "null_frac", "real", + appendNamedArgument(&out, fout, "null_frac", "real", PQgetvalue(res, rownum, i_null_frac)); if (!PQgetisnull(res, rownum, i_avg_width)) - appendNamedArgument(out, fout, "avg_width", "integer", + appendNamedArgument(&out, fout, "avg_width", "integer", PQgetvalue(res, rownum, i_avg_width)); if (!PQgetisnull(res, rownum, i_n_distinct)) - appendNamedArgument(out, fout, "n_distinct", "real", + appendNamedArgument(&out, fout, "n_distinct", "real", PQgetvalue(res, rownum, i_n_distinct)); if (!PQgetisnull(res, rownum, i_most_common_vals)) - appendNamedArgument(out, fout, "most_common_vals", "text", + appendNamedArgument(&out, fout, "most_common_vals", "text", PQgetvalue(res, rownum, i_most_common_vals)); if (!PQgetisnull(res, rownum, i_most_common_freqs)) - appendNamedArgument(out, fout, "most_common_freqs", "real[]", + appendNamedArgument(&out, fout, "most_common_freqs", "real[]", PQgetvalue(res, rownum, i_most_common_freqs)); if (!PQgetisnull(res, rownum, i_histogram_bounds)) - appendNamedArgument(out, fout, "histogram_bounds", "text", + appendNamedArgument(&out, fout, "histogram_bounds", "text", PQgetvalue(res, rownum, i_histogram_bounds)); if (!PQgetisnull(res, rownum, i_correlation)) - appendNamedArgument(out, fout, "correlation", "real", + appendNamedArgument(&out, fout, "correlation", "real", PQgetvalue(res, rownum, i_correlation)); if (!PQgetisnull(res, rownum, i_most_common_elems)) - appendNamedArgument(out, fout, "most_common_elems", "text", + appendNamedArgument(&out, fout, "most_common_elems", "text", PQgetvalue(res, rownum, i_most_common_elems)); if (!PQgetisnull(res, rownum, i_most_common_elem_freqs)) - appendNamedArgument(out, fout, "most_common_elem_freqs", "real[]", + appendNamedArgument(&out, fout, "most_common_elem_freqs", "real[]", PQgetvalue(res, rownum, i_most_common_elem_freqs)); if (!PQgetisnull(res, rownum, i_elem_count_histogram)) - appendNamedArgument(out, fout, "elem_count_histogram", "real[]", + appendNamedArgument(&out, fout, "elem_count_histogram", "real[]", PQgetvalue(res, rownum, i_elem_count_histogram)); if (fout->remoteVersion >= 170000) { if (!PQgetisnull(res, rownum, i_range_length_histogram)) - appendNamedArgument(out, fout, "range_length_histogram", "text", + appendNamedArgument(&out, fout, "range_length_histogram", "text", PQgetvalue(res, rownum, i_range_length_histogram)); if (!PQgetisnull(res, rownum, i_range_empty_frac)) - appendNamedArgument(out, fout, "range_empty_frac", "real", + appendNamedArgument(&out, fout, "range_empty_frac", "real", PQgetvalue(res, rownum, i_range_empty_frac)); if (!PQgetisnull(res, rownum, i_range_bounds_histogram)) - appendNamedArgument(out, fout, "range_bounds_histogram", "text", + appendNamedArgument(&out, fout, "range_bounds_histogram", "text", PQgetvalue(res, rownum, i_range_bounds_histogram)); } - appendPQExpBufferStr(out, "\n);\n"); + appendPQExpBufferStr(&out, "\n);\n"); } PQclear(res); + termPQExpBuffer(&query); + return out.data; +} + +/* + * dumpRelationStats -- + * + * Dump command to import stats into the relation on the new database. + */ +static void +dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo) +{ + const DumpableObject *dobj = &rsinfo->dobj; + + DumpId *deps = NULL; + int ndeps = 0; + + /* nothing to do if we are not dumping statistics */ + if (!fout->dopt->dumpStatistics) + return; + + /* dependent on the relation definition, if doing schema */ + if (fout->dopt->dumpSchema) + { + deps = dobj->dependencies; + ndeps = dobj->nDeps; + } + ArchiveEntry(fout, nilCatalogId, createDumpId(), ARCHIVE_OPTS(.tag = dobj->name, .namespace = dobj->namespace->dobj.name, .description = "STATISTICS DATA", .section = rsinfo->section, - .createStmt = out->data, - .deps = dobj->dependencies, - .nDeps = dobj->nDeps)); - - destroyPQExpBuffer(out); - destroyPQExpBuffer(query); + .createFn = printRelationStats, + .createArg = rsinfo, + .deps = deps, + .nDeps = ndeps)); } /* base-commit: e2809e3a1015697832ee4d37b75ba1cd0caac0f0 -- 2.49.0