v10-0002-Prepare-pg_dump-for-additional-compression-metho.patch
text/x-patch
Filename: v10-0002-Prepare-pg_dump-for-additional-compression-metho.patch
Type: text/x-patch
Part: 2
Message:
Re: Add LZ4 compression in pg_dump
Patch
Format: format-patch
Series: patch v10-0002
Subject: Prepare pg_dump for additional compression methods
| File | + | − |
|---|---|---|
| doc/src/sgml/ref/pg_dump.sgml | 22 | 8 |
| src/bin/pg_dump/compress_io.c | 267 | 164 |
| src/bin/pg_dump/compress_io.h | 10 | 10 |
| src/bin/pg_dump/pg_backup_archiver.c | 87 | 105 |
| src/bin/pg_dump/pg_backup_archiver.h | 3 | 34 |
| src/bin/pg_dump/pg_backup_custom.c | 3 | 3 |
| src/bin/pg_dump/pg_backup_directory.c | 9 | 4 |
| src/bin/pg_dump/pg_backup.h | 5 | 2 |
| src/bin/pg_dump/pg_backup_tar.c | 6 | 6 |
| src/bin/pg_dump/pg_dump.c | 77 | 22 |
| src/bin/pg_dump/t/001_basic.pl | 24 | 3 |
| src/bin/pg_dump/t/002_pg_dump.pl | 1 | 1 |
From 516ea1bb70a4243965e231092ee460bb4ade311d Mon Sep 17 00:00:00 2001
From: Georgios Kokolatos <gkokolatos@pm.me>
Date: Tue, 29 Nov 2022 11:22:44 +0000
Subject: [PATCH v10 2/4] Prepare pg_dump for additional compression methods
This commmit does some of the heavy lifting required for additional compression
methods.
First it is teaching pg_dump.c about the definitions and interfaces found in
common/compression.h. Then it is propagating those throughout the code.
Commit bf9aa490db introduced cfp in compress_io.{c,h} with the intent of
unifying compression related code and allow for the introduction of additional
archive formats. However, pg_backup_archiver.c was not using that API. This
commit teaches pg_backup_archiver.c about cfp and is using it through out.
---
doc/src/sgml/ref/pg_dump.sgml | 30 +-
src/bin/pg_dump/compress_io.c | 431 ++++++++++++++++----------
src/bin/pg_dump/compress_io.h | 20 +-
src/bin/pg_dump/pg_backup.h | 7 +-
src/bin/pg_dump/pg_backup_archiver.c | 192 ++++++------
src/bin/pg_dump/pg_backup_archiver.h | 37 +--
src/bin/pg_dump/pg_backup_custom.c | 6 +-
src/bin/pg_dump/pg_backup_directory.c | 13 +-
src/bin/pg_dump/pg_backup_tar.c | 12 +-
src/bin/pg_dump/pg_dump.c | 99 ++++--
src/bin/pg_dump/t/001_basic.pl | 27 +-
src/bin/pg_dump/t/002_pg_dump.pl | 2 +-
12 files changed, 514 insertions(+), 362 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8b9d9f4cad..3fb8fdce81 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -644,17 +644,31 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
- <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
- <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+ <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+ <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+ <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
<listitem>
<para>
- Specify the compression level to use. Zero means no compression.
+ Specify the compression method and/or the compression level to use.
+ The compression method can be set to <literal>gzip</literal> or
+ <literal>none</literal> for no compression. A compression level can
+ be optionally specified, by appending the level number after a colon
+ (<literal>:</literal>). If no level is specified, the default compression
+ level will be used for the specified method. If only a level is
+ specified without mentioning a method, <literal>gzip</literal> compression
+ will be used.
+ </para>
+
+ <para>
For the custom and directory archive formats, this specifies compression of
- individual table-data segments, and the default is to compress
- at a moderate level.
- For plain text output, setting a nonzero compression level causes
- the entire output file to be compressed, as though it had been
- fed through <application>gzip</application>; but the default is not to compress.
+ individual table-data segments, and the default is to compress using
+ <literal>gzip</literal> at a moderate level. For plain text output,
+ setting a nonzero compression level causes the entire output file to be compressed,
+ as though it had been fed through <application>gzip</application>; but the default
+ is not to compress.
+ </para>
+ <para>
The tar archive format currently does not support compression at all.
</para>
</listitem>
diff --git a/src/bin/pg_dump/compress_io.c b/src/bin/pg_dump/compress_io.c
index 62f940ff7a..4a8fc1e306 100644
--- a/src/bin/pg_dump/compress_io.c
+++ b/src/bin/pg_dump/compress_io.c
@@ -56,6 +56,10 @@
#include "compress_io.h"
#include "pg_backup_utils.h"
+#ifdef HAVE_LIBZ
+#include <zlib.h>
+#endif
+
/*----------------------
* Compressor API
*----------------------
@@ -64,7 +68,7 @@
/* typedef appears in compress_io.h */
struct CompressorState
{
- CompressionAlgorithm comprAlg;
+ pg_compress_algorithm compress_algorithm;
WriteFunc writeF;
#ifdef HAVE_LIBZ
@@ -74,9 +78,6 @@ struct CompressorState
#endif
};
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
- int *level);
-
/* Routines that support zlib compressed data I/O */
#ifdef HAVE_LIBZ
static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +94,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
const char *data, size_t dLen);
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
- if (compression == Z_DEFAULT_COMPRESSION ||
- (compression > 0 && compression <= 9))
- *alg = COMPR_ALG_LIBZ;
- else if (compression == 0)
- *alg = COMPR_ALG_NONE;
- else
- {
- pg_fatal("invalid compression code: %d", compression);
- *alg = COMPR_ALG_NONE; /* keep compiler quiet */
- }
-
- /* The level is just the passed-in value. */
- if (level)
- *level = compression;
-}
-
/* Public interface routines */
/* Allocate a new compressor */
CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF)
{
CompressorState *cs;
- CompressionAlgorithm alg;
- int level;
-
- ParseCompressionOption(compression, &alg, &level);
#ifndef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
+ if (compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_fatal("not built with zlib support");
#endif
cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
cs->writeF = writeF;
- cs->comprAlg = alg;
+ cs->compress_algorithm = compress_spec.algorithm;
/*
* Perform compression algorithm specific initialization.
*/
#ifdef HAVE_LIBZ
- if (alg == COMPR_ALG_LIBZ)
- InitCompressorZlib(cs, level);
+ if (cs->compress_algorithm == PG_COMPRESSION_GZIP)
+ InitCompressorZlib(cs, compress_spec.level);
#endif
return cs;
@@ -154,21 +128,24 @@ AllocateCompressor(int compression, WriteFunc writeF)
* out with ahwrite().
*/
void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compress_spec,
+ ReadFunc readF)
{
- CompressionAlgorithm alg;
-
- ParseCompressionOption(compression, &alg, NULL);
-
- if (alg == COMPR_ALG_NONE)
- ReadDataFromArchiveNone(AH, readF);
- if (alg == COMPR_ALG_LIBZ)
+ switch (compress_spec.algorithm)
{
+ case PG_COMPRESSION_NONE:
+ ReadDataFromArchiveNone(AH, readF);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- ReadDataFromArchiveZlib(AH, readF);
+ ReadDataFromArchiveZlib(AH, readF);
#else
- pg_fatal("not built with zlib support");
+ pg_fatal("not built with zlib support");
#endif
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -179,18 +156,21 @@ void
WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen)
{
- switch (cs->comprAlg)
+ switch (cs->compress_algorithm)
{
- case COMPR_ALG_LIBZ:
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
WriteDataToArchiveZlib(AH, cs, data, dLen);
#else
pg_fatal("not built with zlib support");
#endif
break;
- case COMPR_ALG_NONE:
+ case PG_COMPRESSION_NONE:
WriteDataToArchiveNone(AH, cs, data, dLen);
break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
}
@@ -200,11 +180,23 @@ WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
void
EndCompressor(ArchiveHandle *AH, CompressorState *cs)
{
+ switch (cs->compress_algorithm)
+ {
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (cs->comprAlg == COMPR_ALG_LIBZ)
- EndCompressorZlib(AH, cs);
+ EndCompressorZlib(AH, cs);
+#else
+ pg_fatal("not built with zlib support");
#endif
- free(cs);
+ break;
+ case PG_COMPRESSION_NONE:
+ free(cs);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
}
/* Private routines, specific to each compression method. */
@@ -418,10 +410,8 @@ WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
*/
struct cfp
{
- FILE *uncompressedfp;
-#ifdef HAVE_LIBZ
- gzFile compressedfp;
-#endif
+ pg_compress_algorithm compress_algorithm;
+ void *fp;
};
#ifdef HAVE_LIBZ
@@ -452,21 +442,25 @@ cfp *
cfopen_read(const char *path, const char *mode)
{
cfp *fp;
+ pg_compress_specification compress_spec = {0};
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifdef HAVE_LIBZ
if (hasSuffix(path, ".gz"))
- fp = cfopen(path, mode, 1);
+ fp = cfopen(path, mode, compress_spec);
else
#endif
{
- fp = cfopen(path, mode, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ fp = cfopen(path, mode, compress_spec);
#ifdef HAVE_LIBZ
if (fp == NULL)
{
char *fname;
+ compress_spec.algorithm = PG_COMPRESSION_GZIP;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, 1);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
}
#endif
@@ -479,26 +473,27 @@ cfopen_read(const char *path, const char *mode)
* be a filemode as accepted by fopen() and gzopen() that indicates writing
* ("w", "wb", "a", or "ab").
*
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compress_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compress_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
*
* On failure, return NULL with an error code in errno.
*/
cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
{
cfp *fp;
- if (compression == 0)
- fp = cfopen(path, mode, 0);
+ if (compress_spec.algorithm == PG_COMPRESSION_NONE)
+ fp = cfopen(path, mode, compress_spec);
else
{
#ifdef HAVE_LIBZ
char *fname;
fname = psprintf("%s.gz", path);
- fp = cfopen(fname, mode, compression);
+ fp = cfopen(fname, mode, compress_spec);
free_keep_errno(fname);
#else
pg_fatal("not built with zlib support");
@@ -509,60 +504,96 @@ cfopen_write(const char *path, const char *mode, int compression)
}
/*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
- * is opened with libz gzopen(), otherwise with plain fopen().
+ * This is the workhorse for cfopen() or cfdopen(). It opens file 'path' or
+ * associates a stream 'fd', if 'fd' is a valid descriptor, in 'mode'. The
+ * descriptor is not dup'ed and it is the caller's responsibility to do so.
+ * The caller must verify that the 'compress_algorithm' is supported by the
+ * current build.
*
* On failure, return NULL with an error code in errno.
*/
-cfp *
-cfopen(const char *path, const char *mode, int compression)
+static cfp *
+cfopen_internal(const char *path, int fd, const char *mode,
+ pg_compress_algorithm compress_algorithm, int compressionLevel)
{
cfp *fp = pg_malloc(sizeof(cfp));
- if (compression != 0)
+ fp->compress_algorithm = compress_algorithm;
+
+ switch (compress_algorithm)
{
-#ifdef HAVE_LIBZ
- if (compression != Z_DEFAULT_COMPRESSION)
- {
- /* user has specified a compression level, so tell zlib to use it */
- char mode_compression[32];
+ case PG_COMPRESSION_NONE:
+ if (fd >= 0)
+ fp->fp = fdopen(fd, mode);
+ else
+ fp->fp = fopen(path, mode);
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
- snprintf(mode_compression, sizeof(mode_compression), "%s%d",
- mode, compression);
- fp->compressedfp = gzopen(path, mode_compression);
- }
- else
- {
- /* don't specify a level, just use the zlib default */
- fp->compressedfp = gzopen(path, mode);
- }
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ if (compressionLevel != Z_DEFAULT_COMPRESSION)
+ {
+ /*
+ * user has specified a compression level, so tell zlib to use
+ * it
+ */
+ char mode_compression[32];
+
+ snprintf(mode_compression, sizeof(mode_compression), "%s%d",
+ mode, compressionLevel);
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode_compression);
+ else
+ fp->fp = gzopen(path, mode_compression);
+ }
+ else
+ {
+ /* don't specify a level, just use the zlib default */
+ if (fd >= 0)
+ fp->fp = gzdopen(fd, mode);
+ else
+ fp->fp = gzopen(path, mode);
+ }
- fp->uncompressedfp = NULL;
- if (fp->compressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ if (fp->fp == NULL)
+ {
+ free_keep_errno(fp);
+ fp = NULL;
+ }
#else
- pg_fatal("not built with zlib support");
-#endif
- }
- else
- {
-#ifdef HAVE_LIBZ
- fp->compressedfp = NULL;
+ pg_fatal("not built with zlib support");
#endif
- fp->uncompressedfp = fopen(path, mode);
- if (fp->uncompressedfp == NULL)
- {
- free_keep_errno(fp);
- fp = NULL;
- }
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return fp;
}
+cfp *
+cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(path, -1, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
+
+cfp *
+cfdopen(int fd, const char *mode,
+ const pg_compress_specification compress_spec)
+{
+ return cfopen_internal(NULL, fd, mode,
+ compress_spec.algorithm,
+ compress_spec.level);
+}
int
cfread(void *ptr, int size, cfp *fp)
@@ -572,38 +603,61 @@ cfread(void *ptr, int size, cfp *fp)
if (size == 0)
return 0;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzread(fp->compressedfp, ptr, size);
- if (ret != size && !gzeof(fp->compressedfp))
- {
- int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ case PG_COMPRESSION_NONE:
+ ret = fread(ptr, 1, size, fp->fp);
+ if (ret != size && !feof(fp->fp))
+ READ_ERROR_EXIT(fp->fp);
- pg_fatal("could not read from input file: %s",
- errnum == Z_ERRNO ? strerror(errno) : errmsg);
- }
- }
- else
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzread(fp->fp, ptr, size);
+ if (ret != size && !gzeof(fp->fp))
+ {
+ int errnum;
+ const char *errmsg = gzerror(fp->fp, &errnum);
+
+ pg_fatal("could not read from input file: %s",
+ errnum == Z_ERRNO ? strerror(errno) : errmsg);
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fread(ptr, 1, size, fp->uncompressedfp);
- if (ret != size && !feof(fp->uncompressedfp))
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
return ret;
}
int
cfwrite(const void *ptr, int size, cfp *fp)
{
+ int ret = 0;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fwrite(ptr, 1, size, fp->fp);
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzwrite(fp->compressedfp, ptr, size);
- else
+ ret = gzwrite(fp->fp, ptr, size);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fwrite(ptr, 1, size, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
@@ -611,24 +665,31 @@ cfgetc(cfp *fp)
{
int ret;
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ switch (fp->compress_algorithm)
{
- ret = gzgetc(fp->compressedfp);
- if (ret == EOF)
- {
- if (!gzeof(fp->compressedfp))
- pg_fatal("could not read from input file: %s", strerror(errno));
- else
- pg_fatal("could not read from input file: end of file");
- }
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fgetc(fp->fp);
+ if (ret == EOF)
+ READ_ERROR_EXIT(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzgetc((gzFile) fp->fp);
+ if (ret == EOF)
+ {
+ if (!gzeof(fp->fp))
+ pg_fatal("could not read from input file: %s", strerror(errno));
+ else
+ pg_fatal("could not read from input file: end of file");
+ }
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- ret = fgetc(fp->uncompressedfp);
- if (ret == EOF)
- READ_ERROR_EXIT(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
return ret;
@@ -637,65 +698,107 @@ cfgetc(cfp *fp)
char *
cfgets(cfp *fp, char *buf, int len)
{
+ char *ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = fgets(buf, len, fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzgets(fp->compressedfp, buf, len);
- else
+ ret = gzgets(fp->fp, buf, len);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return fgets(buf, len, fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
int
cfclose(cfp *fp)
{
- int result;
+ int ret;
if (fp == NULL)
{
errno = EBADF;
return EOF;
}
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+
+ switch (fp->compress_algorithm)
{
- result = gzclose(fp->compressedfp);
- fp->compressedfp = NULL;
- }
- else
+ case PG_COMPRESSION_NONE:
+ ret = fclose(fp->fp);
+ fp->fp = NULL;
+
+ break;
+ case PG_COMPRESSION_GZIP:
+#ifdef HAVE_LIBZ
+ ret = gzclose(fp->fp);
+ fp->fp = NULL;
+#else
+ pg_fatal("not built with zlib support");
#endif
- {
- result = fclose(fp->uncompressedfp);
- fp->uncompressedfp = NULL;
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
}
+
free_keep_errno(fp);
- return result;
+ return ret;
}
int
cfeof(cfp *fp)
{
+ int ret;
+
+ switch (fp->compress_algorithm)
+ {
+ case PG_COMPRESSION_NONE:
+ ret = feof(fp->fp);
+
+ break;
+ case PG_COMPRESSION_GZIP:
#ifdef HAVE_LIBZ
- if (fp->compressedfp)
- return gzeof(fp->compressedfp);
- else
+ ret = gzeof(fp->fp);
+#else
+ pg_fatal("not built with zlib support");
#endif
- return feof(fp->uncompressedfp);
+ break;
+ default:
+ pg_fatal("invalid compression method");
+ break;
+ }
+
+ return ret;
}
const char *
get_cfp_error(cfp *fp)
{
-#ifdef HAVE_LIBZ
- if (fp->compressedfp)
+ if (fp->compress_algorithm == PG_COMPRESSION_GZIP)
{
+#ifdef HAVE_LIBZ
int errnum;
- const char *errmsg = gzerror(fp->compressedfp, &errnum);
+ const char *errmsg = gzerror(fp->fp, &errnum);
if (errnum != Z_ERRNO)
return errmsg;
- }
+#else
+ pg_fatal("not built with zlib support");
#endif
+ }
+
return strerror(errno);
}
diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index f635787692..d6335fff02 100644
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -21,12 +21,6 @@
#define ZLIB_OUT_SIZE 4096
#define ZLIB_IN_SIZE 4096
-typedef enum
-{
- COMPR_ALG_NONE,
- COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
/* Prototype for callback function to WriteDataToArchive() */
typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
/* struct definition appears in compress_io.c */
typedef struct CompressorState CompressorState;
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compress_spec,
+ WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+ const pg_compress_specification compress_spec,
ReadFunc readF);
extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
typedef struct cfp cfp;
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+ pg_compress_specification compress_spec);
extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+ const pg_compress_specification compress_spec);
extern int cfread(void *ptr, int size, cfp *fp);
extern int cfwrite(const void *ptr, int size, cfp *fp);
extern int cfgetc(cfp *fp);
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index e8b7898297..61c412c8cb 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -23,6 +23,7 @@
#ifndef PG_BACKUP_H
#define PG_BACKUP_H
+#include "common/compression.h"
#include "fe_utils/simple_list.h"
#include "libpq-fe.h"
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
int noDataForFailedTables;
int exit_on_error;
- int compression;
+ pg_compress_specification compress_spec; /* Specification for
+ * compression */
int suppressDumpWarnings; /* Suppress output of WARNING entries
* to stderr */
bool single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
/* Create a new archive */
extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker);
/* The --list option */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index f39c0fa36f..304cc072ca 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -31,6 +31,7 @@
#endif
#include "common/string.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/string_utils.h"
#include "lib/stringinfo.h"
@@ -43,13 +44,6 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
-/* state needed to save/restore an archive's output target */
-typedef struct _outputContext
-{
- void *OF;
- int gzOut;
-} OutputContext;
-
/*
* State for tracking TocEntrys that are ready to process during a parallel
* restore. (This used to be a list, and we still call it that, though now
@@ -70,7 +64,8 @@ typedef struct _parallelReadyList
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
@@ -98,9 +93,10 @@ static int _discoverArchiveFormat(ArchiveHandle *AH);
static int RestoringToDB(ArchiveHandle *AH);
static void dump_lo_buf(ArchiveHandle *AH);
static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
-static void SetOutput(ArchiveHandle *AH, const char *filename, int compression);
-static OutputContext SaveOutput(ArchiveHandle *AH);
-static void RestoreOutput(ArchiveHandle *AH, OutputContext savedContext);
+static void SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec);
+static cfp *SaveOutput(ArchiveHandle *AH);
+static void RestoreOutput(ArchiveHandle *AH, cfp *savedOutput);
static int restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel);
static void restore_toc_entries_prefork(ArchiveHandle *AH,
@@ -239,12 +235,13 @@ setupRestoreWorker(Archive *AHX)
/* Public */
Archive *
CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupDumpWorker)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, compression, dosync,
- mode, setupDumpWorker);
+ ArchiveHandle *AH = _allocAH(FileSpec, fmt, compress_spec,
+ dosync, mode, setupDumpWorker);
return (Archive *) AH;
}
@@ -254,7 +251,12 @@ CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
Archive *
OpenArchive(const char *FileSpec, const ArchiveFormat fmt)
{
- ArchiveHandle *AH = _allocAH(FileSpec, fmt, 0, true, archModeRead, setupRestoreWorker);
+ ArchiveHandle *AH;
+ pg_compress_specification compress_spec = {0};
+
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH = _allocAH(FileSpec, fmt, compress_spec, true,
+ archModeRead, setupRestoreWorker);
return (Archive *) AH;
}
@@ -269,11 +271,8 @@ CloseArchive(Archive *AHX)
AH->ClosePtr(AH);
/* Close the output */
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else if (AH->OF != stdout)
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
@@ -354,8 +353,9 @@ RestoreArchive(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
bool parallel_mode;
+ bool supports_compression;
TocEntry *te;
- OutputContext sav;
+ cfp *sav;
AH->stage = STAGE_INITIALIZING;
@@ -383,16 +383,23 @@ RestoreArchive(Archive *AHX)
/*
* Make sure we won't need (de)compression we haven't got
*/
-#ifndef HAVE_LIBZ
- if (AH->compression != 0 && AH->PrintTocDataPtr != NULL)
+ supports_compression = true;
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE &&
+ AH->PrintTocDataPtr != NULL)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if (te->hadDumper && (te->reqs & REQ_DATA) != 0)
- pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ {
+#ifndef HAVE_LIBZ
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+ if (supports_compression == false)
+ pg_fatal("cannot restore from compressed archive (compression not supported in this installation)");
+ }
}
}
-#endif
/*
* Prepare index arrays, so we can assume we have them throughout restore.
@@ -459,8 +466,8 @@ RestoreArchive(Archive *AHX)
* Setup the output file if necessary.
*/
sav = SaveOutput(AH);
- if (ropt->filename || ropt->compression)
- SetOutput(AH, ropt->filename, ropt->compression);
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
+ SetOutput(AH, ropt->filename, ropt->compress_spec);
ahprintf(AH, "--\n-- PostgreSQL database dump\n--\n\n");
@@ -739,7 +746,7 @@ RestoreArchive(Archive *AHX)
*/
AH->stage = STAGE_FINALIZING;
- if (ropt->filename || ropt->compression)
+ if (ropt->filename || ropt->compress_spec.algorithm != PG_COMPRESSION_NONE)
RestoreOutput(AH, sav);
if (ropt->useDB)
@@ -969,6 +976,8 @@ NewRestoreOptions(void)
opts->format = archUnknown;
opts->cparams.promptPassword = TRI_DEFAULT;
opts->dumpSections = DUMP_UNSECTIONED;
+ opts->compress_spec.algorithm = PG_COMPRESSION_NONE;
+ opts->compress_spec.level = INT_MIN;
return opts;
}
@@ -1115,23 +1124,28 @@ PrintTOCSummary(Archive *AHX)
ArchiveHandle *AH = (ArchiveHandle *) AHX;
RestoreOptions *ropt = AH->public.ropt;
TocEntry *te;
+ pg_compress_specification out_compress_spec = {0};
teSection curSection;
- OutputContext sav;
+ cfp *sav;
const char *fmtName;
char stamp_str[64];
+ /* TOC is always uncompressed */
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+
sav = SaveOutput(AH);
if (ropt->filename)
- SetOutput(AH, ropt->filename, 0 /* no compression */ );
+ SetOutput(AH, ropt->filename, out_compress_spec);
if (strftime(stamp_str, sizeof(stamp_str), PGDUMP_STRFTIME_FMT,
localtime(&AH->createDate)) == 0)
strcpy(stamp_str, "[unknown]");
ahprintf(AH, ";\n; Archive created at %s\n", stamp_str);
- ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %d\n",
+ ahprintf(AH, "; dbname: %s\n; TOC Entries: %d\n; Compression: %s\n",
sanitize_line(AH->archdbname, false),
- AH->tocCount, AH->compression);
+ AH->tocCount,
+ get_compress_algorithm_name(AH->compress_spec.algorithm));
switch (AH->format)
{
@@ -1485,60 +1499,35 @@ archprintf(Archive *AH, const char *fmt,...)
*******************************/
static void
-SetOutput(ArchiveHandle *AH, const char *filename, int compression)
+SetOutput(ArchiveHandle *AH, const char *filename,
+ const pg_compress_specification compress_spec)
{
- int fn;
+ const char *mode;
+ int fn = -1;
if (filename)
{
if (strcmp(filename, "-") == 0)
fn = fileno(stdout);
- else
- fn = -1;
}
else if (AH->FH)
fn = fileno(AH->FH);
else if (AH->fSpec)
{
- fn = -1;
filename = AH->fSpec;
}
else
fn = fileno(stdout);
- /* If compression explicitly requested, use gzopen */
-#ifdef HAVE_LIBZ
- if (compression != 0)
- {
- char fmode[14];
+ if (AH->mode == archModeAppend)
+ mode = PG_BINARY_A;
+ else
+ mode = PG_BINARY_W;
- /* Don't use PG_BINARY_x since this is zlib */
- sprintf(fmode, "wb%d", compression);
- if (fn >= 0)
- AH->OF = gzdopen(dup(fn), fmode);
- else
- AH->OF = gzopen(filename, fmode);
- AH->gzOut = 1;
- }
+ if (fn >= 0)
+ AH->OF = cfdopen(dup(fn), mode, compress_spec);
else
-#endif
- { /* Use fopen */
- if (AH->mode == archModeAppend)
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_A);
- else
- AH->OF = fopen(filename, PG_BINARY_A);
- }
- else
- {
- if (fn >= 0)
- AH->OF = fdopen(dup(fn), PG_BINARY_W);
- else
- AH->OF = fopen(filename, PG_BINARY_W);
- }
- AH->gzOut = 0;
- }
+ AH->OF = cfopen(filename, mode, compress_spec);
if (!AH->OF)
{
@@ -1549,33 +1538,24 @@ SetOutput(ArchiveHandle *AH, const char *filename, int compression)
}
}
-static OutputContext
+static cfp *
SaveOutput(ArchiveHandle *AH)
{
- OutputContext sav;
-
- sav.OF = AH->OF;
- sav.gzOut = AH->gzOut;
-
- return sav;
+ return (cfp *) AH->OF;
}
static void
-RestoreOutput(ArchiveHandle *AH, OutputContext savedContext)
+RestoreOutput(ArchiveHandle *AH, cfp *savedOutput)
{
int res;
- errno = 0; /* in case gzclose() doesn't set it */
- if (AH->gzOut)
- res = GZCLOSE(AH->OF);
- else
- res = fclose(AH->OF);
+ errno = 0;
+ res = cfclose(AH->OF);
if (res != 0)
pg_fatal("could not close output file: %m");
- AH->gzOut = savedContext.gzOut;
- AH->OF = savedContext.OF;
+ AH->OF = savedOutput;
}
@@ -1699,22 +1679,17 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
bytes_written = size * nmemb;
}
- else if (AH->gzOut)
- bytes_written = GZWRITE(ptr, size, nmemb, AH->OF);
else if (AH->CustomOutPtr)
bytes_written = AH->CustomOutPtr(AH, ptr, size * nmemb);
+ /*
+ * If we're doing a restore, and it's direct to DB, and we're connected
+ * then send it to the DB.
+ */
+ else if (RestoringToDB(AH))
+ bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
else
- {
- /*
- * If we're doing a restore, and it's direct to DB, and we're
- * connected then send it to the DB.
- */
- if (RestoringToDB(AH))
- bytes_written = ExecuteSqlCommandBuf(&AH->public, (const char *) ptr, size * nmemb);
- else
- bytes_written = fwrite(ptr, size, nmemb, AH->OF) * size;
- }
+ bytes_written = cfwrite(ptr, size * nmemb, AH->OF);
if (bytes_written != size * nmemb)
WRITE_ERROR_EXIT;
@@ -2198,10 +2173,12 @@ _discoverArchiveFormat(ArchiveHandle *AH)
*/
static ArchiveHandle *
_allocAH(const char *FileSpec, const ArchiveFormat fmt,
- const int compression, bool dosync, ArchiveMode mode,
+ const pg_compress_specification compress_spec,
+ bool dosync, ArchiveMode mode,
SetupWorkerPtrType setupWorkerPtr)
{
ArchiveHandle *AH;
+ pg_compress_specification out_compress_spec = {0};
pg_log_debug("allocating AH for %s, format %d",
FileSpec ? FileSpec : "(stdio)", fmt);
@@ -2249,14 +2226,14 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->toc->prev = AH->toc;
AH->mode = mode;
- AH->compression = compression;
+ AH->compress_spec = compress_spec;
AH->dosync = dosync;
memset(&(AH->sqlparse), 0, sizeof(AH->sqlparse));
/* Open stdout with no compression for AH output handle */
- AH->gzOut = 0;
- AH->OF = stdout;
+ out_compress_spec.algorithm = PG_COMPRESSION_NONE;
+ AH->OF = cfdopen(dup(fileno(stdout)), PG_BINARY_A, out_compress_spec);
/*
* On Windows, we need to use binary mode to read/write non-text files,
@@ -2264,7 +2241,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
* Force stdin/stdout into binary mode if that is what we are using.
*/
#ifdef WIN32
- if ((fmt != archNull || compression != 0) &&
+ if ((fmt != archNull || compress_spec.algorithm != PG_COMPRESSION_NONE) &&
(AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0))
{
if (mode == archModeWrite)
@@ -3669,7 +3646,7 @@ WriteHead(ArchiveHandle *AH)
AH->WriteBytePtr(AH, AH->intSize);
AH->WriteBytePtr(AH, AH->offSize);
AH->WriteBytePtr(AH, AH->format);
- WriteInt(AH, AH->compression);
+ WriteInt(AH, AH->compress_spec.level);
crtm = *localtime(&AH->createDate);
WriteInt(AH, crtm.tm_sec);
WriteInt(AH, crtm.tm_min);
@@ -3740,21 +3717,26 @@ ReadHead(ArchiveHandle *AH)
pg_fatal("expected format (%d) differs from format found in file (%d)",
AH->format, fmt);
+ AH->compress_spec.algorithm = PG_COMPRESSION_NONE;
if (AH->version >= K_VERS_1_2)
{
if (AH->version < K_VERS_1_4)
- AH->compression = AH->ReadBytePtr(AH);
+ AH->compress_spec.level = AH->ReadBytePtr(AH);
else
- AH->compression = ReadInt(AH);
+ AH->compress_spec.level = ReadInt(AH);
+
+ if (AH->compress_spec.level != 0)
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
}
else
- AH->compression = Z_DEFAULT_COMPRESSION;
+ AH->compress_spec.algorithm = PG_COMPRESSION_GZIP;
#ifndef HAVE_LIBZ
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_GZIP)
pg_log_warning("archive is compressed, but this installation does not support compression -- no data will be available");
#endif
+
if (AH->version >= K_VERS_1_4)
{
struct tm crtm;
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 42687c4ec8..d2930949ab 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -32,30 +32,6 @@
#define LOBBUFSIZE 16384
-#ifdef HAVE_LIBZ
-#include <zlib.h>
-#define GZCLOSE(fh) gzclose(fh)
-#define GZWRITE(p, s, n, fh) gzwrite(fh, p, (n) * (s))
-#define GZREAD(p, s, n, fh) gzread(fh, p, (n) * (s))
-#define GZEOF(fh) gzeof(fh)
-#else
-#define GZCLOSE(fh) fclose(fh)
-#define GZWRITE(p, s, n, fh) (fwrite(p, s, n, fh) * (s))
-#define GZREAD(p, s, n, fh) fread(p, s, n, fh)
-#define GZEOF(fh) feof(fh)
-/* this is just the redefinition of a libz constant */
-#define Z_DEFAULT_COMPRESSION (-1)
-
-typedef struct _z_stream
-{
- void *next_in;
- void *next_out;
- size_t avail_in;
- size_t avail_out;
-} z_stream;
-typedef z_stream *z_streamp;
-#endif
-
/* Data block types */
#define BLK_DATA 1
#define BLK_BLOBS 3
@@ -319,8 +295,7 @@ struct _archiveHandle
char *fSpec; /* Archive File Spec */
FILE *FH; /* General purpose file handle */
- void *OF;
- int gzOut; /* Output file */
+ void *OF; /* Output file */
struct _tocEntry *toc; /* Header of circular list of TOC entries */
int tocCount; /* Number of TOC entries */
@@ -331,14 +306,8 @@ struct _archiveHandle
DumpId *tableDataId; /* TABLE DATA ids, indexed by table dumpId */
struct _tocEntry *currToc; /* Used when dumping data */
- int compression; /*---------
- * Compression requested on open().
- * Possible values for compression:
- * -1 Z_DEFAULT_COMPRESSION
- * 0 COMPRESSION_NONE
- * 1-9 levels for gzip compression
- *---------
- */
+ pg_compress_specification compress_spec; /* Requested specification for
+ * compression */
bool dosync; /* data requested to be synced on sight */
ArchiveMode mode; /* File mode - r or w */
void *formatData; /* Header data specific to file format */
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index a0a55a1edd..6a2112c45f 100644
--- a/src/bin/pg_dump/pg_backup_custom.c
+++ b/src/bin/pg_dump/pg_backup_custom.c
@@ -298,7 +298,7 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
_WriteByte(AH, BLK_DATA); /* Block type */
WriteInt(AH, te->dumpId); /* For sanity check */
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -377,7 +377,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
WriteInt(AH, oid);
- ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+ ctx->cs = AllocateCompressor(AH->compress_spec, _CustomWriteFunc);
}
/*
@@ -566,7 +566,7 @@ _PrintTocData(ArchiveHandle *AH, TocEntry *te)
static void
_PrintData(ArchiveHandle *AH)
{
- ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+ ReadDataFromArchive(AH, AH->compress_spec, _CustomReadFunc);
}
static void
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 798182b6f7..7d2cddbb2c 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -327,7 +327,8 @@ _StartData(ArchiveHandle *AH, TocEntry *te)
setFilePath(AH, fname, tctx->filename);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W,
+ AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -573,6 +574,7 @@ _CloseArchive(ArchiveHandle *AH)
if (AH->mode == archModeWrite)
{
cfp *tocFH;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "toc.dat");
@@ -581,7 +583,8 @@ _CloseArchive(ArchiveHandle *AH)
ctx->pstate = ParallelBackupStart(AH);
/* The TOC is always created uncompressed */
- tocFH = cfopen_write(fname, PG_BINARY_W, 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ tocFH = cfopen_write(fname, PG_BINARY_W, compress_spec);
if (tocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
ctx->dataFH = tocFH;
@@ -639,12 +642,14 @@ static void
_StartBlobs(ArchiveHandle *AH, TocEntry *te)
{
lclContext *ctx = (lclContext *) AH->formatData;
+ pg_compress_specification compress_spec = {0};
char fname[MAXPGPATH];
setFilePath(AH, fname, "blobs.toc");
/* The blob TOC file is never compressed */
- ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
+ compress_spec.algorithm = PG_COMPRESSION_NONE;
+ ctx->blobsTocFH = cfopen_write(fname, "ab", compress_spec);
if (ctx->blobsTocFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
}
@@ -662,7 +667,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
- ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
+ ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compress_spec);
if (ctx->dataFH == NULL)
pg_fatal("could not open output file \"%s\": %m", fname);
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 402b93c610..d773c291c8 100644
--- a/src/bin/pg_dump/pg_backup_tar.c
+++ b/src/bin/pg_dump/pg_backup_tar.c
@@ -35,6 +35,7 @@
#include <unistd.h>
#include "common/file_utils.h"
+#include "compress_io.h"
#include "fe_utils/string_utils.h"
#include "pg_backup_archiver.h"
#include "pg_backup_tar.h"
@@ -194,7 +195,7 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
* possible since gzdopen uses buffered IO which totally screws file
* positioning.
*/
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
}
else
@@ -328,7 +329,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
}
}
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = ctx->tarFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -383,7 +384,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
umask(old_umask);
- if (AH->compression == 0)
+ if (AH->compress_spec.algorithm == PG_COMPRESSION_NONE)
tm->nFH = tm->tmpFH;
else
pg_fatal("compression is not supported by tar archive format");
@@ -401,7 +402,7 @@ tarOpen(ArchiveHandle *AH, const char *filename, char mode)
static void
tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
{
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
if (th->mode == 'w')
@@ -800,7 +801,6 @@ _CloseArchive(ArchiveHandle *AH)
memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
ropt->filename = NULL;
ropt->dropSchema = 1;
- ropt->compression = 0;
ropt->superuser = NULL;
ropt->suppressDumpWarnings = true;
@@ -888,7 +888,7 @@ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
if (oid == 0)
pg_fatal("invalid OID for large object (%u)", oid);
- if (AH->compression != 0)
+ if (AH->compress_spec.algorithm != PG_COMPRESSION_NONE)
pg_fatal("compression is not supported by tar archive format");
sprintf(fname, "blob_%u.dat", oid);
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index da427f4d4a..62c09e6ed4 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -54,8 +54,10 @@
#include "catalog/pg_subscription.h"
#include "catalog/pg_trigger_d.h"
#include "catalog/pg_type_d.h"
+#include "common/compression.h"
#include "common/connect.h"
#include "common/relpath.h"
+#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
#include "fe_utils/string_utils.h"
@@ -164,6 +166,8 @@ static void setup_connection(Archive *AH,
const char *dumpencoding, const char *dumpsnapshot,
char *use_role);
static ArchiveFormat parseArchiveFormat(const char *format, ArchiveMode *mode);
+static bool parse_compression(const char *opt,
+ pg_compress_specification *compress_spec);
static void expand_schema_name_patterns(Archive *fout,
SimpleStringList *patterns,
SimpleOidList *oids,
@@ -340,8 +344,9 @@ main(int argc, char **argv)
const char *dumpsnapshot = NULL;
char *use_role = NULL;
int numWorkers = 1;
- int compressLevel = -1;
int plainText = 0;
+ pg_compress_specification compress_spec = {0};
+ bool user_compression_defined = false;
ArchiveFormat archiveFormat = archUnknown;
ArchiveMode archiveMode;
@@ -561,10 +566,10 @@ main(int argc, char **argv)
dopt.aclsSkip = true;
break;
- case 'Z': /* Compression Level */
- if (!option_parse_int(optarg, "-Z/--compress", 0, 9,
- &compressLevel))
+ case 'Z': /* Compression */
+ if (!parse_compression(optarg, &compress_spec))
exit_nicely(1);
+ user_compression_defined = true;
break;
case 0:
@@ -687,23 +692,20 @@ main(int argc, char **argv)
if (archiveFormat == archNull)
plainText = 1;
- /* Custom and directory formats are compressed by default, others not */
- if (compressLevel == -1)
+ /*
+ * Custom and directory formats are compressed by default (zlib), others
+ * not
+ */
+ if (user_compression_defined == false)
{
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, &compress_spec);
#ifdef HAVE_LIBZ
if (archiveFormat == archCustom || archiveFormat == archDirectory)
- compressLevel = Z_DEFAULT_COMPRESSION;
- else
+ parse_compress_specification(PG_COMPRESSION_GZIP, NULL,
+ &compress_spec);
#endif
- compressLevel = 0;
}
-#ifndef HAVE_LIBZ
- if (compressLevel != 0)
- pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
- compressLevel = 0;
-#endif
-
/*
* If emitting an archive format, we always want to emit a DATABASE item,
* in case --create is specified at pg_restore time.
@@ -716,8 +718,8 @@ main(int argc, char **argv)
pg_fatal("parallel backup only supported by the directory format");
/* Open the output file */
- fout = CreateArchive(filename, archiveFormat, compressLevel, dosync,
- archiveMode, setupDumpWorker);
+ fout = CreateArchive(filename, archiveFormat, compress_spec,
+ dosync, archiveMode, setupDumpWorker);
/* Make dump options accessible right away */
SetArchiveOptions(fout, &dopt, NULL);
@@ -948,10 +950,7 @@ main(int argc, char **argv)
ropt->sequence_data = dopt.sequence_data;
ropt->binary_upgrade = dopt.binary_upgrade;
- if (compressLevel == -1)
- ropt->compression = 0;
- else
- ropt->compression = compressLevel;
+ ropt->compress_spec = compress_spec;
ropt->suppressDumpWarnings = true; /* We've already shown them */
@@ -998,7 +997,8 @@ help(const char *progname)
printf(_(" -j, --jobs=NUM use this many parallel jobs to dump\n"));
printf(_(" -v, --verbose verbose mode\n"));
printf(_(" -V, --version output version information, then exit\n"));
- printf(_(" -Z, --compress=0-9 compression level for compressed formats\n"));
+ printf(_(" -Z, --compress=METHOD[:LEVEL]\n"
+ " compress as specified\n"));
printf(_(" --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -1258,6 +1258,61 @@ get_synchronized_snapshot(Archive *fout)
return result;
}
+/*
+ * Interprets and validates a compression option using the common compression
+ * parsing functions. If the requested compression is not available then the
+ * archives are uncompressed.
+ */
+static bool
+parse_compression(const char *opt, pg_compress_specification *compress_spec)
+{
+ char *algorithm_str = NULL;
+ char *level_str = NULL;
+ char *validation_error = NULL;
+ bool supports_compression = true;
+
+ parse_compress_options(opt, &algorithm_str, &level_str);
+ if (!parse_compress_algorithm(algorithm_str, &(compress_spec->algorithm)))
+ {
+ pg_log_error("invalid compression method: \"%s\" (gzip, none)",
+ algorithm_str);
+ return false;
+ }
+
+ /* Switch off unimplemented or unavailable compressions. */
+ if (compress_spec->algorithm != PG_COMPRESSION_NONE &&
+ compress_spec->algorithm != PG_COMPRESSION_GZIP)
+ supports_compression = false;
+
+#ifndef HAVE_LIBZ
+ if (compress_spec->algorithm == PG_COMPRESSION_GZIP)
+ supports_compression = false;
+#endif
+
+ if (!supports_compression)
+ {
+ pg_log_warning("requested compression not available in this installation -- archive will be uncompressed");
+ parse_compress_specification(PG_COMPRESSION_NONE, NULL, compress_spec);
+
+ pg_free(algorithm_str);
+ pg_free(level_str);
+
+ return true;
+ }
+
+ /* Parse and validate the rest of the options */
+ parse_compress_specification(compress_spec->algorithm, level_str,
+ compress_spec);
+ validation_error = validate_compress_specification(compress_spec);
+ if (validation_error)
+ {
+ pg_log_error("invalid compression specification: %s", validation_error);
+ return false;
+ }
+
+ return true;
+}
+
static ArchiveFormat
parseArchiveFormat(const char *format, ArchiveMode *mode)
{
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index a583c8a6d2..f8d0b2fce5 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -121,16 +121,32 @@ command_fails_like(
'pg_restore: cannot specify both --single-transaction and multiple jobs');
command_fails_like(
- [ 'pg_dump', '-Z', '-1' ],
- qr/\Qpg_dump: error: -Z\/--compress must be in range 0..9\E/,
- 'pg_dump: -Z/--compress must be in range');
+ [ 'pg_dump', '--compress', 'garbage' ],
+ qr/\Qpg_dump: error: invalid compression method: "garbage" (gzip, none)\E/,
+ 'pg_dump: invalid --compress');
+
+command_fails_like(
+ [ 'pg_dump', '--compress', 'none:1' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "none" does not accept a compression level\E/,
+ 'pg_dump: invalid compression specification: compression algorithm "none" does not accept a compression level');
+
if (check_pg_config("#define HAVE_LIBZ 1"))
{
+ command_fails_like(
+ [ 'pg_dump', '-Z', '15' ],
+ qr/\Qpg_dump: error: invalid compression specification: compression algorithm "gzip" expects a compression level between 1 and 9 (default at -1)\E/,
+ 'pg_dump: invalid compression specification: must be in range');
+
command_fails_like(
[ 'pg_dump', '--compress', '1', '--format', 'tar' ],
qr/\Qpg_dump: error: compression is not supported by tar archive format\E/,
'pg_dump: compression is not supported by tar archive format');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt' ],
+ qr/\Qpg_dump: error: invalid compression specification: unrecognized compression option: "nonInt"\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
else
{
@@ -139,6 +155,11 @@ else
[ 'pg_dump', '--compress', '1', '--format', 'tar', '-j3' ],
qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
'pg_dump: warning: compression not available in this installation');
+
+ command_fails_like(
+ [ 'pg_dump', '-Z', 'gzip:nonInt', '--format', 'tar', '-j2' ],
+ qr/\Qpg_dump: warning: requested compression not available in this installation -- archive will be uncompressed\E/,
+ 'pg_dump: invalid compression specification: must be an integer');
}
command_fails_like(
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index fe53ed0f89..d604558f03 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -87,7 +87,7 @@ my %pgdump_runs = (
compile_option => 'gzip',
dump_cmd => [
'pg_dump', '--jobs=2',
- '--format=directory', '--compress=1',
+ '--format=directory', '--compress=gzip:1',
"--file=$tempdir/compression_gzip_dir", 'postgres',
],
# Give coverage for manually compressed blob.toc files during
--
2.34.1