v19-0002-zstd-nodict-support.patch
application/octet-stream
Filename: v19-0002-zstd-nodict-support.patch
Type: application/octet-stream
Part: 0
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: format-patch
Series: patch v19-0002
Subject: zstd nodict support.
| File | + | − |
|---|---|---|
| contrib/amcheck/verify_heapam.c | 1 | 0 |
| src/backend/access/common/detoast.c | 8 | 4 |
| src/backend/access/common/reloptions.c | 12 | 2 |
| src/backend/access/common/toast_compression.c | 216 | 7 |
| src/backend/access/common/toast_internals.c | 4 | 0 |
| src/backend/utils/adt/varlena.c | 3 | 0 |
| src/backend/utils/misc/guc_tables.c | 3 | 0 |
| src/backend/utils/misc/postgresql.conf.sample | 1 | 1 |
| src/bin/psql/describe.c | 3 | 2 |
| src/bin/psql/tab-complete.in.c | 2 | 2 |
| src/include/access/toast_compression.h | 27 | 4 |
| src/include/access/toast_internals.h | 2 | 1 |
| src/include/utils/attoptcache.h | 1 | 0 |
| src/include/varatt.h | 2 | 1 |
| src/test/regress/expected/compression_1.out | 3 | 0 |
| src/test/regress/expected/compression.out | 3 | 2 |
| src/test/regress/sql/compression.sql | 1 | 0 |
From 2594a2b4d22a671dc4776ad0a7ffc214dc3ddf71 Mon Sep 17 00:00:00 2001
From: Nikhil Kumar Veldanda <nikhilkv@amazon.com>
Date: Fri, 25 Apr 2025 06:00:37 +0000
Subject: [PATCH v19 2/2] zstd nodict support.
---
contrib/amcheck/verify_heapam.c | 1 +
src/backend/access/common/detoast.c | 12 +-
src/backend/access/common/reloptions.c | 14 +-
src/backend/access/common/toast_compression.c | 223 +++++++++++++++++-
src/backend/access/common/toast_internals.c | 4 +
src/backend/utils/adt/varlena.c | 3 +
src/backend/utils/misc/guc_tables.c | 3 +
src/backend/utils/misc/postgresql.conf.sample | 2 +-
src/bin/psql/describe.c | 5 +-
src/bin/psql/tab-complete.in.c | 4 +-
src/include/access/toast_compression.h | 31 ++-
src/include/access/toast_internals.h | 3 +-
src/include/utils/attoptcache.h | 1 +
src/include/varatt.h | 3 +-
src/test/regress/expected/compression.out | 5 +-
src/test/regress/expected/compression_1.out | 3 +
src/test/regress/sql/compression.sql | 1 +
17 files changed, 292 insertions(+), 26 deletions(-)
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 3c3faf59579..f1b7e77a322 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1793,6 +1793,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* List of all valid compression method IDs */
case TOAST_PGLZ_COMPRESSION_ID:
case TOAST_LZ4_COMPRESSION_ID:
+ case TOAST_ZSTD_NODICT_COMPRESSION_ID:
valid = true;
break;
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index 01419d1c65f..72b0a2c5672 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -246,10 +246,10 @@ detoast_attr_slice(struct varlena *attr,
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*
- * At least for now, if it's LZ4 data, we'll have to fetch the
- * whole thing, because there doesn't seem to be an API call to
- * determine how much compressed data we need to be sure of being
- * able to decompress the required slice.
+ * At least for now, if it's LZ4 or Zstandard data, we'll have to
+ * fetch the whole thing, because there doesn't seem to be an API
+ * call to determine how much compressed data we need to be sure
+ * of being able to decompress the required slice.
*/
if (VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer) ==
TOAST_PGLZ_COMPRESSION_ID)
@@ -485,6 +485,8 @@ toast_decompress_datum(struct varlena *attr)
return pglz_decompress_datum(attr);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum(attr);
+ case TOAST_ZSTD_NODICT_COMPRESSION_ID:
+ return zstd_decompress_datum(attr);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
@@ -528,6 +530,8 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
return pglz_decompress_datum_slice(attr, slicelength);
case TOAST_LZ4_COMPRESSION_ID:
return lz4_decompress_datum_slice(attr, slicelength);
+ case TOAST_ZSTD_NODICT_COMPRESSION_ID:
+ return zstd_decompress_datum_slice(attr, slicelength);
default:
elog(ERROR, "invalid compression method id %d", cmid);
return NULL; /* keep compiler quiet */
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 46c1dce222d..1267668a242 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -24,6 +24,7 @@
#include "access/nbtree.h"
#include "access/reloptions.h"
#include "access/spgist_private.h"
+#include "access/toast_compression.h"
#include "catalog/pg_type.h"
#include "commands/defrem.h"
#include "commands/tablespace.h"
@@ -381,7 +382,15 @@ static relopt_int intRelOpts[] =
},
-1, 0, 1024
},
-
+ {
+ {
+ "zstd_level",
+ "Set column's ZSTD compression level",
+ RELOPT_KIND_ATTRIBUTE,
+ ShareUpdateExclusiveLock
+ },
+ DEFAULT_ZSTD_LEVEL, MIN_ZSTD_LEVEL, MAX_ZSTD_LEVEL
+ },
/* list terminator */
{{NULL}}
};
@@ -2097,7 +2106,8 @@ attribute_reloptions(Datum reloptions, bool validate)
{
static const relopt_parse_elt tab[] = {
{"n_distinct", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct)},
- {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)}
+ {"n_distinct_inherited", RELOPT_TYPE_REAL, offsetof(AttributeOpts, n_distinct_inherited)},
+ {"zstd_level", RELOPT_TYPE_INT, offsetof(AttributeOpts, zstd_level)},
};
return (bytea *) build_reloptions(reloptions, validate,
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index de31a8dc591..ec1bbffaaf0 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -17,19 +17,36 @@
#include <lz4.h>
#endif
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
+
#include "access/detoast.h"
#include "access/toast_compression.h"
#include "common/pg_lzcompress.h"
#include "varatt.h"
+#include "utils/attoptcache.h"
/* GUC */
int default_toast_compression = TOAST_PGLZ_COMPRESSION;
-#define NO_LZ4_SUPPORT() \
+#ifdef USE_ZSTD
+static ZSTD_CCtx *ZstdCompressionCtx = NULL;
+
+static ZSTD_DCtx *ZstdDecompressionCtx = NULL;
+
+#define ZSTD_CHECK_ERROR(zstd_ret, msg) \
+ do { \
+ if (ZSTD_isError(zstd_ret)) \
+ ereport(ERROR, (errmsg("%s: %s", (msg), ZSTD_getErrorName(zstd_ret)))); \
+ } while (0)
+#endif
+
+#define COMPRESSION_METHOD_NOT_SUPPORTED(method) \
ereport(ERROR, \
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
- errmsg("compression method lz4 not supported"), \
- errdetail("This functionality requires the server to be built with lz4 support.")))
+ errmsg("compression method %s not supported", method), \
+ errdetail("This functionality requires the server to be built with %s support.", method)))
/*
* Compress a varlena using PGLZ.
@@ -139,7 +156,7 @@ struct varlena *
lz4_compress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 valsize;
@@ -182,7 +199,7 @@ struct varlena *
lz4_decompress_datum(const struct varlena *value)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -215,7 +232,7 @@ struct varlena *
lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
return NULL; /* keep compiler quiet */
#else
int32 rawsize;
@@ -245,6 +262,167 @@ lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
#endif
}
+/* Compress datum using ZSTD with optional dictionary (using cdict) */
+struct varlena *
+zstd_compress_datum(const struct varlena *value, CompressionInfo cmp)
+{
+#ifdef USE_ZSTD
+ uint32 valsize = VARSIZE_ANY_EXHDR(value);
+ size_t max_size = ZSTD_compressBound(valsize);
+ struct varlena *compressed;
+ void *dest;
+ size_t cmp_size,
+ ret;
+
+ /* Create the session CCtx if it hasn't been yet */
+ if (ZstdCompressionCtx == NULL)
+ {
+ ZstdCompressionCtx = ZSTD_createCCtx();
+ if (!ZstdCompressionCtx)
+ ereport(ERROR, (errmsg("could not create ZSTD_CCtx")));
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_CCtx_reset(ZstdCompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD CCtx");
+
+ /* Set compression level */
+ ret = ZSTD_CCtx_setParameter(ZstdCompressionCtx, ZSTD_c_compressionLevel, cmp.zstd_level);
+ ZSTD_CHECK_ERROR(ret, "failed to set ZSTD compression level");
+
+ /* Allocate space for the compressed varlena (header + data) */
+ compressed = (struct varlena *) palloc(max_size + VARATT_4BCE_HDRSZ(cmp.cmp_ext));
+ dest = (char *) compressed + VARATT_4BCE_HDRSZ(cmp.cmp_ext);
+
+ cmp_size = ZSTD_compress2(ZstdCompressionCtx,
+ dest,
+ max_size,
+ VARDATA_ANY(value),
+ valsize);
+
+ if (ZSTD_isError(cmp_size))
+ {
+ pfree(compressed);
+ ZSTD_CHECK_ERROR(cmp_size, "ZSTD compression failed");
+ }
+
+ /*
+ * If compression did not reduce size, return NULL so that the
+ * uncompressed data is stored
+ */
+ if (cmp_size > valsize)
+ {
+ pfree(compressed);
+ return NULL;
+ }
+
+ /* Set the compressed size in the varlena header */
+ SET_VARSIZE_COMPRESSED(compressed, cmp_size + VARATT_4BCE_HDRSZ(cmp.cmp_ext));
+ return compressed;
+
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict");
+ return NULL;
+#endif
+}
+
+/* Decompression routine */
+struct varlena *
+zstd_decompress_datum(const struct varlena *value)
+{
+#ifdef USE_ZSTD
+ uint32 actual_size_exhdr = VARDATA_COMPRESSED_GET_EXTSIZE(value);
+ uint32 cmp_size_exhdr = VARATT_4BCE_DATA_SIZE(value);
+ struct varlena *result;
+ size_t uncmp_size,
+ ret;
+
+ if (ZstdDecompressionCtx == NULL)
+ {
+ ZstdDecompressionCtx = ZSTD_createDCtx();
+ if (!ZstdDecompressionCtx)
+ ereport(ERROR, (errmsg("could not create ZSTD_DCtx")));
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_DCtx_reset(ZstdDecompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD DCtx");
+
+ /* Allocate space for the uncompressed data */
+ result = (struct varlena *) palloc(actual_size_exhdr + VARHDRSZ);
+
+ uncmp_size = ZSTD_decompressDCtx(ZstdDecompressionCtx,
+ VARDATA(result),
+ actual_size_exhdr,
+ VARATT_4BCE_DATA_PTR(value),
+ cmp_size_exhdr);
+
+ if (ZSTD_isError(uncmp_size))
+ {
+ pfree(result);
+ ZSTD_CHECK_ERROR(uncmp_size, "ZSTD decompression failed");
+ }
+
+ /* Set final size in the varlena header */
+ SET_VARSIZE(result, uncmp_size + VARHDRSZ);
+ return result;
+
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict");
+ return NULL;
+#endif
+}
+
+/* Decompress a slice of the datum using the streaming API and optional dictionary */
+struct varlena *
+zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength)
+{
+#ifdef USE_ZSTD
+ struct varlena *result;
+ ZSTD_inBuffer inBuf;
+ ZSTD_outBuffer outBuf;
+ size_t ret;
+
+ if (ZstdDecompressionCtx == NULL)
+ {
+ ZstdDecompressionCtx = ZSTD_createDCtx();
+ if (!ZstdDecompressionCtx)
+ elog(ERROR, "could not create ZSTD_DCtx");
+ }
+
+ /* Reset the context to clear any prior state */
+ ret = ZSTD_DCtx_reset(ZstdDecompressionCtx, ZSTD_reset_session_only);
+ ZSTD_CHECK_ERROR(ret, "failed to reset ZSTD_DCtx");
+
+ inBuf.src = VARATT_4BCE_DATA_PTR(value);
+ inBuf.size = VARATT_4BCE_DATA_SIZE(value);
+ inBuf.pos = 0;
+
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+ outBuf.dst = (char *) result + VARHDRSZ;
+ outBuf.size = slicelength;
+ outBuf.pos = 0;
+
+ /* Common decompression loop */
+ while (inBuf.pos < inBuf.size && outBuf.pos < outBuf.size)
+ {
+ ret = ZSTD_decompressStream(ZstdDecompressionCtx, &outBuf, &inBuf);
+ if (ZSTD_isError(ret))
+ {
+ pfree(result);
+ ZSTD_CHECK_ERROR(ret, "zstd decompression failed");
+ }
+ }
+
+ Assert(outBuf.size == slicelength && outBuf.pos == slicelength);
+ SET_VARSIZE(result, outBuf.pos + VARHDRSZ);
+ return result;
+#else
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict");
+ return NULL;
+#endif
+}
+
/*
* Extract compression ID from a varlena.
*
@@ -291,10 +469,17 @@ CompressionNameToMethod(const char *compression)
else if (strcmp(compression, "lz4") == 0)
{
#ifndef USE_LZ4
- NO_LZ4_SUPPORT();
+ COMPRESSION_METHOD_NOT_SUPPORTED("lz4");
#endif
return TOAST_LZ4_COMPRESSION;
}
+ else if (strcmp(compression, "zstd_nodict") == 0)
+ {
+#ifndef USE_ZSTD
+ COMPRESSION_METHOD_NOT_SUPPORTED("zstd_nodict");
+#endif
+ return TOAST_ZSTD_NODICT_COMPRESSION;
+ }
return InvalidCompressionMethod;
}
@@ -311,6 +496,8 @@ GetCompressionMethodName(char method)
return "pglz";
case TOAST_LZ4_COMPRESSION:
return "lz4";
+ case TOAST_ZSTD_NODICT_COMPRESSION:
+ return "zstd_nodict";
default:
elog(ERROR, "invalid compression method %c", method);
return NULL; /* keep compiler quiet */
@@ -324,11 +511,33 @@ setup_compression_info(char cmethod, Form_pg_attribute att)
/* initialize from the attribute’s default settings */
info.cmethod = cmethod;
+ info.zstd_level = DEFAULT_ZSTD_LEVEL;
info.cmp_ext = NULL;
if (!CompressionMethodIsValid(cmethod))
info.cmethod = default_toast_compression;
+ switch (info.cmethod)
+ {
+ case TOAST_PGLZ_COMPRESSION:
+ case TOAST_LZ4_COMPRESSION:
+ break;
+ case TOAST_ZSTD_NODICT_COMPRESSION:
+ {
+ AttributeOpts *aopt = get_attribute_options(att->attrelid, att->attnum);
+
+ if (aopt != NULL)
+ info.zstd_level = aopt->zstd_level;
+
+ info.cmp_ext = palloc(sizeof(varatt_cmp_extended));
+
+ VARATT_4BCE_SET_HDR(info.cmp_ext->ext_hdr, TOAST_ZSTD_NODICT_COMPRESSION_ID, 0);
+ }
+ break;
+ default:
+ elog(ERROR, "invalid compression method %c", info.cmethod);
+ }
+
return info;
}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 21139f20de3..7cc2c2bf8ac 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -68,6 +68,10 @@ toast_compress_datum(Datum value, CompressionInfo cmp)
tmp = lz4_compress_datum((const struct varlena *) value);
cmid = TOAST_LZ4_COMPRESSION_ID;
break;
+ case TOAST_ZSTD_NODICT_COMPRESSION:
+ tmp = zstd_compress_datum((const struct varlena *) value, cmp);
+ cmid = TOAST_ZSTD_NODICT_COMPRESSION_ID;
+ break;
default:
elog(ERROR, "invalid compression method %c", cmp.cmethod);
}
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 3e4d5568bde..5b9151c7e16 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -5301,6 +5301,9 @@ pg_column_compression(PG_FUNCTION_ARGS)
case TOAST_LZ4_COMPRESSION_ID:
result = "lz4";
break;
+ case TOAST_ZSTD_NODICT_COMPRESSION_ID:
+ result = "zstd_nodict";
+ break;
default:
elog(ERROR, "invalid compression method id %d", cmid);
}
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 60b12446a1c..432bb1bd3ab 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -460,6 +460,9 @@ static const struct config_enum_entry default_toast_compression_options[] = {
{"pglz", TOAST_PGLZ_COMPRESSION, false},
#ifdef USE_LZ4
{"lz4", TOAST_LZ4_COMPRESSION, false},
+#endif
+#ifdef USE_ZSTD
+ {"zstd_nodict", TOAST_ZSTD_NODICT_COMPRESSION, false},
#endif
{NULL, 0, false}
};
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 34826d01380..f2d2ca39514 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -756,7 +756,7 @@ autovacuum_worker_slots = 16 # autovacuum worker slots to allocate
#row_security = on
#default_table_access_method = 'heap'
#default_tablespace = '' # a tablespace name, '' uses the default
-#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
+#default_toast_compression = 'pglz' # 'pglz' or 'lz4' or 'zstd_nodict'
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#check_function_bodies = on
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 1d08268393e..3831a7fab03 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -2171,8 +2171,9 @@ describeOneTableDetails(const char *schemaname,
/* these strings are literal in our syntax, so not translated. */
printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
(compression[0] == 'l' ? "lz4" :
- (compression[0] == '\0' ? "" :
- "???"))),
+ (compression[0] == 'n' ? "zstd_nodict" :
+ (compression[0] == '\0' ? "" :
+ "???")))),
false, false);
}
diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c
index c916b9299a8..2441acf41ce 100644
--- a/src/bin/psql/tab-complete.in.c
+++ b/src/bin/psql/tab-complete.in.c
@@ -2875,11 +2875,11 @@ match_previous_words(int pattern_id,
/* ALTER TABLE ALTER [COLUMN] <foo> SET ( */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "("))
- COMPLETE_WITH("n_distinct", "n_distinct_inherited");
+ COMPLETE_WITH("n_distinct", "n_distinct_inherited", "zstd_level");
/* ALTER TABLE ALTER [COLUMN] <foo> SET COMPRESSION */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION"))
- COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4");
+ COMPLETE_WITH("DEFAULT", "PGLZ", "LZ4", "ZSTD_NODICT");
/* ALTER TABLE ALTER [COLUMN] <foo> SET EXPRESSION */
else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "EXPRESSION") ||
Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "EXPRESSION"))
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 8e9d5b44752..9af4a14ebed 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -17,6 +17,10 @@
#include "toast_compression.h"
#include "catalog/pg_attribute.h"
+#ifdef USE_ZSTD
+#include <zstd.h>
+#endif
+
/*
* GUC support.
*
@@ -36,10 +40,11 @@ extern PGDLLIMPORT int default_toast_compression;
*
* The INVALID entry is a sentinel and must remain last.
*/
-#define TOAST_COMPRESSION_LIST \
- X(PGLZ, 0, void) /* PostgreSQL LZ-based */ \
- X(LZ4, 1, void) /* LZ4 algorithm */ \
- X(INVALID, 2, void) /* sentinel, must be last */
+#define TOAST_COMPRESSION_LIST \
+ X(PGLZ, 0, void) /* PostgreSQL LZ-based */ \
+ X(LZ4, 1, void) /* LZ4 algorithm */ \
+ X(ZSTD_NODICT, 2, void) /* ZSTD algorithm, no dictionary */ \
+ X(INVALID, 3, void) /* sentinel, must be last */
typedef enum ToastCompressionId
@@ -54,6 +59,7 @@ typedef enum ToastCompressionId
typedef struct CompressionInfo
{
char cmethod;
+ int zstd_level;
/* Extended compression meta info */
varatt_cmp_extended *cmp_ext;
} CompressionInfo;
@@ -66,10 +72,22 @@ typedef struct CompressionInfo
*/
#define TOAST_PGLZ_COMPRESSION 'p'
#define TOAST_LZ4_COMPRESSION 'l'
+#define TOAST_ZSTD_NODICT_COMPRESSION 'n'
#define InvalidCompressionMethod '\0'
#define CompressionMethodIsValid(cm) ((cm) != InvalidCompressionMethod)
+#define InvalidDictId 0
+
+#ifdef USE_ZSTD
+#define DEFAULT_ZSTD_LEVEL ZSTD_CLEVEL_DEFAULT
+#define MIN_ZSTD_LEVEL (int)-ZSTD_BLOCKSIZE_MAX
+#define MAX_ZSTD_LEVEL 22
+#else
+#define DEFAULT_ZSTD_LEVEL 0
+#define MIN_ZSTD_LEVEL 0
+#define MAX_ZSTD_LEVEL 0
+#endif
/* pglz compression/decompression routines */
extern struct varlena *pglz_compress_datum(const struct varlena *value);
@@ -83,6 +101,11 @@ extern struct varlena *lz4_decompress_datum(const struct varlena *value);
extern struct varlena *lz4_decompress_datum_slice(const struct varlena *value,
int32 slicelength);
+/* zstd compression/decompression routines */
+extern struct varlena *zstd_compress_datum(const struct varlena *value, CompressionInfo cmp);
+extern struct varlena *zstd_decompress_datum(const struct varlena *value);
+extern struct varlena *zstd_decompress_datum_slice(const struct varlena *value, int32 slicelength);
+
/* other stuff */
extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
extern char CompressionNameToMethod(const char *compression);
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index e672766f91a..7ae69792596 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -35,7 +35,8 @@ typedef struct toast_compress_header
do { \
Assert((len) > 0 && (len) <= VARLENA_EXTSIZE_MASK); \
Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm_method) == TOAST_ZSTD_NODICT_COMPRESSION_ID); \
if ((cm_method) <= TOAST_LAST_COMPRESSION_ID_BEFORE_EXT) { \
((toast_compress_header *) (ptr))->tcinfo = \
(len) | ((uint32) (cm_method) << VARLENA_EXTSIZE_BITS); \
diff --git a/src/include/utils/attoptcache.h b/src/include/utils/attoptcache.h
index f684a772af5..51d65ebd646 100644
--- a/src/include/utils/attoptcache.h
+++ b/src/include/utils/attoptcache.h
@@ -21,6 +21,7 @@ typedef struct AttributeOpts
int32 vl_len_; /* varlena header (do not touch directly!) */
float8 n_distinct;
float8 n_distinct_inherited;
+ int zstd_level;
} AttributeOpts;
extern AttributeOpts *get_attribute_options(Oid attrelid, int attnum);
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 72a49c2322e..4e60706744b 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -340,7 +340,8 @@ typedef struct
#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
do { \
Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
+ (cm) == TOAST_LZ4_COMPRESSION_ID || \
+ (cm) == TOAST_ZSTD_NODICT_COMPRESSION_ID); \
if ((cm) <= TOAST_LAST_COMPRESSION_ID_BEFORE_EXT) \
{ \
/* Store the actual method in va_extinfo */ \
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 4dd9ee7200d..c7e108a0f52 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -238,10 +238,11 @@ NOTICE: merging multiple inherited definitions of column "f1"
-- test default_toast_compression GUC
SET default_toast_compression = '';
ERROR: invalid value for parameter "default_toast_compression": ""
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd_nodict.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
-HINT: Available values: pglz, lz4.
+HINT: Available values: pglz, lz4, zstd_nodict.
+SET default_toast_compression = 'zstd_nodict';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
-- test alter compression method
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 7bd7642b4b9..5b10d8c5259 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -233,6 +233,9 @@ HINT: Available values: pglz.
SET default_toast_compression = 'I do not exist compression';
ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
HINT: Available values: pglz.
+SET default_toast_compression = 'zstd_nodict';
+ERROR: invalid value for parameter "default_toast_compression": "zstd_nodict"
+HINT: Available values: pglz.
SET default_toast_compression = 'lz4';
ERROR: invalid value for parameter "default_toast_compression": "lz4"
HINT: Available values: pglz.
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index 490595fcfb2..27979eb7997 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -102,6 +102,7 @@ CREATE TABLE cminh() INHERITS (cmdata, cmdata3);
-- test default_toast_compression GUC
SET default_toast_compression = '';
SET default_toast_compression = 'I do not exist compression';
+SET default_toast_compression = 'zstd_nodict';
SET default_toast_compression = 'lz4';
SET default_toast_compression = 'pglz';
--
2.47.1