v37-0003-Built-in-compression-method.patch
text/x-patch
Filename: v37-0003-Built-in-compression-method.patch
Type: text/x-patch
Part: 1
Patch
Format: format-patch
Series: patch v37-0003
Subject: Built-in compression method
| File | + | − |
|---|---|---|
| configure | 170 | 0 |
| configure.ac | 20 | 0 |
| contrib/amcheck/verify_heapam.c | 1 | 1 |
| doc/src/sgml/catalogs.sgml | 10 | 0 |
| doc/src/sgml/ref/create_table.sgml | 32 | 1 |
| doc/src/sgml/ref/psql-ref.sgml | 11 | 0 |
| src/backend/access/brin/brin_tuple.c | 4 | 1 |
| src/backend/access/common/detoast.c | 68 | 31 |
| src/backend/access/common/indextuple.c | 2 | 1 |
| src/backend/access/common/Makefile | 1 | 0 |
| src/backend/access/common/toast_compression.c | 244 | 0 |
| src/backend/access/common/toast_internals.c | 43 | 32 |
| src/backend/access/common/tupdesc.c | 6 | 0 |
| src/backend/access/table/toast_helper.c | 4 | 1 |
| src/backend/bootstrap/bootstrap.c | 5 | 0 |
| src/backend/catalog/genbki.pl | 3 | 0 |
| src/backend/catalog/heap.c | 4 | 0 |
| src/backend/catalog/index.c | 1 | 0 |
| src/backend/catalog/toasting.c | 6 | 0 |
| src/backend/commands/tablecmds.c | 110 | 0 |
| src/backend/nodes/copyfuncs.c | 1 | 0 |
| src/backend/nodes/equalfuncs.c | 1 | 0 |
| src/backend/nodes/nodeFuncs.c | 2 | 0 |
| src/backend/nodes/outfuncs.c | 1 | 0 |
| src/backend/parser/gram.y | 20 | 6 |
| src/backend/parser/parse_utilcmd.c | 9 | 0 |
| src/backend/replication/logical/reorderbuffer.c | 1 | 1 |
| src/backend/utils/adt/varlena.c | 49 | 0 |
| src/bin/pg_dump/pg_backup.h | 1 | 0 |
| src/bin/pg_dump/pg_dump.c | 39 | 0 |
| src/bin/pg_dump/pg_dump.h | 1 | 0 |
| src/bin/pg_dump/t/002_pg_dump.pl | 6 | 6 |
| src/bin/psql/describe.c | 29 | 2 |
| src/bin/psql/help.c | 2 | 0 |
| src/bin/psql/settings.h | 1 | 0 |
| src/bin/psql/startup.c | 10 | 0 |
| src/include/access/detoast.h | 9 | 9 |
| src/include/access/toast_compression.h | 141 | 0 |
| src/include/access/toast_helper.h | 1 | 0 |
| src/include/access/toast_internals.h | 14 | 9 |
| src/include/catalog/pg_attribute.h | 7 | 1 |
| src/include/catalog/pg_proc.dat | 4 | 0 |
| src/include/nodes/parsenodes.h | 2 | 0 |
| src/include/parser/kwlist.h | 1 | 0 |
| src/include/pg_config.h.in | 3 | 0 |
| src/include/postgres.h | 43 | 3 |
| src/test/regress/expected/compression_1.out | 250 | 0 |
| src/test/regress/expected/compression.out | 261 | 0 |
| src/test/regress/parallel_schedule | 1 | 1 |
| src/test/regress/pg_regress_main.c | 2 | 2 |
| src/test/regress/serial_schedule | 1 | 0 |
| src/test/regress/sql/compression.sql | 104 | 0 |
| src/tools/msvc/Solution.pm | 1 | 0 |
From b55fd3c679bc5ef0af3b6c0a9f8754ea3f3867d1 Mon Sep 17 00:00:00 2001
From: Dilip Kumar <dilipkumar@localhost.localdomain>
Date: Fri, 5 Mar 2021 09:33:08 +0530
Subject: [PATCH v37 3/6] Built-in compression method
Add syntax allowing a compression method to be specified.
As of now there is only 2 option for build-in compression
method (pglz, lz4) which can be set while creating a table
or adding a new column. No option for altering the
compression method for an existing column.
Dilip Kumar based on the patches from Ildus Kurbangaliev.
Design input from Robert Haas and Tomas Vondra.
Reviewed by Robert Haas, Justin Pryzby, Tomas Vondra and Alexander Korotkov
Discussions:
https://www.postgresql.org/message-id/20171213151818.75a20259@postgrespro.ru
https://www.postgresql.org/message-id/CA%2BTgmoaKDW1Oi9V%3Djc9hOGyf77NbkNEABuqgHD1Cq%3D%3D1QsOcxg%40mail.gmail.com
https://www.postgresql.org/message-id/CA%2BTgmobSDVgUage9qQ5P_%3DF_9jaMkCgyKxUQGtFQU7oN4kX-AA%40mail.gmail.com
https://www.postgresql.org/message-id/20201005160355.byp74sh3ejsv7wrj%40development
https://www.postgresql.org/message-id/CAFiTN-tzTTT2oqWdRGLv1dvvS5MC1W%2BLE%2B3bqWPJUZj4GnHOJg%40mail.gmail.com
---
configure | 170 +++++++++++++++
configure.ac | 20 ++
contrib/amcheck/verify_heapam.c | 2 +-
doc/src/sgml/catalogs.sgml | 10 +
doc/src/sgml/ref/create_table.sgml | 33 ++-
doc/src/sgml/ref/psql-ref.sgml | 11 +
src/backend/access/brin/brin_tuple.c | 5 +-
src/backend/access/common/Makefile | 1 +
src/backend/access/common/detoast.c | 99 ++++++---
src/backend/access/common/indextuple.c | 3 +-
src/backend/access/common/toast_compression.c | 244 ++++++++++++++++++++++
src/backend/access/common/toast_internals.c | 75 ++++---
src/backend/access/common/tupdesc.c | 6 +
src/backend/access/table/toast_helper.c | 5 +-
src/backend/bootstrap/bootstrap.c | 5 +
src/backend/catalog/genbki.pl | 3 +
src/backend/catalog/heap.c | 4 +
src/backend/catalog/index.c | 1 +
src/backend/catalog/toasting.c | 6 +
src/backend/commands/tablecmds.c | 110 ++++++++++
src/backend/nodes/copyfuncs.c | 1 +
src/backend/nodes/equalfuncs.c | 1 +
src/backend/nodes/nodeFuncs.c | 2 +
src/backend/nodes/outfuncs.c | 1 +
src/backend/parser/gram.y | 26 ++-
src/backend/parser/parse_utilcmd.c | 9 +
src/backend/replication/logical/reorderbuffer.c | 2 +-
src/backend/utils/adt/varlena.c | 49 +++++
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_dump.c | 39 ++++
src/bin/pg_dump/pg_dump.h | 1 +
src/bin/pg_dump/t/002_pg_dump.pl | 12 +-
src/bin/psql/describe.c | 31 ++-
src/bin/psql/help.c | 2 +
src/bin/psql/settings.h | 1 +
src/bin/psql/startup.c | 10 +
src/include/access/detoast.h | 18 +-
src/include/access/toast_compression.h | 141 +++++++++++++
src/include/access/toast_helper.h | 1 +
src/include/access/toast_internals.h | 23 ++-
src/include/catalog/pg_attribute.h | 8 +-
src/include/catalog/pg_proc.dat | 4 +
src/include/nodes/parsenodes.h | 2 +
src/include/parser/kwlist.h | 1 +
src/include/pg_config.h.in | 3 +
src/include/postgres.h | 46 ++++-
src/test/regress/expected/compression.out | 261 ++++++++++++++++++++++++
src/test/regress/expected/compression_1.out | 250 +++++++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/pg_regress_main.c | 4 +-
src/test/regress/serial_schedule | 1 +
src/test/regress/sql/compression.sql | 104 ++++++++++
src/tools/msvc/Solution.pm | 1 +
53 files changed, 1763 insertions(+), 108 deletions(-)
create mode 100644 src/backend/access/common/toast_compression.c
create mode 100644 src/include/access/toast_compression.h
create mode 100644 src/test/regress/expected/compression.out
create mode 100644 src/test/regress/expected/compression_1.out
create mode 100644 src/test/regress/sql/compression.sql
diff --git a/configure b/configure
index 3fd4cec..8176e99 100755
--- a/configure
+++ b/configure
@@ -699,6 +699,9 @@ with_gnu_ld
LD
LDFLAGS_SL
LDFLAGS_EX
+LZ4_LIBS
+LZ4_CFLAGS
+with_lz4
with_zlib
with_system_tzdata
with_libxslt
@@ -864,6 +867,7 @@ with_libxml
with_libxslt
with_system_tzdata
with_zlib
+with_lz4
with_gnu_ld
with_ssl
with_openssl
@@ -891,6 +895,8 @@ ICU_LIBS
XML2_CONFIG
XML2_CFLAGS
XML2_LIBS
+LZ4_CFLAGS
+LZ4_LIBS
LDFLAGS_EX
LDFLAGS_SL
PERL
@@ -1569,6 +1575,7 @@ Optional Packages:
--with-system-tzdata=DIR
use system time zone data in DIR
--without-zlib do not use Zlib
+ --with-lz4 build with LZ4 support
--with-gnu-ld assume the C compiler uses GNU ld [default=no]
--with-ssl=LIB use LIB for SSL/TLS support (openssl)
--with-openssl obsolete spelling of --with-ssl=openssl
@@ -1596,6 +1603,8 @@ Some influential environment variables:
XML2_CONFIG path to xml2-config utility
XML2_CFLAGS C compiler flags for XML2, overriding pkg-config
XML2_LIBS linker flags for XML2, overriding pkg-config
+ LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config
+ LZ4_LIBS linker flags for LZ4, overriding pkg-config
LDFLAGS_EX extra linker flags for linking executables only
LDFLAGS_SL extra linker flags for linking shared libraries only
PERL Perl program
@@ -8564,6 +8573,137 @@ fi
#
+# LZ4
+#
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with LZ4 support" >&5
+$as_echo_n "checking whether to build with LZ4 support... " >&6; }
+
+
+
+# Check whether --with-lz4 was given.
+if test "${with_lz4+set}" = set; then :
+ withval=$with_lz4;
+ case $withval in
+ yes)
+
+$as_echo "#define USE_LZ4 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --with-lz4 option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ with_lz4=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_lz4" >&5
+$as_echo "$with_lz4" >&6; }
+
+
+if test "$with_lz4" = yes; then
+
+pkg_failed=no
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5
+$as_echo_n "checking for liblz4... " >&6; }
+
+if test -n "$LZ4_CFLAGS"; then
+ pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+if test -n "$LZ4_LIBS"; then
+ pkg_cv_LZ4_LIBS="$LZ4_LIBS"
+ elif test -n "$PKG_CONFIG"; then
+ if test -n "$PKG_CONFIG" && \
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5
+ ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; then
+ pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null`
+ test "x$?" != "x0" && pkg_failed=yes
+else
+ pkg_failed=yes
+fi
+ else
+ pkg_failed=untried
+fi
+
+
+
+if test $pkg_failed = yes; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then
+ _pkg_short_errors_supported=yes
+else
+ _pkg_short_errors_supported=no
+fi
+ if test $_pkg_short_errors_supported = yes; then
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1`
+ else
+ LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1`
+ fi
+ # Put the nasty error message in config.log where it belongs
+ echo "$LZ4_PKG_ERRORS" >&5
+
+ as_fn_error $? "Package requirements (liblz4) were not met:
+
+$LZ4_PKG_ERRORS
+
+Consider adjusting the PKG_CONFIG_PATH environment variable if you
+installed software in a non-standard prefix.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details." "$LINENO" 5
+elif test $pkg_failed = untried; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it
+is in your PATH or set the PKG_CONFIG environment variable to the full
+path to pkg-config.
+
+Alternatively, you may set the environment variables LZ4_CFLAGS
+and LZ4_LIBS to avoid the need to call pkg-config.
+See the pkg-config man page for more details.
+
+To get pkg-config, see <http://pkg-config.freedesktop.org/>.
+See \`config.log' for more details" "$LINENO" 5; }
+else
+ LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS
+ LZ4_LIBS=$pkg_cv_LZ4_LIBS
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+fi
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
+#
# Assignments
#
@@ -13381,6 +13521,36 @@ fi
fi
+if test "$with_lz4" = yes; then
+ for ac_header in lz4/lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_LZ4_H 1
+_ACEOF
+
+else
+ for ac_header in lz4.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default"
+if test "x$ac_cv_header_lz4_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_LZ4_H 1
+_ACEOF
+
+else
+ as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5
+fi
+
+done
+
+fi
+
+done
+
+fi
+
if test "$with_gssapi" = yes ; then
for ac_header in gssapi/gssapi.h
do :
diff --git a/configure.ac b/configure.ac
index 2f1585a..54efbb2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -987,6 +987,21 @@ PGAC_ARG_BOOL(with, zlib, yes,
AC_SUBST(with_zlib)
#
+# LZ4
+#
+AC_MSG_CHECKING([whether to build with LZ4 support])
+PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support],
+ [AC_DEFINE([USE_LZ4], 1, [Define to 1 to build with LZ4 support. (--with-lz4)])])
+AC_MSG_RESULT([$with_lz4])
+AC_SUBST(with_lz4)
+
+if test "$with_lz4" = yes; then
+ PKG_CHECK_MODULES(LZ4, liblz4)
+ LIBS="$LZ4_LIBS $LIBS"
+ CFLAGS="$LZ4_CFLAGS $CFLAGS"
+fi
+
+#
# Assignments
#
@@ -1410,6 +1425,11 @@ failure. It is possible the compiler isn't looking in the proper directory.
Use --without-zlib to disable zlib support.])])
fi
+if test "$with_lz4" = yes; then
+ AC_CHECK_HEADERS(lz4/lz4.h, [],
+ [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])])
+fi
+
if test "$with_gssapi" = yes ; then
AC_CHECK_HEADERS(gssapi/gssapi.h, [],
[AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])])
diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index e614c12..6f972e6 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -1069,7 +1069,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
*/
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- ctx->attrsize = toast_pointer.va_extsize;
+ ctx->attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
ctx->endchunk = (ctx->attrsize - 1) / TOAST_MAX_CHUNK_SIZE;
ctx->totalchunks = ctx->endchunk + 1;
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 5c9f4af..de5f02a 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -1357,6 +1357,16 @@
<row>
<entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>attcompression</structfield> <type>char</type>
+ </para>
+ <para>
+ The current compression method of the column. Must be <literal>InvalidCompressionMethod</literal>
+ if and only if typstorage is 'plain' or 'external'.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
<structfield>attacl</structfield> <type>aclitem[]</type>
</para>
<para>
diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml
index ff1b642..a731239 100644
--- a/doc/src/sgml/ref/create_table.sgml
+++ b/doc/src/sgml/ref/create_table.sgml
@@ -22,7 +22,7 @@ PostgreSQL documentation
<refsynopsisdiv>
<synopsis>
CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] <replaceable class="parameter">table_name</replaceable> ( [
- { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ]
+ { <replaceable class="parameter">column_name</replaceable> <replaceable class="parameter">data_type</replaceable> [ COLLATE <replaceable>collation</replaceable> ] [ COMPRESSION <replaceable>compression_method</replaceable> ] [ <replaceable class="parameter">column_constraint</replaceable> [ ... ] ]
| <replaceable>table_constraint</replaceable>
| LIKE <replaceable>source_table</replaceable> [ <replaceable>like_option</replaceable> ... ] }
[, ... ]
@@ -289,6 +289,26 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
</varlistentry>
<varlistentry>
+ <term><literal>COMPRESSION <replaceable class="parameter">compression_method</replaceable></literal></term>
+ <listitem>
+ <para>
+ The <literal>COMPRESSION</literal> clause sets the compression method
+ for a column. Compression is supported only for variable-width data
+ types, and is used only for columns whose storage type is main or
+ extended. (See <xref linkend="sql-altertable"/> for information on
+ column storage types.) Setting this property for a partitioned table
+ has no direct effect, because such tables have no storage of their own,
+ but the configured value is inherited by newly-created partitions.
+ The supported compression methods are <literal>pglz</literal> and
+ <literal>lz4</literal>. <literal>lz4</literal> is available only if
+ <literal>--with-lz4</literal> was used when building
+ <productname>PostgreSQL</productname>. The default
+ is <literal>pglz</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><literal>INHERITS ( <replaceable>parent_table</replaceable> [, ... ] )</literal></term>
<listitem>
<para>
@@ -606,6 +626,17 @@ WITH ( MODULUS <replaceable class="parameter">numeric_literal</replaceable>, REM
</varlistentry>
<varlistentry>
+ <term><literal>INCLUDING COMPRESSION</literal></term>
+ <listitem>
+ <para>
+ Compression method of the columns will be copied. The default
+ behavior is to exclude compression methods, resulting in columns
+ having the default compression method.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><literal>INCLUDING CONSTRAINTS</literal></term>
<listitem>
<para>
diff --git a/doc/src/sgml/ref/psql-ref.sgml b/doc/src/sgml/ref/psql-ref.sgml
index 13c1edf..01ec9b8 100644
--- a/doc/src/sgml/ref/psql-ref.sgml
+++ b/doc/src/sgml/ref/psql-ref.sgml
@@ -3864,6 +3864,17 @@ bar
</varlistentry>
<varlistentry>
+ <term><varname>HIDE_TOAST_COMPRESSION</varname></term>
+ <listitem>
+ <para>
+ If this variable is set to <literal>true</literal>, column
+ compression method details are not displayed. This is mainly
+ useful for regression tests.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><varname>HIDE_TABLEAM</varname></term>
<listitem>
<para>
diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c
index a7eb1c9..0ab5712 100644
--- a/src/backend/access/brin/brin_tuple.c
+++ b/src/backend/access/brin/brin_tuple.c
@@ -213,7 +213,10 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple,
(atttype->typstorage == TYPSTORAGE_EXTENDED ||
atttype->typstorage == TYPSTORAGE_MAIN))
{
- Datum cvalue = toast_compress_datum(value);
+ Form_pg_attribute att = TupleDescAttr(brdesc->bd_tupdesc,
+ keyno);
+ Datum cvalue = toast_compress_datum(value,
+ att->attcompression);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/Makefile b/src/backend/access/common/Makefile
index 5a007d6..b9aff0c 100644
--- a/src/backend/access/common/Makefile
+++ b/src/backend/access/common/Makefile
@@ -25,6 +25,7 @@ OBJS = \
scankey.o \
session.o \
syncscan.o \
+ toast_compression.o \
toast_internals.o \
tupconvert.o \
tupdesc.o
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index d1cdbaf..fc930b7 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -240,14 +240,15 @@ detoast_attr_slice(struct varlena *attr,
*/
if (slicelimit >= 0)
{
- int32 max_size;
+ int32 max_size = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
/*
* Determine maximum amount of compressed data needed for a prefix
* of a given length (after decompression).
*/
- max_size = pglz_maximum_compressed_size(slicelimit,
- toast_pointer.va_extsize);
+ if (VARATT_EXTERNAL_GET_COMPRESSION(toast_pointer) ==
+ TOAST_PGLZ_COMPRESSION_ID)
+ max_size = pglz_maximum_compressed_size(slicelimit, max_size);
/*
* Fetch enough compressed slices (compressed marker will get set
@@ -347,7 +348,7 @@ toast_fetch_datum(struct varlena *attr)
/* Must copy to access aligned fields */
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- attrsize = toast_pointer.va_extsize;
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
result = (struct varlena *) palloc(attrsize + VARHDRSZ);
@@ -408,7 +409,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
*/
Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
- attrsize = toast_pointer.va_extsize;
+ attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
if (sliceoffset >= attrsize)
{
@@ -459,26 +460,60 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset,
/* ----------
* toast_decompress_datum -
*
+ * Returns compression method id stored in the compressed data. Otherwise,
+ * returns TOAST_INVALID_COMPRESSION_ID for the uncompressed data.
+ */
+ToastCompressionId
+toast_get_compression_id(struct varlena *attr)
+{
+ ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
+
+ /*
+ * Get the compression method id stored in compressed data. If it is
+ * stored externally then fetch it from the external toast pointer
+ * otherwise for inline compressed data fetch it from the compression
+ * header.
+ */
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
+ {
+ struct varatt_external toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+
+ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ cmid = VARATT_EXTERNAL_GET_COMPRESSION(toast_pointer);
+ }
+ else if (VARATT_IS_COMPRESSED(attr))
+ cmid = VARCOMPRESS_4B_C(attr);
+
+ return cmid;
+}
+/* ----------
+ * toast_decompress_datum -
+ *
* Decompress a compressed version of a varlena datum
*/
static struct varlena *
toast_decompress_datum(struct varlena *attr)
{
- struct varlena *result;
+ ToastCompressionId cmid;
Assert(VARATT_IS_COMPRESSED(attr));
- result = (struct varlena *)
- palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
- SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ);
-
- if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
- TOAST_COMPRESS_SIZE(attr),
- VARDATA(result),
- TOAST_COMPRESS_RAWSIZE(attr), true) < 0)
- elog(ERROR, "compressed data is corrupted");
-
- return result;
+ /*
+ * Fetch the compression method id stored in the compression header and
+ * decompress the data using the respective decompression routine.
+ */
+ cmid = TOAST_COMPRESS_METHOD(attr);
+ switch (cmid)
+ {
+ case TOAST_PGLZ_COMPRESSION_ID:
+ return pglz_cmdecompress(attr);
+ case TOAST_LZ4_COMPRESSION_ID:
+ return lz4_cmdecompress(attr);
+ default:
+ elog(ERROR, "invalid compression method id %d", cmid);
+ }
}
@@ -492,22 +527,24 @@ toast_decompress_datum(struct varlena *attr)
static struct varlena *
toast_decompress_datum_slice(struct varlena *attr, int32 slicelength)
{
- struct varlena *result;
- int32 rawsize;
+ ToastCompressionId cmid;
Assert(VARATT_IS_COMPRESSED(attr));
- result = (struct varlena *) palloc(slicelength + VARHDRSZ);
-
- rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr),
- VARSIZE(attr) - TOAST_COMPRESS_HDRSZ,
- VARDATA(result),
- slicelength, false);
- if (rawsize < 0)
- elog(ERROR, "compressed data is corrupted");
-
- SET_VARSIZE(result, rawsize + VARHDRSZ);
- return result;
+ /*
+ * Fetch the compression method id stored in the compression header and
+ * decompress the data slice using the respective decompression routine.
+ */
+ cmid = TOAST_COMPRESS_METHOD(attr);
+ switch (cmid)
+ {
+ case TOAST_PGLZ_COMPRESSION_ID:
+ return pglz_cmdecompress_slice(attr, slicelength);
+ case TOAST_LZ4_COMPRESSION_ID:
+ return lz4_cmdecompress_slice(attr, slicelength);
+ default:
+ elog(ERROR, "invalid compression method id %d", cmid);
+ }
}
/* ----------
@@ -589,7 +626,7 @@ toast_datum_size(Datum value)
struct varatt_external toast_pointer;
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_extsize;
+ result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer);
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index b72a138..1f6b7b7 100644
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -103,7 +103,8 @@ index_form_tuple(TupleDesc tupleDescriptor,
(att->attstorage == TYPSTORAGE_EXTENDED ||
att->attstorage == TYPSTORAGE_MAIN))
{
- Datum cvalue = toast_compress_datum(untoasted_values[i]);
+ Datum cvalue = toast_compress_datum(untoasted_values[i],
+ att->attcompression);
if (DatumGetPointer(cvalue) != NULL)
{
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
new file mode 100644
index 0000000..22a8d54
--- /dev/null
+++ b/src/backend/access/common/toast_compression.c
@@ -0,0 +1,244 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_compression.c
+ * Functions for toast compression.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/access/common/toast_compression.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#ifdef USE_LZ4
+#include <lz4.h>
+#endif
+
+#include "access/toast_compression.h"
+#include "common/pg_lzcompress.h"
+#include "fmgr.h"
+#include "utils/builtins.h"
+
+/*
+ * pglz_cmcompress - compression routine for pglz compression method
+ *
+ * Compresses source into dest using the default strategy. Returns the
+ * compressed varlena, or NULL if compression fails.
+ */
+struct varlena *
+pglz_cmcompress(const struct varlena *value)
+{
+ int32 valsize,
+ len;
+ struct varlena *tmp = NULL;
+
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+
+ /*
+ * No point in wasting a palloc cycle if value size is outside the allowed
+ * range for compression.
+ */
+ if (valsize < PGLZ_strategy_default->min_input_size ||
+ valsize > PGLZ_strategy_default->max_input_size)
+ return NULL;
+
+ /*
+ * Figure out the maximum possible size of the pglz output, add the bytes
+ * that will be needed for varlena overhead, and allocate that amount.
+ */
+ tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
+ VARHDRSZ_COMPRESS);
+
+ len = pglz_compress(VARDATA_ANY(value),
+ valsize,
+ (char *) tmp + VARHDRSZ_COMPRESS,
+ NULL);
+ if (len < 0)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS);
+
+ return tmp;
+}
+
+/*
+ * pglz_cmdecompress - decompression routine for pglz compression method
+ *
+ * Returns the decompressed varlena.
+ */
+struct varlena *
+pglz_cmdecompress(const struct varlena *value)
+{
+ struct varlena *result;
+ int32 rawsize;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS,
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARRAWSIZE_4B_C(value), true);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed pglz data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+}
+
+/*
+ * pglz_decompress - slice decompression routine for pglz compression method
+ *
+ * Decompresses part of the data. Returns the decompressed varlena.
+ */
+struct varlena *
+pglz_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength)
+{
+ struct varlena *result;
+ int32 rawsize;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESS,
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ slicelength, false);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed pglz data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+}
+
+/*
+ * lz4_cmcompress - compression routine for lz4 compression method
+ *
+ * Compresses source into dest using the LZ4 defaults. Returns the
+ * compressed varlena, or NULL if compression fails.
+ */
+struct varlena *
+lz4_cmcompress(const struct varlena *value)
+{
+#ifndef USE_LZ4
+ NO_LZ4_SUPPORT();
+#else
+ int32 valsize;
+ int32 len;
+ int32 max_size;
+ struct varlena *tmp = NULL;
+
+ valsize = VARSIZE_ANY_EXHDR(value);
+
+ /*
+ * Figure out the maximum possible size of the LZ4 output, add the bytes
+ * that will be needed for varlena overhead, and allocate that amount.
+ */
+ max_size = LZ4_compressBound(valsize);
+ tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESS);
+
+ len = LZ4_compress_default(VARDATA_ANY(value),
+ (char *) tmp + VARHDRSZ_COMPRESS,
+ valsize, max_size);
+ if (len <= 0)
+ elog(ERROR, "could not compress data with lz4");
+
+ /* data is incompressible so just free the memory and return NULL */
+ if (len > valsize)
+ {
+ pfree(tmp);
+ return NULL;
+ }
+
+ SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESS);
+
+ return tmp;
+#endif
+}
+
+/*
+ * lz4_cmdecompress - decompression routine for lz4 compression method
+ *
+ * Returns the decompressed varlena.
+ */
+struct varlena *
+lz4_cmdecompress(const struct varlena *value)
+{
+#ifndef USE_LZ4
+ NO_LZ4_SUPPORT();
+#else
+ int32 rawsize;
+ struct varlena *result;
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ VARRAWSIZE_4B_C(value));
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed lz4 data is corrupt")));
+
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif
+}
+
+/*
+ * lz4_cmdecompress_slice - slice decompression routine for lz4 compression
+ *
+ * Decompresses part of the data. Returns the decompressed varlena.
+ */
+struct varlena *
+lz4_cmdecompress_slice(const struct varlena *value, int32 slicelength)
+{
+#ifndef USE_LZ4
+ NO_LZ4_SUPPORT();
+#else
+ int32 rawsize;
+ struct varlena *result;
+
+ /* slice decompression not supported prior to 1.8.3 */
+ if (LZ4_versionNumber() < 10803)
+ return lz4_cmdecompress(value);
+
+ /* allocate memory for the uncompressed data */
+ result = (struct varlena *) palloc(slicelength + VARHDRSZ);
+
+ /* decompress the data */
+ rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESS,
+ VARDATA(result),
+ VARSIZE(value) - VARHDRSZ_COMPRESS,
+ slicelength,
+ slicelength);
+ if (rawsize < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATA_CORRUPTED),
+ errmsg_internal("compressed lz4 data is corrupt")));
+
+ SET_VARSIZE(result, rawsize + VARHDRSZ);
+
+ return result;
+#endif
+}
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 9b9da0f..c84521e 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -44,48 +44,56 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid);
* ----------
*/
Datum
-toast_compress_datum(Datum value)
+toast_compress_datum(Datum value, char cmethod)
{
- struct varlena *tmp;
- int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
- int32 len;
+ struct varlena *tmp = NULL;
+ int32 valsize;
Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value)));
Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value)));
+ Assert(CompressionMethodIsValid(cmethod));
+
+ valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
+
/*
- * No point in wasting a palloc cycle if value size is out of the allowed
- * range for compression
+ * Call respective compression routine for the compression method and also
+ * set the size and compression method id in the compressed data header.
*/
- if (valsize < PGLZ_strategy_default->min_input_size ||
- valsize > PGLZ_strategy_default->max_input_size)
- return PointerGetDatum(NULL);
+ switch (cmethod)
+ {
+ case TOAST_PGLZ_COMPRESSION:
+ tmp = pglz_cmcompress((const struct varlena *) value);
+ if (tmp == NULL)
+ return PointerGetDatum(NULL);
- tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
- TOAST_COMPRESS_HDRSZ);
+ TOAST_COMPRESS_SET_SIZE_AND_METHOD(tmp, valsize,
+ TOAST_PGLZ_COMPRESSION_ID);
+ break;
+ case TOAST_LZ4_COMPRESSION:
+ tmp = lz4_cmcompress((const struct varlena *) value);
+ if (tmp == NULL)
+ return PointerGetDatum(NULL);
+ TOAST_COMPRESS_SET_SIZE_AND_METHOD(tmp, valsize,
+ TOAST_LZ4_COMPRESSION_ID);
+ break;
+ default:
+ elog(ERROR, "invalid compression method %c", cmethod);
+ }
/*
- * We recheck the actual size even if pglz_compress() reports success,
- * because it might be satisfied with having saved as little as one byte
- * in the compressed data --- which could turn into a net loss once you
- * consider header and alignment padding. Worst case, the compressed
- * format might require three padding bytes (plus header, which is
- * included in VARSIZE(tmp)), whereas the uncompressed format would take
- * only one header byte and no padding if the value is short enough. So
- * we insist on a savings of more than 2 bytes to ensure we have a gain.
+ * We recheck the actual size even if compression reports success, because
+ * it might be satisfied with having saved as little as one byte in the
+ * compressed data --- which could turn into a net loss once you consider
+ * header and alignment padding. Worst case, the compressed format might
+ * require three padding bytes (plus header, which is included in
+ * VARSIZE(tmp)), whereas the uncompressed format would take only one
+ * header byte and no padding if the value is short enough. So we insist
+ * on a savings of more than 2 bytes to ensure we have a gain.
*/
- len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)),
- valsize,
- TOAST_COMPRESS_RAWDATA(tmp),
- PGLZ_strategy_default);
- if (len >= 0 &&
- len + TOAST_COMPRESS_HDRSZ < valsize - 2)
- {
- TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize);
- SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ);
+ if (VARSIZE(tmp) < valsize - 2)
/* successful compression */
return PointerGetDatum(tmp);
- }
else
{
/* incompressible data */
@@ -164,7 +172,7 @@ toast_save_datum(Relation rel, Datum value,
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
- toast_pointer.va_extsize = data_todo;
+ toast_pointer.va_extinfo = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
@@ -172,7 +180,10 @@ toast_save_datum(Relation rel, Datum value,
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
toast_pointer.va_rawsize = VARRAWSIZE_4B_C(dval) + VARHDRSZ;
- toast_pointer.va_extsize = data_todo;
+
+ /* set external size and compression method */
+ VARATT_EXTERNAL_SET_SIZE_AND_COMPRESSION(toast_pointer, data_todo,
+ VARCOMPRESS_4B_C(dval));
/* Assert that the numbers look like it's compressed */
Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer));
}
@@ -181,7 +192,7 @@ toast_save_datum(Relation rel, Datum value,
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extsize = data_todo;
+ toast_pointer.va_extinfo = data_todo;
}
/*
diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c
index 902f594..c612e3c 100644
--- a/src/backend/access/common/tupdesc.c
+++ b/src/backend/access/common/tupdesc.c
@@ -20,6 +20,7 @@
#include "postgres.h"
#include "access/htup_details.h"
+#include "access/toast_compression.h"
#include "access/tupdesc_details.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
@@ -664,6 +665,11 @@ TupleDescInitEntry(TupleDesc desc,
att->attstorage = typeForm->typstorage;
att->attcollation = typeForm->typcollation;
+ if (IsStorageCompressible(typeForm->typstorage))
+ att->attcompression = DefaultCompressionMethod;
+ else
+ att->attcompression = InvalidCompressionMethod;
+
ReleaseSysCache(tuple);
}
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index fb36151..53f78f9 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -54,6 +54,7 @@ toast_tuple_init(ToastTupleContext *ttc)
ttc->ttc_attr[i].tai_colflags = 0;
ttc->ttc_attr[i].tai_oldexternal = NULL;
+ ttc->ttc_attr[i].tai_compression = att->attcompression;
if (ttc->ttc_oldvalues != NULL)
{
@@ -226,9 +227,11 @@ void
toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
{
Datum *value = &ttc->ttc_values[attribute];
- Datum new_value = toast_compress_datum(*value);
+ Datum new_value;
ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+ new_value = toast_compress_datum(*value, attr->tai_compression);
+
if (DatumGetPointer(new_value) != NULL)
{
/* successful compression */
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index 41da0c5..c57086c 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -21,6 +21,7 @@
#include "access/heapam.h"
#include "access/htup_details.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "access/xlog_internal.h"
#include "bootstrap/bootstrap.h"
@@ -733,6 +734,10 @@ DefineAttr(char *name, char *type, int attnum, int nullness)
attrtypes[attnum]->attcacheoff = -1;
attrtypes[attnum]->atttypmod = -1;
attrtypes[attnum]->attislocal = true;
+ if (IsStorageCompressible(attrtypes[attnum]->attstorage))
+ attrtypes[attnum]->attcompression = DefaultCompressionMethod;
+ else
+ attrtypes[attnum]->attcompression = InvalidCompressionMethod;
if (nullness == BOOTCOL_NULL_FORCE_NOT_NULL)
{
diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl
index b159958..9586c29 100644
--- a/src/backend/catalog/genbki.pl
+++ b/src/backend/catalog/genbki.pl
@@ -906,6 +906,9 @@ sub morph_row_for_pgattr
$row->{attcollation} =
$type->{typcollation} ne '0' ? $C_COLLATION_OID : 0;
+ $row->{attcompression} =
+ $type->{typstorage} ne 'p' && $type->{typstorage} ne 'e' ? 'p' : '\0';
+
if (defined $attr->{forcenotnull})
{
$row->{attnotnull} = 't';
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index 9abc4a1..d0ec44b 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -36,6 +36,7 @@
#include "access/sysattr.h"
#include "access/table.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/transam.h"
#include "access/xact.h"
#include "access/xlog.h"
@@ -789,6 +790,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel,
slot[slotCount]->tts_values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(attrs->attislocal);
slot[slotCount]->tts_values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(attrs->attinhcount);
slot[slotCount]->tts_values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(attrs->attcollation);
+ slot[slotCount]->tts_values[Anum_pg_attribute_attcompression - 1] = CharGetDatum(attrs->attcompression);
if (attoptions && attoptions[natts] != (Datum) 0)
slot[slotCount]->tts_values[Anum_pg_attribute_attoptions - 1] = attoptions[natts];
else
@@ -1715,6 +1717,8 @@ RemoveAttributeById(Oid relid, AttrNumber attnum)
/* Unset this so no one tries to look up the generation expression */
attStruct->attgenerated = '\0';
+ attStruct->attcompression = InvalidCompressionMethod;
+
/*
* Change the column name to something that isn't likely to conflict
*/
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 4ef61b5..397d70d 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -348,6 +348,7 @@ ConstructTupleDescriptor(Relation heapRelation,
to->attbyval = from->attbyval;
to->attstorage = from->attstorage;
to->attalign = from->attalign;
+ to->attcompression = from->attcompression;
}
else
{
diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c
index d7b8060..933a073 100644
--- a/src/backend/catalog/toasting.c
+++ b/src/backend/catalog/toasting.c
@@ -15,6 +15,7 @@
#include "postgres.h"
#include "access/heapam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "catalog/binary_upgrade.h"
#include "catalog/catalog.h"
@@ -220,6 +221,11 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
TupleDescAttr(tupdesc, 1)->attstorage = TYPSTORAGE_PLAIN;
TupleDescAttr(tupdesc, 2)->attstorage = TYPSTORAGE_PLAIN;
+ /* Toast field should not be compressed */
+ TupleDescAttr(tupdesc, 0)->attcompression = InvalidCompressionMethod;
+ TupleDescAttr(tupdesc, 1)->attcompression = InvalidCompressionMethod;
+ TupleDescAttr(tupdesc, 2)->attcompression = InvalidCompressionMethod;
+
/*
* Toast tables for regular relations go in pg_toast; those for temp
* relations go into the per-backend temp-toast-table namespace.
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index ffb1308..08558f8 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -23,6 +23,7 @@
#include "access/relscan.h"
#include "access/sysattr.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "catalog/catalog.h"
@@ -558,6 +559,7 @@ static void refuseDupeIndexAttach(Relation parentIdx, Relation partIdx,
static List *GetParentedForeignKeyRefs(Relation partition);
static void ATDetachCheckNoForeignKeyRefs(Relation partition);
static void ATExecAlterCollationRefreshVersion(Relation rel, List *coll);
+static char GetAttributeCompression(Form_pg_attribute att, char *compression);
/* ----------------------------------------------------------------
@@ -852,6 +854,18 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
if (colDef->generated)
attr->attgenerated = colDef->generated;
+
+ /*
+ * lookup attribute's compression method and store it in the
+ * attr->attcompression.
+ */
+ if (relkind == RELKIND_RELATION ||
+ relkind == RELKIND_PARTITIONED_TABLE ||
+ relkind == RELKIND_MATVIEW)
+ attr->attcompression =
+ GetAttributeCompression(attr, colDef->compression);
+ else
+ attr->attcompression = InvalidCompressionMethod;
}
/*
@@ -2396,6 +2410,22 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
storage_name(def->storage),
storage_name(attribute->attstorage))));
+ /* Copy/check compression parameter */
+ if (CompressionMethodIsValid(attribute->attcompression))
+ {
+ const char *compression =
+ GetCompressionMethodName(attribute->attcompression);
+
+ if (def->compression == NULL)
+ def->compression = pstrdup(compression);
+ else if (strcmp(def->compression, compression) != 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("column \"%s\" has a compression method conflict",
+ attributeName),
+ errdetail("%s versus %s", def->compression, compression)));
+ }
+
def->inhcount++;
/* Merge of NOT NULL constraints = OR 'em together */
def->is_not_null |= attribute->attnotnull;
@@ -2430,6 +2460,11 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
def->collOid = attribute->attcollation;
def->constraints = NIL;
def->location = -1;
+ if (CompressionMethodIsValid(attribute->attcompression))
+ def->compression = pstrdup(GetCompressionMethodName(
+ attribute->attcompression));
+ else
+ def->compression = NULL;
inhSchema = lappend(inhSchema, def);
newattmap->attnums[parent_attno - 1] = ++child_attno;
}
@@ -2675,6 +2710,19 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
storage_name(def->storage),
storage_name(newdef->storage))));
+ /* Copy compression parameter */
+ if (def->compression == NULL)
+ def->compression = newdef->compression;
+ else if (newdef->compression != NULL)
+ {
+ if (strcmp(def->compression, newdef->compression) != 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("column \"%s\" has a compression method conflict",
+ attributeName),
+ errdetail("%s versus %s", def->compression, newdef->compression)));
+ }
+
/* Mark the column as locally defined */
def->is_local = true;
/* Merge of NOT NULL constraints = OR 'em together */
@@ -6340,6 +6388,18 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
attribute.attislocal = colDef->is_local;
attribute.attinhcount = colDef->inhcount;
attribute.attcollation = collOid;
+
+ /*
+ * lookup attribute's compression method and store it in the
+ * attr->attcompression.
+ */
+ if (rel->rd_rel->relkind == RELKIND_RELATION ||
+ rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ attribute.attcompression = GetAttributeCompression(&attribute,
+ colDef->compression);
+ else
+ attribute.attcompression = InvalidCompressionMethod;
+
/* attribute.attacl is handled by InsertPgAttributeTuples() */
ReleaseSysCache(typeTuple);
@@ -11859,6 +11919,23 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
ReleaseSysCache(typeTuple);
+ /* Setup attribute compression */
+ if (rel->rd_rel->relkind == RELKIND_RELATION ||
+ rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+ {
+ /*
+ * No compression for plain/external storage, otherwise, default
+ * compression method if it is not already set, refer comments atop
+ * attcompression parameter in pg_attribute.h.
+ */
+ if (!IsStorageCompressible(tform->typstorage))
+ attTup->attcompression = InvalidCompressionMethod;
+ else if (!CompressionMethodIsValid(attTup->attcompression))
+ attTup->attcompression = DefaultCompressionMethod;
+ }
+ else
+ attTup->attcompression = InvalidCompressionMethod;
+
CatalogTupleUpdate(attrelation, &heapTup->t_self, heapTup);
table_close(attrelation, RowExclusiveLock);
@@ -17641,3 +17718,36 @@ ATExecAlterCollationRefreshVersion(Relation rel, List *coll)
index_update_collation_versions(rel->rd_id, get_collation_oid(coll, false));
CacheInvalidateRelcache(rel);
}
+
+/*
+ * resolve column compression specification to compression method.
+ */
+static char
+GetAttributeCompression(Form_pg_attribute att, char *compression)
+{
+ char typstorage = get_typstorage(att->atttypid);
+ char cmethod;
+
+ /*
+ * No compression for plain/external storage, refer comments atop
+ * attcompression parameter in pg_attribute.h
+ */
+ if (!IsStorageCompressible(typstorage))
+ {
+ if (compression == NULL)
+ return InvalidCompressionMethod;
+
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("column data type %s does not support compression",
+ format_type_be(att->atttypid))));
+ }
+
+ /* fallback to default compression if it's not specified */
+ if (compression == NULL)
+ return DefaultCompressionMethod;
+
+ cmethod = CompressionNameToMethod(compression);
+
+ return cmethod;
+}
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index da91cbd..0d3b923 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -2988,6 +2988,7 @@ _copyColumnDef(const ColumnDef *from)
COPY_STRING_FIELD(colname);
COPY_NODE_FIELD(typeName);
+ COPY_STRING_FIELD(compression);
COPY_SCALAR_FIELD(inhcount);
COPY_SCALAR_FIELD(is_local);
COPY_SCALAR_FIELD(is_not_null);
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index c2d7362..f359200 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -2599,6 +2599,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b)
{
COMPARE_STRING_FIELD(colname);
COMPARE_NODE_FIELD(typeName);
+ COMPARE_STRING_FIELD(compression);
COMPARE_SCALAR_FIELD(inhcount);
COMPARE_SCALAR_FIELD(is_local);
COMPARE_SCALAR_FIELD(is_not_null);
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 49357ac..3822653 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -3897,6 +3897,8 @@ raw_expression_tree_walker(Node *node,
if (walker(coldef->typeName, context))
return true;
+ if (walker(coldef->compression, context))
+ return true;
if (walker(coldef->raw_default, context))
return true;
if (walker(coldef->collClause, context))
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 6493a03..75343b8 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -2876,6 +2876,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node)
WRITE_STRING_FIELD(colname);
WRITE_NODE_FIELD(typeName);
+ WRITE_STRING_FIELD(compression);
WRITE_INT_FIELD(inhcount);
WRITE_BOOL_FIELD(is_local);
WRITE_BOOL_FIELD(is_not_null);
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 652be0b..9d923b5 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -596,6 +596,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound
%type <defelt> hash_partbound_elem
+%type <str> optColumnCompression
+
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -631,9 +633,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT
- CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE
- CROSS CSV CUBE CURRENT_P
+ COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
+ COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
@@ -3421,11 +3423,12 @@ TypedTableElement:
| TableConstraint { $$ = $1; }
;
-columnDef: ColId Typename create_generic_options ColQualList
+columnDef: ColId Typename optColumnCompression create_generic_options ColQualList
{
ColumnDef *n = makeNode(ColumnDef);
n->colname = $1;
n->typeName = $2;
+ n->compression = $3;
n->inhcount = 0;
n->is_local = true;
n->is_not_null = false;
@@ -3434,8 +3437,8 @@ columnDef: ColId Typename create_generic_options ColQualList
n->raw_default = NULL;
n->cooked_default = NULL;
n->collOid = InvalidOid;
- n->fdwoptions = $3;
- SplitColQualList($4, &n->constraints, &n->collClause,
+ n->fdwoptions = $4;
+ SplitColQualList($5, &n->constraints, &n->collClause,
yyscanner);
n->location = @1;
$$ = (Node *)n;
@@ -3480,6 +3483,14 @@ columnOptions: ColId ColQualList
}
;
+optColumnCompression:
+ COMPRESSION name
+ {
+ $$ = $2;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
ColQualList:
ColQualList ColConstraint { $$ = lappend($1, $2); }
| /*EMPTY*/ { $$ = NIL; }
@@ -3710,6 +3721,7 @@ TableLikeOption:
| INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; }
| STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; }
| STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; }
+ | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; }
| ALL { $$ = CREATE_TABLE_LIKE_ALL; }
;
@@ -15296,6 +15308,7 @@ unreserved_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -15816,6 +15829,7 @@ bare_label_keyword:
| COMMENTS
| COMMIT
| COMMITTED
+ | COMPRESSION
| CONCURRENTLY
| CONFIGURATION
| CONFLICT
diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c
index d56f81c..aa6c19a 100644
--- a/src/backend/parser/parse_utilcmd.c
+++ b/src/backend/parser/parse_utilcmd.c
@@ -31,6 +31,7 @@
#include "access/relation.h"
#include "access/reloptions.h"
#include "access/table.h"
+#include "access/toast_compression.h"
#include "catalog/dependency.h"
#include "catalog/heap.h"
#include "catalog/index.h"
@@ -1092,6 +1093,14 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla
else
def->storage = 0;
+ /* Likewise, copy compression if requested */
+ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION) != 0
+ && CompressionMethodIsValid(attribute->attcompression))
+ def->compression =
+ pstrdup(GetCompressionMethodName(attribute->attcompression));
+ else
+ def->compression = NULL;
+
/* Likewise, copy comment if requested */
if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) &&
(comment = GetComment(attribute->attrelid,
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 91600ac..c291b05 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -4641,7 +4641,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == toast_pointer.va_extsize);
+ Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer));
/* make sure its marked as compressed or not */
if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 479ed9a..cbb3fd3 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -18,6 +18,7 @@
#include <limits.h>
#include "access/detoast.h"
+#include "access/toast_compression.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
#include "common/hashfn.h"
@@ -5300,6 +5301,54 @@ pg_column_size(PG_FUNCTION_ARGS)
}
/*
+ * Return the compression method stored in the compressed attribute. Return
+ * NULL for non varlena type or uncompressed data.
+ */
+Datum
+pg_column_compression(PG_FUNCTION_ARGS)
+{
+ int typlen;
+ ToastCompressionId cmid;
+
+ /* On first call, get the input type's typlen, and save at *fn_extra */
+ if (fcinfo->flinfo->fn_extra == NULL)
+ {
+ /* Lookup the datatype of the supplied argument */
+ Oid argtypeid = get_fn_expr_argtype(fcinfo->flinfo, 0);
+
+ typlen = get_typlen(argtypeid);
+ if (typlen == 0) /* should not happen */
+ elog(ERROR, "cache lookup failed for type %u", argtypeid);
+
+ fcinfo->flinfo->fn_extra = MemoryContextAlloc(fcinfo->flinfo->fn_mcxt,
+ sizeof(int));
+ *((int *) fcinfo->flinfo->fn_extra) = typlen;
+ }
+ else
+ typlen = *((int *) fcinfo->flinfo->fn_extra);
+
+ if (typlen != -1)
+ PG_RETURN_NULL();
+
+ /* get the compression method id stored in the compressed varlena */
+ cmid = toast_get_compression_id((struct varlena *)
+ DatumGetPointer(PG_GETARG_DATUM(0)));
+ if (cmid == TOAST_INVALID_COMPRESSION_ID)
+ PG_RETURN_NULL();
+
+ /* convert compression method id to compression method name */
+ switch (cmid)
+ {
+ case TOAST_PGLZ_COMPRESSION_ID:
+ PG_RETURN_TEXT_P(cstring_to_text("pglz"));
+ case TOAST_LZ4_COMPRESSION_ID:
+ PG_RETURN_TEXT_P(cstring_to_text("lz4"));
+ default:
+ elog(ERROR, "invalid compression method id %d", cmid);
+ }
+}
+
+/*
* string_agg - Concatenates values and returns string.
*
* Syntax: string_agg(value text, delimiter text) RETURNS text
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index eea9f30..0296b9b 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -160,6 +160,7 @@ typedef struct _dumpOptions
int no_subscriptions;
int no_synchronized_snapshots;
int no_unlogged_table_data;
+ int no_toast_compression;
int serializable_deferrable;
int disable_triggers;
int outputNoTablespaces;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index eb988d7..cef9bbd 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -387,6 +387,7 @@ main(int argc, char **argv)
{"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
+ {"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-sync", no_argument, NULL, 7},
{"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1},
{"rows-per-insert", required_argument, NULL, 10},
@@ -1047,6 +1048,7 @@ help(const char *progname)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
+ printf(_(" --no-toast-compression do not dump toast compression methods\n"));
printf(_(" --no-synchronized-snapshots do not use synchronized snapshots in parallel jobs\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
printf(_(" --no-unlogged-table-data do not dump unlogged table data\n"));
@@ -8617,6 +8619,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
{
DumpOptions *dopt = fout->dopt;
PQExpBuffer q = createPQExpBuffer();
+ bool createWithCompression;
for (int i = 0; i < numTables; i++)
{
@@ -8702,6 +8705,15 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
appendPQExpBufferStr(q,
"'' AS attidentity,\n");
+ createWithCompression = (fout->remoteVersion >= 140000);
+
+ if (createWithCompression)
+ appendPQExpBuffer(q,
+ "a.attcompression AS attcompression,\n");
+ else
+ appendPQExpBuffer(q,
+ "NULL AS attcmname,\n");
+
if (fout->remoteVersion >= 110000)
appendPQExpBufferStr(q,
"CASE WHEN a.atthasmissing AND NOT a.attisdropped "
@@ -8747,6 +8759,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid));
tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *));
tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *));
+ tbinfo->attcompression = (char *) pg_malloc(ntups * sizeof(char *));
tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *));
@@ -8775,6 +8788,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, PQfnumber(res, "attcollation")));
tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attfdwoptions")));
tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, PQfnumber(res, "attmissingval")));
+ tbinfo->attcompression[j] = *(PQgetvalue(res, j, PQfnumber(res, "attcompression")));
tbinfo->attrdefs[j] = NULL; /* fix below */
if (PQgetvalue(res, j, PQfnumber(res, "atthasdef"))[0] == 't')
hasdefaults = true;
@@ -15891,6 +15905,31 @@ dumpTableSchema(Archive *fout, const TableInfo *tbinfo)
tbinfo->atttypnames[j]);
}
+ /*
+ * Attribute compression
+ */
+ if (!dopt->no_toast_compression &&
+ tbinfo->attcompression[j] &&
+ tbinfo->attcompression[j] != '\0')
+ {
+ char *cmname;
+
+ switch (tbinfo->attcompression[j])
+ {
+ case 'p':
+ cmname = "pglz";
+ break;
+ case 'l':
+ cmname = "lz4";
+ break;
+ default:
+ cmname = NULL;
+ }
+
+ if (cmname != NULL)
+ appendPQExpBuffer(q, " COMPRESSION %s", cmname);
+ }
+
if (print_default)
{
if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED)
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 0a2213f..453f946 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -326,6 +326,7 @@ typedef struct _tableInfo
char *partbound; /* partition bound definition */
bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */
char *amname; /* relation access method */
+ char *attcompression; /* per-attribute current compression method */
/*
* Stuff computed only for dumpable tables.
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 737e464..bc91bb1 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -2284,9 +2284,9 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text,\E\n
- \s+\Qcol3 text,\E\n
- \s+\Qcol4 text,\E\n
+ \s+\Qcol2 text COMPRESSION\E\D*,\n
+ \s+\Qcol3 text COMPRESSION\E\D*,\n
+ \s+\Qcol4 text COMPRESSION\E\D*,\n
\s+\QCONSTRAINT test_table_col1_check CHECK ((col1 <= 1000))\E\n
\Q)\E\n
\QWITH (autovacuum_enabled='false', fillfactor='80');\E\n/xm,
@@ -2326,7 +2326,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_second_table (\E
\n\s+\Qcol1 integer,\E
- \n\s+\Qcol2 text\E
+ \n\s+\Qcol2 text COMPRESSION\E\D*
\n\);
/xm,
like =>
@@ -2441,7 +2441,7 @@ my %tests = (
\n\s+\Qcol1 integer,\E
\n\s+\Qcol2 boolean,\E
\n\s+\Qcol3 boolean,\E
- \n\s+\Qcol4 bit(5),\E
+ \n\s+\Qcol4 bit(5) COMPRESSION\E\D*,
\n\s+\Qcol5 double precision\E
\n\);
/xm,
@@ -2459,7 +2459,7 @@ my %tests = (
regexp => qr/^
\QCREATE TABLE dump_test.test_table_identity (\E\n
\s+\Qcol1 integer NOT NULL,\E\n
- \s+\Qcol2 text\E\n
+ \s+\Qcol2 text COMPRESSION\E\D*\n
\);
.*
\QALTER TABLE dump_test.test_table_identity ALTER COLUMN col1 ADD GENERATED ALWAYS AS IDENTITY (\E\n
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 20af5a9..b284113 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -1459,7 +1459,7 @@ describeOneTableDetails(const char *schemaname,
bool printTableInitialized = false;
int i;
char *view_def = NULL;
- char *headers[11];
+ char *headers[12];
PQExpBufferData title;
PQExpBufferData tmpbuf;
int cols;
@@ -1475,7 +1475,8 @@ describeOneTableDetails(const char *schemaname,
fdwopts_col = -1,
attstorage_col = -1,
attstattarget_col = -1,
- attdescr_col = -1;
+ attdescr_col = -1,
+ attcompression_col = -1;
int numrows;
struct
{
@@ -1892,6 +1893,17 @@ describeOneTableDetails(const char *schemaname,
appendPQExpBufferStr(&buf, ",\n a.attstorage");
attstorage_col = cols++;
+ /* compresssion info */
+ if (pset.sversion >= 140000 &&
+ !pset.hide_compression &&
+ (tableinfo.relkind == RELKIND_RELATION ||
+ tableinfo.relkind == RELKIND_PARTITIONED_TABLE ||
+ tableinfo.relkind == RELKIND_MATVIEW))
+ {
+ appendPQExpBufferStr(&buf, ",\n a.attcompression AS attcompression");
+ attcompression_col = cols++;
+ }
+
/* stats target, if relevant to relkind */
if (tableinfo.relkind == RELKIND_RELATION ||
tableinfo.relkind == RELKIND_INDEX ||
@@ -2018,6 +2030,8 @@ describeOneTableDetails(const char *schemaname,
headers[cols++] = gettext_noop("FDW options");
if (attstorage_col >= 0)
headers[cols++] = gettext_noop("Storage");
+ if (attcompression_col >= 0)
+ headers[cols++] = gettext_noop("Compression");
if (attstattarget_col >= 0)
headers[cols++] = gettext_noop("Stats target");
if (attdescr_col >= 0)
@@ -2097,6 +2111,19 @@ describeOneTableDetails(const char *schemaname,
false, false);
}
+ /* Column compression. */
+ if (attcompression_col >= 0)
+ {
+ char *compression = PQgetvalue(res, i, attcompression_col);
+
+ /* these strings are literal in our syntax, so not translated. */
+ printTableAddCell(&cont, (compression[0] == 'p' ? "pglz" :
+ (compression[0] == 'l' ? "lz4" :
+ (compression[0] == '\0' ? "" :
+ "???"))),
+ false, false);
+ }
+
/* Statistics target, if the relkind supports this feature */
if (attstattarget_col >= 0)
printTableAddCell(&cont, PQgetvalue(res, i, attstattarget_col),
diff --git a/src/bin/psql/help.c b/src/bin/psql/help.c
index daa5081..99a5947 100644
--- a/src/bin/psql/help.c
+++ b/src/bin/psql/help.c
@@ -372,6 +372,8 @@ helpVariables(unsigned short int pager)
" true if last query failed, else false\n"));
fprintf(output, _(" FETCH_COUNT\n"
" the number of result rows to fetch and display at a time (0 = unlimited)\n"));
+ fprintf(output, _(" HIDE_TOAST_COMPRESSION\n"
+ " if set, compression methods are not displayed\n"));
fprintf(output, _(" HIDE_TABLEAM\n"
" if set, table access methods are not displayed\n"));
fprintf(output, _(" HISTCONTROL\n"
diff --git a/src/bin/psql/settings.h b/src/bin/psql/settings.h
index d659900..83f2e6f 100644
--- a/src/bin/psql/settings.h
+++ b/src/bin/psql/settings.h
@@ -134,6 +134,7 @@ typedef struct _psqlSettings
bool quiet;
bool singleline;
bool singlestep;
+ bool hide_compression;
bool hide_tableam;
int fetch_count;
int histsize;
diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c
index 780479c..110906a 100644
--- a/src/bin/psql/startup.c
+++ b/src/bin/psql/startup.c
@@ -1160,6 +1160,13 @@ show_context_hook(const char *newval)
}
static bool
+hide_compression_hook(const char *newval)
+{
+ return ParseVariableBool(newval, "HIDE_TOAST_COMPRESSION",
+ &pset.hide_compression);
+}
+
+static bool
hide_tableam_hook(const char *newval)
{
return ParseVariableBool(newval, "HIDE_TABLEAM", &pset.hide_tableam);
@@ -1227,6 +1234,9 @@ EstablishVariableSpace(void)
SetVariableHooks(pset.vars, "SHOW_CONTEXT",
show_context_substitute_hook,
show_context_hook);
+ SetVariableHooks(pset.vars, "HIDE_TOAST_COMPRESSION",
+ bool_substitute_hook,
+ hide_compression_hook);
SetVariableHooks(pset.vars, "HIDE_TABLEAM",
bool_substitute_hook,
hide_tableam_hook);
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index 0adf53c..4bfff21 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -12,15 +12,7 @@
#ifndef DETOAST_H
#define DETOAST_H
-/*
- * Testing whether an externally-stored value is compressed now requires
- * comparing extsize (the actual length of the external data) to rawsize
- * (the original uncompressed datum's size). The latter includes VARHDRSZ
- * overhead, the former doesn't. We never use compression unless it actually
- * saves space, so we expect either equality or less-than.
- */
-#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
- ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
+#include "access/toast_compression.h"
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
@@ -89,4 +81,12 @@ extern Size toast_raw_datum_size(Datum value);
*/
extern Size toast_datum_size(Datum value);
+/* ----------
+ * toast_get_compression_id -
+ *
+ * Return the compression method id from the compressed value
+ * ----------
+ */
+extern ToastCompressionId toast_get_compression_id(struct varlena *attr);
+
#endif /* DETOAST_H */
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
new file mode 100644
index 0000000..c6d27a4
--- /dev/null
+++ b/src/include/access/toast_compression.h
@@ -0,0 +1,141 @@
+/*-------------------------------------------------------------------------
+ *
+ * toast_compression.h
+ * Functions for toast compression.
+ *
+ * Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/toast_compression.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TOAST_COMPRESSION_H
+#define TOAST_COMPRESSION_H
+
+/*
+ * Built-in compression method-id. The toast compression header will store
+ * this in the first 2 bits of the raw length. These built-in compression
+ * method-id are directly mapped to the built-in compression methods.
+ */
+typedef enum ToastCompressionId
+{
+ TOAST_PGLZ_COMPRESSION_ID = 0,
+ TOAST_LZ4_COMPRESSION_ID = 1,
+ TOAST_INVALID_COMPRESSION_ID = 2
+} ToastCompressionId;
+
+/*
+ * Built-in compression methods. pg_attribute will store this in the
+ * attcompression column.
+ */
+#define TOAST_PGLZ_COMPRESSION 'p'
+#define TOAST_LZ4_COMPRESSION 'l'
+
+#define InvalidCompressionMethod '\0'
+#define CompressionMethodIsValid(cm) ((bool) ((cm) != InvalidCompressionMethod))
+
+#define NO_LZ4_SUPPORT() \
+ ereport(ERROR, \
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
+ errmsg("unsupported LZ4 compression method"), \
+ errdetail("This functionality requires the server to be built with lz4 support."), \
+ errhint("You need to rebuild PostgreSQL using --with-lz4.")))
+
+/* use default compression method if it is not specified. */
+#define DefaultCompressionMethod TOAST_PGLZ_COMPRESSION
+#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod)
+
+#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \
+ (storage) != TYPSTORAGE_EXTERNAL)
+
+/*
+ * CompressionMethodToId - Convert compression method to compression id.
+ *
+ * For more details refer comment atop ToastCompressionId.
+ */
+static inline ToastCompressionId
+CompressionMethodToId(char method)
+{
+ switch (method)
+ {
+ case TOAST_PGLZ_COMPRESSION:
+ return TOAST_PGLZ_COMPRESSION_ID;
+ case TOAST_LZ4_COMPRESSION:
+ return TOAST_LZ4_COMPRESSION_ID;
+ default:
+ elog(ERROR, "invalid compression method %c", method);
+ }
+}
+
+/*
+ * CompressionIdToMethod - Convert compression id to compression method
+ *
+ * For more details refer comment atop ToastCompressionId.
+ */
+static inline char
+CompressionIdToMethod(ToastCompressionId cmid)
+{
+ switch (cmid)
+ {
+ case TOAST_PGLZ_COMPRESSION_ID:
+ return TOAST_PGLZ_COMPRESSION;
+ case TOAST_LZ4_COMPRESSION_ID:
+ return TOAST_LZ4_COMPRESSION;
+ default:
+ elog(ERROR, "invalid compression method id %d", cmid);
+ }
+}
+
+/*
+ * GetCompressionMethodName - Get compression method name
+ */
+static inline const char *
+GetCompressionMethodName(char method)
+{
+ switch (method)
+ {
+ case TOAST_PGLZ_COMPRESSION:
+ return "pglz";
+ case TOAST_LZ4_COMPRESSION:
+ return "lz4";
+ default:
+ elog(ERROR, "invalid compression method %c", method);
+ }
+}
+
+/*
+ * CompressionNameToMethod - Get compression method from compression name
+ *
+ * Search in the available built-in methods. If the compression not found
+ * in the built-in methods then return InvalidCompressionMethod.
+ */
+static inline char
+CompressionNameToMethod(char *compression)
+{
+ if (strcmp(compression, "pglz") == 0)
+ return TOAST_PGLZ_COMPRESSION;
+ else if (strcmp(compression, "lz4") == 0)
+ {
+#ifndef USE_LZ4
+ NO_LZ4_SUPPORT();
+#endif
+ return TOAST_LZ4_COMPRESSION;
+ }
+
+ return InvalidCompressionMethod;
+}
+
+/* pglz compression/decompression routines */
+extern struct varlena *pglz_cmcompress(const struct varlena *value);
+extern struct varlena *pglz_cmdecompress(const struct varlena *value);
+extern struct varlena *pglz_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength);
+
+/* lz4 compression/decompression routines */
+extern struct varlena *lz4_cmcompress(const struct varlena *value);
+extern struct varlena *lz4_cmdecompress(const struct varlena *value);
+extern struct varlena *lz4_cmdecompress_slice(const struct varlena *value,
+ int32 slicelength);
+
+#endif /* TOAST_COMPRESSION_H */
diff --git a/src/include/access/toast_helper.h b/src/include/access/toast_helper.h
index a9a6d64..05104ce 100644
--- a/src/include/access/toast_helper.h
+++ b/src/include/access/toast_helper.h
@@ -32,6 +32,7 @@ typedef struct
struct varlena *tai_oldexternal;
int32 tai_size;
uint8 tai_colflags;
+ char tai_compression;
} ToastAttrInfo;
/*
diff --git a/src/include/access/toast_internals.h b/src/include/access/toast_internals.h
index cedfb89..b4d0684 100644
--- a/src/include/access/toast_internals.h
+++ b/src/include/access/toast_internals.h
@@ -12,6 +12,7 @@
#ifndef TOAST_INTERNALS_H
#define TOAST_INTERNALS_H
+#include "access/toast_compression.h"
#include "storage/lockdefs.h"
#include "utils/relcache.h"
#include "utils/snapshot.h"
@@ -22,22 +23,26 @@
typedef struct toast_compress_header
{
int32 vl_len_; /* varlena header (do not touch directly!) */
- int32 rawsize;
+ uint32 tcinfo; /* 2 bits for compression method and 30 bits
+ * rawsize */
} toast_compress_header;
/*
* Utilities for manipulation of header information for compressed
* toast entries.
*/
-#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header))
-#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize)
-#define TOAST_COMPRESS_SIZE(ptr) ((int32) VARSIZE_ANY(ptr) - TOAST_COMPRESS_HDRSZ)
-#define TOAST_COMPRESS_RAWDATA(ptr) \
- (((char *) (ptr)) + TOAST_COMPRESS_HDRSZ)
-#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \
- (((toast_compress_header *) (ptr))->rawsize = (len))
+#define TOAST_COMPRESS_METHOD(ptr) \
+ (((toast_compress_header *) (ptr))->tcinfo >> VARLENA_RAWSIZE_BITS)
+#define TOAST_COMPRESS_SET_SIZE_AND_METHOD(ptr, len, cm_method) \
+ do { \
+ Assert((len) > 0 && (len) <= VARLENA_RAWSIZE_MASK); \
+ Assert((cm_method) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm_method) == TOAST_LZ4_COMPRESSION_ID); \
+ ((toast_compress_header *) (ptr))->tcinfo = \
+ ((len) | (cm_method) << VARLENA_RAWSIZE_BITS); \
+ } while (0)
-extern Datum toast_compress_datum(Datum value);
+extern Datum toast_compress_datum(Datum value, char cmethod);
extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock);
extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h
index 3db42ab..560f8f0 100644
--- a/src/include/catalog/pg_attribute.h
+++ b/src/include/catalog/pg_attribute.h
@@ -160,6 +160,12 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75,
/* attribute's collation, if any */
Oid attcollation BKI_LOOKUP_OPT(pg_collation);
+ /*
+ * compression method. Must be InvalidCompressionMethod if and only if
+ * typstorage is 'plain' or 'external'.
+ */
+ char attcompression BKI_DEFAULT('\0');
+
#ifdef CATALOG_VARLEN /* variable-length fields start here */
/* NOTE: The following fields are not present in tuple descriptors. */
@@ -187,7 +193,7 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75,
* can access fields beyond attcollation except in a real tuple!
*/
#define ATTRIBUTE_FIXED_PART_SIZE \
- (offsetof(FormData_pg_attribute,attcollation) + sizeof(Oid))
+ (offsetof(FormData_pg_attribute,attcompression) + sizeof(char))
/* ----------------
* Form_pg_attribute corresponds to a pointer to a tuple with
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fc..e259531 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -7103,6 +7103,10 @@
descr => 'bytes required to store the value, perhaps with compression',
proname => 'pg_column_size', provolatile => 's', prorettype => 'int4',
proargtypes => 'any', prosrc => 'pg_column_size' },
+{ oid => '2121',
+ descr => 'compression method for the compressed datum',
+ proname => 'pg_column_compression', provolatile => 's', prorettype => 'text',
+ proargtypes => 'any', prosrc => 'pg_column_compression' },
{ oid => '2322',
descr => 'total disk space usage for the specified tablespace',
proname => 'pg_tablespace_size', provolatile => 'v', prorettype => 'int8',
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 236832a..19d2ba2 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -646,6 +646,7 @@ typedef struct ColumnDef
NodeTag type;
char *colname; /* name of column */
TypeName *typeName; /* type of column */
+ char *compression; /* compression method for column */
int inhcount; /* number of times column is inherited */
bool is_local; /* column has local (non-inherited) def'n */
bool is_not_null; /* NOT NULL constraint specified? */
@@ -685,6 +686,7 @@ typedef enum TableLikeOption
CREATE_TABLE_LIKE_INDEXES = 1 << 5,
CREATE_TABLE_LIKE_STATISTICS = 1 << 6,
CREATE_TABLE_LIKE_STORAGE = 1 << 7,
+ CREATE_TABLE_LIKE_COMPRESSION = 1 << 8,
CREATE_TABLE_LIKE_ALL = PG_INT32_MAX
} TableLikeOption;
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 28083aa..ca1f950 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -88,6 +88,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7a7cc21..0a6422d 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -899,6 +899,9 @@
/* Define to 1 to build with LLVM based JIT support. (--with-llvm) */
#undef USE_LLVM
+/* Define to 1 to build with LZ4 support (--with-lz4) */
+#undef USE_LZ4
+
/* Define to select named POSIX semaphores. */
#undef USE_NAMED_POSIX_SEMAPHORES
diff --git a/src/include/postgres.h b/src/include/postgres.h
index 2ed5720..9dfa85b9 100644
--- a/src/include/postgres.h
+++ b/src/include/postgres.h
@@ -67,7 +67,8 @@
typedef struct varatt_external
{
int32 va_rawsize; /* Original data size (includes header) */
- int32 va_extsize; /* External saved size (doesn't) */
+ uint32 va_extinfo; /* External saved size (without header) and
+ * compression method */
Oid va_valueid; /* Unique ID of value within TOAST table */
Oid va_toastrelid; /* RelID of TOAST table containing it */
} varatt_external;
@@ -145,7 +146,8 @@ typedef union
struct /* Compressed-in-line format */
{
uint32 va_header;
- uint32 va_rawsize; /* Original data size (excludes header) */
+ uint32 va_tcinfo; /* Original data size (excludes header) and
+ * compression method */
char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */
} va_compressed;
} varattrib_4b;
@@ -274,14 +276,23 @@ typedef struct
(VARSIZE(PTR) - VARHDRSZ + VARHDRSZ_SHORT)
#define VARHDRSZ_EXTERNAL offsetof(varattrib_1b_e, va_data)
+#define VARHDRSZ_COMPRESS offsetof(varattrib_4b, va_compressed.va_data)
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
+#define VARLENA_RAWSIZE_BITS 30
+#define VARLENA_RAWSIZE_MASK ((1U << VARLENA_RAWSIZE_BITS) - 1)
+
+/*
+ * va_tcinfo in va_compress contains raw size of datum and compression method.
+ */
#define VARRAWSIZE_4B_C(PTR) \
- (((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
+ (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo & VARLENA_RAWSIZE_MASK)
+#define VARCOMPRESS_4B_C(PTR) \
+ (((varattrib_4b *) (PTR))->va_compressed.va_tcinfo >> VARLENA_RAWSIZE_BITS)
/* Externally visible macros */
@@ -323,6 +334,35 @@ typedef struct
(VARATT_IS_EXTERNAL(PTR) && VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
#define VARATT_IS_EXTERNAL_NON_EXPANDED(PTR) \
(VARATT_IS_EXTERNAL(PTR) && !VARTAG_IS_EXPANDED(VARTAG_EXTERNAL(PTR)))
+
+/*
+ * va_extinfo in varatt_external contains actual length of the external data
+ * and compression method if external data is compressed.
+ */
+#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \
+ ((toast_pointer).va_extinfo & VARLENA_RAWSIZE_MASK)
+
+#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESSION(toast_pointer, len, cm) \
+ do { \
+ Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
+ (cm) == TOAST_LZ4_COMPRESSION_ID); \
+ ((toast_pointer).va_extinfo = (len) | (cm) << VARLENA_RAWSIZE_BITS); \
+ } while (0)
+
+#define VARATT_EXTERNAL_GET_COMPRESSION(PTR) \
+ ((toast_pointer).va_extinfo >> VARLENA_RAWSIZE_BITS)
+
+/*
+ * Testing whether an externally-stored value is compressed now requires
+ * comparing size stored in va_extinfo (the actual length of the external data)
+ * to rawsize (the original uncompressed datum's size). The latter includes
+ * VARHDRSZ overhead, the former doesn't. We never use compression unless it
+ * actually saves space, so we expect either equality or less-than.
+ */
+#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
+ (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \
+ (toast_pointer).va_rawsize - VARHDRSZ)
+
#define VARATT_IS_SHORT(PTR) VARATT_IS_1B(PTR)
#define VARATT_IS_EXTENDED(PTR) (!VARATT_IS_4B_U(PTR))
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
new file mode 100644
index 0000000..3115829
--- /dev/null
+++ b/src/test/regress/expected/compression.out
@@ -0,0 +1,261 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+ Table "public.cmdata"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+Indexes:
+ "idx" btree (f1)
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+\d+ cmdata1
+ Table "public.cmdata1"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | lz4 | |
+
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+ERROR: column data type integer does not support compression
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+(1 row)
+
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+ substr
+--------
+ 01234
+(1 row)
+
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ substr
+----------------------------------------------------
+ 01234567890123456789012345678901234567890123456789
+(1 row)
+
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ lz4
+(1 row)
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove3;
+ pg_column_compression
+-----------------------
+ pglz
+ lz4
+(2 rows)
+
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+SELECT SUBSTR(f1, 200, 5) FROM cmdata1;
+ substr
+--------
+ 01234
+ 8f14e
+(2 rows)
+
+SELECT SUBSTR(f1, 200, 5) FROM cmdata2;
+ substr
+--------
+ 8f14e
+(1 row)
+
+DROP TABLE cmdata2;
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | lz4 | |
+
+DROP TABLE cmdata2;
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | character varying | | | | plain | pglz | |
+
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+\d+ mv
+ Materialized view "public.mv"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ x | text | | | | extended | pglz | |
+View definition:
+ SELECT cmdata1.f1 AS x
+ FROM cmdata1;
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+SELECT pg_column_compression(x) FROM mv;
+ pg_column_compression
+-----------------------
+ lz4
+ lz4
+(2 rows)
+
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+SELECT pg_column_compression(f1) FROM cmpart;
+ pg_column_compression
+-----------------------
+ lz4
+ pglz
+(2 rows)
+
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+NOTICE: merging multiple inherited definitions of column "f1"
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+NOTICE: merging column "f1" with inherited definition
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmdata1;
+ length
+--------
+ 10040
+ 12449
+(2 rows)
+
+SELECT length(f1) FROM cmmove1;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove2;
+ length
+--------
+ 10040
+(1 row)
+
+SELECT length(f1) FROM cmmove3;
+ length
+--------
+ 10000
+ 10040
+(2 rows)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
new file mode 100644
index 0000000..e6bd377
--- /dev/null
+++ b/src/test/regress/expected/compression_1.out
@@ -0,0 +1,250 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+ Table "public.cmdata"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+Indexes:
+ "idx" btree (f1)
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+ERROR: unsupported LZ4 compression method
+DETAIL: This functionality requires the server to be built with lz4 support.
+HINT: You need to rebuild PostgreSQL using --with-lz4.
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+ ^
+\d+ cmdata1
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+ERROR: column data type integer does not support compression
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+ substr
+--------
+ 01234
+(1 row)
+
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+ ^
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+ ^
+SELECT pg_column_compression(f1) FROM cmmove2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmmove3 SELECT * FROM cmdata1;
+ ^
+SELECT pg_column_compression(f1) FROM cmmove3;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+ pglz
+(1 row)
+
+INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000);
+ERROR: relation "cmdata1" does not exist
+LINE 1: INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000);
+ ^
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+SELECT SUBSTR(f1, 200, 5) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT SUBSTR(f1, 200, 5) FROM cmdata1;
+ ^
+SELECT SUBSTR(f1, 200, 5) FROM cmdata2;
+ substr
+--------
+ 8f14e
+(1 row)
+
+DROP TABLE cmdata2;
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+ERROR: relation "cmdata1" does not exist
+LINE 1: CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+ ^
+\d+ cmdata2
+DROP TABLE cmdata2;
+ERROR: table "cmdata2" does not exist
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+---------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | integer | | | | plain | | |
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | character varying | | | | extended | pglz | |
+
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+-------------------+-----------+----------+---------+---------+-------------+--------------+-------------
+ f1 | character varying | | | | plain | pglz | |
+
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+ pg_column_compression
+-----------------------
+
+(1 row)
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+ ^
+\d+ mv
+SELECT pg_column_compression(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmdata1;
+ ^
+SELECT pg_column_compression(x) FROM mv;
+ERROR: relation "mv" does not exist
+LINE 1: SELECT pg_column_compression(x) FROM mv;
+ ^
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+ERROR: unsupported LZ4 compression method
+DETAIL: This functionality requires the server to be built with lz4 support.
+HINT: You need to rebuild PostgreSQL using --with-lz4.
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+ERROR: relation "cmpart" does not exist
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+ERROR: relation "cmpart" does not exist
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+ERROR: relation "cmpart" does not exist
+LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',1004));
+ ^
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+ERROR: relation "cmpart" does not exist
+LINE 1: INSERT INTO cmpart VALUES (repeat('123456789',4004));
+ ^
+SELECT pg_column_compression(f1) FROM cmpart;
+ERROR: relation "cmpart" does not exist
+LINE 1: SELECT pg_column_compression(f1) FROM cmpart;
+ ^
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+ERROR: relation "cmdata1" does not exist
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+NOTICE: merging column "f1" with inherited definition
+ERROR: column "f1" has a compression method conflict
+DETAIL: pglz versus lz4
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmdata1;
+ERROR: relation "cmdata1" does not exist
+LINE 1: SELECT length(f1) FROM cmdata1;
+ ^
+SELECT length(f1) FROM cmmove1;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove2;
+ length
+--------
+ 10000
+(1 row)
+
+SELECT length(f1) FROM cmmove3;
+ length
+--------
+ 10000
+(1 row)
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index e280198..70c3830 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -115,7 +115,7 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion tr
# ----------
# Another group of parallel tests
# ----------
-test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain
+test: partition_join partition_prune reloptions hash_part indexing partition_aggregate partition_info tuplesort explain compression
# event triggers cannot run concurrently with any test that runs DDL
# oidjoins is read-only, though, and should run late for best coverage
diff --git a/src/test/regress/pg_regress_main.c b/src/test/regress/pg_regress_main.c
index 8dc4941..1524676 100644
--- a/src/test/regress/pg_regress_main.c
+++ b/src/test/regress/pg_regress_main.c
@@ -78,11 +78,11 @@ psql_start_test(const char *testname,
* against different AMs without unnecessary differences.
*/
offset += snprintf(psql_cmd + offset, sizeof(psql_cmd) - offset,
- "\"%s%spsql\" -X -a -q -d \"%s\" -v %s < \"%s\" > \"%s\" 2>&1",
+ "\"%s%spsql\" -X -a -q -d \"%s\" %s < \"%s\" > \"%s\" 2>&1",
bindir ? bindir : "",
bindir ? "/" : "",
dblist->str,
- "HIDE_TABLEAM=\"on\"",
+ "-v HIDE_TABLEAM=on -v HIDE_TOAST_COMPRESSION=on",
infile,
outfile);
if (offset >= sizeof(psql_cmd))
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index 6a57e88..d81d041 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -201,6 +201,7 @@ test: partition_aggregate
test: partition_info
test: tuplesort
test: explain
+test: compression
test: event_trigger
test: oidjoins
test: fast_default
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
new file mode 100644
index 0000000..f56c24d
--- /dev/null
+++ b/src/test/regress/sql/compression.sql
@@ -0,0 +1,104 @@
+\set HIDE_TOAST_COMPRESSION false
+-- test creating table with compression method
+CREATE TABLE cmdata(f1 text COMPRESSION pglz);
+CREATE INDEX idx ON cmdata(f1);
+INSERT INTO cmdata VALUES(repeat('1234567890',1000));
+\d+ cmdata
+
+CREATE TABLE cmdata1(f1 TEXT COMPRESSION lz4);
+INSERT INTO cmdata1 VALUES(repeat('1234567890',1004));
+\d+ cmdata1
+
+-- try setting compression for incompressible data type
+CREATE TABLE cmdata2 (f1 int COMPRESSION pglz);
+
+-- verify stored compression method
+SELECT pg_column_compression(f1) FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmdata1;
+
+-- decompress data slice
+SELECT SUBSTR(f1, 200, 5) FROM cmdata;
+SELECT SUBSTR(f1, 2000, 50) FROM cmdata1;
+
+-- copy with table creation
+SELECT * INTO cmmove1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove1;
+
+-- update using datum from different table
+CREATE TABLE cmmove2(f1 text COMPRESSION pglz);
+INSERT INTO cmmove2 VALUES (repeat('1234567890',1004));
+SELECT pg_column_compression(f1) FROM cmmove2;
+
+UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata;
+SELECT pg_column_compression(f1) FROM cmmove2;
+UPDATE cmmove2 SET f1 = cmdata1.f1 FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove2;
+
+-- copy to existing table
+CREATE TABLE cmmove3(f1 text COMPRESSION pglz);
+INSERT INTO cmmove3 SELECT * FROM cmdata;
+INSERT INTO cmmove3 SELECT * FROM cmdata1;
+SELECT pg_column_compression(f1) FROM cmmove3;
+
+-- test external compressed data
+CREATE OR REPLACE FUNCTION large_val() RETURNS TEXT LANGUAGE SQL AS
+'select array_agg(md5(g::text))::text from generate_series(1, 256) g';
+CREATE TABLE cmdata2 (f1 text COMPRESSION pglz);
+INSERT INTO cmdata2 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata2;
+INSERT INTO cmdata1 SELECT large_val() || repeat('a', 4000);
+SELECT pg_column_compression(f1) FROM cmdata1;
+SELECT SUBSTR(f1, 200, 5) FROM cmdata1;
+SELECT SUBSTR(f1, 200, 5) FROM cmdata2;
+DROP TABLE cmdata2;
+
+-- test LIKE INCLUDING COMPRESSION
+CREATE TABLE cmdata2 (LIKE cmdata1 INCLUDING COMPRESSION);
+\d+ cmdata2
+DROP TABLE cmdata2;
+
+--test column type update varlena/non-varlena
+CREATE TABLE cmdata2 (f1 int);
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE int USING f1::integer;
+\d+ cmdata2
+
+--changing column storage should not impact the compression method
+--but the data should not be compressed
+ALTER TABLE cmdata2 ALTER COLUMN f1 TYPE varchar;
+\d+ cmdata2
+ALTER TABLE cmdata2 ALTER COLUMN f1 SET STORAGE plain;
+\d+ cmdata2
+INSERT INTO cmdata2 VALUES (repeat('123456789', 800));
+SELECT pg_column_compression(f1) FROM cmdata2;
+
+-- test compression with materialized view
+CREATE MATERIALIZED VIEW mv(x) AS SELECT * FROM cmdata1;
+\d+ mv
+SELECT pg_column_compression(f1) FROM cmdata1;
+SELECT pg_column_compression(x) FROM mv;
+
+-- test compression with partition
+CREATE TABLE cmpart(f1 text COMPRESSION lz4) PARTITION BY HASH(f1);
+CREATE TABLE cmpart1 PARTITION OF cmpart FOR VALUES WITH (MODULUS 2, REMAINDER 0);
+CREATE TABLE cmpart2(f1 text COMPRESSION pglz);
+
+ALTER TABLE cmpart ATTACH PARTITION cmpart2 FOR VALUES WITH (MODULUS 2, REMAINDER 1);
+INSERT INTO cmpart VALUES (repeat('123456789',1004));
+INSERT INTO cmpart VALUES (repeat('123456789',4004));
+SELECT pg_column_compression(f1) FROM cmpart;
+
+-- test compression with inheritence, error
+CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
+CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+
+-- check data is ok
+SELECT length(f1) FROM cmdata;
+SELECT length(f1) FROM cmdata1;
+SELECT length(f1) FROM cmmove1;
+SELECT length(f1) FROM cmmove2;
+SELECT length(f1) FROM cmmove3;
+
+\set HIDE_TOAST_COMPRESSION true
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index a4f5cc4..1460537 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -485,6 +485,7 @@ sub GenerateFiles
USE_ICU => $self->{options}->{icu} ? 1 : undef,
USE_LIBXML => undef,
USE_LIBXSLT => undef,
+ USE_LZ4 => undef,
USE_LDAP => $self->{options}->{ldap} ? 1 : undef,
USE_LLVM => undef,
USE_NAMED_POSIX_SEMAPHORES => undef,
--
1.8.3.1