From 1d8b9f8e605d9efb34ccec4a4ef9f04626d1dd22 Mon Sep 17 00:00:00 2001 From: dilipkumar Date: Fri, 16 Oct 2020 19:56:14 +0530 Subject: [PATCH v7 5/5] new compression method extension for lz4 --- contrib/Makefile | 1 + contrib/cm_lz4/.gitignore | 4 + contrib/cm_lz4/Makefile | 26 ++++++ contrib/cm_lz4/cm_lz4--1.0.sql | 13 +++ contrib/cm_lz4/cm_lz4.c | 139 +++++++++++++++++++++++++++++ contrib/cm_lz4/cm_lz4.control | 5 ++ contrib/cm_lz4/expected/cm_lz4.out | 45 ++++++++++ contrib/cm_lz4/sql/cm_lz4.sql | 21 +++++ 8 files changed, 254 insertions(+) create mode 100644 contrib/cm_lz4/.gitignore create mode 100644 contrib/cm_lz4/Makefile create mode 100644 contrib/cm_lz4/cm_lz4--1.0.sql create mode 100644 contrib/cm_lz4/cm_lz4.c create mode 100644 contrib/cm_lz4/cm_lz4.control create mode 100644 contrib/cm_lz4/expected/cm_lz4.out create mode 100644 contrib/cm_lz4/sql/cm_lz4.sql diff --git a/contrib/Makefile b/contrib/Makefile index 7a4866e338..7d01c58678 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -12,6 +12,7 @@ SUBDIRS = \ bloom \ btree_gin \ btree_gist \ + cm_lz4 \ citext \ cube \ dblink \ diff --git a/contrib/cm_lz4/.gitignore b/contrib/cm_lz4/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/cm_lz4/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/cm_lz4/Makefile b/contrib/cm_lz4/Makefile new file mode 100644 index 0000000000..4fd281f50b --- /dev/null +++ b/contrib/cm_lz4/Makefile @@ -0,0 +1,26 @@ +# contrib/cm_lz4/Makefile + +MODULE_big = cm_lz4 +OBJS = \ + $(WIN32RES) \ + cm_lz4.o + +EXTENSION = cm_lz4 +DATA = cm_lz4--1.0.sql +PGFILEDESC = "lz4 compression method " + +SHLIB_LINK += $(filter -llz4, $(LIBS)) + +REGRESS = cm_lz4 + + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/cm_lz4 +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/cm_lz4/cm_lz4--1.0.sql b/contrib/cm_lz4/cm_lz4--1.0.sql new file mode 100644 index 0000000000..4229c4cced --- /dev/null +++ b/contrib/cm_lz4/cm_lz4--1.0.sql @@ -0,0 +1,13 @@ +/* contrib/cm_lz4/cm_lz4--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cm_lz4" to load this file. \quit + +CREATE FUNCTION lz4handler(internal) +RETURNS compression_handler +AS 'MODULE_PATHNAME' +LANGUAGE C; + +-- Compression method +CREATE COMPRESSION METHOD lz4 HANDLER lz4handler; +COMMENT ON COMPRESSION METHOD lz4 IS 'lz4 compression method'; diff --git a/contrib/cm_lz4/cm_lz4.c b/contrib/cm_lz4/cm_lz4.c new file mode 100644 index 0000000000..f2cab6421f --- /dev/null +++ b/contrib/cm_lz4/cm_lz4.c @@ -0,0 +1,139 @@ +/*------------------------------------------------------------------------- + * + * cm_lz4.c + * lz4 compression method. + * + * Portions Copyright (c) 2016-2020, PostgreSQL Global Development Group + * Portions Copyright (c) 1990-1993, Regents of the University of California + * + * IDENTIFICATION + * contrib/cm_lz4/cm_lz4.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "postgres.h" +#include "access/compressionapi.h" +#include "access/toast_internals.h" + +#include "fmgr.h" +#include "lz4.h" +#include "utils/builtins.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(lz4handler); + +void _PG_init(void); + +//typedef int int32; +/* + * Module initialize function: initialize info about lz4 + */ +void +_PG_init(void) +{ + +} + +/* + * lz4_cmcompress - compression routine for lz4 compression method + * + * Compresses source into dest using the default strategy. Returns the + * compressed varlena, or NULL if compression fails. + */ +static struct varlena * +lz4_cmcompress(const struct varlena *value, int32 header_size) +{ + int32 valsize; + int32 len; + int32 max_size; + struct varlena *tmp = NULL; + + valsize = VARSIZE_ANY_EXHDR(value); + + max_size = LZ4_compressBound(VARSIZE_ANY_EXHDR(value)); + tmp = (struct varlena *) palloc(max_size + header_size); + + len = LZ4_compress_default(VARDATA_ANY(value), + (char *) tmp + header_size, + valsize, max_size); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + header_size); + return tmp; + } + + pfree(tmp); + + return NULL; +} + +/* + * lz4_cmdecompress - decompression routine for lz4 compression method + * + * Returns the decompressed varlena. + */ +static struct varlena * +lz4_cmdecompress(const struct varlena *value, int32 header_size) +{ + int32 rawsize; + struct varlena *result; + + result = (struct varlena *) palloc(TOAST_COMPRESS_RAWSIZE(value) + VARHDRSZ); + SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(value) + VARHDRSZ); + + rawsize = LZ4_decompress_safe((char *)value + header_size, + VARDATA(result), + VARSIZE(value) - header_size, + TOAST_COMPRESS_RAWSIZE(value)); + if (rawsize < 0) + elog(ERROR, "lz4: compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +} + +/* + * lz4_cmdecompress_slice - slice decompression routine for lz4 compression + * + * Decompresses part of the data. Returns the decompressed varlena. + */ +static struct varlena * +lz4_cmdecompress_slice(const struct varlena *value, int32 header_size, + int32 slicelength) +{ + int32 rawsize; + struct varlena *result; + + result = (struct varlena *)palloc(slicelength + VARHDRSZ); + SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(value) + VARHDRSZ); + + rawsize = LZ4_decompress_safe_partial((char *)value + header_size, + VARDATA(result), + VARSIZE(value) - header_size, + slicelength, + slicelength); + if (rawsize < 0) + elog(ERROR, "lz4: compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + + return result; +} + +/* lz4 is the default compression method */ +Datum +lz4handler(PG_FUNCTION_ARGS) +{ + CompressionRoutine *routine = makeNode(CompressionRoutine); + + routine->cmcompress = lz4_cmcompress; + routine->cmdecompress = lz4_cmdecompress; + routine->cmdecompress_slice = lz4_cmdecompress_slice; + + PG_RETURN_POINTER(routine); +} diff --git a/contrib/cm_lz4/cm_lz4.control b/contrib/cm_lz4/cm_lz4.control new file mode 100644 index 0000000000..7f2b0aff00 --- /dev/null +++ b/contrib/cm_lz4/cm_lz4.control @@ -0,0 +1,5 @@ +# cm_lz4 extension +comment = 'cm_lz4 compression method' +default_version = '1.0' +module_pathname = '$libdir/cm_lz4' +relocatable = true diff --git a/contrib/cm_lz4/expected/cm_lz4.out b/contrib/cm_lz4/expected/cm_lz4.out new file mode 100644 index 0000000000..3d0d10476d --- /dev/null +++ b/contrib/cm_lz4/expected/cm_lz4.out @@ -0,0 +1,45 @@ +CREATE EXTENSION cm_lz4; +-- zlib compression +CREATE TABLE lz4test(f1 TEXT COMPRESSION lz4); +INSERT INTO lz4test VALUES(repeat('1234567890',1004)); +INSERT INTO lz4test VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM lz4test; + length +-------- + 10040 + 24096 +(2 rows) + +-- alter compression method with rewrite +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ lz4test + Table "public.lz4test" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zlib | | + +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION lz4; +\d+ lz4test + Table "public.lz4test" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | lz4 | | + +-- preserve old compression method +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION zlib PRESERVE (lz4); +INSERT INTO lz4test VALUES (repeat('1234567890',1004)); +\d+ lz4test + Table "public.lz4test" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | zlib | | + +SELECT length(f1) FROM lz4test; + length +-------- + 10040 + 24096 + 10040 +(3 rows) + +DROP TABLE lz4test; diff --git a/contrib/cm_lz4/sql/cm_lz4.sql b/contrib/cm_lz4/sql/cm_lz4.sql new file mode 100644 index 0000000000..ccde7f73ca --- /dev/null +++ b/contrib/cm_lz4/sql/cm_lz4.sql @@ -0,0 +1,21 @@ +CREATE EXTENSION cm_lz4; + +-- zlib compression +CREATE TABLE lz4test(f1 TEXT COMPRESSION lz4); +INSERT INTO lz4test VALUES(repeat('1234567890',1004)); +INSERT INTO lz4test VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM lz4test; + +-- alter compression method with rewrite +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION zlib; +\d+ lz4test +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION lz4; +\d+ lz4test + +-- preserve old compression method +ALTER TABLE lz4test ALTER COLUMN f1 SET COMPRESSION zlib PRESERVE (lz4); +INSERT INTO lz4test VALUES (repeat('1234567890',1004)); +\d+ lz4test +SELECT length(f1) FROM lz4test; + +DROP TABLE lz4test; -- 2.23.0