├── .gitignore ├── Makefile ├── README.md ├── pg_lz4--1.0.sql ├── pg_lz4.c └── pg_lz4.control /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | *.bc 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULE_big = pg_lz4 2 | OBJS = pg_lz4.o $(WIN32RES) 3 | 4 | EXTENSION = pg_lz4 5 | EXTVERSION = 1.0 6 | DATA = $(EXTENSION)--$(EXTVERSION).sql 7 | PGFILEDESC = "lz4 compression method" 8 | 9 | REGRESS = pg_lz4 10 | 11 | PG_CONFIG = pg_config 12 | PGXS := $(shell $(PG_CONFIG) --pgxs) 13 | include $(PGXS) 14 | 15 | LDFLAGS += -llz4 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pg_lz4 extension 2 | 3 | lz4 compression for PostgreSQL 4 | 5 | ## Requirements 6 | 7 | This extension only works with developement version of PostgreSQL with custom compression methods patch applied: 8 | https://www.postgresql.org/message-id/20190315125203.5da43edb%40ildus-work.localdomain 9 | 10 | Also `liblz4` is required to build and use this extension. 11 | 12 | ## Installation 13 | 14 | To install `pg_lz4` extension run: 15 | 16 | ``` 17 | make install 18 | ``` 19 | 20 | or if your PostgreSQL bin directory isn't in PATH: 21 | 22 | ``` 23 | make install PG_CONFIG=/path/to/your/postgres/bin/pg_config 24 | ``` 25 | 26 | ## Usage 27 | 28 | First, create pg_lz4 extension: 29 | 30 | ``` 31 | CREATE EXTENSION pg_lz4; 32 | ``` 33 | 34 | Now you can specify pg_lz4 compression when defining new tables: 35 | 36 | ``` 37 | CREATE TABLE mytable ( 38 | id SERIAL, 39 | data1 TEXT COMPRESSION pg_lz4, 40 | data2 TEXT 41 | ); 42 | ``` 43 | 44 | or enable compression for existing table columns: 45 | 46 | ``` 47 | ALTER TABLE mytable ALTER COLUMN data2 SET COMPRESSION pg_lz4; 48 | ``` 49 | 50 | You can also specify acceleration parameter for lz4 algorithm (see [lz4 documentation](https://github.com/lz4/lz4) for more information): 51 | 52 | ``` 53 | ALTER TABLE mytable ALTER COLUMN data2 SET COMPRESSION pg_lz4 WITH (acceleration '8'); 54 | ``` 55 | -------------------------------------------------------------------------------- /pg_lz4--1.0.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION lz4_handler(INTERNAL) 2 | RETURNS compression_am_handler 3 | AS 'pg_lz4', 'lz4_handler' 4 | LANGUAGE C STRICT; 5 | 6 | CREATE ACCESS METHOD pg_lz4 7 | TYPE COMPRESSION 8 | HANDLER lz4_handler; 9 | -------------------------------------------------------------------------------- /pg_lz4.c: -------------------------------------------------------------------------------- 1 | #include "postgres.h" 2 | #include "fmgr.h" 3 | #include "lz4.h" 4 | #include "access/cmapi.h" 5 | #include "commands/defrem.h" 6 | #include "nodes/parsenodes.h" 7 | #include "utils/builtins.h" 8 | #include "utils/datum.h" 9 | 10 | 11 | PG_MODULE_MAGIC; 12 | 13 | typedef struct 14 | { 15 | int acceleration; 16 | } lz4_option; 17 | 18 | PG_FUNCTION_INFO_V1( lz4_handler ); 19 | 20 | static void 21 | lz4_check(Form_pg_attribute att, List *options) 22 | { 23 | ListCell *lc; 24 | 25 | foreach (lc, options) 26 | { 27 | DefElem *def = (DefElem *) lfirst(lc); 28 | 29 | if (strcmp(def->defname, "acceleration") == 0) 30 | { 31 | int accel = pg_atoi(defGetString(def), 4, 0); 32 | 33 | if (accel < 1) 34 | elog(WARNING, "Acceleration value <= 0 will be replaced by 1 (default)"); 35 | } 36 | else 37 | elog(ERROR, "Unknown option '%s'", def->defname); 38 | } 39 | } 40 | 41 | static void * 42 | lz4_initstate(Oid acoid, List *options) 43 | { 44 | ListCell *lc; 45 | lz4_option *opt = (lz4_option *) palloc(sizeof(lz4_option)); 46 | 47 | /* default acceleration */ 48 | opt->acceleration = 1; 49 | 50 | /* iterate through user options */ 51 | foreach (lc, options) 52 | { 53 | DefElem *def = (DefElem *) lfirst(lc); 54 | 55 | if (strcmp(def->defname, "acceleration") == 0) 56 | opt->acceleration = pg_atoi(defGetString(def), 4, 0); 57 | else 58 | elog(ERROR, "Unknown option '%s'", def->defname); 59 | } 60 | 61 | return (void *) opt; 62 | } 63 | 64 | static bytea * 65 | lz4_compress(CompressionAmOptions *cmoptions, const bytea *value) 66 | { 67 | lz4_option *opt = (lz4_option *) cmoptions->acstate; 68 | int src_len = (Size) VARSIZE_ANY_EXHDR(value); 69 | int dst_len; 70 | int len; 71 | bytea *ret; 72 | 73 | dst_len = LZ4_compressBound(src_len); 74 | ret = (bytea *) palloc(dst_len + VARHDRSZ_CUSTOM_COMPRESSED); 75 | 76 | len = LZ4_compress_fast((char *) VARDATA_ANY(value), 77 | (char *) ret + VARHDRSZ_CUSTOM_COMPRESSED, 78 | src_len, dst_len, 79 | opt->acceleration); 80 | 81 | if (len >= 0) 82 | { 83 | SET_VARSIZE_COMPRESSED(ret, len + VARHDRSZ_CUSTOM_COMPRESSED); 84 | return ret; 85 | } 86 | 87 | pfree(ret); 88 | return NULL; 89 | } 90 | 91 | static bytea * 92 | lz4_decompress(CompressionAmOptions *cmoptions, const bytea *value) 93 | { 94 | int src_len = VARSIZE_ANY(value); 95 | int dst_len = VARRAWSIZE_4B_C(value); 96 | int len; 97 | bytea *ret; 98 | 99 | ret = (bytea *) palloc(dst_len + VARHDRSZ); 100 | SET_VARSIZE(ret, dst_len + VARHDRSZ); 101 | len = LZ4_decompress_safe((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, 102 | (char *) VARDATA(ret), 103 | src_len - VARHDRSZ_CUSTOM_COMPRESSED, 104 | dst_len); 105 | 106 | if (len != dst_len) 107 | elog(ERROR, "Decompression error"); 108 | 109 | return ret; 110 | } 111 | 112 | Datum 113 | lz4_handler(PG_FUNCTION_ARGS) 114 | { 115 | CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); 116 | 117 | routine->cmcheck = lz4_check; 118 | routine->cminitstate = lz4_initstate; 119 | routine->cmcompress = lz4_compress; 120 | routine->cmdecompress = lz4_decompress; 121 | 122 | PG_RETURN_POINTER(routine); 123 | } 124 | -------------------------------------------------------------------------------- /pg_lz4.control: -------------------------------------------------------------------------------- 1 | # pg_lz4 extension 2 | comment 'lz4 compression method' 3 | default_version = '1.0' 4 | module_pathname='$libdir/pg_lz4' 5 | --------------------------------------------------------------------------------