├── debian ├── compat ├── edgefs.manpages ├── rules ├── edgefs.install ├── changelog └── control ├── tests ├── conf ├── unlink │ ├── 13.t │ ├── 04.t │ ├── 02.t │ └── 09.t ├── chmod │ ├── 10.t │ ├── 02.t │ └── 08.t ├── open │ ├── 21.t │ ├── 02.t │ ├── 17.t │ ├── 20.t │ ├── 23.t │ ├── 18.t │ ├── 10.t │ └── 11.t ├── chown │ ├── 10.t │ ├── 02.t │ └── 08.t ├── chflags │ ├── 13.t │ └── 02.t ├── rename │ ├── 01.t │ └── 17.t └── misc.sh ├── CHANGELOG.md ├── cachemap ├── uint128.h ├── filemap.h ├── Makefile ├── cachemap.h ├── cachemap_test.c ├── cachemap.c ├── midl.h ├── filemap.c ├── midl.c ├── lz4.h └── lz4.c ├── .gitignore ├── edgefs.spec ├── edgefs.1.txt ├── Makefile ├── README.md └── fstest.c /debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /debian/edgefs.manpages: -------------------------------------------------------------------------------- 1 | edgefs.1 2 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | %: 4 | dh ${@} 5 | -------------------------------------------------------------------------------- /debian/edgefs.install: -------------------------------------------------------------------------------- 1 | #!/usr/bin/dh-exec 2 | edgefs => /usr/bin 3 | cachemap/libcachemap.so.0.0 => /usr/lib 4 | -------------------------------------------------------------------------------- /tests/conf: -------------------------------------------------------------------------------- 1 | # $FreeBSD: src/tools/regression/fstest/tests/conf,v 1.1 2007/01/17 01:42:08 pjd Exp $ 2 | # fstest configuration file 3 | 4 | # Known operating systems: FreeBSD, SunOS, Linux 5 | os=`uname` 6 | 7 | # Known file systems: UFS, ZFS, ext3, ntfs-3g, xfs 8 | fs="ext3" 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 1.1.0 / 01/24/2018 2 | ================== 3 | 4 | * support for L2 extended cache on SSD/NVMe 5 | * bug fixes 6 | 7 | 1.0.1 / 11/13/2017 8 | ================== 9 | 10 | * support for v4 authentication 11 | * support for RPM packaging 12 | * bug fixes 13 | 14 | 1.0.0 / 11/05/2017 15 | ================== 16 | 17 | * First release 18 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | edgefs (1.1.0-1) testing; urgency=medium 2 | 3 | * Support for L2 extended cache on SSD/NVMe 4 | 5 | -- Nexenta Systems Wed, 24 Oct 2018 17:16:08 +0200 6 | 7 | edgefs (1.0.1-1) testing; urgency=medium 8 | 9 | * Initial release 10 | 11 | -- Nexenta Systems Fri, 15 Oct 2017 17:16:08 +0200 12 | 13 | -------------------------------------------------------------------------------- /tests/unlink/13.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/unlink/13.t,v 1.1 2007/01/17 01:42:12 pjd Exp $ 3 | 4 | desc="unlink returns EFAULT if the path argument points outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..2" 10 | 11 | expect EFAULT unlink NULL 12 | expect EFAULT unlink DEADCODE 13 | -------------------------------------------------------------------------------- /tests/chmod/10.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chmod/10.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chmod returns EFAULT if the path argument points outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..2" 10 | 11 | expect EFAULT chmod NULL 0644 12 | expect EFAULT chmod DEADCODE 0644 13 | -------------------------------------------------------------------------------- /tests/open/21.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/21.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns EFAULT if the path argument points outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..2" 10 | 11 | expect EFAULT open NULL O_RDONLY 12 | expect EFAULT open DEADCODE O_RDONLY 13 | -------------------------------------------------------------------------------- /tests/chown/10.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chown/10.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chown returns EFAULT if the path argument points outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..2" 10 | 11 | expect EFAULT chown NULL 65534 65534 12 | expect EFAULT chown DEADCODE 65534 65534 13 | -------------------------------------------------------------------------------- /tests/chflags/13.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chflags/13.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chflags returns EFAULT if the path argument points outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..2" 12 | 13 | expect EFAULT chflags NULL UF_IMMUTABLE 14 | expect EFAULT chflags DEADCODE UF_IMMUTABLE 15 | -------------------------------------------------------------------------------- /tests/unlink/04.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/unlink/04.t,v 1.1 2007/01/17 01:42:12 pjd Exp $ 3 | 4 | desc="unlink returns ENOENT if the named file does not exist" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..4" 10 | 11 | n0=`namegen` 12 | n1=`namegen` 13 | 14 | expect 0 create ${n0} 0644 15 | expect 0 unlink ${n0} 16 | expect ENOENT unlink ${n0} 17 | expect ENOENT unlink ${n1} 18 | -------------------------------------------------------------------------------- /tests/unlink/02.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/unlink/02.t,v 1.1 2007/01/17 01:42:12 pjd Exp $ 3 | 4 | desc="unlink returns ENAMETOOLONG if a component of a pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..4" 10 | 11 | expect 0 create ${name255} 0644 12 | expect 0 unlink ${name255} 13 | expect ENOENT unlink ${name255} 14 | expect ENAMETOOLONG unlink ${name256} 15 | -------------------------------------------------------------------------------- /tests/open/02.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/02.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns ENAMETOOLONG if a component of a pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..4" 10 | 11 | expect 0 open ${name255} O_CREAT 0620 12 | expect 0620 stat ${name255} mode 13 | expect 0 unlink ${name255} 14 | expect ENAMETOOLONG open ${name256} O_CREAT 0620 15 | -------------------------------------------------------------------------------- /tests/open/17.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/17.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns ENXIO when O_NONBLOCK is set, the named file is a fifo, O_WRONLY is set, and no process has the file open for reading" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..3" 10 | 11 | n0=`namegen` 12 | 13 | expect 0 mkfifo ${n0} 0644 14 | expect ENXIO open ${n0} O_WRONLY,O_NONBLOCK 15 | expect 0 unlink ${n0} 16 | -------------------------------------------------------------------------------- /tests/chmod/02.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chmod/02.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chmod returns ENAMETOOLONG if a component of a pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..5" 10 | 11 | expect 0 create ${name255} 0644 12 | expect 0 chmod ${name255} 0620 13 | expect 0620 stat ${name255} mode 14 | expect 0 unlink ${name255} 15 | expect ENAMETOOLONG chmod ${name256} 0620 16 | -------------------------------------------------------------------------------- /tests/chown/02.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chown/02.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chown returns ENAMETOOLONG if a component of a pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..5" 10 | 11 | expect 0 create ${name255} 0644 12 | expect 0 chown ${name255} 65534 65534 13 | expect 65534,65534 stat ${name255} uid,gid 14 | expect 0 unlink ${name255} 15 | expect ENAMETOOLONG chown ${name256} 65533 65533 16 | -------------------------------------------------------------------------------- /cachemap/uint128.h: -------------------------------------------------------------------------------- 1 | #ifndef UINT128_H 2 | #define UINT128_H 3 | 4 | typedef struct { uint64_t u; uint64_t l; } uint128_t; 5 | 6 | static inline void 7 | FNV_hash(const void *key, int length, uint64_t *out) 8 | { 9 | unsigned char* p = (unsigned char *)key; 10 | uint64_t fnv64offset = 14695981039346656037ULL; 11 | uint64_t fnv64prime = 0x100000001b3ULL; 12 | uint64_t h = fnv64offset; 13 | int i; 14 | 15 | for (i = 0; i < length; i++) { 16 | h = h ^ p[i]; 17 | h *= fnv64prime; 18 | } 19 | 20 | *out = h; 21 | } 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /tests/chflags/02.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chflags/02.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chflags returns ENAMETOOLONG if a component of a pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..6" 12 | 13 | expect 0 create ${name255} 0644 14 | expect 0 chflags ${name255} UF_IMMUTABLE 15 | expect UF_IMMUTABLE stat ${name255} flags 16 | expect 0 chflags ${name255} none 17 | expect 0 unlink ${name255} 18 | expect ENAMETOOLONG chflags ${name256} UF_IMMUTABLE 19 | -------------------------------------------------------------------------------- /tests/rename/01.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/rename/01.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="rename returns ENAMETOOLONG if a component of either pathname exceeded 255 characters" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..8" 10 | 11 | n0=`namegen` 12 | 13 | expect 0 create ${name255} 0644 14 | expect 0 rename ${name255} ${n0} 15 | expect 0 rename ${n0} ${name255} 16 | expect 0 unlink ${name255} 17 | 18 | expect 0 create ${n0} 0644 19 | expect ENAMETOOLONG rename ${n0} ${name256} 20 | expect 0 unlink ${n0} 21 | expect ENAMETOOLONG rename ${name256} ${n0} 22 | -------------------------------------------------------------------------------- /tests/rename/17.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/rename/17.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="rename returns EFAULT if one of the pathnames specified is outside the process's allocated address space" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..8" 10 | 11 | n0=`namegen` 12 | 13 | expect 0 create ${n0} 0644 14 | expect EFAULT rename ${n0} NULL 15 | expect EFAULT rename ${n0} DEADCODE 16 | expect 0 unlink ${n0} 17 | expect EFAULT rename NULL ${n0} 18 | expect EFAULT rename DEADCODE ${n0} 19 | expect EFAULT rename NULL DEADCODE 20 | expect EFAULT rename DEADCODE NULL 21 | -------------------------------------------------------------------------------- /tests/open/20.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/20.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns ETXTBSY when the file is a pure procedure (shared text) file that is being executed and the open() system call requests write access" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | case "${os}" in 10 | FreeBSD) 11 | echo "1..4" 12 | 13 | n0=`namegen` 14 | 15 | cp -pf `which sleep` ${n0} 16 | ./${n0} 3 & 17 | expect ETXTBSY open ${n0} O_WRONLY 18 | expect ETXTBSY open ${n0} O_RDWR 19 | expect ETXTBSY open ${n0} O_RDONLY,O_TRUNC 20 | expect 0 unlink ${n0} 21 | ;; 22 | *) 23 | quick_exit 24 | ;; 25 | esac 26 | -------------------------------------------------------------------------------- /tests/open/23.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/23.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns EINVAL when an attempt was made to open a descriptor with an illegal combination of O_RDONLY, O_WRONLY, and O_RDWR" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | echo "1..4" 10 | 11 | n0=`namegen` 12 | 13 | expect 0 create ${n0} 0644 14 | case "${os}" in 15 | Linux) 16 | expect 0 open ${n0} O_WRONLY,O_RDWR 17 | expect 0 open ${n0} O_RDONLY,O_WRONLY,O_RDWR 18 | ;; 19 | *) 20 | expect EINVAL open ${n0} O_WRONLY,O_RDWR 21 | expect EINVAL open ${n0} O_RDONLY,O_WRONLY,O_RDWR 22 | ;; 23 | esac 24 | expect 0 unlink ${n0} 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Object files 5 | *.o 6 | *.ko 7 | *.obj 8 | *.elf 9 | 10 | # Linker output 11 | *.ilk 12 | *.map 13 | *.exp 14 | 15 | # Precompiled Headers 16 | *.gch 17 | *.pch 18 | 19 | # Libraries 20 | *.lib 21 | *.a 22 | *.la 23 | *.lo 24 | 25 | # Shared objects (inc. Windows DLLs) 26 | *.dll 27 | *.so 28 | *.so.* 29 | *.dylib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | *.i*86 36 | *.x86_64 37 | *.hex 38 | 39 | # Debug files 40 | *.dSYM/ 41 | *.su 42 | *.idb 43 | *.pdb 44 | 45 | # Kernel Module Compile Results 46 | *.mod* 47 | *.cmd 48 | .tmp_versions/ 49 | modules.order 50 | Module.symvers 51 | Mkfile.old 52 | dkms.conf 53 | edgefs 54 | edgefs.1 55 | fstest 56 | cachemap/cachemap_test 57 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: edgefs 2 | Section: web 3 | Priority: extra 4 | Maintainer: Nexenta 5 | Build-Depends: 6 | debhelper (>= 9~), dh-exec, 7 | asciidoc, xmlto, 8 | libfuse-dev (>> 2.6), pkg-config, 9 | libgnutls-dev (>= 2.10~) | libgnutls28-dev, 10 | Standards-Version: 3.8.4 11 | Homepage: http://github.com/Nexenta/edge-fuse/ 12 | 13 | Package: edgefs 14 | Architecture: any 15 | Depends: ${misc:Depends}, ${shlibs:Depends}, 16 | Recommends: fuse-utils | fuse (>= 2.9.3~) 17 | Description: FUSE filesystem for mounting files from http servers 18 | edgefs is a FUSE based filesystem for mounting http or https URLS as files in 19 | the filesystem. There is no notion of listable directories in Edge API so only 20 | a single URL can be mounted. 21 | -------------------------------------------------------------------------------- /tests/open/18.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/18.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns EWOULDBLOCK when O_NONBLOCK and one of O_SHLOCK or O_EXLOCK is specified and the file is locked" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | case "${os}" in 10 | FreeBSD) 11 | echo "1..6" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 open ${n0} O_RDONLY,O_SHLOCK : open ${n0} O_RDONLY,O_SHLOCK,O_NONBLOCK 17 | expect "EWOULDBLOCK|EAGAIN" open ${n0} O_RDONLY,O_EXLOCK : open ${n0} O_RDONLY,O_EXLOCK,O_NONBLOCK 18 | expect "EWOULDBLOCK|EAGAIN" open ${n0} O_RDONLY,O_SHLOCK : open ${n0} O_RDONLY,O_EXLOCK,O_NONBLOCK 19 | expect "EWOULDBLOCK|EAGAIN" open ${n0} O_RDONLY,O_EXLOCK : open ${n0} O_RDONLY,O_SHLOCK,O_NONBLOCK 20 | expect 0 unlink ${n0} 21 | ;; 22 | *) 23 | quick_exit 24 | ;; 25 | esac 26 | -------------------------------------------------------------------------------- /cachemap/filemap.h: -------------------------------------------------------------------------------- 1 | #ifndef FILEMAP_H 2 | #define FILEMAP_H 3 | 4 | #include 5 | #include "uint128.h" 6 | 7 | #define FILEMAP_SHARD_NUM 32 8 | #define FILEMAP_SHARD_FACTOR 1024 9 | 10 | struct filemap { 11 | uint64_t n; // number of buckets 12 | int compress; // enable compression (LZ4 acceleration) 13 | int bsize; // cached block size 14 | uint64_t pshift; // cached block size associated page shift 15 | char destdir[2048]; // location where cache files should be stored 16 | MDB_env *env[FILEMAP_SHARD_NUM]; 17 | }; 18 | 19 | struct filemap *filemap_create(char *destdir, uint64_t n, int compress_accel, 20 | int pshift); 21 | void filemap_free(struct filemap *m); 22 | 23 | void filemap_set(struct filemap *m, uint128_t *key, void *value, uint64_t attr); 24 | void filemap_unset(struct filemap *m, uint128_t *key); 25 | 26 | void *filemap_get(struct filemap *m, uint128_t *key); 27 | int filemap_get_rand(struct filemap *m, uint128_t *key, uint64_t *ts); 28 | 29 | uint64_t filemap_entries(struct filemap *m); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /cachemap/Makefile: -------------------------------------------------------------------------------- 1 | TARGET_LIB = libcachemap.so.0.0 2 | TARGET_SRCS = mdb.c midl.c lz4.c filemap.c cachemap.c 3 | # 4 | # Activate by exporing this env variable: 5 | # 6 | # export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_core=1:alloc_dealloc_mismatch=0:detect_leaks=1 7 | # 8 | #ASAN_CPPFLAGS=-fsanitize=address -fno-omit-frame-pointer -fno-common 9 | #ASAN_LDFLAGS=-fsanitize=address -fno-omit-frame-pointer -fno-common -lasan 10 | CFLAGS += -std=c99 -D_FILE_OFFSET_BITS=64 -fPIC -Wall -Wextra -Wno-unused -O3 -pthread -g -D_XOPEN_SOURCE=700 -D_ISOC99_SOURCE -I. -L. $(ASAN_CPPFLAGS) 11 | LDFLAGS += -L. $(ASAN_LDFLAGS) 12 | 13 | TARGET_OBJS = $(TARGET_SRCS:.c=.o) 14 | 15 | .PHONY: all 16 | all: ${TARGET_LIB} cachemap_test 17 | 18 | $(TARGET_LIB): $(TARGET_OBJS) 19 | gcc -shared -o $@ $^ $(LDFLAGS) 20 | gcc -shared -Wl,-soname,$(TARGET_LIB) $^ $(LDFLAGS) -o $@ 21 | ln -sf libcachemap.so.0.0 libcachemap.so 22 | 23 | cachemap_test: cachemap_test.c $(TARGET_LIB) 24 | gcc $(CFLAGS) -o $@ $^ -lcachemap 25 | 26 | pktrecv: pktrecv.c $(TARGET_LIB) 27 | gcc $(CFLAGS) -o $@ $^ -lcachemap 28 | 29 | clean: 30 | rm -f $(TARGET_LIB) cachemap_test *.o *.so 31 | -------------------------------------------------------------------------------- /cachemap/cachemap.h: -------------------------------------------------------------------------------- 1 | #ifndef CACHEMAP_H 2 | #define CACHEMAP_H 3 | 4 | #include 5 | #include "filemap.h" 6 | 7 | #define PUT_THREADS 4 8 | 9 | struct addr_work { 10 | uint128_t addr; 11 | void *page; 12 | uint64_t ts; 13 | }; 14 | 15 | struct addr_node { 16 | struct addr_node *next; 17 | struct addr_work work; 18 | }; 19 | 20 | struct cachemap { 21 | struct filemap *pages; 22 | struct addr_node *front; 23 | struct addr_node *rear; 24 | pthread_t cm_thread[PUT_THREADS]; 25 | pthread_cond_t step_condvar; 26 | pthread_mutex_t cm_mutex; 27 | int cm_thread_stop; 28 | uint64_t capacity; 29 | uint64_t requests; 30 | uint64_t hits; 31 | }; 32 | 33 | struct cachemap * cachemap_create (char *destdir, uint64_t capacity, int comp_accel, 34 | int pshift); 35 | 36 | void cachemap_free(struct cachemap *cm); 37 | 38 | void * cachemap_get(struct cachemap *cm, uint64_t offset, 39 | uint64_t nhid_small, uint32_t genid); 40 | 41 | void cachemap_put(struct cachemap *cm, uint64_t offset, 42 | uint64_t nhid_small, uint32_t genid, const void *page); 43 | 44 | void cachemap_put_async(struct cachemap *cm, uint64_t offset, 45 | uint64_t nhid_small, uint32_t genid, const void *page); 46 | 47 | void cachemap_print_stats(struct cachemap *cm); 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /tests/open/10.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/10.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns EPERM when the named file has its immutable flag set and the file is to be modified" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..28" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 chflags ${n0} SF_IMMUTABLE 17 | expect EPERM open ${n0} O_WRONLY 18 | expect EPERM open ${n0} O_RDWR 19 | expect EPERM open ${n0} O_RDONLY,O_TRUNC 20 | expect 0 chflags ${n0} none 21 | expect 0 unlink ${n0} 22 | 23 | expect 0 create ${n0} 0644 24 | expect 0 chflags ${n0} UF_IMMUTABLE 25 | expect EPERM open ${n0} O_WRONLY 26 | expect EPERM open ${n0} O_RDWR 27 | expect EPERM open ${n0} O_RDONLY,O_TRUNC 28 | expect 0 chflags ${n0} none 29 | expect 0 unlink ${n0} 30 | 31 | expect 0 create ${n0} 0644 32 | expect 0 chflags ${n0} SF_NOUNLINK 33 | expect 0 open ${n0} O_WRONLY 34 | expect 0 open ${n0} O_RDWR 35 | expect 0 open ${n0} O_RDONLY,O_TRUNC 36 | expect 0 chflags ${n0} none 37 | expect 0 unlink ${n0} 38 | 39 | expect 0 create ${n0} 0644 40 | expect 0 chflags ${n0} UF_NOUNLINK 41 | expect 0 open ${n0} O_WRONLY 42 | expect 0 open ${n0} O_RDWR 43 | expect 0 open ${n0} O_RDONLY,O_TRUNC 44 | expect 0 chflags ${n0} none 45 | expect 0 unlink ${n0} 46 | -------------------------------------------------------------------------------- /tests/unlink/09.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/unlink/09.t,v 1.1 2007/01/17 01:42:12 pjd Exp $ 3 | 4 | desc="unlink returns EPERM if the named file has its immutable, undeletable or append-only flag set" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..30" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 chflags ${n0} SF_IMMUTABLE 17 | expect EPERM unlink ${n0} 18 | expect 0 chflags ${n0} none 19 | expect 0 unlink ${n0} 20 | 21 | expect 0 create ${n0} 0644 22 | expect 0 chflags ${n0} UF_IMMUTABLE 23 | expect EPERM unlink ${n0} 24 | expect 0 chflags ${n0} none 25 | expect 0 unlink ${n0} 26 | 27 | expect 0 create ${n0} 0644 28 | expect 0 chflags ${n0} SF_NOUNLINK 29 | expect EPERM unlink ${n0} 30 | expect 0 chflags ${n0} none 31 | expect 0 unlink ${n0} 32 | 33 | expect 0 create ${n0} 0644 34 | expect 0 chflags ${n0} UF_NOUNLINK 35 | expect EPERM unlink ${n0} 36 | expect 0 chflags ${n0} none 37 | expect 0 unlink ${n0} 38 | 39 | expect 0 create ${n0} 0644 40 | expect 0 chflags ${n0} SF_APPEND 41 | expect EPERM unlink ${n0} 42 | expect 0 chflags ${n0} none 43 | expect 0 unlink ${n0} 44 | 45 | expect 0 create ${n0} 0644 46 | expect 0 chflags ${n0} UF_APPEND 47 | expect EPERM unlink ${n0} 48 | expect 0 chflags ${n0} none 49 | expect 0 unlink ${n0} 50 | -------------------------------------------------------------------------------- /tests/open/11.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/open/11.t,v 1.1 2007/01/17 01:42:10 pjd Exp $ 3 | 4 | desc="open returns EPERM when the named file has its append-only flag set, the file is to be modified, and O_TRUNC is specified or O_APPEND is not specified" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..24" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 chflags ${n0} SF_APPEND 17 | expect 0 open ${n0} O_WRONLY,O_APPEND 18 | expect 0 open ${n0} O_RDWR,O_APPEND 19 | expect EPERM open ${n0} O_WRONLY 20 | expect EPERM open ${n0} O_RDWR 21 | expect EPERM open ${n0} O_RDONLY,O_TRUNC 22 | expect EPERM open ${n0} O_RDONLY,O_APPEND,O_TRUNC 23 | expect EPERM open ${n0} O_WRONLY,O_APPEND,O_TRUNC 24 | expect EPERM open ${n0} O_RDWR,O_APPEND,O_TRUNC 25 | expect 0 chflags ${n0} none 26 | expect 0 unlink ${n0} 27 | 28 | expect 0 create ${n0} 0644 29 | expect 0 chflags ${n0} UF_APPEND 30 | expect 0 open ${n0} O_WRONLY,O_APPEND 31 | expect 0 open ${n0} O_RDWR,O_APPEND 32 | expect EPERM open ${n0} O_WRONLY 33 | expect EPERM open ${n0} O_RDWR 34 | expect EPERM open ${n0} O_RDONLY,O_TRUNC 35 | expect EPERM open ${n0} O_RDONLY,O_APPEND,O_TRUNC 36 | expect EPERM open ${n0} O_WRONLY,O_APPEND,O_TRUNC 37 | expect EPERM open ${n0} O_RDWR,O_APPEND,O_TRUNC 38 | expect 0 chflags ${n0} none 39 | expect 0 unlink ${n0} 40 | -------------------------------------------------------------------------------- /tests/chown/08.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chown/08.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chown returns EPERM if the named file has its immutable or append-only flag set" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..34" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 chflags ${n0} SF_IMMUTABLE 17 | expect EPERM chown ${n0} 65534 65534 18 | expect 0 chflags ${n0} none 19 | expect 0 chown ${n0} 65534 65534 20 | expect 0 unlink ${n0} 21 | 22 | expect 0 create ${n0} 0644 23 | expect 0 chflags ${n0} UF_IMMUTABLE 24 | expect EPERM chown ${n0} 65534 65534 25 | expect 0 chflags ${n0} none 26 | expect 0 chown ${n0} 65534 65534 27 | expect 0 unlink ${n0} 28 | 29 | expect 0 create ${n0} 0644 30 | expect 0 chflags ${n0} SF_APPEND 31 | expect EPERM chown ${n0} 65534 65534 32 | expect 0 chflags ${n0} none 33 | expect 0 chown ${n0} 65534 65534 34 | expect 0 unlink ${n0} 35 | 36 | expect 0 create ${n0} 0644 37 | expect 0 chflags ${n0} UF_APPEND 38 | expect EPERM chown ${n0} 65534 65534 39 | expect 0 chflags ${n0} none 40 | expect 0 chown ${n0} 65534 65534 41 | expect 0 unlink ${n0} 42 | 43 | expect 0 create ${n0} 0644 44 | expect 0 chflags ${n0} SF_NOUNLINK 45 | expect 0 chown ${n0} 65534 65534 46 | expect 0 chflags ${n0} none 47 | expect 0 unlink ${n0} 48 | 49 | expect 0 create ${n0} 0644 50 | expect 0 chflags ${n0} UF_NOUNLINK 51 | expect 0 chown ${n0} 65534 65534 52 | expect 0 chflags ${n0} none 53 | expect 0 unlink ${n0} 54 | -------------------------------------------------------------------------------- /edgefs.spec: -------------------------------------------------------------------------------- 1 | Name: edgefs 2 | Version: 1.1.0 3 | Release: 1%{?dist} 4 | Summary: FUSE-based file system backed by NexentaEdge Extended S3 API 5 | Group: System Environment/Base 6 | 7 | License: GPLv2 8 | URL: https://github.com/Nexenta/edge-fuse 9 | Source0: https://github.com/Nexenta/edge-fuse/archive/%{name}-%{version}.tar.gz 10 | 11 | Requires: fuse >= 2.8.4 12 | Requires: fuse-libs >= 2.8.4 13 | Requires: gnutls >= 3.3 14 | 15 | BuildRequires: fuse-devel, gnutls-devel 16 | BuildRequires: gcc, asciidoc 17 | 18 | %description 19 | edgefs is a FUSE based filesystem for mounting http or https URLS as files in 20 | the filesystem. There is no notion of listable directories in Edge API so only 21 | a single URL can be mounted. 22 | 23 | %global debug_package %{nil} 24 | 25 | 26 | %prep 27 | %setup -q 28 | 29 | 30 | %build 31 | make %{?_smp_mflags} all 32 | 33 | 34 | %install 35 | mkdir -p %{buildroot}%{_bindir} 36 | mkdir -p %{buildroot}%{_libdir} 37 | mkdir -p %{buildroot}%{_mandir}/man1 38 | cp edgefs %{buildroot}%{_bindir}/ 39 | cp libcachemap.so.0.0 %{buildroot}%{_libdir}/libcachemap.so.0.0 40 | cp edgefs.1 %{buildroot}%{_mandir}/man1/ 41 | 42 | 43 | %files 44 | %{_bindir}/edgefs 45 | %{_libdir}/libcachemap.so.0.0 46 | %{_mandir}/man1/edgefs.1* 47 | %doc README.md 48 | 49 | 50 | %changelog 51 | * Wed Jan 24 2018 Dmitry Yusupov - 1.1.0-1 52 | - Support for L2 extended cache on SSD/NVMe 53 | 54 | * Fri Nov 10 2017 Dmitry Yusupov - 1.0.1-1 55 | - Initial build of 1.0.1 from https://github.com/Nexenta/edge-fuse 56 | -------------------------------------------------------------------------------- /tests/chmod/08.t: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # $FreeBSD: src/tools/regression/fstest/tests/chmod/08.t,v 1.1 2007/01/17 01:42:08 pjd Exp $ 3 | 4 | desc="chmod returns EPERM if the named file has its immutable or append-only flag set" 5 | 6 | dir=`dirname $0` 7 | . ${dir}/../misc.sh 8 | 9 | require chflags 10 | 11 | echo "1..40" 12 | 13 | n0=`namegen` 14 | 15 | expect 0 create ${n0} 0644 16 | expect 0 chflags ${n0} SF_IMMUTABLE 17 | expect EPERM chmod ${n0} 0600 18 | expect 0644 stat ${n0} mode 19 | expect 0 chflags ${n0} none 20 | expect 0 chmod ${n0} 0600 21 | expect 0 unlink ${n0} 22 | 23 | expect 0 create ${n0} 0644 24 | expect 0 chflags ${n0} UF_IMMUTABLE 25 | expect EPERM chmod ${n0} 0600 26 | expect 0644 stat ${n0} mode 27 | expect 0 chflags ${n0} none 28 | expect 0 chmod ${n0} 0600 29 | expect 0 unlink ${n0} 30 | 31 | expect 0 create ${n0} 0644 32 | expect 0 chflags ${n0} SF_APPEND 33 | expect EPERM chmod ${n0} 0600 34 | expect 0644 stat ${n0} mode 35 | expect 0 chflags ${n0} none 36 | expect 0 chmod ${n0} 0600 37 | expect 0 unlink ${n0} 38 | 39 | expect 0 create ${n0} 0644 40 | expect 0 chflags ${n0} UF_APPEND 41 | expect EPERM chmod ${n0} 0600 42 | expect 0644 stat ${n0} mode 43 | expect 0 chflags ${n0} none 44 | expect 0 chmod ${n0} 0600 45 | expect 0 unlink ${n0} 46 | 47 | expect 0 create ${n0} 0644 48 | expect 0 chflags ${n0} SF_NOUNLINK 49 | expect 0 chmod ${n0} 0600 50 | expect 0600 stat ${n0} mode 51 | expect 0 chflags ${n0} none 52 | expect 0 unlink ${n0} 53 | 54 | expect 0 create ${n0} 0644 55 | expect 0 chflags ${n0} UF_NOUNLINK 56 | expect 0 chmod ${n0} 0600 57 | expect 0600 stat ${n0} mode 58 | expect 0 chflags ${n0} none 59 | expect 0 unlink ${n0} 60 | -------------------------------------------------------------------------------- /tests/misc.sh: -------------------------------------------------------------------------------- 1 | # $FreeBSD: src/tools/regression/fstest/tests/misc.sh,v 1.1 2007/01/17 01:42:08 pjd Exp $ 2 | 3 | ntest=1 4 | 5 | name253="_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_123456789_12" 6 | name255="${name253}34" 7 | name256="${name255}5" 8 | path1021="${name255}/${name255}/${name255}/${name253}" 9 | path1023="${path1021}/x" 10 | path1024="${path1023}x" 11 | 12 | echo ${dir} | egrep '^/' >/dev/null 2>&1 13 | if [ $? -eq 0 ]; then 14 | maindir="${dir}/../.." 15 | else 16 | maindir="`pwd`/${dir}/../.." 17 | fi 18 | fstest="${maindir}/fstest" 19 | . ${maindir}/tests/conf 20 | 21 | run_getconf() 22 | { 23 | if val=$(getconf "${1}" .); then 24 | if [ "$value" = "undefined" ]; then 25 | echo "${1} is undefined" 26 | exit 1 27 | fi 28 | else 29 | echo "Failed to get ${1}" 30 | exit 1 31 | fi 32 | 33 | echo $val 34 | } 35 | 36 | name_max_val=$(run_getconf NAME_MAX) 37 | path_max_val=$(run_getconf PATH_MAX) 38 | 39 | name_max="_" 40 | i=1 41 | while test $i -lt $name_max_val ; do 42 | name_max="${name_max}x" 43 | i=$(($i+1)) 44 | done 45 | 46 | num_of_dirs=$(( ($path_max_val + $name_max_val) / ($name_max_val + 1) - 1 )) 47 | 48 | long_dir="${name_max}" 49 | i=1 50 | while test $i -lt $num_of_dirs ; do 51 | long_dir="${long_dir}/${name_max}" 52 | i=$(($i+1)) 53 | done 54 | long_dir="${long_dir}/x" 55 | 56 | too_long="${long_dir}/${name_max}" 57 | 58 | create_too_long() 59 | { 60 | mkdir -p ${long_dir} 61 | } 62 | 63 | unlink_too_long() 64 | { 65 | rm -rf ${name_max} 66 | } 67 | 68 | expect() 69 | { 70 | e="${1}" 71 | shift 72 | r=`${fstest} $* 2>/dev/null | tail -1` 73 | echo "${r}" | egrep '^'${e}'$' >/dev/null 2>&1 74 | if [ $? -eq 0 ]; then 75 | echo "ok ${ntest}" 76 | else 77 | echo "not ok ${ntest}" 78 | fi 79 | ntest=`expr $ntest + 1` 80 | } 81 | 82 | jexpect() 83 | { 84 | s="${1}" 85 | d="${2}" 86 | e="${3}" 87 | shift 3 88 | r=`jail -s ${s} / fstest 127.0.0.1 /bin/sh -c "cd ${d} && ${fstest} $* 2>/dev/null" | tail -1` 89 | echo "${r}" | egrep '^'${e}'$' >/dev/null 2>&1 90 | if [ $? -eq 0 ]; then 91 | echo "ok ${ntest}" 92 | else 93 | echo "not ok ${ntest}" 94 | fi 95 | ntest=`expr $ntest + 1` 96 | } 97 | 98 | test_check() 99 | { 100 | if [ $* ]; then 101 | echo "ok ${ntest}" 102 | else 103 | echo "not ok ${ntest}" 104 | fi 105 | ntest=`expr $ntest + 1` 106 | } 107 | 108 | namegen() 109 | { 110 | echo "fstest_`dd if=/dev/urandom bs=1k count=1 2>/dev/null | md5sum | cut -f1 -d' '`" 111 | } 112 | 113 | quick_exit() 114 | { 115 | echo "1..1" 116 | echo "ok 1" 117 | exit 0 118 | } 119 | 120 | supported() 121 | { 122 | case "${1}" in 123 | chflags) 124 | if [ ${os} != "FreeBSD" -o ${fs} != "UFS" ]; then 125 | return 1 126 | fi 127 | ;; 128 | lchmod) 129 | if [ ${os} != "FreeBSD" ]; then 130 | return 1 131 | fi 132 | ;; 133 | esac 134 | return 0 135 | } 136 | 137 | require() 138 | { 139 | if supported ${1}; then 140 | return 141 | fi 142 | quick_exit 143 | } 144 | -------------------------------------------------------------------------------- /cachemap/cachemap_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "cachemap.h" 7 | 8 | #define MAXOBJ 32768 9 | #define COMP_ACCEL 12 10 | #define PAGE_SHIFT 15 11 | #define PAGE_SIZE (1<offset = i * 4096UL; 39 | objs[i]->genid = i; 40 | objs[i]->nhid_small = i * random(); 41 | objs[i]->page[0] = i; 42 | bytes += PAGE_SIZE; 43 | } 44 | 45 | printf("Now running test on %ldMB of data\n", bytes/1024UL/1024UL); 46 | 47 | t1 = clock(); 48 | for (int i = 0; i < MAXOBJ; ++i) { 49 | cachemap_put_async(cm, objs[i]->offset, objs[i]->nhid_small, 50 | objs[i]->genid, objs[i]->page); 51 | } 52 | t2 = clock(); 53 | 54 | sleep(1); 55 | 56 | printf("Inserted %d objects in %.6fs\n", MAXOBJ, ((double)(t2-t1))/CLOCKS_PER_SEC); 57 | cachemap_print_stats(cm); 58 | 59 | t1 = clock(); 60 | for (int i = 0; i < MAXOBJ; ++i) { 61 | void *p = cachemap_get(cm, objs[i]->offset, objs[i]->nhid_small, 62 | objs[i]->genid); 63 | free(p); 64 | } 65 | t2 = clock(); 66 | 67 | printf("Read %d objects in %.6fs\n", MAXOBJ, ((double)(t2-t1))/CLOCKS_PER_SEC); 68 | cachemap_print_stats(cm); 69 | 70 | t1 = clock(); 71 | for (int i = 0; i < MAXOBJ; ++i) { 72 | void *p = cachemap_get(cm, objs[i]->offset, objs[i]->nhid_small, 73 | objs[i]->genid); 74 | free(p); 75 | } 76 | t2 = clock(); 77 | 78 | printf("Re-read %d objects in %.6fs\n", MAXOBJ, ((double)(t2-t1))/CLOCKS_PER_SEC); 79 | cachemap_print_stats(cm); 80 | 81 | t1 = clock(); 82 | for (int i = 0; i < MAXOBJ/2; ++i) { 83 | objs[i]->genid = random(); 84 | cachemap_put_async(cm, objs[i]->offset, objs[i]->nhid_small, 85 | objs[i]->genid, objs[i]->page); 86 | } 87 | t2 = clock(); 88 | 89 | printf("Added %d objects in %.6fs\n", MAXOBJ/2, ((double)(t2-t1))/CLOCKS_PER_SEC); 90 | cachemap_print_stats(cm); 91 | 92 | t1 = clock(); 93 | for (int i = 0; i < MAXOBJ; ++i) { 94 | void *p = cachemap_get(cm, objs[i]->offset, objs[i]->nhid_small, 95 | objs[i]->genid); 96 | free(p); 97 | } 98 | t2 = clock(); 99 | 100 | printf("Read %d objects in %.6fs\n", MAXOBJ, ((double)(t2-t1))/CLOCKS_PER_SEC); 101 | cachemap_print_stats(cm); 102 | 103 | t1 = clock(); 104 | for (int i = 0; i < MAXOBJ; ++i) { 105 | void *p = cachemap_get(cm, objs[i]->offset, objs[i]->nhid_small, 106 | objs[i]->genid); 107 | free(p); 108 | } 109 | t2 = clock(); 110 | 111 | printf("Re-read %d objects in %.6fs\n", MAXOBJ, ((double)(t2-t1))/CLOCKS_PER_SEC); 112 | cachemap_print_stats(cm); 113 | 114 | cachemap_free(cm); 115 | for (int i = 0; i < MAXOBJ; ++i) { 116 | free(objs[i]); 117 | } 118 | 119 | return 0; 120 | } 121 | 122 | int 123 | main (int argc, char **argv) 124 | { 125 | if (argc != 2) { 126 | printf("Usage: cachemap_test \n"); 127 | return -1; 128 | } 129 | test(argv[1]); 130 | return 0; 131 | } 132 | -------------------------------------------------------------------------------- /edgefs.1.txt: -------------------------------------------------------------------------------- 1 | EDGEFS(1) 2 | =========== 3 | Michal Suchanek 4 | Modified by Nexenta Systems, Inc to add support for EdgeX API 5 | 6 | 7 | NAME 8 | ---- 9 | edgefs - mount a file from a http server into the filesystem 10 | 11 | 12 | SYNOPSIS 13 | -------- 14 | *edgefs* ['OPTIONS'] 'URL' 'FUSE-OPTIONS' 15 | 16 | 17 | DESCRIPTION 18 | ----------- 19 | edgefs is a *FUSE* based filesystem for mounting http or https URLS as files in 20 | the filesystem. There is no notion of listable directories in API so only a 21 | single URL can be mounted. 22 | 23 | 24 | OPTIONS 25 | ------- 26 | 27 | *-c 'console'*:: 28 | Attempt to use the file or device 'console' for output after fork. 29 | The default is '/dev/console'. 30 | 31 | *-f*:: 32 | Do not fork, stay in foreground. 33 | 34 | *-r 'retries'*:: 35 | (if 'ECONNRESET' is defined) Retry connecting to server after receiving 36 | 'ECONNRESET'. Defaults to 8 times which gives about 30s timeout. 37 | 38 | *-t 'timeout'*:: 39 | Use different timeout for connections. Default '30's. 40 | 41 | *-d 'debug'*:: 42 | Set debug level (default 0). 43 | 44 | *-D 'direct'*:: 45 | Enable FUSE direct_io mode 46 | 47 | *-b 'chunksize'*:: 48 | Setup CCOW chunk size in bytes for newly created files (default 131072, 49 | power of 2. Range: 4096 - 4194304). Use this parameter to set most 50 | optimal I/O alignment with an application. 51 | 52 | *-o 'order'*:: 53 | Setup CCOW btree order for newly created files (default 256. Range 32-640). 54 | 55 | *-R 'sockrecv'*:: 56 | Override socket receive buffer size 57 | 58 | *-S 'socksend'*:: 59 | Override socket send buffer size 60 | 61 | Example of running in foreground with debug on: 62 | 63 | edgefs -d 1 -f -c - http://user:password@server.com/MySharedBucket /mnt/MySharedBucket -d 64 | 65 | SSL OPTIONS 66 | ~~~~~~~~~~~ 67 | For usage with HTTPS protocol URLs. 68 | 69 | *-2*:: 70 | Allow server certificates signed with RSA-MD2 (strongly discouraged) 71 | 72 | *-5*:: 73 | Allow server certificates signed with RSA-MD5 (discouraged) 74 | 75 | *-a 'CA file'*:: 76 | Specify a file to load trusted CA root certificates from. 77 | A default location is set at build time. 78 | 79 | *-d 'n'*:: 80 | Set GnuTLS debug level to numeric value n. 81 | 82 | *'URL'* 83 | ~~~~~~~ 84 | The url should specify the protocol as http or https, and it may specify basic 85 | authentication username and password. Currently special characters like 86 | whitespace are not handled so the URL cannot contain them. See a sample URL 87 | below: 88 | 89 | http://user:password@server.com/MySharedBucket 90 | https://user:password@server.com/MySharedBucket 91 | 92 | V4 AUTH OPTIONS 93 | ~~~~~~~~~~~~~~~ 94 | 95 | *-k 'key_secret'*:: 96 | Enable AWS Signature v4 Authentication. Format: KEY:SECRET 97 | 98 | *-T 'region'*:: 99 | Use specific S3 region. The default is us-west-1 100 | 101 | L2 SSD/NVMe cache 102 | ~~~~~~~~~~~~~~~~~ 103 | 104 | *-C 'cachedir'*:: 105 | Enable use of local SSD/NVMe devices as high-performance block level cache. 106 | 107 | *'FUSE-OPTIONS'* 108 | ~~~~~~~~~~~~~~~~ 109 | These options are passed to the *FUSE* library. At the very least the mount point should be specified. 110 | 111 | 112 | EXIT STATUS 113 | ----------- 114 | *0*:: 115 | Successfully connected to the server 116 | 117 | *other*:: 118 | Failure (url parsing error, server error, FUSE setup error). 119 | Some FUSE errors may happen only after the process forks so they will not be returned in exit value. 120 | 121 | 122 | BUGS 123 | ---- 124 | The process can be stopped by typing ^Z on the terminal which may not be desirable under some circumstances. 125 | 126 | 127 | 128 | AUTHORS 129 | ------- 130 | Miklos Szeredi 131 | hmb marionraven at users.sourceforge.net 132 | Michal Suchanek 133 | Dmitry Yusupov - Nexenta Systems, Inc 134 | 135 | 136 | COPYING 137 | ------- 138 | Free use of this software is granted under the terms of the GNU General Public 139 | License (GPL). 140 | 141 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Determine the platform first 3 | UNAME := $(shell uname -s) 4 | DISTID := $(shell echo $$(. /etc/os-release; echo $$ID | tr '[A-Z]' '[a-z]')) 5 | 6 | # 7 | # Activate ASAN by exporing this env variable: 8 | # 9 | # export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_core=1:alloc_dealloc_mismatch=0:detect_leaks=1 10 | # 11 | #ASAN_CPPFLAGS=-fsanitize=address -fno-omit-frame-pointer -fno-common 12 | #ASAN_LDFLAGS=-fsanitize=address -fno-omit-frame-pointer -fno-common -lasan 13 | 14 | ifeq ($(UNAME), Darwin) 15 | 16 | MAIN_CFLAGS := -g -O2 -Wall -std=c99 -D_FILE_OFFSET_BITS=64 -Icachemap $(ASAN_CPPFLAGS) 17 | CC = gcc 18 | CFLAGS += -I/usr/local/opt/openssl/include 19 | LDFLAGS := -lgnutls -lfuse 20 | MAIN_LDFLAGS := -L. -lcachemap 21 | 22 | else 23 | 24 | MAIN_CFLAGS := -g -O2 -Wall -std=c99 -Icachemap $(shell pkg-config fuse --cflags) $(ASAN_CPPFLAGS) 25 | MAIN_CPPFLAGS := -Wall -Wno-unused-function -Wconversion -Wtype-limits -DUSE_AUTH -D_XOPEN_SOURCE=700 -D_ISOC99_SOURCE $(ASAN_LDFLAGS) 26 | THR_LDFLAGS := -lpthread 27 | GNUTLS_VERSION := 2.10 28 | MAIN_LDFLAGS := $(shell pkg-config fuse --libs | sed -e s/-lrt// -e s/-ldl// -e s/-pthread// -e "s/ / /g") -L. -lcachemap 29 | intermediates = 30 | 31 | ifeq ($(shell pkg-config --atleast-version $(GNUTLS_VERSION) gnutls ; echo $$?), 0) 32 | CERT_STORE := /etc/ssl/certs/ca-certificates.crt 33 | CPPFLAGS := -DUSE_SSL $(shell pkg-config gnutls --cflags) -DCERT_STORE=\"$(CERT_STORE)\" 34 | LDFLAGS := $(shell pkg-config gnutls --libs) 35 | else 36 | $(info GNUTLS version at least $(GNUTLS_VERSION) required for SSL support.) 37 | endif 38 | 39 | endif 40 | 41 | binbase = edgefs 42 | 43 | binaries = $(binbase) 44 | 45 | manpages = $(addsuffix .1,$(binaries)) 46 | 47 | intermediates += $(addsuffix .xml,$(manpages)) 48 | 49 | targets = $(binaries) $(manpages) 50 | 51 | all: libcachemap.so.0.0 $(targets) 52 | 53 | cachemap: 54 | mkdir -p $@ 55 | 56 | libcachemap.so.0.0: cachemap 57 | $(MAKE) -C cachemap 58 | ln -sf cachemap/libcachemap.so.0.0 libcachemap.so.0.0 59 | ln -sf cachemap/libcachemap.so.0.0 libcachemap.so 60 | 61 | edgefs: libcachemap.so.0.0 edgefs.c 62 | $(CC) $(MAIN_CPPFLAGS) $(CPPFLAGS) $(MAIN_CFLAGS) $(CFLAGS) edgefs.c $(MAIN_LDFLAGS) $(THR_LDFLAGS) $(LDFLAGS) -o $@ 63 | 64 | edgefs%.1: edgefs.1 65 | ln -sf edgefs.1 $@ 66 | 67 | clean: 68 | make -C cachemap clean 69 | rm -f libcachemap.so* 70 | rm -f $(targets) $(intermediates) 71 | rm -rf ./$(pkg_dir) ./BUILD ./BUILDROOT ./RPMS ./SOURCES ./SPECS ./SRPMS 72 | 73 | %.1: %.1.txt 74 | a2x -f manpage $< 75 | 76 | fstest: fstest.c 77 | $(CC) -Wall ${CFLAGS} fstest.c -o fstest 78 | 79 | ifeq ($(DISTID), ubuntu) 80 | 81 | # Rules to automatically make a Debian package 82 | 83 | pkg_dir = pkgdeb 84 | package = $(shell dpkg-parsechangelog | grep ^Source: | sed -e s,'^Source: ',,) 85 | version = $(shell dpkg-parsechangelog | grep ^Version: | sed -e s,'^Version: ',, -e 's,-.*,,') 86 | revision = $(shell dpkg-parsechangelog | grep ^Version: | sed -e -e 's,.*-,,') 87 | architecture = $(shell dpkg --print-architecture) 88 | tar_dir = $(package)-$(version) 89 | tar_gz = $(tar_dir).tar.gz 90 | unpack_dir = $(pkg_dir)/$(tar_dir) 91 | orig_tar_gz = $(pkg_dir)/$(package)_$(version).orig.tar.gz 92 | pkg_deb_src = $(pkg_dir)/$(package)_$(version)-$(revision)_source.changes 93 | pkg_deb_bin = $(pkg_dir)/$(package)_$(version)-$(revision)_$(architecture).changes 94 | 95 | deb_pkg_key = CB8C5858 96 | 97 | debclean: 98 | rm -rf $(pkg_dir) 99 | 100 | deb: debsrc debbin 101 | 102 | debbin: $(unpack_dir) 103 | cd $(unpack_dir) && dpkg-buildpackage -b -k$(deb_pkg_key) 104 | 105 | debsrc: $(unpack_dir) 106 | cd $(unpack_dir) && dpkg-buildpackage -S -k$(deb_pkg_key) 107 | 108 | $(unpack_dir): $(orig_tar_gz) 109 | tar -zxf $(orig_tar_gz) -C $(pkg_dir) 110 | 111 | $(pkg_dir): 112 | mkdir $(pkg_dir) 113 | 114 | $(pkg_dir)/$(tar_gz): $(pkg_dir) 115 | git archive --format=tar.gz --prefix=$(package)-$(version)/ -o $(pkg_dir)/$(tar_gz) HEAD 116 | 117 | $(orig_tar_gz): $(pkg_dir)/$(tar_gz) 118 | ln -s $(tar_gz) $(orig_tar_gz) 119 | 120 | else 121 | 122 | pkg_dir = SOURCES 123 | package = $(binbase) 124 | version = $(shell cat edgefs.spec |awk '/Version:/{print $$2}') 125 | tar_gz = $(package)-$(version).tar.gz 126 | 127 | rpm: rpmbin 128 | 129 | rpmbin: $(pkg_dir)/$(tar_gz) 130 | rpmbuild --quiet --define "_topdir `pwd`" -ba 'edgefs.spec' 131 | 132 | $(pkg_dir): 133 | mkdir $(pkg_dir) 134 | 135 | $(pkg_dir)/$(tar_gz): $(pkg_dir) 136 | git archive --format=tar.gz --prefix=$(package)-$(version)/ -o $(pkg_dir)/$(tar_gz) HEAD 137 | 138 | endif 139 | 140 | .PHONY: all $(SUBDIRS) 141 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NexentaEdge Edge-X S3 POSIX compatible file system 2 | 3 | High-Performance FUSE library to access Edge-X S3 API. 4 | 5 | Tested and supported platforms: 6 | 7 | - Linux x86 64-bit 8 | - Mac OSX x86 64-bit 9 | - Linux ARM 64-bit, mobile platforms (coming soon) 10 | - Windows x86 64-bit (coming soon) 11 | 12 | NexentaEdge Extended S3 API provides unique benefits which can be useful for Machine Deep Learning, Big Data and IoT frameworks: 13 | 14 | * Mount S3 objects for fast File/POSIX access avoid unnecessary copy, fetch only needed datasets 15 | * Optimized for local acces with fast Level-2 cache on SSD/NVMe 16 | * Extended S3 feature set: Append, Range Writes, Object/Bucket snapshots, Key-Value Object access 17 | * Data Reduction with global inline de-duplication, compression and erasure encoding 18 | * Cost Reduction File/Block/DB access with S3 economics 19 | 20 | ## Use cases details 21 | 22 | * Advanced Versioned S3 Object Append and RW "Object as File" access 23 | * S3 Object as a Key-Value database, including integrations w/ Caffe, TensorFlow, Spark, Kafka, etc 24 | * High-performance Versioned S3 Object Stream Session (RW), including FUSE library to mount an object 25 | * Management API for Snapshots and Clones, including Bucket instantaneous snapshots 26 | * Transparent NFS to/from S3 bucket access, “ingest via NFS, read via S3” or vice-versa 27 | 28 | Comparision to existing cloud object storage APIs: 29 | 30 | ![fig1: EdgeVsS3](https://raw.githubusercontent.com/nexenta/nedge-dev/master/images/EdgeVsS3.png) 31 | 32 | ## Quick start 33 | 34 | Give Edge-X S3 a try in easy to run single command installation: 35 | 36 | ```bash 37 | # location where to keep blobs 38 | mkdir /var/tmp/data 39 | 40 | # start nexenta/nedge daemon and Edge-X S3 compatible service 41 | docker run --name s3data -v /etc/localtime:/etc/localtime:ro -v /var/tmp/data:/data -d \ 42 | nexenta/nedge start -j ccowserv -j ccowgws3 43 | 44 | ``` 45 | 46 | Follow up with our Community! Please join us at the [NexentaEdge Devops community](https://community.nexenta.com/s/topic/0TOU0000000brtXOAQ/nexentaedge) site. 47 | 48 | * [Register DevOps account and obtain license key here](https://community.nexenta.com/s/devops-edition) 49 | * Use e-mailed ACTIVATION_KEY to activate installation 50 | 51 | The following are the steps to initialize, setup region namespace, tenant, service: 52 | 53 | ```bash 54 | # setup alias for easy CLI style management 55 | alias neadm="docker exec -it s3data neadm" 56 | 57 | # verify that service is running 58 | neadm system status 59 | 60 | # initialize and setup devops license 61 | neadm system init 62 | neadm system license set online ACTIVATION_KEY 63 | 64 | # setup simple Edge-X S3 service 65 | neadm cluster create region1 66 | neadm tenant create region1/tenant1 67 | neadm bucket create region1/tenant1/bk1 68 | neadm service create s3 s3svc 69 | neadm service serve s3svc region1/tenant1 70 | neadm service add s3svc SID # use neadm system status to find out server id 71 | neadm service restart s3svc 72 | neadm service show s3svc 73 | 74 | # assuming that default Docker bridge address asigned to container 75 | # is 172.17.0.3 verify that Edge-X S3 port is listening 76 | curl http://172.17.0.3:9982 77 | ``` 78 | 79 | Setup GUI for easy on-going management and monitoring: 80 | 81 | ```bash 82 | docker run -e API_ENDPOINT=http://172.17.0.3:8080 -p 3000:3000 \ 83 | nexenta/nedgeui:2.1.0 84 | ``` 85 | 86 | * Point browser to the host's port 3000 87 | * Default user/password: admin/nexenta 88 | * You know show be able to manage and monitor your simple single node cluster! 89 | 90 | ![fig2: gui-s3svc](https://raw.githubusercontent.com/nexenta/nedge-dev/master/images/nedgeui-s3svc.png) 91 | 92 | # Mount Edge-X S3 bucket for R/W access 93 | 94 | While mounted, objects remain versioned, searchable and globally accessible (multi-site replication case). 95 | 96 | ## Compile edgefs binary 97 | 98 | ```bash 99 | make all 100 | ``` 101 | 102 | ## Build Debian package 103 | 104 | ```bash 105 | make deb 106 | ``` 107 | 108 | ## Build RPM package 109 | 110 | ```bash 111 | make rpm 112 | ``` 113 | 114 | ## Install package or binary and mount bucket 115 | 116 | ```bash 117 | mkdir /mnt/bk1 118 | edgefs -c - -f http://172.17.0.3:9982/bk1 /mnt/bk1 119 | ``` 120 | 121 | At this point EdgeFS module emulates POSIX access to S3 bucket and would use Extended Edge API to enable high performance R/W access at /mnt/bk1 mount point. At the moment, we only emulate flat bucket operations. 122 | 123 | ## Regression tests 124 | 125 | Build fstest utility and execute TAP tests from the mount point, example: 126 | 127 | ```bash 128 | cd /mnt/bk1 129 | prove -r /path/to/edge-fuse/tests 130 | ``` 131 | 132 | Learn more about [Edge-X S3 API here](https://edgex.docs.apiary.io). 133 | 134 | Ask immediate question on [NexentaEdge Developers Channel](https://nexentaedge.slack.com/messages/general/) 135 | 136 | **Note:** The full documentation for NexentaEdge Enterprise Edition is [available here](https://nexenta.com/products/nexentaedge). 137 | -------------------------------------------------------------------------------- /cachemap/cachemap.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "cachemap.h" 9 | 10 | static inline uint64_t 11 | get_time_ns() { 12 | struct timespec tp; 13 | (void)clock_gettime(CLOCK_REALTIME_COARSE, &tp); 14 | return ((tp.tv_sec * 1000000000UL) + tp.tv_nsec); 15 | } 16 | 17 | static void 18 | cachemap_put_work(struct cachemap *cm, struct addr_work *w) 19 | { 20 | // is the cachemap full, evict 21 | if (filemap_entries(cm->pages) >= cm->capacity) { 22 | uint64_t a, b, c; 23 | uint128_t addra, addrb, addrc; 24 | 25 | // select MIN from 3 random timestamps 26 | filemap_get_rand(cm->pages, &addra, &a); 27 | filemap_get_rand(cm->pages, &addrb, &b); 28 | filemap_get_rand(cm->pages, &addrc, &c); 29 | 30 | uint128_t *smallest_addr; 31 | if (a < b) { 32 | if (a > c) { 33 | smallest_addr = &addrc; 34 | } else { // a <= c 35 | smallest_addr = &addra; 36 | } 37 | } else { // a >= b 38 | if (b > c) { 39 | smallest_addr = &addrc; 40 | } else { // b <= c 41 | smallest_addr = &addrb; 42 | } 43 | } 44 | filemap_unset(cm->pages, smallest_addr); 45 | } 46 | 47 | filemap_set(cm->pages, &w->addr, w->page, w->ts); 48 | } 49 | 50 | static void 51 | addr_enqueue(struct cachemap *cm, struct addr_work *w) 52 | { 53 | struct addr_node *temp = (struct addr_node *)malloc(sizeof(struct addr_node)); 54 | temp->work = *w; 55 | temp->next = NULL; 56 | 57 | if (cm->front == NULL && cm->rear == NULL) { 58 | cm->front = cm->rear = temp; 59 | return; 60 | } 61 | cm->rear->next = temp; 62 | cm->rear = temp; 63 | } 64 | 65 | static int 66 | addr_dequeue(struct cachemap *cm, struct addr_work *w_out) 67 | { 68 | struct addr_node *temp = cm->front; 69 | 70 | if (cm->front == NULL) { 71 | return 0; 72 | } 73 | 74 | if (cm->front == cm->rear) { 75 | cm->front = cm->rear = NULL; 76 | } else { 77 | cm->front = cm->front->next; 78 | } 79 | *w_out = temp->work; 80 | free(temp); 81 | return 1; 82 | } 83 | 84 | static void 85 | cachemap_thread(void *arg) 86 | { 87 | struct cachemap *cm = arg; 88 | struct addr_work work; 89 | 90 | pthread_mutex_lock(&cm->cm_mutex); 91 | while (cm->front || !cm->cm_thread_stop) { 92 | if (!cm->front) 93 | pthread_cond_wait(&cm->step_condvar, &cm->cm_mutex); 94 | if (!addr_dequeue(cm, &work)) 95 | continue; 96 | pthread_mutex_unlock(&cm->cm_mutex); 97 | 98 | cachemap_put_work(cm, &work); 99 | free(work.page); 100 | 101 | pthread_mutex_lock(&cm->cm_mutex); 102 | } 103 | pthread_mutex_unlock(&cm->cm_mutex); 104 | pthread_exit(0); 105 | } 106 | 107 | struct cachemap * 108 | cachemap_create(char *destdir, uint64_t capacity, int comp_accel, int pshift) 109 | { 110 | struct stat sb; 111 | int err; 112 | 113 | if (!(stat(destdir, &sb) == 0) || !S_ISDIR(sb.st_mode)) 114 | return NULL; 115 | 116 | struct cachemap *cm = calloc(1, sizeof(struct cachemap)); 117 | 118 | cm->pages = filemap_create(destdir, capacity, comp_accel, pshift); 119 | if (!cm->pages) { 120 | goto _exit; 121 | } 122 | 123 | for (int i = 0; i < PUT_THREADS; i++) { 124 | err = pthread_create(&cm->cm_thread[i], NULL, 125 | (void *)&cachemap_thread, (void *)cm); 126 | if (err) { 127 | goto _exit; 128 | } 129 | } 130 | 131 | err = pthread_mutex_init(&cm->cm_mutex, NULL); 132 | if (err) { 133 | goto _exit; 134 | } 135 | 136 | err = pthread_cond_init(&cm->step_condvar, NULL); 137 | if (err) { 138 | goto _exit; 139 | } 140 | 141 | cm->capacity = capacity; 142 | 143 | return cm; 144 | _exit: 145 | if (cm->pages) 146 | filemap_free(cm->pages); 147 | free(cm); 148 | return NULL; 149 | } 150 | 151 | static inline int 152 | to_uint128_addr(struct cachemap *cm, uint64_t offset, uint64_t nhid_small, 153 | uint32_t genid, uint128_t *addr_out) 154 | { 155 | #define PNUM_SHIFT 44 156 | uint64_t l = offset >> cm->pages->pshift; 157 | 158 | if (l >> PNUM_SHIFT) 159 | return -1; 160 | 161 | l |= ((uint64_t)genid << PNUM_SHIFT); 162 | 163 | addr_out->l = l; 164 | addr_out->u = nhid_small; 165 | return 0; 166 | } 167 | 168 | void * 169 | cachemap_get(struct cachemap *cm, uint64_t offset, uint64_t nhid_small, 170 | uint32_t genid) 171 | { 172 | uint128_t addr; 173 | if (to_uint128_addr(cm, offset, nhid_small, genid, &addr) != 0) 174 | return NULL; 175 | 176 | cm->requests++; 177 | 178 | // is page already in cachemap - hit 179 | void *page = filemap_get(cm->pages, &addr); 180 | if (page) { 181 | cm->hits++; 182 | } 183 | return page; 184 | } 185 | 186 | void 187 | cachemap_put(struct cachemap *cm, uint64_t offset, uint64_t nhid_small, 188 | uint32_t genid, const void *page) 189 | { 190 | struct addr_work work; 191 | if (to_uint128_addr(cm, offset, nhid_small, genid, &work.addr) != 0) 192 | return; 193 | 194 | work.page = (char*)page; 195 | work.ts = get_time_ns(); 196 | cachemap_put_work(cm, &work); 197 | } 198 | 199 | void 200 | cachemap_put_async(struct cachemap *cm, uint64_t offset, uint64_t nhid_small, 201 | uint32_t genid, const void *page) 202 | { 203 | struct addr_work work; 204 | if (to_uint128_addr(cm, offset, nhid_small, genid, &work.addr) != 0) 205 | return; 206 | 207 | work.page = malloc(cm->pages->bsize); 208 | memcpy(work.page, page, cm->pages->bsize); 209 | 210 | work.ts = get_time_ns(); 211 | 212 | pthread_mutex_lock(&cm->cm_mutex); 213 | addr_enqueue(cm, &work); 214 | pthread_cond_broadcast(&cm->step_condvar); 215 | pthread_mutex_unlock(&cm->cm_mutex); 216 | } 217 | 218 | void 219 | cachemap_free(struct cachemap *cm) 220 | { 221 | pthread_mutex_lock(&cm->cm_mutex); 222 | cm->cm_thread_stop = 1; 223 | pthread_cond_broadcast(&cm->step_condvar); 224 | pthread_mutex_unlock(&cm->cm_mutex); 225 | 226 | for (int i = 0; i < PUT_THREADS; i++) { 227 | pthread_join(cm->cm_thread[i], NULL); 228 | } 229 | pthread_mutex_destroy(&cm->cm_mutex); 230 | filemap_free(cm->pages); 231 | free(cm); 232 | } 233 | 234 | void 235 | cachemap_print_stats(struct cachemap *cm) 236 | { 237 | printf("requests: %lu, hits: %lu, ratio: %5.2f\n", 238 | cm->requests, cm->hits, cm->hits*100/(float)cm->requests); 239 | } 240 | -------------------------------------------------------------------------------- /cachemap/midl.h: -------------------------------------------------------------------------------- 1 | /** @file midl.h 2 | * @brief LMDB ID List header file. 3 | * 4 | * This file was originally part of back-bdb but has been 5 | * modified for use in libmdb. Most of the macros defined 6 | * in this file are unused, just left over from the original. 7 | * 8 | * This file is only used internally in libmdb and its definitions 9 | * are not exposed publicly. 10 | */ 11 | /* $OpenLDAP$ */ 12 | /* This work is part of OpenLDAP Software . 13 | * 14 | * Copyright 2000-2016 The OpenLDAP Foundation. 15 | * Portions Copyright 2001-2017 Howard Chu, Symas Corp. 16 | * All rights reserved. 17 | * 18 | * Redistribution and use in source and binary forms, with or without 19 | * modification, are permitted only as authorized by the OpenLDAP 20 | * Public License. 21 | * 22 | * A copy of this license is available in the file LICENSE in the 23 | * top-level directory of the distribution or, alternatively, at 24 | * . 25 | */ 26 | 27 | #ifndef _MDB_MIDL_H_ 28 | #define _MDB_MIDL_H_ 29 | 30 | #include 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | /** @defgroup internal LMDB Internals 37 | * @{ 38 | */ 39 | 40 | /** @defgroup idls ID List Management 41 | * @{ 42 | */ 43 | /** A generic unsigned ID number. These were entryIDs in back-bdb. 44 | * Preferably it should have the same size as a pointer. 45 | */ 46 | typedef size_t MDB_ID; 47 | 48 | /** An IDL is an ID List, a sorted array of IDs. The first 49 | * element of the array is a counter for how many actual 50 | * IDs are in the list. In the original back-bdb code, IDLs are 51 | * sorted in ascending order. For libmdb IDLs are sorted in 52 | * descending order. 53 | */ 54 | typedef MDB_ID *MDB_IDL; 55 | 56 | /* IDL sizes - likely should be even bigger 57 | * limiting factors: sizeof(ID), thread stack size 58 | */ 59 | #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ 60 | #define MDB_IDL_DB_SIZE (1< 2 | #include 3 | #include 4 | #include 5 | 6 | #include "filemap.h" 7 | #include "lz4.h" 8 | 9 | struct data_prefix { 10 | uint128_t addr; 11 | int compressed_length; 12 | }; 13 | 14 | #define DEBUG_ON 0 15 | #define DBG_TRACE() \ 16 | if (DEBUG_ON) printf("%s:%d : err %d\n", __func__, __LINE__, err) 17 | 18 | static inline uint64_t 19 | addr_key(uint128_t *addr) 20 | { 21 | uint64_t hv; 22 | FNV_hash(addr, sizeof(uint128_t), &hv); 23 | return hv; 24 | } 25 | 26 | static inline MDB_env * 27 | filemap_get_shard(struct filemap *m, uint128_t *addr, uint64_t *hv_out) 28 | { 29 | uint64_t hv = addr_key(addr); 30 | int i = hv & (FILEMAP_SHARD_NUM - 1); 31 | *hv_out = hv; 32 | return m->env[i]; 33 | } 34 | 35 | struct filemap * 36 | filemap_create(char *destdir, uint64_t n, int compress_accel, int pshift) 37 | { 38 | int err, i; 39 | MDB_dbi dbi = 0; 40 | MDB_txn *txn = NULL; 41 | struct filemap *m = (struct filemap*)calloc(1, sizeof(struct filemap)); 42 | m->n = n; 43 | m->compress = compress_accel; 44 | m->bsize = 1 << pshift; 45 | m->pshift = pshift; 46 | 47 | strcpy(m->destdir, destdir); 48 | 49 | uint64_t shard_n = n / FILEMAP_SHARD_NUM; 50 | 51 | if (n < FILEMAP_SHARD_FACTOR) 52 | goto _exit; 53 | 54 | for (i = 0; i < FILEMAP_SHARD_NUM; i++) { 55 | char dbpath[2048]; 56 | 57 | sprintf(dbpath, "%s/filemap.%d", m->destdir, i); 58 | 59 | err = mdb_env_create(&m->env[i]); 60 | if (err) 61 | goto _exit; 62 | 63 | err = mdb_env_set_maxreaders(m->env[i], 32); 64 | if (err) 65 | goto _exit; 66 | 67 | err = mdb_env_set_mapsize(m->env[i], 4 * shard_n * m->bsize); 68 | if (err) 69 | goto _exit; 70 | 71 | err = mdb_env_open(m->env[i], dbpath, 72 | MDB_NOTLS | MDB_NOSYNC | MDB_NOSUBDIR | MDB_NORDAHEAD | MDB_NOMEMINIT, 0664); 73 | if (err) 74 | goto _exit; 75 | 76 | err = mdb_txn_begin(m->env[i], NULL, 0, &txn); 77 | if (err) 78 | goto _exit; 79 | 80 | err = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_INTEGERKEY, &dbi); 81 | if (err) 82 | goto _exit; 83 | 84 | err = mdb_txn_commit(txn); 85 | if (err) 86 | goto _exit; 87 | 88 | dbi = 0; 89 | txn = NULL; 90 | } 91 | return m; 92 | 93 | _exit: 94 | if (dbi) 95 | mdb_dbi_close(m->env[i], dbi); 96 | if (txn) 97 | mdb_txn_abort(txn); 98 | filemap_free(m); 99 | DBG_TRACE(); 100 | return NULL; 101 | } 102 | 103 | void 104 | filemap_free(struct filemap *m) 105 | { 106 | for (int i = 0; i < FILEMAP_SHARD_NUM; i++) { 107 | mdb_env_close(m->env[i]); 108 | } 109 | free(m); 110 | } 111 | 112 | void 113 | filemap_set(struct filemap *m, uint128_t *addr, void *value, uint64_t attr) 114 | { 115 | int err; 116 | uint64_t key; 117 | MDB_env *env = filemap_get_shard(m, addr, &key); 118 | MDB_txn *txn = NULL; 119 | char dest[m->bsize + 1024]; 120 | int actual; 121 | char *value_ptr; 122 | size_t value_size; 123 | 124 | if (m->compress) { 125 | /* compress outside of transaction to keep it scope short */ 126 | actual = LZ4_compress_fast(value, &dest[0], m->bsize, m->bsize + 1024, 127 | m->compress); 128 | value_ptr = &dest[0]; 129 | value_size = actual; 130 | } else { 131 | actual = 0; 132 | value_ptr = value; 133 | value_size = m->bsize; 134 | } 135 | 136 | err = mdb_txn_begin(env, NULL, 0, &txn); 137 | if (err) 138 | goto _exit; 139 | 140 | struct data_prefix prefix = { .addr = *addr, .compressed_length = actual }; 141 | MDB_val k = { .mv_size = sizeof(uint64_t), .mv_data = &key }; 142 | MDB_val page = { .mv_size = sizeof(struct data_prefix) + value_size, .mv_data = NULL }; 143 | err = mdb_put_attr(txn, 1, &k, &page, attr, MDB_RESERVE); 144 | if (err) 145 | goto _exit; 146 | memcpy(page.mv_data, &prefix, sizeof(struct data_prefix)); 147 | memcpy((char*)page.mv_data + sizeof(struct data_prefix), value_ptr, value_size); 148 | 149 | err = mdb_txn_commit(txn); 150 | if (err) 151 | goto _exit; 152 | 153 | return; 154 | _exit: 155 | if (txn) 156 | mdb_txn_abort(txn); 157 | DBG_TRACE(); 158 | } 159 | 160 | void 161 | filemap_set_attr(struct filemap *m, uint128_t *addr, uint64_t attr) 162 | { 163 | int err; 164 | uint64_t key; 165 | MDB_env *env = filemap_get_shard(m, addr, &key); 166 | MDB_txn *txn = NULL; 167 | 168 | err = mdb_txn_begin(env, NULL, 0, &txn); 169 | if (err) 170 | goto _exit; 171 | 172 | MDB_val k = { .mv_size = sizeof(uint64_t), .mv_data = &key }; 173 | err = mdb_set_attr(txn, 1, &k, NULL, attr); 174 | if (err) 175 | goto _exit; 176 | 177 | err = mdb_txn_commit(txn); 178 | if (err) 179 | goto _exit; 180 | 181 | return; 182 | _exit: 183 | if (txn) 184 | mdb_txn_abort(txn); 185 | DBG_TRACE(); 186 | } 187 | 188 | void 189 | filemap_unset(struct filemap *m, uint128_t *addr) 190 | { 191 | int err; 192 | uint64_t key; 193 | MDB_env *env = filemap_get_shard(m, addr, &key); 194 | MDB_txn *txn = NULL; 195 | 196 | err = mdb_txn_begin(env, NULL, 0, &txn); 197 | if (err) 198 | goto _exit; 199 | 200 | MDB_val k = { .mv_size = sizeof(uint64_t), .mv_data = &key }; 201 | err = mdb_del(txn, 1, &k, NULL); 202 | if (err) 203 | goto _exit; 204 | 205 | err = mdb_txn_commit(txn); 206 | if (err) 207 | goto _exit; 208 | 209 | return; 210 | _exit: 211 | if (txn) 212 | mdb_txn_abort(txn); 213 | if (err != MDB_NOTFOUND) 214 | DBG_TRACE(); 215 | } 216 | 217 | void * 218 | filemap_get(struct filemap *m, uint128_t *addr) 219 | { 220 | int err; 221 | uint64_t key; 222 | MDB_env *env = filemap_get_shard(m, addr, &key); 223 | MDB_txn *txn = NULL; 224 | 225 | err = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); 226 | if (err) 227 | goto _exit; 228 | 229 | MDB_val k = { .mv_size = sizeof(uint64_t), .mv_data = &key }; 230 | MDB_val page; 231 | err = mdb_get(txn, 1, &k, &page); 232 | if (err) 233 | goto _exit; 234 | struct data_prefix *prefix = (struct data_prefix *)page.mv_data; 235 | 236 | if (memcmp(&prefix->addr, addr, sizeof(uint128_t)) != 0) { 237 | printf("bad entry\n"); 238 | err = MDB_NOTFOUND; 239 | goto _exit; 240 | } 241 | 242 | void *data = malloc(m->bsize); 243 | if (prefix->compressed_length) { 244 | int actual = LZ4_decompress_fast((char *)page.mv_data + sizeof(struct data_prefix), data, m->bsize); 245 | if (actual != prefix->compressed_length) { 246 | free(data); 247 | goto _exit; 248 | } 249 | } else { 250 | memcpy(data, (char *)page.mv_data + sizeof(struct data_prefix), m->bsize); 251 | } 252 | 253 | mdb_txn_abort(txn); 254 | 255 | return data; 256 | _exit: 257 | if (txn) 258 | mdb_txn_abort(txn); 259 | if (err != MDB_NOTFOUND) 260 | DBG_TRACE(); 261 | return NULL; 262 | } 263 | 264 | int 265 | filemap_get_rand(struct filemap *m, uint128_t *addr, uint64_t *attrp) 266 | { 267 | int err; 268 | MDB_txn *txn = NULL; 269 | MDB_cursor *cursor = NULL; 270 | 271 | uint64_t key = 0; 272 | for (int i = 0; i < 64; i += 30) { 273 | key = key * ((uint64_t)RAND_MAX + 1) + rand(); 274 | } 275 | 276 | int i = key & (FILEMAP_SHARD_NUM - 1); 277 | MDB_env *env = m->env[i]; 278 | 279 | err = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); 280 | if (err) 281 | goto _exit; 282 | 283 | err = mdb_cursor_open(txn, 1, &cursor); 284 | if (err) 285 | goto _exit; 286 | 287 | MDB_val k = { .mv_size = sizeof(uint64_t), .mv_data = &key }; 288 | err = mdb_cursor_get(cursor, &k, NULL, MDB_SET_RANGE); 289 | if (err == MDB_NOTFOUND) 290 | err = mdb_cursor_get(cursor, &k, NULL, MDB_PREV); 291 | if (err) 292 | goto _exit; 293 | key = *(uint64_t *)k.mv_data; 294 | k.mv_data = &key; 295 | 296 | MDB_val v; 297 | err = mdb_cursor_get_attr(cursor, &k, &v, attrp); 298 | if (err) 299 | goto _exit; 300 | *addr = *(uint128_t *)v.mv_data; 301 | 302 | mdb_cursor_close(cursor); 303 | 304 | mdb_txn_abort(txn); 305 | 306 | return 1; 307 | _exit: 308 | if (cursor) 309 | mdb_cursor_close(cursor); 310 | if (txn) 311 | mdb_txn_abort(txn); 312 | DBG_TRACE(); 313 | return 0; 314 | } 315 | 316 | uint64_t 317 | filemap_entries(struct filemap *m) 318 | { 319 | int err; 320 | MDB_stat stat; 321 | uint64_t entries = 0; 322 | 323 | for (int i = 0; i < FILEMAP_SHARD_NUM; i++) { 324 | err = mdb_env_stat(m->env[i], &stat); 325 | if (err) 326 | return 0; 327 | entries += stat.ms_entries; 328 | } 329 | return entries; 330 | } 331 | -------------------------------------------------------------------------------- /cachemap/midl.c: -------------------------------------------------------------------------------- 1 | /** @file midl.c 2 | * @brief ldap bdb back-end ID List functions */ 3 | /* $OpenLDAP$ */ 4 | /* This work is part of OpenLDAP Software . 5 | * 6 | * Copyright 2000-2016 The OpenLDAP Foundation. 7 | * Portions Copyright 2001-2017 Howard Chu, Symas Corp. 8 | * All rights reserved. 9 | * 10 | * Redistribution and use in source and binary forms, with or without 11 | * modification, are permitted only as authorized by the OpenLDAP 12 | * Public License. 13 | * 14 | * A copy of this license is available in the file LICENSE in the 15 | * top-level directory of the distribution or, alternatively, at 16 | * . 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #ifdef USE_JEMALLOC 25 | #include 26 | #else 27 | #define je_malloc malloc 28 | #define je_free free 29 | #define je_calloc calloc 30 | #define je_realloc realloc 31 | #endif 32 | #include "midl.h" 33 | 34 | /** @defgroup internal LMDB Internals 35 | * @{ 36 | */ 37 | /** @defgroup idls ID List Management 38 | * @{ 39 | */ 40 | #define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) 41 | 42 | unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) 43 | { 44 | /* 45 | * binary search of id in ids 46 | * if found, returns position of id 47 | * if not found, returns first position greater than id 48 | */ 49 | unsigned base = 0; 50 | unsigned cursor = 1; 51 | int val = 0; 52 | unsigned n = ids[0]; 53 | 54 | while( 0 < n ) { 55 | unsigned pivot = n >> 1; 56 | cursor = base + pivot + 1; 57 | val = CMP( ids[cursor], id ); 58 | 59 | if( val < 0 ) { 60 | n = pivot; 61 | 62 | } else if ( val > 0 ) { 63 | base = cursor; 64 | n -= pivot + 1; 65 | 66 | } else { 67 | return cursor; 68 | } 69 | } 70 | 71 | if( val > 0 ) { 72 | ++cursor; 73 | } 74 | return cursor; 75 | } 76 | 77 | #if 0 /* superseded by append/sort */ 78 | int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) 79 | { 80 | unsigned x, i; 81 | 82 | x = mdb_midl_search( ids, id ); 83 | assert( x > 0 ); 84 | 85 | if( x < 1 ) { 86 | /* internal error */ 87 | return -2; 88 | } 89 | 90 | if ( x <= ids[0] && ids[x] == id ) { 91 | /* duplicate */ 92 | assert(0); 93 | return -1; 94 | } 95 | 96 | if ( ++ids[0] >= MDB_IDL_DB_MAX ) { 97 | /* no room */ 98 | --ids[0]; 99 | return -2; 100 | 101 | } else { 102 | /* insert id */ 103 | for (i=ids[0]; i>x; i--) 104 | ids[i] = ids[i-1]; 105 | ids[x] = id; 106 | } 107 | 108 | return 0; 109 | } 110 | #endif 111 | 112 | MDB_IDL mdb_midl_alloc(int num) 113 | { 114 | MDB_IDL ids = je_malloc((num+2) * sizeof(MDB_ID)); 115 | if (ids) { 116 | *ids++ = num; 117 | *ids = 0; 118 | } 119 | return ids; 120 | } 121 | 122 | void mdb_midl_free(MDB_IDL ids) 123 | { 124 | if (ids) 125 | je_free(ids-1); 126 | } 127 | 128 | void mdb_midl_shrink( MDB_IDL *idp ) 129 | { 130 | MDB_IDL ids = *idp; 131 | if (*(--ids) > MDB_IDL_UM_MAX && 132 | (ids = je_realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) 133 | { 134 | *ids++ = MDB_IDL_UM_MAX; 135 | *idp = ids; 136 | } 137 | } 138 | 139 | static int mdb_midl_grow( MDB_IDL *idp, int num ) 140 | { 141 | MDB_IDL idn = *idp-1; 142 | /* grow it */ 143 | idn = je_realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); 144 | if (!idn) 145 | return ENOMEM; 146 | *idn++ += num; 147 | *idp = idn; 148 | return 0; 149 | } 150 | 151 | int mdb_midl_need( MDB_IDL *idp, unsigned num ) 152 | { 153 | MDB_IDL ids = *idp; 154 | num += ids[0]; 155 | if (num > ids[-1]) { 156 | num = (num + num/4 + (256 + 2)) & -256; 157 | if (!(ids = je_realloc(ids-1, num * sizeof(MDB_ID)))) 158 | return ENOMEM; 159 | *ids++ = num - 2; 160 | *idp = ids; 161 | } 162 | return 0; 163 | } 164 | 165 | int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) 166 | { 167 | MDB_IDL ids = *idp; 168 | /* Too big? */ 169 | if (ids[0] >= ids[-1]) { 170 | if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) 171 | return ENOMEM; 172 | ids = *idp; 173 | } 174 | ids[0]++; 175 | ids[ids[0]] = id; 176 | return 0; 177 | } 178 | 179 | int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) 180 | { 181 | MDB_IDL ids = *idp; 182 | /* Too big? */ 183 | if (ids[0] + app[0] >= ids[-1]) { 184 | if (mdb_midl_grow(idp, app[0])) 185 | return ENOMEM; 186 | ids = *idp; 187 | } 188 | memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); 189 | ids[0] += app[0]; 190 | return 0; 191 | } 192 | 193 | int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) 194 | { 195 | MDB_ID *ids = *idp, len = ids[0]; 196 | /* Too big? */ 197 | if (len + n > ids[-1]) { 198 | if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) 199 | return ENOMEM; 200 | ids = *idp; 201 | } 202 | ids[0] = len + n; 203 | ids += len; 204 | while (n) 205 | ids[n--] = id++; 206 | return 0; 207 | } 208 | 209 | void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) 210 | { 211 | MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; 212 | idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ 213 | old_id = idl[j]; 214 | while (i) { 215 | merge_id = merge[i--]; 216 | for (; old_id < merge_id; old_id = idl[--j]) 217 | idl[k--] = old_id; 218 | idl[k--] = merge_id; 219 | } 220 | idl[0] = total; 221 | } 222 | 223 | /* Quicksort + Insertion sort for small arrays */ 224 | 225 | #define SMALL 8 226 | #define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } 227 | 228 | void 229 | mdb_midl_sort( MDB_IDL ids ) 230 | { 231 | /* Max possible depth of int-indexed tree * 2 items/level */ 232 | int istack[sizeof(int)*CHAR_BIT * 2]; 233 | int i,j,k,l,ir,jstack; 234 | MDB_ID a, itmp; 235 | 236 | ir = (int)ids[0]; 237 | l = 1; 238 | jstack = 0; 239 | for(;;) { 240 | if (ir - l < SMALL) { /* Insertion sort */ 241 | for (j=l+1;j<=ir;j++) { 242 | a = ids[j]; 243 | for (i=j-1;i>=1;i--) { 244 | if (ids[i] >= a) break; 245 | ids[i+1] = ids[i]; 246 | } 247 | ids[i+1] = a; 248 | } 249 | if (jstack == 0) break; 250 | ir = istack[jstack--]; 251 | l = istack[jstack--]; 252 | } else { 253 | k = (l + ir) >> 1; /* Choose median of left, center, right */ 254 | MIDL_SWAP(ids[k], ids[l+1]); 255 | if (ids[l] < ids[ir]) { 256 | MIDL_SWAP(ids[l], ids[ir]); 257 | } 258 | if (ids[l+1] < ids[ir]) { 259 | MIDL_SWAP(ids[l+1], ids[ir]); 260 | } 261 | if (ids[l] < ids[l+1]) { 262 | MIDL_SWAP(ids[l], ids[l+1]); 263 | } 264 | i = l+1; 265 | j = ir; 266 | a = ids[l+1]; 267 | for(;;) { 268 | do i++; while(ids[i] > a); 269 | do j--; while(ids[j] < a); 270 | if (j < i) break; 271 | MIDL_SWAP(ids[i],ids[j]); 272 | } 273 | ids[l+1] = ids[j]; 274 | ids[j] = a; 275 | jstack += 2; 276 | if (ir-i+1 >= j-l) { 277 | istack[jstack] = ir; 278 | istack[jstack-1] = i; 279 | ir = j-1; 280 | } else { 281 | istack[jstack] = j-1; 282 | istack[jstack-1] = l; 283 | l = i; 284 | } 285 | } 286 | } 287 | } 288 | 289 | unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) 290 | { 291 | /* 292 | * binary search of id in ids 293 | * if found, returns position of id 294 | * if not found, returns first position greater than id 295 | */ 296 | unsigned base = 0; 297 | unsigned cursor = 1; 298 | int val = 0; 299 | unsigned n = (unsigned)ids[0].mid; 300 | 301 | while( 0 < n ) { 302 | unsigned pivot = n >> 1; 303 | cursor = base + pivot + 1; 304 | val = CMP( id, ids[cursor].mid ); 305 | 306 | if( val < 0 ) { 307 | n = pivot; 308 | 309 | } else if ( val > 0 ) { 310 | base = cursor; 311 | n -= pivot + 1; 312 | 313 | } else { 314 | return cursor; 315 | } 316 | } 317 | 318 | if( val > 0 ) { 319 | ++cursor; 320 | } 321 | return cursor; 322 | } 323 | 324 | int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) 325 | { 326 | unsigned x, i; 327 | 328 | x = mdb_mid2l_search( ids, id->mid ); 329 | 330 | if( x < 1 ) { 331 | /* internal error */ 332 | return -2; 333 | } 334 | 335 | if ( x <= ids[0].mid && ids[x].mid == id->mid ) { 336 | /* duplicate */ 337 | return -1; 338 | } 339 | 340 | if ( ids[0].mid >= MDB_IDL_UM_MAX ) { 341 | /* too big */ 342 | return -2; 343 | 344 | } else { 345 | /* insert id */ 346 | ids[0].mid++; 347 | for (i=(unsigned)ids[0].mid; i>x; i--) 348 | ids[i] = ids[i-1]; 349 | ids[x] = *id; 350 | } 351 | 352 | return 0; 353 | } 354 | 355 | int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) 356 | { 357 | /* Too big? */ 358 | if (ids[0].mid >= MDB_IDL_UM_MAX) { 359 | return -2; 360 | } 361 | ids[0].mid++; 362 | ids[ids[0].mid] = *id; 363 | return 0; 364 | } 365 | 366 | /** @} */ 367 | /** @} */ 368 | -------------------------------------------------------------------------------- /fstest.c: -------------------------------------------------------------------------------- 1 | /*- 2 | * Copyright (c) 2006-2007 Pawel Jakub Dawidek 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 | * SUCH DAMAGE. 25 | * 26 | * $FreeBSD: src/tools/regression/fstest/fstest.c,v 1.1 2007/01/17 01:42:07 pjd Exp $ 27 | */ 28 | 29 | #ifdef linux 30 | #define _GNU_SOURCE 31 | #endif 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | #ifndef HAS_TRUNCATE64 45 | #define truncate64 truncate 46 | #endif 47 | #ifndef HAS_STAT64 48 | #define stat64 stat 49 | #define lstat64 lstat 50 | #endif 51 | 52 | #ifndef ALLPERMS 53 | #define ALLPERMS (S_ISUID|S_ISGID|S_ISVTX|S_IRWXU|S_IRWXG|S_IRWXO) 54 | #endif 55 | 56 | enum action { 57 | ACTION_OPEN, 58 | ACTION_CREATE, 59 | ACTION_UNLINK, 60 | ACTION_MKDIR, 61 | ACTION_RMDIR, 62 | ACTION_LINK, 63 | ACTION_SYMLINK, 64 | ACTION_RENAME, 65 | ACTION_MKFIFO, 66 | ACTION_CHMOD, 67 | #ifdef HAS_LCHMOD 68 | ACTION_LCHMOD, 69 | #endif 70 | ACTION_CHOWN, 71 | ACTION_LCHOWN, 72 | #ifdef HAS_CHFLAGS 73 | ACTION_CHFLAGS, 74 | #endif 75 | #ifdef HAS_LCHFLAGS 76 | ACTION_LCHFLAGS, 77 | #endif 78 | ACTION_TRUNCATE, 79 | ACTION_STAT, 80 | ACTION_LSTAT, 81 | }; 82 | 83 | #define TYPE_NONE 0x0000 84 | #define TYPE_STRING 0x0001 85 | #define TYPE_NUMBER 0x0002 86 | 87 | #define TYPE_OPTIONAL 0x0100 88 | 89 | #define MAX_ARGS 8 90 | 91 | struct syscall_desc { 92 | char *sd_name; 93 | enum action sd_action; 94 | int sd_args[MAX_ARGS]; 95 | }; 96 | 97 | static struct syscall_desc syscalls[] = { 98 | { "open", ACTION_OPEN, { TYPE_STRING, TYPE_STRING, TYPE_NUMBER | TYPE_OPTIONAL, TYPE_NONE } }, 99 | { "create", ACTION_CREATE, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 100 | { "unlink", ACTION_UNLINK, { TYPE_STRING, TYPE_NONE } }, 101 | { "mkdir", ACTION_MKDIR, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 102 | { "rmdir", ACTION_RMDIR, { TYPE_STRING, TYPE_NONE } }, 103 | { "link", ACTION_LINK, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 104 | { "symlink", ACTION_SYMLINK, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 105 | { "rename", ACTION_RENAME, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 106 | { "mkfifo", ACTION_MKFIFO, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 107 | { "chmod", ACTION_CHMOD, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 108 | #ifdef HAS_LCHMOD 109 | { "lchmod", ACTION_LCHMOD, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 110 | #endif 111 | { "chown", ACTION_CHOWN, { TYPE_STRING, TYPE_NUMBER, TYPE_NUMBER, TYPE_NONE } }, 112 | { "lchown", ACTION_LCHOWN, { TYPE_STRING, TYPE_NUMBER, TYPE_NUMBER, TYPE_NONE } }, 113 | #ifdef HAS_CHFLAGS 114 | { "chflags", ACTION_CHFLAGS, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 115 | #endif 116 | #ifdef HAS_LCHFLAGS 117 | { "lchflags", ACTION_LCHFLAGS, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 118 | #endif 119 | { "truncate", ACTION_TRUNCATE, { TYPE_STRING, TYPE_NUMBER, TYPE_NONE } }, 120 | { "stat", ACTION_STAT, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 121 | { "lstat", ACTION_LSTAT, { TYPE_STRING, TYPE_STRING, TYPE_NONE } }, 122 | { NULL, -1, { TYPE_NONE } } 123 | }; 124 | 125 | struct flag { 126 | long long f_flag; 127 | char *f_str; 128 | }; 129 | 130 | static struct flag open_flags[] = { 131 | #ifdef O_RDONLY 132 | { O_RDONLY, "O_RDONLY" }, 133 | #endif 134 | #ifdef O_WRONLY 135 | { O_WRONLY, "O_WRONLY" }, 136 | #endif 137 | #ifdef O_RDWR 138 | { O_RDWR, "O_RDWR" }, 139 | #endif 140 | #ifdef O_NONBLOCK 141 | { O_NONBLOCK, "O_NONBLOCK" }, 142 | #endif 143 | #ifdef O_APPEND 144 | { O_APPEND, "O_APPEND" }, 145 | #endif 146 | #ifdef O_CREAT 147 | { O_CREAT, "O_CREAT" }, 148 | #endif 149 | #ifdef O_TRUNC 150 | { O_TRUNC, "O_TRUNC" }, 151 | #endif 152 | #ifdef O_EXCL 153 | { O_EXCL, "O_EXCL" }, 154 | #endif 155 | #ifdef O_SHLOCK 156 | { O_SHLOCK, "O_SHLOCK" }, 157 | #endif 158 | #ifdef O_EXLOCK 159 | { O_EXLOCK, "O_EXLOCK" }, 160 | #endif 161 | #ifdef O_DIRECT 162 | { O_DIRECT, "O_DIRECT" }, 163 | #endif 164 | #ifdef O_FSYNC 165 | { O_FSYNC, "O_FSYNC" }, 166 | #endif 167 | #ifdef O_SYNC 168 | { O_SYNC, "O_SYNC" }, 169 | #endif 170 | #ifdef O_NOFOLLOW 171 | { O_NOFOLLOW, "O_NOFOLLOW" }, 172 | #endif 173 | #ifdef O_NOCTTY 174 | { O_NOCTTY, "O_NOCTTY" }, 175 | #endif 176 | { 0, NULL } 177 | }; 178 | 179 | #ifdef HAS_CHFLAGS 180 | static struct flag chflags_flags[] = { 181 | #ifdef UF_NODUMP 182 | { UF_NODUMP, "UF_NODUMP" }, 183 | #endif 184 | #ifdef UF_IMMUTABLE 185 | { UF_IMMUTABLE, "UF_IMMUTABLE" }, 186 | #endif 187 | #ifdef UF_APPEND 188 | { UF_APPEND, "UF_APPEND" }, 189 | #endif 190 | #ifdef UF_NOUNLINK 191 | { UF_NOUNLINK, "UF_NOUNLINK" }, 192 | #endif 193 | #ifdef UF_OPAQUE 194 | { UF_OPAQUE, "UF_OPAQUE" }, 195 | #endif 196 | #ifdef SF_ARCHIVED 197 | { SF_ARCHIVED, "SF_ARCHIVED" }, 198 | #endif 199 | #ifdef SF_IMMUTABLE 200 | { SF_IMMUTABLE, "SF_IMMUTABLE" }, 201 | #endif 202 | #ifdef SF_APPEND 203 | { SF_APPEND, "SF_APPEND" }, 204 | #endif 205 | #ifdef SF_NOUNLINK 206 | { SF_NOUNLINK, "SF_NOUNLINK" }, 207 | #endif 208 | #ifdef SF_SNAPSHOT 209 | { SF_SNAPSHOT, "SF_SNAPSHOT" }, 210 | #endif 211 | { 0, NULL } 212 | }; 213 | #endif 214 | 215 | static const char *err2str(int error); 216 | 217 | static void 218 | usage(void) 219 | { 220 | 221 | fprintf(stderr, "usage: fstest [-u uid] [-g gid1[,gid2[...]]] syscall args ...\n"); 222 | exit(1); 223 | } 224 | 225 | static long long 226 | str2flags(struct flag *tflags, char *sflags) 227 | { 228 | long long flags = 0; 229 | unsigned int i; 230 | char *f; 231 | 232 | for (f = strtok(sflags, ","); f != NULL; f = strtok(NULL, ",")) { 233 | /* Support magic 'none' flag which just reset all flags. */ 234 | if (strcmp(f, "none") == 0) 235 | return (0); 236 | for (i = 0; tflags[i].f_str != NULL; i++) { 237 | if (strcmp(tflags[i].f_str, f) == 0) 238 | break; 239 | } 240 | if (tflags[i].f_str == NULL) { 241 | fprintf(stderr, "unknown flag '%s'\n", f); 242 | exit(1); 243 | } 244 | flags |= tflags[i].f_flag; 245 | } 246 | return (flags); 247 | } 248 | 249 | #ifdef HAS_CHFLAGS 250 | static char * 251 | flags2str(struct flag *tflags, long long flags) 252 | { 253 | static char sflags[1024]; 254 | unsigned int i; 255 | 256 | sflags[0] = '\0'; 257 | for (i = 0; tflags[i].f_str != NULL; i++) { 258 | if (flags & tflags[i].f_flag) { 259 | if (sflags[0] != '\0') 260 | strlcat(sflags, ",", sizeof(sflags)); 261 | strlcat(sflags, tflags[i].f_str, sizeof(sflags)); 262 | } 263 | } 264 | if (sflags[0] == '\0') 265 | strlcpy(sflags, "none", sizeof(sflags)); 266 | return (sflags); 267 | } 268 | #endif 269 | 270 | static struct syscall_desc * 271 | find_syscall(const char *name) 272 | { 273 | int i; 274 | 275 | for (i = 0; syscalls[i].sd_name != NULL; i++) { 276 | if (strcmp(syscalls[i].sd_name, name) == 0) 277 | return (&syscalls[i]); 278 | } 279 | return (NULL); 280 | } 281 | 282 | static void 283 | show_stat(struct stat64 *sp, const char *what) 284 | { 285 | 286 | if (strcmp(what, "mode") == 0) 287 | printf("0%o", (unsigned int)(sp->st_mode & ALLPERMS)); 288 | else if (strcmp(what, "inode") == 0) 289 | printf("%lld", (long long)sp->st_ino); 290 | else if (strcmp(what, "nlink") == 0) 291 | printf("%lld", (long long)sp->st_nlink); 292 | else if (strcmp(what, "uid") == 0) 293 | printf("%d", (int)sp->st_uid); 294 | else if (strcmp(what, "gid") == 0) 295 | printf("%d", (int)sp->st_gid); 296 | else if (strcmp(what, "size") == 0) 297 | printf("%lld", (long long)sp->st_size); 298 | else if (strcmp(what, "blocks") == 0) 299 | printf("%lld", (long long)sp->st_blocks); 300 | else if (strcmp(what, "atime") == 0) 301 | printf("%lld", (long long)sp->st_atime); 302 | else if (strcmp(what, "mtime") == 0) 303 | printf("%lld", (long long)sp->st_mtime); 304 | else if (strcmp(what, "ctime") == 0) 305 | printf("%lld", (long long)sp->st_ctime); 306 | #ifdef HAS_CHFLAGS 307 | else if (strcmp(what, "flags") == 0) 308 | printf("%s", flags2str(chflags_flags, sp->st_flags)); 309 | #endif 310 | else if (strcmp(what, "type") == 0) { 311 | switch (sp->st_mode & S_IFMT) { 312 | case S_IFIFO: 313 | printf("fifo"); 314 | break; 315 | case S_IFCHR: 316 | printf("char"); 317 | break; 318 | case S_IFDIR: 319 | printf("dir"); 320 | break; 321 | case S_IFBLK: 322 | printf("block"); 323 | break; 324 | case S_IFREG: 325 | printf("regular"); 326 | break; 327 | case S_IFLNK: 328 | printf("symlink"); 329 | break; 330 | case S_IFSOCK: 331 | printf("socket"); 332 | break; 333 | default: 334 | printf("unknown"); 335 | break; 336 | } 337 | } else { 338 | printf("unknown"); 339 | } 340 | } 341 | 342 | static void 343 | show_stats(struct stat64 *sp, char *what) 344 | { 345 | const char *s = ""; 346 | char *w; 347 | 348 | for (w = strtok(what, ","); w != NULL; w = strtok(NULL, ",")) { 349 | printf("%s", s); 350 | show_stat(sp, w); 351 | s = ","; 352 | } 353 | printf("\n"); 354 | } 355 | 356 | static unsigned int 357 | call_syscall(struct syscall_desc *scall, char *argv[]) 358 | { 359 | struct stat64 sb; 360 | long long flags; 361 | unsigned int i; 362 | char *endp; 363 | int rval; 364 | union { 365 | char *str; 366 | long long num; 367 | } args[MAX_ARGS]; 368 | 369 | /* 370 | * Verify correctness of the arguments. 371 | */ 372 | for (i = 0; i < sizeof(args)/sizeof(args[0]); i++) { 373 | if (scall->sd_args[i] == TYPE_NONE) { 374 | if (argv[i] == NULL || strcmp(argv[i], ":") == 0) 375 | break; 376 | fprintf(stderr, "too many arguments [%s]\n", argv[i]); 377 | exit(1); 378 | } else { 379 | if (argv[i] == NULL || strcmp(argv[i], ":") == 0) { 380 | if (scall->sd_args[i] & TYPE_OPTIONAL) 381 | break; 382 | fprintf(stderr, "too few arguments\n"); 383 | exit(1); 384 | } 385 | if (scall->sd_args[i] & TYPE_STRING) { 386 | if (strcmp(argv[i], "NULL") == 0) 387 | args[i].str = NULL; 388 | else if (strcmp(argv[i], "DEADCODE") == 0) 389 | args[i].str = (void *)0xdeadc0de; 390 | else 391 | args[i].str = argv[i]; 392 | } else if (scall->sd_args[i] & TYPE_NUMBER) { 393 | args[i].num = strtoll(argv[i], &endp, 0); 394 | if (*endp != '\0' && !isspace((unsigned char)*endp)) { 395 | fprintf(stderr, "invalid argument %u, number expected [%s]\n", i, endp); 396 | exit(1); 397 | } 398 | } 399 | } 400 | } 401 | /* 402 | * Call the given syscall. 403 | */ 404 | #define NUM(n) (args[(n)].num) 405 | #define STR(n) (args[(n)].str) 406 | switch (scall->sd_action) { 407 | case ACTION_OPEN: 408 | flags = str2flags(open_flags, STR(1)); 409 | if (flags & O_CREAT) { 410 | if (i == 2) { 411 | fprintf(stderr, "too few arguments\n"); 412 | exit(1); 413 | } 414 | rval = open(STR(0), flags, (mode_t)NUM(2)); 415 | } else { 416 | if (i == 3) { 417 | fprintf(stderr, "too many arguments\n"); 418 | exit(1); 419 | } 420 | rval = open(STR(0), flags); 421 | } 422 | break; 423 | case ACTION_CREATE: 424 | rval = open(STR(0), O_CREAT | O_EXCL, NUM(1)); 425 | if (rval >= 0) 426 | close(rval); 427 | break; 428 | case ACTION_UNLINK: 429 | rval = unlink(STR(0)); 430 | break; 431 | case ACTION_MKDIR: 432 | rval = mkdir(STR(0), NUM(1)); 433 | break; 434 | case ACTION_RMDIR: 435 | rval = rmdir(STR(0)); 436 | break; 437 | case ACTION_LINK: 438 | rval = link(STR(0), STR(1)); 439 | break; 440 | case ACTION_SYMLINK: 441 | rval = symlink(STR(0), STR(1)); 442 | break; 443 | case ACTION_RENAME: 444 | rval = rename(STR(0), STR(1)); 445 | break; 446 | case ACTION_MKFIFO: 447 | rval = mkfifo(STR(0), NUM(1)); 448 | break; 449 | case ACTION_CHMOD: 450 | rval = chmod(STR(0), NUM(1)); 451 | break; 452 | #ifdef HAS_LCHMOD 453 | case ACTION_LCHMOD: 454 | rval = lchmod(STR(0), NUM(1)); 455 | break; 456 | #endif 457 | case ACTION_CHOWN: 458 | rval = chown(STR(0), NUM(1), NUM(2)); 459 | break; 460 | case ACTION_LCHOWN: 461 | rval = lchown(STR(0), NUM(1), NUM(2)); 462 | break; 463 | #ifdef HAS_CHFLAGS 464 | case ACTION_CHFLAGS: 465 | rval = chflags(STR(0), str2flags(chflags_flags, STR(1))); 466 | break; 467 | #endif 468 | #ifdef HAS_LCHFLAGS 469 | case ACTION_LCHFLAGS: 470 | rval = lchflags(STR(0), str2flags(chflags_flags, STR(1))); 471 | break; 472 | #endif 473 | case ACTION_TRUNCATE: 474 | rval = truncate64(STR(0), NUM(1)); 475 | break; 476 | case ACTION_STAT: 477 | rval = stat64(STR(0), &sb); 478 | if (rval == 0) { 479 | show_stats(&sb, STR(1)); 480 | return (i); 481 | } 482 | break; 483 | case ACTION_LSTAT: 484 | rval = lstat64(STR(0), &sb); 485 | if (rval == 0) { 486 | show_stats(&sb, STR(1)); 487 | return (i); 488 | } 489 | break; 490 | default: 491 | fprintf(stderr, "unsupported syscall\n"); 492 | exit(1); 493 | } 494 | #undef STR 495 | #undef NUM 496 | if (rval < 0) { 497 | const char *serrno; 498 | 499 | serrno = err2str(errno); 500 | fprintf(stderr, "%s returned %d\n", scall->sd_name, rval); 501 | printf("%s\n", serrno); 502 | exit(1); 503 | } 504 | printf("0\n"); 505 | return (i); 506 | } 507 | 508 | static void 509 | set_gids(char *gids) 510 | { 511 | gid_t *gidset; 512 | long ngroups; 513 | char *g, *endp; 514 | unsigned i; 515 | 516 | ngroups = sysconf(_SC_NGROUPS_MAX); 517 | assert(ngroups > 0); 518 | gidset = malloc(sizeof(*gidset) * ngroups); 519 | assert(gidset != NULL); 520 | for (i = 0, g = strtok(gids, ","); g != NULL; g = strtok(NULL, ","), i++) { 521 | if (i >= ngroups) { 522 | fprintf(stderr, "too many gids\n"); 523 | exit(1); 524 | } 525 | gidset[i] = strtol(g, &endp, 0); 526 | if (*endp != '\0' && !isspace((unsigned char)*endp)) { 527 | fprintf(stderr, "invalid gid '%s' - number expected\n", 528 | g); 529 | exit(1); 530 | } 531 | } 532 | if (setgroups(i, gidset) < 0) { 533 | fprintf(stderr, "cannot change groups: %s\n", strerror(errno)); 534 | exit(1); 535 | } 536 | if (setegid(gidset[0]) < 0) { 537 | fprintf(stderr, "cannot change effective gid: %s\n", strerror(errno)); 538 | exit(1); 539 | } 540 | free(gidset); 541 | } 542 | 543 | int 544 | main(int argc, char *argv[]) 545 | { 546 | struct syscall_desc *scall; 547 | unsigned int n; 548 | char *gids, *endp; 549 | int uid, umsk, ch; 550 | 551 | uid = -1; 552 | gids = NULL; 553 | umsk = 0; 554 | 555 | while ((ch = getopt(argc, argv, "g:u:U:")) != -1) { 556 | switch(ch) { 557 | case 'g': 558 | gids = optarg; 559 | break; 560 | case 'u': 561 | uid = (int)strtol(optarg, &endp, 0); 562 | if (*endp != '\0' && !isspace((unsigned char)*endp)) { 563 | fprintf(stderr, "invalid uid '%s' - number " 564 | "expected\n", optarg); 565 | exit(1); 566 | } 567 | break; 568 | case 'U': 569 | umsk = (int)strtol(optarg, &endp, 0); 570 | if (*endp != '\0' && !isspace((unsigned char)*endp)) { 571 | fprintf(stderr, "invalid umask '%s' - number " 572 | "expected\n", optarg); 573 | exit(1); 574 | } 575 | break; 576 | default: 577 | usage(); 578 | } 579 | } 580 | argc -= optind; 581 | argv += optind; 582 | 583 | if (argc < 1) { 584 | fprintf(stderr, "too few arguments\n"); 585 | usage(); 586 | } 587 | 588 | if (gids != NULL) { 589 | fprintf(stderr, "changing groups to %s\n", gids); 590 | set_gids(gids); 591 | } 592 | if (uid != -1) { 593 | fprintf(stderr, "changing uid to %d\n", uid); 594 | if (setuid(uid) < 0) { 595 | fprintf(stderr, "cannot change uid: %s\n", 596 | strerror(errno)); 597 | exit(1); 598 | } 599 | } 600 | 601 | /* Change umask to requested value or to 0, if not requested. */ 602 | umask(umsk); 603 | 604 | for (;;) { 605 | scall = find_syscall(argv[0]); 606 | if (scall == NULL) { 607 | fprintf(stderr, "syscall '%s' not supported\n", argv[0]); 608 | exit(1); 609 | } 610 | argc++; 611 | argv++; 612 | n = call_syscall(scall, argv); 613 | argc += n; 614 | argv += n; 615 | if (argv[0] == NULL) 616 | break; 617 | argc++; 618 | argv++; 619 | } 620 | 621 | exit(0); 622 | } 623 | 624 | static const char * 625 | err2str(int error) 626 | { 627 | static char errnum[8]; 628 | 629 | switch (error) { 630 | #ifdef EPERM 631 | case EPERM: 632 | return ("EPERM"); 633 | #endif 634 | #ifdef ENOENT 635 | case ENOENT: 636 | return ("ENOENT"); 637 | #endif 638 | #ifdef ESRCH 639 | case ESRCH: 640 | return ("ESRCH"); 641 | #endif 642 | #ifdef EINTR 643 | case EINTR: 644 | return ("EINTR"); 645 | #endif 646 | #ifdef EIO 647 | case EIO: 648 | return ("EIO"); 649 | #endif 650 | #ifdef ENXIO 651 | case ENXIO: 652 | return ("ENXIO"); 653 | #endif 654 | #ifdef E2BIG 655 | case E2BIG: 656 | return ("E2BIG"); 657 | #endif 658 | #ifdef ENOEXEC 659 | case ENOEXEC: 660 | return ("ENOEXEC"); 661 | #endif 662 | #ifdef EBADF 663 | case EBADF: 664 | return ("EBADF"); 665 | #endif 666 | #ifdef ECHILD 667 | case ECHILD: 668 | return ("ECHILD"); 669 | #endif 670 | #ifdef EDEADLK 671 | case EDEADLK: 672 | return ("EDEADLK"); 673 | #endif 674 | #ifdef ENOMEM 675 | case ENOMEM: 676 | return ("ENOMEM"); 677 | #endif 678 | #ifdef EACCES 679 | case EACCES: 680 | return ("EACCES"); 681 | #endif 682 | #ifdef EFAULT 683 | case EFAULT: 684 | return ("EFAULT"); 685 | #endif 686 | #ifdef ENOTBLK 687 | case ENOTBLK: 688 | return ("ENOTBLK"); 689 | #endif 690 | #ifdef EBUSY 691 | case EBUSY: 692 | return ("EBUSY"); 693 | #endif 694 | #ifdef EEXIST 695 | case EEXIST: 696 | return ("EEXIST"); 697 | #endif 698 | #ifdef EXDEV 699 | case EXDEV: 700 | return ("EXDEV"); 701 | #endif 702 | #ifdef ENODEV 703 | case ENODEV: 704 | return ("ENODEV"); 705 | #endif 706 | #ifdef ENOTDIR 707 | case ENOTDIR: 708 | return ("ENOTDIR"); 709 | #endif 710 | #ifdef EISDIR 711 | case EISDIR: 712 | return ("EISDIR"); 713 | #endif 714 | #ifdef EINVAL 715 | case EINVAL: 716 | return ("EINVAL"); 717 | #endif 718 | #ifdef ENFILE 719 | case ENFILE: 720 | return ("ENFILE"); 721 | #endif 722 | #ifdef EMFILE 723 | case EMFILE: 724 | return ("EMFILE"); 725 | #endif 726 | #ifdef ENOTTY 727 | case ENOTTY: 728 | return ("ENOTTY"); 729 | #endif 730 | #ifdef ETXTBSY 731 | case ETXTBSY: 732 | return ("ETXTBSY"); 733 | #endif 734 | #ifdef EFBIG 735 | case EFBIG: 736 | return ("EFBIG"); 737 | #endif 738 | #ifdef ENOSPC 739 | case ENOSPC: 740 | return ("ENOSPC"); 741 | #endif 742 | #ifdef ESPIPE 743 | case ESPIPE: 744 | return ("ESPIPE"); 745 | #endif 746 | #ifdef EROFS 747 | case EROFS: 748 | return ("EROFS"); 749 | #endif 750 | #ifdef EMLINK 751 | case EMLINK: 752 | return ("EMLINK"); 753 | #endif 754 | #ifdef EPIPE 755 | case EPIPE: 756 | return ("EPIPE"); 757 | #endif 758 | #ifdef EDOM 759 | case EDOM: 760 | return ("EDOM"); 761 | #endif 762 | #ifdef ERANGE 763 | case ERANGE: 764 | return ("ERANGE"); 765 | #endif 766 | #ifdef EAGAIN 767 | case EAGAIN: 768 | return ("EAGAIN"); 769 | #endif 770 | #ifdef EINPROGRESS 771 | case EINPROGRESS: 772 | return ("EINPROGRESS"); 773 | #endif 774 | #ifdef EALREADY 775 | case EALREADY: 776 | return ("EALREADY"); 777 | #endif 778 | #ifdef ENOTSOCK 779 | case ENOTSOCK: 780 | return ("ENOTSOCK"); 781 | #endif 782 | #ifdef EDESTADDRREQ 783 | case EDESTADDRREQ: 784 | return ("EDESTADDRREQ"); 785 | #endif 786 | #ifdef EMSGSIZE 787 | case EMSGSIZE: 788 | return ("EMSGSIZE"); 789 | #endif 790 | #ifdef EPROTOTYPE 791 | case EPROTOTYPE: 792 | return ("EPROTOTYPE"); 793 | #endif 794 | #ifdef ENOPROTOOPT 795 | case ENOPROTOOPT: 796 | return ("ENOPROTOOPT"); 797 | #endif 798 | #ifdef EPROTONOSUPPORT 799 | case EPROTONOSUPPORT: 800 | return ("EPROTONOSUPPORT"); 801 | #endif 802 | #ifdef ESOCKTNOSUPPORT 803 | case ESOCKTNOSUPPORT: 804 | return ("ESOCKTNOSUPPORT"); 805 | #endif 806 | #ifdef EOPNOTSUPP 807 | case EOPNOTSUPP: 808 | return ("EOPNOTSUPP"); 809 | #endif 810 | #ifdef EPFNOSUPPORT 811 | case EPFNOSUPPORT: 812 | return ("EPFNOSUPPORT"); 813 | #endif 814 | #ifdef EAFNOSUPPORT 815 | case EAFNOSUPPORT: 816 | return ("EAFNOSUPPORT"); 817 | #endif 818 | #ifdef EADDRINUSE 819 | case EADDRINUSE: 820 | return ("EADDRINUSE"); 821 | #endif 822 | #ifdef EADDRNOTAVAIL 823 | case EADDRNOTAVAIL: 824 | return ("EADDRNOTAVAIL"); 825 | #endif 826 | #ifdef ENETDOWN 827 | case ENETDOWN: 828 | return ("ENETDOWN"); 829 | #endif 830 | #ifdef ENETUNREACH 831 | case ENETUNREACH: 832 | return ("ENETUNREACH"); 833 | #endif 834 | #ifdef ENETRESET 835 | case ENETRESET: 836 | return ("ENETRESET"); 837 | #endif 838 | #ifdef ECONNABORTED 839 | case ECONNABORTED: 840 | return ("ECONNABORTED"); 841 | #endif 842 | #ifdef ECONNRESET 843 | case ECONNRESET: 844 | return ("ECONNRESET"); 845 | #endif 846 | #ifdef ENOBUFS 847 | case ENOBUFS: 848 | return ("ENOBUFS"); 849 | #endif 850 | #ifdef EISCONN 851 | case EISCONN: 852 | return ("EISCONN"); 853 | #endif 854 | #ifdef ENOTCONN 855 | case ENOTCONN: 856 | return ("ENOTCONN"); 857 | #endif 858 | #ifdef ESHUTDOWN 859 | case ESHUTDOWN: 860 | return ("ESHUTDOWN"); 861 | #endif 862 | #ifdef ETOOMANYREFS 863 | case ETOOMANYREFS: 864 | return ("ETOOMANYREFS"); 865 | #endif 866 | #ifdef ETIMEDOUT 867 | case ETIMEDOUT: 868 | return ("ETIMEDOUT"); 869 | #endif 870 | #ifdef ECONNREFUSED 871 | case ECONNREFUSED: 872 | return ("ECONNREFUSED"); 873 | #endif 874 | #ifdef ELOOP 875 | case ELOOP: 876 | return ("ELOOP"); 877 | #endif 878 | #ifdef ENAMETOOLONG 879 | case ENAMETOOLONG: 880 | return ("ENAMETOOLONG"); 881 | #endif 882 | #ifdef EHOSTDOWN 883 | case EHOSTDOWN: 884 | return ("EHOSTDOWN"); 885 | #endif 886 | #ifdef EHOSTUNREACH 887 | case EHOSTUNREACH: 888 | return ("EHOSTUNREACH"); 889 | #endif 890 | #ifdef ENOTEMPTY 891 | case ENOTEMPTY: 892 | return ("ENOTEMPTY"); 893 | #endif 894 | #ifdef EPROCLIM 895 | case EPROCLIM: 896 | return ("EPROCLIM"); 897 | #endif 898 | #ifdef EUSERS 899 | case EUSERS: 900 | return ("EUSERS"); 901 | #endif 902 | #ifdef EDQUOT 903 | case EDQUOT: 904 | return ("EDQUOT"); 905 | #endif 906 | #ifdef ESTALE 907 | case ESTALE: 908 | return ("ESTALE"); 909 | #endif 910 | #ifdef EREMOTE 911 | case EREMOTE: 912 | return ("EREMOTE"); 913 | #endif 914 | #ifdef EBADRPC 915 | case EBADRPC: 916 | return ("EBADRPC"); 917 | #endif 918 | #ifdef ERPCMISMATCH 919 | case ERPCMISMATCH: 920 | return ("ERPCMISMATCH"); 921 | #endif 922 | #ifdef EPROGUNAVAIL 923 | case EPROGUNAVAIL: 924 | return ("EPROGUNAVAIL"); 925 | #endif 926 | #ifdef EPROGMISMATCH 927 | case EPROGMISMATCH: 928 | return ("EPROGMISMATCH"); 929 | #endif 930 | #ifdef EPROCUNAVAIL 931 | case EPROCUNAVAIL: 932 | return ("EPROCUNAVAIL"); 933 | #endif 934 | #ifdef ENOLCK 935 | case ENOLCK: 936 | return ("ENOLCK"); 937 | #endif 938 | #ifdef ENOSYS 939 | case ENOSYS: 940 | return ("ENOSYS"); 941 | #endif 942 | #ifdef EFTYPE 943 | case EFTYPE: 944 | return ("EFTYPE"); 945 | #endif 946 | #ifdef EAUTH 947 | case EAUTH: 948 | return ("EAUTH"); 949 | #endif 950 | #ifdef ENEEDAUTH 951 | case ENEEDAUTH: 952 | return ("ENEEDAUTH"); 953 | #endif 954 | #ifdef EIDRM 955 | case EIDRM: 956 | return ("EIDRM"); 957 | #endif 958 | #ifdef ENOMSG 959 | case ENOMSG: 960 | return ("ENOMSG"); 961 | #endif 962 | #ifdef EOVERFLOW 963 | case EOVERFLOW: 964 | return ("EOVERFLOW"); 965 | #endif 966 | #ifdef ECANCELED 967 | case ECANCELED: 968 | return ("ECANCELED"); 969 | #endif 970 | #ifdef EILSEQ 971 | case EILSEQ: 972 | return ("EILSEQ"); 973 | #endif 974 | #ifdef ENOATTR 975 | case ENOATTR: 976 | return ("ENOATTR"); 977 | #endif 978 | #ifdef EDOOFUS 979 | case EDOOFUS: 980 | return ("EDOOFUS"); 981 | #endif 982 | #ifdef EBADMSG 983 | case EBADMSG: 984 | return ("EBADMSG"); 985 | #endif 986 | #ifdef EMULTIHOP 987 | case EMULTIHOP: 988 | return ("EMULTIHOP"); 989 | #endif 990 | #ifdef ENOLINK 991 | case ENOLINK: 992 | return ("ENOLINK"); 993 | #endif 994 | #ifdef EPROTO 995 | case EPROTO: 996 | return ("EPROTO"); 997 | #endif 998 | default: 999 | snprintf(errnum, sizeof(errnum), "%d", error); 1000 | return (errnum); 1001 | } 1002 | } 1003 | -------------------------------------------------------------------------------- /cachemap/lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | * LZ4 - Fast LZ compression algorithm 3 | * Header File 4 | * Copyright (C) 2011-2017, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 homepage : http://www.lz4.org 33 | - LZ4 source repository : https://github.com/lz4/lz4 34 | */ 35 | #if defined (__cplusplus) 36 | extern "C" { 37 | #endif 38 | 39 | #ifndef LZ4_H_2983827168210 40 | #define LZ4_H_2983827168210 41 | 42 | /* --- Dependency --- */ 43 | #include /* size_t */ 44 | 45 | 46 | /** 47 | Introduction 48 | 49 | LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, 50 | scalable with multi-cores CPU. It features an extremely fast decoder, with speed in 51 | multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. 52 | 53 | The LZ4 compression library provides in-memory compression and decompression functions. 54 | Compression can be done in: 55 | - a single step (described as Simple Functions) 56 | - a single step, reusing a context (described in Advanced Functions) 57 | - unbounded multiple steps (described as Streaming compression) 58 | 59 | lz4.h provides block compression functions. It gives full buffer control to user. 60 | Decompressing an lz4-compressed block also requires metadata (such as compressed size). 61 | Each application is free to encode such metadata in whichever way it wants. 62 | 63 | An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), 64 | take care of encoding standard metadata alongside LZ4-compressed blocks. 65 | If your application requires interoperability, it's recommended to use it. 66 | A library is provided to take care of it, see lz4frame.h. 67 | */ 68 | 69 | /*^*************************************************************** 70 | * Export parameters 71 | *****************************************************************/ 72 | /* 73 | * LZ4_DLL_EXPORT : 74 | * Enable exporting of functions when building a Windows DLL 75 | * LZ4LIB_VISIBILITY : 76 | * Control library symbols visibility. 77 | */ 78 | #ifndef LZ4LIB_VISIBILITY 79 | # if defined(__GNUC__) && (__GNUC__ >= 4) 80 | # define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default"))) 81 | # else 82 | # define LZ4LIB_VISIBILITY 83 | # endif 84 | #endif 85 | #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) 86 | # define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY 87 | #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) 88 | # define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 89 | #else 90 | # define LZ4LIB_API LZ4LIB_VISIBILITY 91 | #endif 92 | 93 | /*------ Version ------*/ 94 | #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 95 | #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ 96 | #define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ 97 | 98 | #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) 99 | 100 | #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE 101 | #define LZ4_QUOTE(str) #str 102 | #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) 103 | #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) 104 | 105 | LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ 106 | LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ 107 | 108 | 109 | /*-************************************ 110 | * Tuning parameter 111 | **************************************/ 112 | /*! 113 | * LZ4_MEMORY_USAGE : 114 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 115 | * Increasing memory usage improves compression ratio 116 | * Reduced memory usage can improve speed, due to cache effect 117 | * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache 118 | */ 119 | #ifndef LZ4_MEMORY_USAGE 120 | # define LZ4_MEMORY_USAGE 14 121 | #endif 122 | 123 | /*-************************************ 124 | * Simple Functions 125 | **************************************/ 126 | /*! LZ4_compress_default() : 127 | Compresses 'srcSize' bytes from buffer 'src' 128 | into already allocated 'dst' buffer of size 'dstCapacity'. 129 | Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). 130 | It also runs faster, so it's a recommended setting. 131 | If the function cannot compress 'src' into a limited 'dst' budget, 132 | compression stops *immediately*, and the function result is zero. 133 | As a consequence, 'dst' content is not valid. 134 | This function never writes outside 'dst' buffer, nor read outside 'source' buffer. 135 | srcSize : supported max value is LZ4_MAX_INPUT_VALUE 136 | dstCapacity : full or partial size of buffer 'dst' (which must be already allocated) 137 | return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) 138 | or 0 if compression fails */ 139 | LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity); 140 | 141 | /*! LZ4_decompress_safe() : 142 | compressedSize : is the exact complete size of the compressed block. 143 | dstCapacity : is the size of destination buffer, which must be already allocated. 144 | return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) 145 | If destination buffer is not large enough, decoding will stop and output an error code (negative value). 146 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 147 | This function is protected against buffer overflow exploits, including malicious data packets. 148 | It never writes outside output buffer, nor reads outside input buffer. 149 | */ 150 | LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity); 151 | 152 | 153 | /*-************************************ 154 | * Advanced Functions 155 | **************************************/ 156 | #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ 157 | #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 158 | 159 | /*! 160 | LZ4_compressBound() : 161 | Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) 162 | This function is primarily useful for memory allocation purposes (destination buffer size). 163 | Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). 164 | Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) 165 | inputSize : max supported value is LZ4_MAX_INPUT_SIZE 166 | return : maximum output size in a "worst case" scenario 167 | or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) 168 | */ 169 | LZ4LIB_API int LZ4_compressBound(int inputSize); 170 | 171 | /*! 172 | LZ4_compress_fast() : 173 | Same as LZ4_compress_default(), but allows to select an "acceleration" factor. 174 | The larger the acceleration value, the faster the algorithm, but also the lesser the compression. 175 | It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. 176 | An acceleration value of "1" is the same as regular LZ4_compress_default() 177 | Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. 178 | */ 179 | LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 180 | 181 | 182 | /*! 183 | LZ4_compress_fast_extState() : 184 | Same compression function, just using an externally allocated memory space to store compression state. 185 | Use LZ4_sizeofState() to know how much memory must be allocated, 186 | and allocate it on 8-bytes boundaries (using malloc() typically). 187 | Then, provide it as 'void* state' to compression function. 188 | */ 189 | LZ4LIB_API int LZ4_sizeofState(void); 190 | LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 191 | 192 | 193 | /*! 194 | LZ4_compress_destSize() : 195 | Reverse the logic : compresses as much data as possible from 'src' buffer 196 | into already allocated buffer 'dst' of size 'targetDestSize'. 197 | This function either compresses the entire 'src' content into 'dst' if it's large enough, 198 | or fill 'dst' buffer completely with as much data as possible from 'src'. 199 | *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'. 200 | New value is necessarily <= old value. 201 | return : Nb bytes written into 'dst' (necessarily <= targetDestSize) 202 | or 0 if compression fails 203 | */ 204 | LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize); 205 | 206 | 207 | /*! 208 | LZ4_decompress_fast() : (unsafe!!) 209 | originalSize : is the original uncompressed size 210 | return : the number of bytes read from the source buffer (in other words, the compressed size) 211 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 212 | Destination buffer must be already allocated. Its size must be >= 'originalSize' bytes. 213 | note : This function respects memory boundaries for *properly formed* compressed data. 214 | It is a bit faster than LZ4_decompress_safe(). 215 | However, it does not provide any protection against intentionally modified data stream (malicious input). 216 | Use this function in trusted environment only (data to decode comes from a trusted source). 217 | */ 218 | LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize); 219 | 220 | /*! 221 | LZ4_decompress_safe_partial() : 222 | This function decompress a compressed block of size 'srcSize' at position 'src' 223 | into destination buffer 'dst' of size 'dstCapacity'. 224 | The function will decompress a minimum of 'targetOutputSize' bytes, and stop after that. 225 | However, it's not accurate, and may write more than 'targetOutputSize' (but <= dstCapacity). 226 | @return : the number of bytes decoded in the destination buffer (necessarily <= dstCapacity) 227 | Note : this number can be < 'targetOutputSize' should the compressed block contain less data. 228 | Always control how many bytes were decoded. 229 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 230 | This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets. 231 | */ 232 | LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity); 233 | 234 | 235 | /*-********************************************* 236 | * Streaming Compression Functions 237 | ***********************************************/ 238 | typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ 239 | 240 | /*! LZ4_createStream() and LZ4_freeStream() : 241 | * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. 242 | * LZ4_freeStream() releases its memory. 243 | */ 244 | LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); 245 | LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); 246 | 247 | /*! LZ4_resetStream() : 248 | * An LZ4_stream_t structure can be allocated once and re-used multiple times. 249 | * Use this function to start compressing a new stream. 250 | */ 251 | LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); 252 | 253 | /*! LZ4_loadDict() : 254 | * Use this function to load a static dictionary into LZ4_stream_t. 255 | * Any previous data will be forgotten, only 'dictionary' will remain in memory. 256 | * Loading a size of 0 is allowed, and is the same as reset. 257 | * @return : dictionary size, in bytes (necessarily <= 64 KB) 258 | */ 259 | LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); 260 | 261 | /*! LZ4_compress_fast_continue() : 262 | * Compress content into 'src' using data from previously compressed blocks, improving compression ratio. 263 | * 'dst' buffer must be already allocated. 264 | * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. 265 | * 266 | * Important : Up to 64KB of previously compressed data is assumed to remain present and unmodified in memory ! 267 | * Special 1 : If input buffer is a double-buffer, it can have any size, including < 64 KB. 268 | * Special 2 : If input buffer is a ring-buffer, it can have any size, including < 64 KB. 269 | * 270 | * @return : size of compressed block 271 | * or 0 if there is an error (typically, compressed data cannot fit into 'dst') 272 | * After an error, the stream status is invalid, it can only be reset or freed. 273 | */ 274 | LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 275 | 276 | /*! LZ4_saveDict() : 277 | * If previously compressed data block is not guaranteed to remain available at its current memory location, 278 | * save it into a safer place (char* safeBuffer). 279 | * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. 280 | * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. 281 | */ 282 | LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); 283 | 284 | 285 | /*-********************************************** 286 | * Streaming Decompression Functions 287 | * Bufferless synchronous API 288 | ************************************************/ 289 | typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ 290 | 291 | /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : 292 | * creation / destruction of streaming decompression tracking structure. 293 | * A tracking structure can be re-used multiple times sequentially. */ 294 | LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); 295 | LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); 296 | 297 | /*! LZ4_setStreamDecode() : 298 | * An LZ4_streamDecode_t structure can be allocated once and re-used multiple times. 299 | * Use this function to start decompression of a new stream of blocks. 300 | * A dictionary can optionnally be set. Use NULL or size 0 for a simple reset order. 301 | * @return : 1 if OK, 0 if error 302 | */ 303 | LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); 304 | 305 | /*! LZ4_decompress_*_continue() : 306 | * These decoding functions allow decompression of consecutive blocks in "streaming" mode. 307 | * A block is an unsplittable entity, it must be presented entirely to a decompression function. 308 | * Decompression functions only accept one block at a time. 309 | * Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB). 310 | * 311 | * Special : if application sets a ring buffer for decompression, it must respect one of the following conditions : 312 | * - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) 313 | * In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). 314 | * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. 315 | * maxBlockSize is implementation dependent. It's the maximum size of any single block. 316 | * In which case, encoding and decoding buffers do not need to be synchronized, 317 | * and encoding ring buffer can have any size, including small ones ( < 64 KB). 318 | * - _At least_ 64 KB + 8 bytes + maxBlockSize. 319 | * In which case, encoding and decoding buffers do not need to be synchronized, 320 | * and encoding ring buffer can have any size, including larger than decoding buffer. 321 | * Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, 322 | * and indicate where it is saved using LZ4_setStreamDecode() before decompressing next block. 323 | */ 324 | LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity); 325 | LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize); 326 | 327 | 328 | /*! LZ4_decompress_*_usingDict() : 329 | * These decoding functions work the same as 330 | * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() 331 | * They are stand-alone, and don't need an LZ4_streamDecode_t structure. 332 | */ 333 | LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize); 334 | LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize); 335 | 336 | 337 | /*^********************************************** 338 | * !!!!!! STATIC LINKING ONLY !!!!!! 339 | ***********************************************/ 340 | /*-************************************ 341 | * Private definitions 342 | ************************************** 343 | * Do not use these definitions. 344 | * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. 345 | * Using these definitions will expose code to API and/or ABI break in future versions of the library. 346 | **************************************/ 347 | #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) 348 | #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) 349 | #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ 350 | 351 | #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 352 | #include 353 | 354 | typedef struct { 355 | uint32_t hashTable[LZ4_HASH_SIZE_U32]; 356 | uint32_t currentOffset; 357 | uint32_t initCheck; 358 | const uint8_t* dictionary; 359 | uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ 360 | uint32_t dictSize; 361 | } LZ4_stream_t_internal; 362 | 363 | typedef struct { 364 | const uint8_t* externalDict; 365 | size_t extDictSize; 366 | const uint8_t* prefixEnd; 367 | size_t prefixSize; 368 | } LZ4_streamDecode_t_internal; 369 | 370 | #else 371 | 372 | typedef struct { 373 | unsigned int hashTable[LZ4_HASH_SIZE_U32]; 374 | unsigned int currentOffset; 375 | unsigned int initCheck; 376 | const unsigned char* dictionary; 377 | unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ 378 | unsigned int dictSize; 379 | } LZ4_stream_t_internal; 380 | 381 | typedef struct { 382 | const unsigned char* externalDict; 383 | size_t extDictSize; 384 | const unsigned char* prefixEnd; 385 | size_t prefixSize; 386 | } LZ4_streamDecode_t_internal; 387 | 388 | #endif 389 | 390 | /*! 391 | * LZ4_stream_t : 392 | * information structure to track an LZ4 stream. 393 | * init this structure before first use. 394 | * note : only use in association with static linking ! 395 | * this definition is not API/ABI safe, 396 | * it may change in a future version ! 397 | */ 398 | #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) 399 | #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) 400 | union LZ4_stream_u { 401 | unsigned long long table[LZ4_STREAMSIZE_U64]; 402 | LZ4_stream_t_internal internal_donotuse; 403 | } ; /* previously typedef'd to LZ4_stream_t */ 404 | 405 | 406 | /*! 407 | * LZ4_streamDecode_t : 408 | * information structure to track an LZ4 stream during decompression. 409 | * init this structure using LZ4_setStreamDecode (or memset()) before first use 410 | * note : only use in association with static linking ! 411 | * this definition is not API/ABI safe, 412 | * and may change in a future version ! 413 | */ 414 | #define LZ4_STREAMDECODESIZE_U64 4 415 | #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) 416 | union LZ4_streamDecode_u { 417 | unsigned long long table[LZ4_STREAMDECODESIZE_U64]; 418 | LZ4_streamDecode_t_internal internal_donotuse; 419 | } ; /* previously typedef'd to LZ4_streamDecode_t */ 420 | 421 | 422 | /*-************************************ 423 | * Obsolete Functions 424 | **************************************/ 425 | 426 | /*! Deprecation warnings 427 | Should deprecation warnings be a problem, 428 | it is generally possible to disable them, 429 | typically with -Wno-deprecated-declarations for gcc 430 | or _CRT_SECURE_NO_WARNINGS in Visual. 431 | Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ 432 | #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS 433 | # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ 434 | #else 435 | # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 436 | # if defined(__clang__) /* clang doesn't handle mixed C++11 and CNU attributes */ 437 | # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) 438 | # elif defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ 439 | # define LZ4_DEPRECATED(message) [[deprecated(message)]] 440 | # elif (LZ4_GCC_VERSION >= 405) 441 | # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) 442 | # elif (LZ4_GCC_VERSION >= 301) 443 | # define LZ4_DEPRECATED(message) __attribute__((deprecated)) 444 | # elif defined(_MSC_VER) 445 | # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) 446 | # else 447 | # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") 448 | # define LZ4_DEPRECATED(message) 449 | # endif 450 | #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ 451 | 452 | /* Obsolete compression functions */ 453 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); 454 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 455 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 456 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 457 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 458 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 459 | 460 | /* Obsolete decompression functions */ 461 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); 462 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); 463 | 464 | /* Obsolete streaming functions; use new streaming interface whenever possible */ 465 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); 466 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); 467 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); 468 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); 469 | 470 | /* Obsolete streaming decoding functions */ 471 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); 472 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); 473 | 474 | #endif /* LZ4_H_2983827168210 */ 475 | 476 | 477 | #if defined (__cplusplus) 478 | } 479 | #endif 480 | -------------------------------------------------------------------------------- /cachemap/lz4.c: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Copyright (C) 2011-2017, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 homepage : http://www.lz4.org 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | */ 34 | 35 | 36 | /*-************************************ 37 | * Tuning parameters 38 | **************************************/ 39 | /* 40 | * LZ4_HEAPMODE : 41 | * Select how default compression functions will allocate memory for their hash table, 42 | * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). 43 | */ 44 | #ifndef LZ4_HEAPMODE 45 | # define LZ4_HEAPMODE 0 46 | #endif 47 | 48 | /* 49 | * ACCELERATION_DEFAULT : 50 | * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 51 | */ 52 | #define ACCELERATION_DEFAULT 1 53 | 54 | 55 | /*-************************************ 56 | * CPU Feature Detection 57 | **************************************/ 58 | /* LZ4_FORCE_MEMORY_ACCESS 59 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 60 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 61 | * The below switch allow to select different access method for improved performance. 62 | * Method 0 (default) : use `memcpy()`. Safe and portable. 63 | * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). 64 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 65 | * Method 2 : direct access. This method is portable but violate C standard. 66 | * It can generate buggy code on targets which assembly generation depends on alignment. 67 | * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) 68 | * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 69 | * Prefer these methods in priority order (0 > 1 > 2) 70 | */ 71 | #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally */ 72 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 73 | # define LZ4_FORCE_MEMORY_ACCESS 2 74 | # elif defined(__INTEL_COMPILER) || defined(__GNUC__) 75 | # define LZ4_FORCE_MEMORY_ACCESS 1 76 | # endif 77 | #endif 78 | 79 | /* 80 | * LZ4_FORCE_SW_BITCOUNT 81 | * Define this parameter if your target system or compiler does not support hardware bit count 82 | */ 83 | #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ 84 | # define LZ4_FORCE_SW_BITCOUNT 85 | #endif 86 | 87 | 88 | 89 | /*-************************************ 90 | * Dependency 91 | **************************************/ 92 | #include "lz4.h" 93 | /* see also "memory routines" below */ 94 | 95 | 96 | /*-************************************ 97 | * Compiler Options 98 | **************************************/ 99 | #ifdef _MSC_VER /* Visual Studio */ 100 | # include 101 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 102 | # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ 103 | #endif /* _MSC_VER */ 104 | 105 | #ifndef LZ4_FORCE_INLINE 106 | # ifdef _MSC_VER /* Visual Studio */ 107 | # define LZ4_FORCE_INLINE static __forceinline 108 | # else 109 | # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 110 | # ifdef __GNUC__ 111 | # define LZ4_FORCE_INLINE static inline __attribute__((always_inline)) 112 | # else 113 | # define LZ4_FORCE_INLINE static inline 114 | # endif 115 | # else 116 | # define LZ4_FORCE_INLINE static 117 | # endif /* __STDC_VERSION__ */ 118 | # endif /* _MSC_VER */ 119 | #endif /* LZ4_FORCE_INLINE */ 120 | 121 | /* LZ4_FORCE_O2_GCC_PPC64LE and LZ4_FORCE_O2_INLINE_GCC_PPC64LE 122 | * Gcc on ppc64le generates an unrolled SIMDized loop for LZ4_wildCopy, 123 | * together with a simple 8-byte copy loop as a fall-back path. 124 | * However, this optimization hurts the decompression speed by >30%, 125 | * because the execution does not go to the optimized loop 126 | * for typical compressible data, and all of the preamble checks 127 | * before going to the fall-back path become useless overhead. 128 | * This optimization happens only with the -O3 flag, and -O2 generates 129 | * a simple 8-byte copy loop. 130 | * With gcc on ppc64le, all of the LZ4_decompress_* and LZ4_wildCopy 131 | * functions are annotated with __attribute__((optimize("O2"))), 132 | * and also LZ4_wildCopy is forcibly inlined, so that the O2 attribute 133 | * of LZ4_wildCopy does not affect the compression speed. 134 | */ 135 | #if defined(__PPC64__) && defined(__LITTLE_ENDIAN__) && defined(__GNUC__) 136 | # define LZ4_FORCE_O2_GCC_PPC64LE __attribute__((optimize("O2"))) 137 | # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE __attribute__((optimize("O2"))) LZ4_FORCE_INLINE 138 | #else 139 | # define LZ4_FORCE_O2_GCC_PPC64LE 140 | # define LZ4_FORCE_O2_INLINE_GCC_PPC64LE static 141 | #endif 142 | 143 | #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) 144 | # define expect(expr,value) (__builtin_expect ((expr),(value)) ) 145 | #else 146 | # define expect(expr,value) (expr) 147 | #endif 148 | 149 | #define likely(expr) expect((expr) != 0, 1) 150 | #define unlikely(expr) expect((expr) != 0, 0) 151 | 152 | 153 | /*-************************************ 154 | * Memory routines 155 | **************************************/ 156 | #include /* malloc, calloc, free */ 157 | #define ALLOCATOR(n,s) calloc(n,s) 158 | #define FREEMEM free 159 | #include /* memset, memcpy */ 160 | #define MEM_INIT memset 161 | 162 | 163 | /*-************************************ 164 | * Basic Types 165 | **************************************/ 166 | #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 167 | # include 168 | typedef uint8_t BYTE; 169 | typedef uint16_t U16; 170 | typedef uint32_t U32; 171 | typedef int32_t S32; 172 | typedef uint64_t U64; 173 | typedef uintptr_t uptrval; 174 | #else 175 | typedef unsigned char BYTE; 176 | typedef unsigned short U16; 177 | typedef unsigned int U32; 178 | typedef signed int S32; 179 | typedef unsigned long long U64; 180 | typedef size_t uptrval; /* generally true, except OpenVMS-64 */ 181 | #endif 182 | 183 | #if defined(__x86_64__) 184 | typedef U64 reg_t; /* 64-bits in x32 mode */ 185 | #else 186 | typedef size_t reg_t; /* 32-bits in x32 mode */ 187 | #endif 188 | 189 | /*-************************************ 190 | * Reading and writing into memory 191 | **************************************/ 192 | static unsigned LZ4_isLittleEndian(void) 193 | { 194 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 195 | return one.c[0]; 196 | } 197 | 198 | 199 | #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) 200 | /* lie to the compiler about data alignment; use with caution */ 201 | 202 | static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } 203 | static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } 204 | static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } 205 | 206 | static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 207 | static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 208 | 209 | #elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) 210 | 211 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 212 | /* currently only defined for gcc and icc */ 213 | typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; 214 | 215 | static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } 216 | static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } 217 | static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } 218 | 219 | static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } 220 | static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } 221 | 222 | #else /* safe and portable access through memcpy() */ 223 | 224 | static U16 LZ4_read16(const void* memPtr) 225 | { 226 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 227 | } 228 | 229 | static U32 LZ4_read32(const void* memPtr) 230 | { 231 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 232 | } 233 | 234 | static reg_t LZ4_read_ARCH(const void* memPtr) 235 | { 236 | reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; 237 | } 238 | 239 | static void LZ4_write16(void* memPtr, U16 value) 240 | { 241 | memcpy(memPtr, &value, sizeof(value)); 242 | } 243 | 244 | static void LZ4_write32(void* memPtr, U32 value) 245 | { 246 | memcpy(memPtr, &value, sizeof(value)); 247 | } 248 | 249 | #endif /* LZ4_FORCE_MEMORY_ACCESS */ 250 | 251 | 252 | static U16 LZ4_readLE16(const void* memPtr) 253 | { 254 | if (LZ4_isLittleEndian()) { 255 | return LZ4_read16(memPtr); 256 | } else { 257 | const BYTE* p = (const BYTE*)memPtr; 258 | return (U16)((U16)p[0] + (p[1]<<8)); 259 | } 260 | } 261 | 262 | static void LZ4_writeLE16(void* memPtr, U16 value) 263 | { 264 | if (LZ4_isLittleEndian()) { 265 | LZ4_write16(memPtr, value); 266 | } else { 267 | BYTE* p = (BYTE*)memPtr; 268 | p[0] = (BYTE) value; 269 | p[1] = (BYTE)(value>>8); 270 | } 271 | } 272 | 273 | static void LZ4_copy8(void* dst, const void* src) 274 | { 275 | memcpy(dst,src,8); 276 | } 277 | 278 | /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ 279 | LZ4_FORCE_O2_INLINE_GCC_PPC64LE 280 | void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) 281 | { 282 | BYTE* d = (BYTE*)dstPtr; 283 | const BYTE* s = (const BYTE*)srcPtr; 284 | BYTE* const e = (BYTE*)dstEnd; 285 | 286 | do { LZ4_copy8(d,s); d+=8; s+=8; } while (d=1) 317 | # include 318 | #else 319 | # ifndef assert 320 | # define assert(condition) ((void)0) 321 | # endif 322 | #endif 323 | 324 | #define LZ4_STATIC_ASSERT(c) { enum { LZ4_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ 325 | 326 | #if defined(LZ4_DEBUG) && (LZ4_DEBUG>=2) 327 | # include 328 | static int g_debuglog_enable = 1; 329 | # define DEBUGLOG(l, ...) { \ 330 | if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \ 331 | fprintf(stderr, __FILE__ ": "); \ 332 | fprintf(stderr, __VA_ARGS__); \ 333 | fprintf(stderr, " \n"); \ 334 | } } 335 | #else 336 | # define DEBUGLOG(l, ...) {} /* disabled */ 337 | #endif 338 | 339 | 340 | /*-************************************ 341 | * Common functions 342 | **************************************/ 343 | static unsigned LZ4_NbCommonBytes (reg_t val) 344 | { 345 | if (LZ4_isLittleEndian()) { 346 | if (sizeof(val)==8) { 347 | # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) 348 | unsigned long r = 0; 349 | _BitScanForward64( &r, (U64)val ); 350 | return (int)(r>>3); 351 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 352 | return (__builtin_ctzll((U64)val) >> 3); 353 | # else 354 | static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 355 | 0, 3, 1, 3, 1, 4, 2, 7, 356 | 0, 2, 3, 6, 1, 5, 3, 5, 357 | 1, 3, 4, 4, 2, 5, 6, 7, 358 | 7, 0, 1, 2, 3, 3, 4, 6, 359 | 2, 6, 5, 5, 3, 4, 5, 6, 360 | 7, 1, 2, 4, 6, 4, 4, 5, 361 | 7, 2, 6, 5, 7, 6, 7, 7 }; 362 | return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; 363 | # endif 364 | } else /* 32 bits */ { 365 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 366 | unsigned long r; 367 | _BitScanForward( &r, (U32)val ); 368 | return (int)(r>>3); 369 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 370 | return (__builtin_ctz((U32)val) >> 3); 371 | # else 372 | static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 373 | 3, 2, 2, 1, 3, 2, 0, 1, 374 | 3, 3, 1, 2, 2, 2, 2, 0, 375 | 3, 1, 2, 0, 1, 0, 1, 1 }; 376 | return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; 377 | # endif 378 | } 379 | } else /* Big Endian CPU */ { 380 | if (sizeof(val)==8) { /* 64-bits */ 381 | # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) 382 | unsigned long r = 0; 383 | _BitScanReverse64( &r, val ); 384 | return (unsigned)(r>>3); 385 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 386 | return (__builtin_clzll((U64)val) >> 3); 387 | # else 388 | static const U32 by32 = sizeof(val)*4; /* 32 on 64 bits (goal), 16 on 32 bits. 389 | Just to avoid some static analyzer complaining about shift by 32 on 32-bits target. 390 | Note that this code path is never triggered in 32-bits mode. */ 391 | unsigned r; 392 | if (!(val>>by32)) { r=4; } else { r=0; val>>=by32; } 393 | if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } 394 | r += (!val); 395 | return r; 396 | # endif 397 | } else /* 32 bits */ { 398 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 399 | unsigned long r = 0; 400 | _BitScanReverse( &r, (unsigned long)val ); 401 | return (unsigned)(r>>3); 402 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 403 | return (__builtin_clz((U32)val) >> 3); 404 | # else 405 | unsigned r; 406 | if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } 407 | r += (!val); 408 | return r; 409 | # endif 410 | } 411 | } 412 | } 413 | 414 | #define STEPSIZE sizeof(reg_t) 415 | LZ4_FORCE_INLINE 416 | unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) 417 | { 418 | const BYTE* const pStart = pIn; 419 | 420 | if (likely(pIn < pInLimit-(STEPSIZE-1))) { 421 | reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); 422 | if (!diff) { 423 | pIn+=STEPSIZE; pMatch+=STEPSIZE; 424 | } else { 425 | return LZ4_NbCommonBytes(diff); 426 | } } 427 | 428 | while (likely(pIn < pInLimit-(STEPSIZE-1))) { 429 | reg_t const diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn); 430 | if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; } 431 | pIn += LZ4_NbCommonBytes(diff); 432 | return (unsigned)(pIn - pStart); 433 | } 434 | 435 | if ((STEPSIZE==8) && (pIn<(pInLimit-3)) && (LZ4_read32(pMatch) == LZ4_read32(pIn))) { pIn+=4; pMatch+=4; } 436 | if ((pIn<(pInLimit-1)) && (LZ4_read16(pMatch) == LZ4_read16(pIn))) { pIn+=2; pMatch+=2; } 437 | if ((pIn compression run slower on incompressible data */ 448 | 449 | 450 | /*-************************************ 451 | * Local Structures and types 452 | **************************************/ 453 | typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; 454 | typedef enum { byPtr, byU32, byU16 } tableType_t; 455 | 456 | typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; 457 | typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; 458 | 459 | typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; 460 | typedef enum { full = 0, partial = 1 } earlyEnd_directive; 461 | 462 | 463 | /*-************************************ 464 | * Local Utils 465 | **************************************/ 466 | int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } 467 | const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } 468 | int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } 469 | int LZ4_sizeofState() { return LZ4_STREAMSIZE; } 470 | 471 | 472 | /*-****************************** 473 | * Compression functions 474 | ********************************/ 475 | static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) 476 | { 477 | if (tableType == byU16) 478 | return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); 479 | else 480 | return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); 481 | } 482 | 483 | static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) 484 | { 485 | static const U64 prime5bytes = 889523592379ULL; 486 | static const U64 prime8bytes = 11400714785074694791ULL; 487 | const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; 488 | if (LZ4_isLittleEndian()) 489 | return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); 490 | else 491 | return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); 492 | } 493 | 494 | LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) 495 | { 496 | if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); 497 | return LZ4_hash4(LZ4_read32(p), tableType); 498 | } 499 | 500 | static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) 501 | { 502 | switch (tableType) 503 | { 504 | case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } 505 | case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } 506 | case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } 507 | } 508 | } 509 | 510 | LZ4_FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 511 | { 512 | U32 const h = LZ4_hashPosition(p, tableType); 513 | LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); 514 | } 515 | 516 | static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) 517 | { 518 | if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } 519 | if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } 520 | { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ 521 | } 522 | 523 | LZ4_FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 524 | { 525 | U32 const h = LZ4_hashPosition(p, tableType); 526 | return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); 527 | } 528 | 529 | 530 | /** LZ4_compress_generic() : 531 | inlined, to ensure branches are decided at compilation time */ 532 | LZ4_FORCE_INLINE int LZ4_compress_generic( 533 | LZ4_stream_t_internal* const cctx, 534 | const char* const source, 535 | char* const dest, 536 | const int inputSize, 537 | const int maxOutputSize, 538 | const limitedOutput_directive outputLimited, 539 | const tableType_t tableType, 540 | const dict_directive dict, 541 | const dictIssue_directive dictIssue, 542 | const U32 acceleration) 543 | { 544 | const BYTE* ip = (const BYTE*) source; 545 | const BYTE* base; 546 | const BYTE* lowLimit; 547 | const BYTE* const lowRefLimit = ip - cctx->dictSize; 548 | const BYTE* const dictionary = cctx->dictionary; 549 | const BYTE* const dictEnd = dictionary + cctx->dictSize; 550 | const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; 551 | const BYTE* anchor = (const BYTE*) source; 552 | const BYTE* const iend = ip + inputSize; 553 | const BYTE* const mflimit = iend - MFLIMIT; 554 | const BYTE* const matchlimit = iend - LASTLITERALS; 555 | 556 | BYTE* op = (BYTE*) dest; 557 | BYTE* const olimit = op + maxOutputSize; 558 | 559 | U32 forwardH; 560 | 561 | /* Init conditions */ 562 | if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ 563 | switch(dict) 564 | { 565 | case noDict: 566 | default: 567 | base = (const BYTE*)source; 568 | lowLimit = (const BYTE*)source; 569 | break; 570 | case withPrefix64k: 571 | base = (const BYTE*)source - cctx->currentOffset; 572 | lowLimit = (const BYTE*)source - cctx->dictSize; 573 | break; 574 | case usingExtDict: 575 | base = (const BYTE*)source - cctx->currentOffset; 576 | lowLimit = (const BYTE*)source; 577 | break; 578 | } 579 | if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 580 | if (inputSizehashTable, tableType, base); 584 | ip++; forwardH = LZ4_hashPosition(ip, tableType); 585 | 586 | /* Main Loop */ 587 | for ( ; ; ) { 588 | ptrdiff_t refDelta = 0; 589 | const BYTE* match; 590 | BYTE* token; 591 | 592 | /* Find a match */ 593 | { const BYTE* forwardIp = ip; 594 | unsigned step = 1; 595 | unsigned searchMatchNb = acceleration << LZ4_skipTrigger; 596 | do { 597 | U32 const h = forwardH; 598 | ip = forwardIp; 599 | forwardIp += step; 600 | step = (searchMatchNb++ >> LZ4_skipTrigger); 601 | 602 | if (unlikely(forwardIp > mflimit)) goto _last_literals; 603 | 604 | match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); 605 | if (dict==usingExtDict) { 606 | if (match < (const BYTE*)source) { 607 | refDelta = dictDelta; 608 | lowLimit = dictionary; 609 | } else { 610 | refDelta = 0; 611 | lowLimit = (const BYTE*)source; 612 | } } 613 | forwardH = LZ4_hashPosition(forwardIp, tableType); 614 | LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); 615 | 616 | } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) 617 | || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) 618 | || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); 619 | } 620 | 621 | /* Catch up */ 622 | while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } 623 | 624 | /* Encode Literals */ 625 | { unsigned const litLength = (unsigned)(ip - anchor); 626 | token = op++; 627 | if ((outputLimited) && /* Check output buffer overflow */ 628 | (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) 629 | return 0; 630 | if (litLength >= RUN_MASK) { 631 | int len = (int)litLength-RUN_MASK; 632 | *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; 634 | *op++ = (BYTE)len; 635 | } 636 | else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; 655 | matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); 656 | ip += MINMATCH + matchCode; 657 | if (ip==limit) { 658 | unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); 659 | matchCode += more; 660 | ip += more; 661 | } 662 | } else { 663 | matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); 664 | ip += MINMATCH + matchCode; 665 | } 666 | 667 | if ( outputLimited && /* Check output buffer overflow */ 668 | (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) 669 | return 0; 670 | if (matchCode >= ML_MASK) { 671 | *token += ML_MASK; 672 | matchCode -= ML_MASK; 673 | LZ4_write32(op, 0xFFFFFFFF); 674 | while (matchCode >= 4*255) { 675 | op+=4; 676 | LZ4_write32(op, 0xFFFFFFFF); 677 | matchCode -= 4*255; 678 | } 679 | op += matchCode / 255; 680 | *op++ = (BYTE)(matchCode % 255); 681 | } else 682 | *token += (BYTE)(matchCode); 683 | } 684 | 685 | anchor = ip; 686 | 687 | /* Test end of chunk */ 688 | if (ip > mflimit) break; 689 | 690 | /* Fill table */ 691 | LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); 692 | 693 | /* Test next position */ 694 | match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); 695 | if (dict==usingExtDict) { 696 | if (match < (const BYTE*)source) { 697 | refDelta = dictDelta; 698 | lowLimit = dictionary; 699 | } else { 700 | refDelta = 0; 701 | lowLimit = (const BYTE*)source; 702 | } } 703 | LZ4_putPosition(ip, cctx->hashTable, tableType, base); 704 | if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) 705 | && (match+MAX_DISTANCE>=ip) 706 | && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) 707 | { token=op++; *token=0; goto _next_match; } 708 | 709 | /* Prepare next loop */ 710 | forwardH = LZ4_hashPosition(++ip, tableType); 711 | } 712 | 713 | _last_literals: 714 | /* Encode Last Literals */ 715 | { size_t const lastRun = (size_t)(iend - anchor); 716 | if ( (outputLimited) && /* Check output buffer overflow */ 717 | ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) 718 | return 0; 719 | if (lastRun >= RUN_MASK) { 720 | size_t accumulator = lastRun - RUN_MASK; 721 | *op++ = RUN_MASK << ML_BITS; 722 | for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; 723 | *op++ = (BYTE) accumulator; 724 | } else { 725 | *op++ = (BYTE)(lastRun<internal_donotuse; 739 | LZ4_resetStream((LZ4_stream_t*)state); 740 | if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; 741 | 742 | if (maxOutputSize >= LZ4_compressBound(inputSize)) { 743 | if (inputSize < LZ4_64Klimit) 744 | return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); 745 | else 746 | return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); 747 | } else { 748 | if (inputSize < LZ4_64Klimit) 749 | return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); 750 | else 751 | return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); 752 | } 753 | } 754 | 755 | 756 | int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 757 | { 758 | #if (LZ4_HEAPMODE) 759 | void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ 760 | #else 761 | LZ4_stream_t ctx; 762 | void* const ctxPtr = &ctx; 763 | #endif 764 | 765 | int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); 766 | 767 | #if (LZ4_HEAPMODE) 768 | FREEMEM(ctxPtr); 769 | #endif 770 | return result; 771 | } 772 | 773 | 774 | int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) 775 | { 776 | return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); 777 | } 778 | 779 | 780 | /* hidden debug function */ 781 | /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ 782 | int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 783 | { 784 | LZ4_stream_t ctx; 785 | LZ4_resetStream(&ctx); 786 | 787 | if (inputSize < LZ4_64Klimit) 788 | return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); 789 | else 790 | return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); 791 | } 792 | 793 | 794 | /*-****************************** 795 | * *_destSize() variant 796 | ********************************/ 797 | 798 | static int LZ4_compress_destSize_generic( 799 | LZ4_stream_t_internal* const ctx, 800 | const char* const src, 801 | char* const dst, 802 | int* const srcSizePtr, 803 | const int targetDstSize, 804 | const tableType_t tableType) 805 | { 806 | const BYTE* ip = (const BYTE*) src; 807 | const BYTE* base = (const BYTE*) src; 808 | const BYTE* lowLimit = (const BYTE*) src; 809 | const BYTE* anchor = ip; 810 | const BYTE* const iend = ip + *srcSizePtr; 811 | const BYTE* const mflimit = iend - MFLIMIT; 812 | const BYTE* const matchlimit = iend - LASTLITERALS; 813 | 814 | BYTE* op = (BYTE*) dst; 815 | BYTE* const oend = op + targetDstSize; 816 | BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; 817 | BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); 818 | BYTE* const oMaxSeq = oMaxLit - 1 /* token */; 819 | 820 | U32 forwardH; 821 | 822 | 823 | /* Init conditions */ 824 | if (targetDstSize < 1) return 0; /* Impossible to store anything */ 825 | if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ 826 | if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 827 | if (*srcSizePtrhashTable, tableType, base); 832 | ip++; forwardH = LZ4_hashPosition(ip, tableType); 833 | 834 | /* Main Loop */ 835 | for ( ; ; ) { 836 | const BYTE* match; 837 | BYTE* token; 838 | 839 | /* Find a match */ 840 | { const BYTE* forwardIp = ip; 841 | unsigned step = 1; 842 | unsigned searchMatchNb = 1 << LZ4_skipTrigger; 843 | 844 | do { 845 | U32 h = forwardH; 846 | ip = forwardIp; 847 | forwardIp += step; 848 | step = (searchMatchNb++ >> LZ4_skipTrigger); 849 | 850 | if (unlikely(forwardIp > mflimit)) goto _last_literals; 851 | 852 | match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); 853 | forwardH = LZ4_hashPosition(forwardIp, tableType); 854 | LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); 855 | 856 | } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) 857 | || (LZ4_read32(match) != LZ4_read32(ip)) ); 858 | } 859 | 860 | /* Catch up */ 861 | while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } 862 | 863 | /* Encode Literal length */ 864 | { unsigned litLength = (unsigned)(ip - anchor); 865 | token = op++; 866 | if (op + ((litLength+240)/255) + litLength > oMaxLit) { 867 | /* Not enough space for a last match */ 868 | op--; 869 | goto _last_literals; 870 | } 871 | if (litLength>=RUN_MASK) { 872 | unsigned len = litLength - RUN_MASK; 873 | *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; 875 | *op++ = (BYTE)len; 876 | } 877 | else *token = (BYTE)(litLength< oMaxMatch) { 892 | /* Match description too long : reduce it */ 893 | matchLength = (15-1) + (oMaxMatch-op) * 255; 894 | } 895 | ip += MINMATCH + matchLength; 896 | 897 | if (matchLength>=ML_MASK) { 898 | *token += ML_MASK; 899 | matchLength -= ML_MASK; 900 | while (matchLength >= 255) { matchLength-=255; *op++ = 255; } 901 | *op++ = (BYTE)matchLength; 902 | } 903 | else *token += (BYTE)(matchLength); 904 | } 905 | 906 | anchor = ip; 907 | 908 | /* Test end of block */ 909 | if (ip > mflimit) break; 910 | if (op > oMaxSeq) break; 911 | 912 | /* Fill table */ 913 | LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); 914 | 915 | /* Test next position */ 916 | match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); 917 | LZ4_putPosition(ip, ctx->hashTable, tableType, base); 918 | if ( (match+MAX_DISTANCE>=ip) 919 | && (LZ4_read32(match)==LZ4_read32(ip)) ) 920 | { token=op++; *token=0; goto _next_match; } 921 | 922 | /* Prepare next loop */ 923 | forwardH = LZ4_hashPosition(++ip, tableType); 924 | } 925 | 926 | _last_literals: 927 | /* Encode Last Literals */ 928 | { size_t lastRunSize = (size_t)(iend - anchor); 929 | if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { 930 | /* adapt lastRunSize to fill 'dst' */ 931 | lastRunSize = (oend-op) - 1; 932 | lastRunSize -= (lastRunSize+240)/255; 933 | } 934 | ip = anchor + lastRunSize; 935 | 936 | if (lastRunSize >= RUN_MASK) { 937 | size_t accumulator = lastRunSize - RUN_MASK; 938 | *op++ = RUN_MASK << ML_BITS; 939 | for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; 940 | *op++ = (BYTE) accumulator; 941 | } else { 942 | *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ 959 | return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); 960 | } else { 961 | if (*srcSizePtr < LZ4_64Klimit) 962 | return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); 963 | else 964 | return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); 965 | } 966 | } 967 | 968 | 969 | int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) 970 | { 971 | #if (LZ4_HEAPMODE) 972 | LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ 973 | #else 974 | LZ4_stream_t ctxBody; 975 | LZ4_stream_t* ctx = &ctxBody; 976 | #endif 977 | 978 | int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); 979 | 980 | #if (LZ4_HEAPMODE) 981 | FREEMEM(ctx); 982 | #endif 983 | return result; 984 | } 985 | 986 | 987 | 988 | /*-****************************** 989 | * Streaming functions 990 | ********************************/ 991 | 992 | LZ4_stream_t* LZ4_createStream(void) 993 | { 994 | LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); 995 | LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ 996 | LZ4_resetStream(lz4s); 997 | return lz4s; 998 | } 999 | 1000 | void LZ4_resetStream (LZ4_stream_t* LZ4_stream) 1001 | { 1002 | DEBUGLOG(4, "LZ4_resetStream"); 1003 | MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); 1004 | } 1005 | 1006 | int LZ4_freeStream (LZ4_stream_t* LZ4_stream) 1007 | { 1008 | if (!LZ4_stream) return 0; /* support free on NULL */ 1009 | FREEMEM(LZ4_stream); 1010 | return (0); 1011 | } 1012 | 1013 | 1014 | #define HASH_UNIT sizeof(reg_t) 1015 | int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) 1016 | { 1017 | LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; 1018 | const BYTE* p = (const BYTE*)dictionary; 1019 | const BYTE* const dictEnd = p + dictSize; 1020 | const BYTE* base; 1021 | 1022 | if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ 1023 | LZ4_resetStream(LZ4_dict); 1024 | 1025 | if (dictSize < (int)HASH_UNIT) { 1026 | dict->dictionary = NULL; 1027 | dict->dictSize = 0; 1028 | return 0; 1029 | } 1030 | 1031 | if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; 1032 | dict->currentOffset += 64 KB; 1033 | base = p - dict->currentOffset; 1034 | dict->dictionary = p; 1035 | dict->dictSize = (U32)(dictEnd - p); 1036 | dict->currentOffset += dict->dictSize; 1037 | 1038 | while (p <= dictEnd-HASH_UNIT) { 1039 | LZ4_putPosition(p, dict->hashTable, byU32, base); 1040 | p+=3; 1041 | } 1042 | 1043 | return dict->dictSize; 1044 | } 1045 | 1046 | 1047 | static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) 1048 | { 1049 | if ((LZ4_dict->currentOffset > 0x80000000) || 1050 | ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ 1051 | /* rescale hash table */ 1052 | U32 const delta = LZ4_dict->currentOffset - 64 KB; 1053 | const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; 1054 | int i; 1055 | for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; 1057 | else LZ4_dict->hashTable[i] -= delta; 1058 | } 1059 | LZ4_dict->currentOffset = 64 KB; 1060 | if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; 1061 | LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; 1062 | } 1063 | } 1064 | 1065 | 1066 | int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 1067 | { 1068 | LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; 1069 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 1070 | 1071 | const BYTE* smallest = (const BYTE*) source; 1072 | if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ 1073 | if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; 1074 | LZ4_renormDictT(streamPtr, smallest); 1075 | if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; 1076 | 1077 | /* Check overlapping input/dictionary space */ 1078 | { const BYTE* sourceEnd = (const BYTE*) source + inputSize; 1079 | if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { 1080 | streamPtr->dictSize = (U32)(dictEnd - sourceEnd); 1081 | if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; 1082 | if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; 1083 | streamPtr->dictionary = dictEnd - streamPtr->dictSize; 1084 | } 1085 | } 1086 | 1087 | /* prefix mode : source data follows dictionary */ 1088 | if (dictEnd == (const BYTE*)source) { 1089 | int result; 1090 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 1091 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); 1092 | else 1093 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); 1094 | streamPtr->dictSize += (U32)inputSize; 1095 | streamPtr->currentOffset += (U32)inputSize; 1096 | return result; 1097 | } 1098 | 1099 | /* external dictionary mode */ 1100 | { int result; 1101 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 1102 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); 1103 | else 1104 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); 1105 | streamPtr->dictionary = (const BYTE*)source; 1106 | streamPtr->dictSize = (U32)inputSize; 1107 | streamPtr->currentOffset += (U32)inputSize; 1108 | return result; 1109 | } 1110 | } 1111 | 1112 | 1113 | /* Hidden debug function, to force external dictionary mode */ 1114 | int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) 1115 | { 1116 | LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; 1117 | int result; 1118 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 1119 | 1120 | const BYTE* smallest = dictEnd; 1121 | if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; 1122 | LZ4_renormDictT(streamPtr, smallest); 1123 | 1124 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); 1125 | 1126 | streamPtr->dictionary = (const BYTE*)source; 1127 | streamPtr->dictSize = (U32)inputSize; 1128 | streamPtr->currentOffset += (U32)inputSize; 1129 | 1130 | return result; 1131 | } 1132 | 1133 | 1134 | /*! LZ4_saveDict() : 1135 | * If previously compressed data block is not guaranteed to remain available at its memory location, 1136 | * save it into a safer place (char* safeBuffer). 1137 | * Note : you don't need to call LZ4_loadDict() afterwards, 1138 | * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). 1139 | * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. 1140 | */ 1141 | int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) 1142 | { 1143 | LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; 1144 | const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; 1145 | 1146 | if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ 1147 | if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; 1148 | 1149 | memmove(safeBuffer, previousDictEnd - dictSize, dictSize); 1150 | 1151 | dict->dictionary = (const BYTE*)safeBuffer; 1152 | dict->dictSize = (U32)dictSize; 1153 | 1154 | return dictSize; 1155 | } 1156 | 1157 | 1158 | 1159 | /*-***************************** 1160 | * Decompression functions 1161 | *******************************/ 1162 | /*! LZ4_decompress_generic() : 1163 | * This generic decompression function covers all use cases. 1164 | * It shall be instantiated several times, using different sets of directives. 1165 | * Note that it is important for performance that this function really get inlined, 1166 | * in order to remove useless branches during compilation optimization. 1167 | */ 1168 | LZ4_FORCE_O2_GCC_PPC64LE 1169 | LZ4_FORCE_INLINE int LZ4_decompress_generic( 1170 | const char* const src, 1171 | char* const dst, 1172 | int srcSize, 1173 | int outputSize, /* If endOnInput==endOnInputSize, this value is `dstCapacity` */ 1174 | 1175 | int endOnInput, /* endOnOutputSize, endOnInputSize */ 1176 | int partialDecoding, /* full, partial */ 1177 | int targetOutputSize, /* only used if partialDecoding==partial */ 1178 | int dict, /* noDict, withPrefix64k, usingExtDict */ 1179 | const BYTE* const lowPrefix, /* always <= dst, == dst when no prefix */ 1180 | const BYTE* const dictStart, /* only if dict==usingExtDict */ 1181 | const size_t dictSize /* note : = 0 if noDict */ 1182 | ) 1183 | { 1184 | const BYTE* ip = (const BYTE*) src; 1185 | const BYTE* const iend = ip + srcSize; 1186 | 1187 | BYTE* op = (BYTE*) dst; 1188 | BYTE* const oend = op + outputSize; 1189 | BYTE* cpy; 1190 | BYTE* oexit = op + targetOutputSize; 1191 | 1192 | const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; 1193 | const unsigned inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; 1194 | const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; 1195 | 1196 | const int safeDecode = (endOnInput==endOnInputSize); 1197 | const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); 1198 | 1199 | 1200 | /* Special cases */ 1201 | if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => just decode everything */ 1202 | if ((endOnInput) && (unlikely(outputSize==0))) return ((srcSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ 1203 | if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); 1204 | 1205 | /* Main Loop : decode sequences */ 1206 | while (1) { 1207 | size_t length; 1208 | const BYTE* match; 1209 | size_t offset; 1210 | 1211 | unsigned const token = *ip++; 1212 | 1213 | /* shortcut for common case : 1214 | * in most circumstances, we expect to decode small matches (<= 18 bytes) separated by few literals (<= 14 bytes). 1215 | * this shortcut was tested on x86 and x64, where it improves decoding speed. 1216 | * it has not yet been benchmarked on ARM, Power, mips, etc. */ 1217 | if (((ip + 14 /*maxLL*/ + 2 /*offset*/ <= iend) 1218 | & (op + 14 /*maxLL*/ + 18 /*maxML*/ <= oend)) 1219 | & ((token < (15<> ML_BITS; 1221 | size_t const off = LZ4_readLE16(ip+ll); 1222 | const BYTE* const matchPtr = op + ll - off; /* pointer underflow risk ? */ 1223 | if ((off >= 18) /* do not deal with overlapping matches */ & (matchPtr >= lowPrefix)) { 1224 | size_t const ml = (token & ML_MASK) + MINMATCH; 1225 | memcpy(op, ip, 16); op += ll; ip += ll + 2 /*offset*/; 1226 | memcpy(op, matchPtr, 18); op += ml; 1227 | continue; 1228 | } 1229 | } 1230 | 1231 | /* decode literal length */ 1232 | if ((length=(token>>ML_BITS)) == RUN_MASK) { 1233 | unsigned s; 1234 | do { 1235 | s = *ip++; 1236 | length += s; 1237 | } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) 1245 | || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) 1246 | { 1247 | if (partialDecoding) { 1248 | if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ 1249 | if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ 1250 | } else { 1251 | if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ 1252 | if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ 1253 | } 1254 | memcpy(op, ip, length); 1255 | ip += length; 1256 | op += length; 1257 | break; /* Necessarily EOF, due to parsing restrictions */ 1258 | } 1259 | LZ4_wildCopy(op, ip, cpy); 1260 | ip += length; op = cpy; 1261 | 1262 | /* get offset */ 1263 | offset = LZ4_readLE16(ip); ip+=2; 1264 | match = op - offset; 1265 | if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) goto _output_error; /* Error : offset outside buffers */ 1266 | LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ 1267 | 1268 | /* get matchlength */ 1269 | length = token & ML_MASK; 1270 | if (length == ML_MASK) { 1271 | unsigned s; 1272 | do { 1273 | s = *ip++; 1274 | if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; 1275 | length += s; 1276 | } while (s==255); 1277 | if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ 1278 | } 1279 | length += MINMATCH; 1280 | 1281 | /* check external dictionary */ 1282 | if ((dict==usingExtDict) && (match < lowPrefix)) { 1283 | if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ 1284 | 1285 | if (length <= (size_t)(lowPrefix-match)) { 1286 | /* match can be copied as a single segment from external dictionary */ 1287 | memmove(op, dictEnd - (lowPrefix-match), length); 1288 | op += length; 1289 | } else { 1290 | /* match encompass external dictionary and current block */ 1291 | size_t const copySize = (size_t)(lowPrefix-match); 1292 | size_t const restSize = length - copySize; 1293 | memcpy(op, dictEnd - copySize, copySize); 1294 | op += copySize; 1295 | if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ 1296 | BYTE* const endOfMatch = op + restSize; 1297 | const BYTE* copyFrom = lowPrefix; 1298 | while (op < endOfMatch) *op++ = *copyFrom++; 1299 | } else { 1300 | memcpy(op, lowPrefix, restSize); 1301 | op += restSize; 1302 | } } 1303 | continue; 1304 | } 1305 | 1306 | /* copy match within block */ 1307 | cpy = op + length; 1308 | if (unlikely(offset<8)) { 1309 | op[0] = match[0]; 1310 | op[1] = match[1]; 1311 | op[2] = match[2]; 1312 | op[3] = match[3]; 1313 | match += inc32table[offset]; 1314 | memcpy(op+4, match, 4); 1315 | match -= dec64table[offset]; 1316 | } else { LZ4_copy8(op, match); match+=8; } 1317 | op += 8; 1318 | 1319 | if (unlikely(cpy>oend-12)) { 1320 | BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); 1321 | if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ 1322 | if (op < oCopyLimit) { 1323 | LZ4_wildCopy(op, match, oCopyLimit); 1324 | match += oCopyLimit - op; 1325 | op = oCopyLimit; 1326 | } 1327 | while (op16) LZ4_wildCopy(op+8, match+8, cpy); 1331 | } 1332 | op = cpy; /* correction */ 1333 | } 1334 | 1335 | /* end of decoding */ 1336 | if (endOnInput) 1337 | return (int) (((char*)op)-dst); /* Nb of output bytes decoded */ 1338 | else 1339 | return (int) (((const char*)ip)-src); /* Nb of input bytes read */ 1340 | 1341 | /* Overflow error detected */ 1342 | _output_error: 1343 | return (int) (-(((const char*)ip)-src))-1; 1344 | } 1345 | 1346 | 1347 | LZ4_FORCE_O2_GCC_PPC64LE 1348 | int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) 1349 | { 1350 | return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); 1351 | } 1352 | 1353 | LZ4_FORCE_O2_GCC_PPC64LE 1354 | int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) 1355 | { 1356 | return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); 1357 | } 1358 | 1359 | LZ4_FORCE_O2_GCC_PPC64LE 1360 | int LZ4_decompress_fast(const char* source, char* dest, int originalSize) 1361 | { 1362 | return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); 1363 | } 1364 | 1365 | 1366 | /*===== streaming decompression functions =====*/ 1367 | 1368 | LZ4_streamDecode_t* LZ4_createStreamDecode(void) 1369 | { 1370 | LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); 1371 | return lz4s; 1372 | } 1373 | 1374 | int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) 1375 | { 1376 | if (!LZ4_stream) return 0; /* support free on NULL */ 1377 | FREEMEM(LZ4_stream); 1378 | return 0; 1379 | } 1380 | 1381 | /*! 1382 | * LZ4_setStreamDecode() : 1383 | * Use this function to instruct where to find the dictionary. 1384 | * This function is not necessary if previous data is still available where it was decoded. 1385 | * Loading a size of 0 is allowed (same effect as no dictionary). 1386 | * Return : 1 if OK, 0 if error 1387 | */ 1388 | int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) 1389 | { 1390 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1391 | lz4sd->prefixSize = (size_t) dictSize; 1392 | lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; 1393 | lz4sd->externalDict = NULL; 1394 | lz4sd->extDictSize = 0; 1395 | return 1; 1396 | } 1397 | 1398 | /* 1399 | *_continue() : 1400 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 1401 | Previously decoded blocks must still be available at the memory position where they were decoded. 1402 | If it's not possible, save the relevant part of decoded data into a safe buffer, 1403 | and indicate where it stands using LZ4_setStreamDecode() 1404 | */ 1405 | LZ4_FORCE_O2_GCC_PPC64LE 1406 | int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) 1407 | { 1408 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1409 | int result; 1410 | 1411 | if (lz4sd->prefixEnd == (BYTE*)dest) { 1412 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1413 | endOnInputSize, full, 0, 1414 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1415 | if (result <= 0) return result; 1416 | lz4sd->prefixSize += result; 1417 | lz4sd->prefixEnd += result; 1418 | } else { 1419 | lz4sd->extDictSize = lz4sd->prefixSize; 1420 | lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; 1421 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1422 | endOnInputSize, full, 0, 1423 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1424 | if (result <= 0) return result; 1425 | lz4sd->prefixSize = result; 1426 | lz4sd->prefixEnd = (BYTE*)dest + result; 1427 | } 1428 | 1429 | return result; 1430 | } 1431 | 1432 | LZ4_FORCE_O2_GCC_PPC64LE 1433 | int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) 1434 | { 1435 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1436 | int result; 1437 | 1438 | if (lz4sd->prefixEnd == (BYTE*)dest) { 1439 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1440 | endOnOutputSize, full, 0, 1441 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1442 | if (result <= 0) return result; 1443 | lz4sd->prefixSize += originalSize; 1444 | lz4sd->prefixEnd += originalSize; 1445 | } else { 1446 | lz4sd->extDictSize = lz4sd->prefixSize; 1447 | lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; 1448 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1449 | endOnOutputSize, full, 0, 1450 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1451 | if (result <= 0) return result; 1452 | lz4sd->prefixSize = originalSize; 1453 | lz4sd->prefixEnd = (BYTE*)dest + originalSize; 1454 | } 1455 | 1456 | return result; 1457 | } 1458 | 1459 | 1460 | /* 1461 | Advanced decoding functions : 1462 | *_usingDict() : 1463 | These decoding functions work the same as "_continue" ones, 1464 | the dictionary must be explicitly provided within parameters 1465 | */ 1466 | 1467 | LZ4_FORCE_O2_GCC_PPC64LE 1468 | LZ4_FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) 1469 | { 1470 | if (dictSize==0) 1471 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); 1472 | if (dictStart+dictSize == dest) { 1473 | if (dictSize >= (int)(64 KB - 1)) 1474 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); 1475 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); 1476 | } 1477 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); 1478 | } 1479 | 1480 | LZ4_FORCE_O2_GCC_PPC64LE 1481 | int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1482 | { 1483 | return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); 1484 | } 1485 | 1486 | LZ4_FORCE_O2_GCC_PPC64LE 1487 | int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) 1488 | { 1489 | return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); 1490 | } 1491 | 1492 | /* debug function */ 1493 | LZ4_FORCE_O2_GCC_PPC64LE 1494 | int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1495 | { 1496 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); 1497 | } 1498 | 1499 | 1500 | /*=************************************************* 1501 | * Obsolete Functions 1502 | ***************************************************/ 1503 | /* obsolete compression functions */ 1504 | int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } 1505 | int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } 1506 | int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } 1507 | int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } 1508 | int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } 1509 | int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } 1510 | 1511 | /* 1512 | These function names are deprecated and should no longer be used. 1513 | They are only provided here for compatibility with older user programs. 1514 | - LZ4_uncompress is totally equivalent to LZ4_decompress_fast 1515 | - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe 1516 | */ 1517 | int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } 1518 | int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } 1519 | 1520 | 1521 | /* Obsolete Streaming functions */ 1522 | 1523 | int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } 1524 | 1525 | static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) 1526 | { 1527 | MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); 1528 | lz4ds->internal_donotuse.bufferStart = base; 1529 | } 1530 | 1531 | int LZ4_resetStreamState(void* state, char* inputBuffer) 1532 | { 1533 | if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ 1534 | LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); 1535 | return 0; 1536 | } 1537 | 1538 | void* LZ4_create (char* inputBuffer) 1539 | { 1540 | LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); 1541 | LZ4_init (lz4ds, (BYTE*)inputBuffer); 1542 | return lz4ds; 1543 | } 1544 | 1545 | char* LZ4_slideInputBuffer (void* LZ4_Data) 1546 | { 1547 | LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; 1548 | int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); 1549 | return (char*)(ctx->bufferStart + dictSize); 1550 | } 1551 | 1552 | /* Obsolete streaming decompression functions */ 1553 | 1554 | int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) 1555 | { 1556 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1557 | } 1558 | 1559 | int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) 1560 | { 1561 | return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1562 | } 1563 | 1564 | #endif /* LZ4_COMMONDEFS_ONLY */ 1565 | --------------------------------------------------------------------------------