├── .gitignore ├── bin └── .gitignore ├── tmp └── .gitignore ├── lz4 ├── .DS_Store ├── LICENSE ├── lz4frame_static.h ├── xxhash.h ├── lz4hc.h ├── lz4opt.h ├── lz4frame.h ├── lz4.h └── xxhash.c ├── Makefile ├── src ├── readfiles.h ├── logger.h ├── config.h ├── pixie-timer.h ├── packetdump.h ├── logger.c ├── pixie-threads.h ├── string_s.c ├── readfiles.c ├── string_s.h ├── rawsock-pcapfile.h ├── test-compress.c ├── pixie-threads.c ├── pixie-timer.c ├── rawsock-pcap.h ├── rawsock-pcap.c ├── config.c └── main.c └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | -------------------------------------------------------------------------------- /bin/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | */ 3 | !.gitignore 4 | 5 | -------------------------------------------------------------------------------- /tmp/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | */ 3 | !.gitignore 4 | 5 | -------------------------------------------------------------------------------- /lz4/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robertdavidgraham/packetdump/HEAD/lz4/.DS_Store -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | CC = gcc 3 | CFLAGS = -I. 4 | 5 | bin/packetdump: src/*.c lz4/*.c 6 | $(CC) $(CFLAGS) -o $@ $^ 7 | -------------------------------------------------------------------------------- /src/readfiles.h: -------------------------------------------------------------------------------- 1 | #ifndef readfiles_h 2 | #define readfiles_h 3 | #include "packetdump.h" 4 | 5 | void 6 | read_files(const struct PacketDump *conf); 7 | 8 | #endif /* readfiles_h */ 9 | -------------------------------------------------------------------------------- /src/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef LOGGER_H 2 | #define LOGGER_H 3 | 4 | 5 | void LOG(int level, const char *fmt, ...); 6 | void LOGip(int level, unsigned ip, unsigned port, const char *fmt, ...); 7 | 8 | void LOG_add_level(int level); 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /src/config.h: -------------------------------------------------------------------------------- 1 | #ifndef config_h 2 | #define config_h 3 | #include 4 | #include "packetdump.h" 5 | 6 | void 7 | read_configuration(int argc, char **argv, struct PacketDump *conf); 8 | 9 | void 10 | echo_configuration(FILE *fp, struct PacketDump *conf); 11 | 12 | void 13 | configuration_help(void); 14 | 15 | #endif /* config_h */ 16 | -------------------------------------------------------------------------------- /src/pixie-timer.h: -------------------------------------------------------------------------------- 1 | #ifndef PIXIE_TIMER_H 2 | #define PIXIE_TIMER_H 3 | #include 4 | 5 | /** 6 | * The current time, in microseconds 7 | */ 8 | uint64_t pixie_gettime(void); 9 | 10 | /** 11 | * The current time, in nanoseconds 12 | */ 13 | uint64_t pixie_nanotime(void); 14 | 15 | /** 16 | * Wait the specified number of microseconds 17 | */ 18 | void pixie_usleep(uint64_t usec); 19 | 20 | /** 21 | * Wait the specified number of milliseconds 22 | */ 23 | void pixie_mssleep(unsigned milliseconds); 24 | 25 | /** 26 | * Do a self-test. Note that in some cases, this may 27 | * actaully fail when there is no problem. So far it hasn't, but I should 28 | * probably add some code to fix this. 29 | */ 30 | int pixie_time_selftest(void); 31 | 32 | 33 | 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /lz4/LICENSE: -------------------------------------------------------------------------------- 1 | LZ4 Library 2 | Copyright (c) 2011-2016, Yann Collet 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | list of conditions and the following disclaimer in the documentation and/or 13 | other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | packetdump - a high-speed compressing packet-sniffer 2 | ============================================= 3 | 4 | This is a high-speed (10gig) packet-sniffer that compresses packets as it 5 | writes to disk. I wrote it because the packet logging features of `tcpdump` 6 | are slow, but extra special slow when attempting to compress files after 7 | they've been written uncompressed to the disk when rotating files. 8 | 9 | Typical usage is simply something like this: 10 | 11 | packetdump -i eth0 -G 3600 -w foo-%y%m%d-%H%M%S.pcap.lz4 12 | 13 | This will recognize the `.lz4` extension, and automatically use the LZ4 14 | compression method. The files can then be expanded using the default `lz4` 15 | tool from the command-line. 16 | 17 | The LZ4 algorithm isn't as good at compression as alternatives, but is a lot 18 | faster. 19 | 20 | Because of the `-G 3600` option, it will rotate files on an hourly basis. 21 | When it creates a new file, it'll use the timestamp specifiers to (the 22 | options with %) to use the current timestamp as the filename. 23 | 24 | Unlike `tcdump`, this program aligns its rotations. In other words, it rotates 25 | on the hour (when 3600 seconds is specified), such as 1:00pm, 2:00pm, 3:00pm, 26 | etc. In contrast, `tcpdump` would rotate exaclty one hour from program 27 | startup, and every hour after. So if the program started at 1:47pm, then 28 | `tcpdump` would rotate at 2:47pm, 3:47pm, and so forth. This means that the 29 | first file will be (likley) shorter than the others, since it will have less 30 | than an hour's data. 31 | 32 | For higher speed sniffing, the PF_RING/ZC drivers can be used for Intel 10gig 33 | cards. Simply put the letters `zc:` in front of the interface name, and it 34 | should be automatically selected, as long as PF_RING is installed. 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/packetdump.h: -------------------------------------------------------------------------------- 1 | #ifndef packetdump_h 2 | #define packetdump_h 3 | #include 4 | 5 | /*************************************************************************** 6 | ***************************************************************************/ 7 | struct PacketDump { 8 | 9 | /** 10 | * The netwrok interface to start packet-sniffing on 11 | */ 12 | const char *ifname; 13 | 14 | /** 15 | * The filename to write packets into. This can also be a filename 16 | * spec, that allows adding data/timestamp parameters to the 17 | * filename. 18 | */ 19 | const char *filename; 20 | 21 | /** 22 | * Instead of reading from the network, we can read from one or more 23 | * files. This is useful for decompressing previously captured files, 24 | * splitting a large files into smaller parts, or combining many 25 | * smaller captures into a larger capture. 26 | */ 27 | const char **readfiles; 28 | 29 | /** 30 | * Maximum number of bytes in a file before it is rotated. For 31 | * compressed files, this is the maximum number after 32 | * compression 33 | * [packetdump -C size] 34 | * [packetdump --filesize size] 35 | */ 36 | uint64_t rotate_size; 37 | 38 | /** 39 | * Maximum number of seconds before rotating a file. 40 | */ 41 | uint64_t rotate_seconds; 42 | 43 | /** 44 | * Maximum number of files to write before deleting 45 | * old ones 46 | */ 47 | uint64_t rotate_filecount; 48 | 49 | /** 50 | * user account for dropping priviledges 51 | */ 52 | const char *drop_user; 53 | 54 | /** 55 | * BPF rule specified on commandline 56 | */ 57 | const char *bpf_rule; 58 | 59 | const char *bpf_file; 60 | 61 | char is_monitor_mode; 62 | char is_promiscuous_mode; 63 | char is_compression; 64 | char is_gmt; 65 | 66 | char is_help; 67 | char is_version; 68 | char is_iflist; 69 | char is_echo; 70 | }; 71 | typedef struct PacketDump PacketDump; 72 | 73 | #endif /* packetdump_h */ 74 | -------------------------------------------------------------------------------- /src/logger.c: -------------------------------------------------------------------------------- 1 | /* 2 | log messages to console, depending on verbose level 3 | 4 | Use -d to get more verbose output. The more -v you add, the 5 | more verbose the output becomes. 6 | 7 | Details about the running of the program go to . 8 | Details about scan results go to , so that they can easily 9 | be redirected to a file. 10 | */ 11 | #include "logger.h" 12 | #include "string_s.h" 13 | #include 14 | #include 15 | 16 | static int global_debug_level = 0; /* yea! a global variable!! */ 17 | void LOG_add_level(int x) 18 | { 19 | global_debug_level += x; 20 | } 21 | 22 | /*************************************************************************** 23 | ***************************************************************************/ 24 | static void 25 | vLOG(int level, const char *fmt, va_list marker) 26 | { 27 | if (level <= global_debug_level) { 28 | vfprintf(stderr, fmt, marker); 29 | fflush(stderr); 30 | } 31 | } 32 | 33 | 34 | /*************************************************************************** 35 | * Prints the message if the global "verbosity" flag exceeds this level. 36 | ***************************************************************************/ 37 | void 38 | LOG(int level, const char *fmt, ...) 39 | { 40 | va_list marker; 41 | 42 | va_start(marker, fmt); 43 | vLOG(level, fmt, marker); 44 | va_end(marker); 45 | } 46 | 47 | /*************************************************************************** 48 | ***************************************************************************/ 49 | static void 50 | vLOGip(int level, unsigned ip, unsigned port, const char *fmt, va_list marker) 51 | { 52 | if (level <= global_debug_level) { 53 | char sz_ip[16]; 54 | 55 | sprintf_s(sz_ip, sizeof(sz_ip), "%u.%u.%u.%u", 56 | (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF); 57 | fprintf(stderr, "%-15s:%5u: ", sz_ip, port); 58 | vfprintf(stderr, fmt, marker); 59 | fflush(stderr); 60 | } 61 | } 62 | 63 | 64 | /*************************************************************************** 65 | ***************************************************************************/ 66 | void 67 | LOGip(int level, unsigned ip, unsigned port, const char *fmt, ...) 68 | { 69 | va_list marker; 70 | 71 | va_start(marker, fmt); 72 | vLOGip(level, ip, port, fmt, marker); 73 | va_end(marker); 74 | } 75 | 76 | -------------------------------------------------------------------------------- /src/pixie-threads.h: -------------------------------------------------------------------------------- 1 | #ifndef PORT_THREADS_H 2 | #define PORT_THREADS_H 3 | #include 4 | #include 5 | #if defined(_MSC_VER) 6 | #include 7 | #endif 8 | 9 | /** 10 | * Returns the number of CPUs in the system, including virtual CPUs. 11 | * On a single processor system, the number returned will be '1'. 12 | * On a dual socket, dual-core per socket, hyperthreaded system, the 13 | * count will be '8'. 14 | */ 15 | unsigned pixie_cpu_get_count(void); 16 | 17 | /** 18 | * Launch a thread 19 | */ 20 | size_t pixie_begin_thread(void (*worker_thread)(void*), 21 | unsigned flags, 22 | void *worker_data); 23 | 24 | void pixie_thread_join(size_t thread_handle); 25 | 26 | void pixie_cpu_set_affinity(unsigned processor); 27 | void pixie_cpu_raise_priority(void); 28 | 29 | void pixie_locked_subtract_u32(unsigned *lhs, unsigned rhs); 30 | 31 | 32 | 33 | #if defined(_MSC_VER) 34 | #define pixie_locked_add_u32(dst, src) _InterlockedExchangeAdd((volatile long*)(dst), (src)) 35 | #define pixie_locked_CAS32(dst, src, expected) (_InterlockedCompareExchange((volatile long*)dst, src, expected) == (expected)) 36 | #define pixie_locked_CAS64(dst, src, expected) (_InterlockedCompareExchange64((volatile long long*)dst, src, expected) == (expected)) 37 | #define rte_atomic32_cmpset(dst, exp, src) (_InterlockedCompareExchange((volatile long *)dst, (long)src, (long)exp)==(long)(exp)) 38 | 39 | #elif defined(__GNUC__) 40 | #define pixie_locked_add_u32(dst, src) __sync_add_and_fetch((volatile int*)(dst), (int)(src)); 41 | #define rte_atomic32_cmpset(dst, expected, src) __sync_bool_compare_and_swap((volatile int*)(dst),(int)expected,(int)src) 42 | #define pixie_locked_CAS32(dst, src, expected) __sync_bool_compare_and_swap((volatile int*)(dst),(int)expected,(int)src); 43 | #define pixie_locked_CAS64(dst, src, expected) __sync_bool_compare_and_swap((volatile long long int*)(dst),(long long int)expected,(long long int)src); 44 | 45 | #if !defined(__x86_64__) && !defined(__i386__) 46 | #define rte_wmb() __sync_synchronize() 47 | #define rte_rmb() __sync_synchronize() 48 | #define rte_pause() 49 | #else 50 | #define rte_wmb() asm volatile("sfence;" : : : "memory") 51 | #define rte_rmb() asm volatile("lfence;" : : : "memory") 52 | #define rte_pause() asm volatile ("pause") 53 | #endif 54 | #else 55 | unsigned pixie_locked_add_u32(volatile unsigned *lhs, unsigned rhs); 56 | int pixie_locked_CAS32(volatile unsigned *dst, unsigned src, unsigned expected); 57 | int pixie_locked_CAS64(volatile uint64_t *dst, uint64_t src, uint64_t expected); 58 | #endif 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/string_s.c: -------------------------------------------------------------------------------- 1 | /* 2 | safe C library functions 3 | 4 | This upgrades unsafe C functions like "strcpy()" to safer equivelents, 5 | like "strcpy_s()". 6 | 7 | NOTE: This is for maintaining a policy of "no unsafe functions" 8 | */ 9 | #include "string_s.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | /** 16 | * fopen_s 17 | */ 18 | #if defined(__GNUC__) || _MSC_VER == 1200 19 | errno_t fopen_s(FILE **pFile, const char *filename, const char *mode) 20 | { 21 | if (pFile == NULL || filename == NULL || mode == NULL) 22 | return EINVAL; 23 | *pFile = fopen(filename, mode); 24 | if (*pFile != NULL) 25 | return 0; 26 | else 27 | return errno; 28 | } 29 | #endif 30 | 31 | /** 32 | * Case-insensitive memcmp() 33 | */ 34 | #ifdef __GNUC__ 35 | int 36 | memcasecmp(const void *lhs, const void *rhs, int length) 37 | { 38 | int i; 39 | for (i=0; i= sizeof_dst) { 68 | dst[0] = 0; 69 | return ERANGE; 70 | } else 71 | dst[i] = src[i]; 72 | } 73 | if (i >= sizeof_dst) { 74 | dst[0] = 0; 75 | return ERANGE; 76 | } else 77 | dst[i] = src[i]; 78 | 79 | return 0; 80 | } 81 | 82 | errno_t localtime_s(struct tm* _tm, const time_t *time) 83 | { 84 | struct tm *x; 85 | 86 | x = localtime(time); 87 | if (x == NULL) { 88 | memset(_tm, 0, sizeof(*_tm)); 89 | return -1; 90 | } 91 | memcpy(_tm, x, sizeof(*_tm)); 92 | 93 | return 0; 94 | } 95 | errno_t gmtime_s(struct tm* _tm, const time_t *time) 96 | { 97 | struct tm *x; 98 | 99 | x = gmtime(time); 100 | if (x == NULL) { 101 | memset(_tm, 0, sizeof(*_tm)); 102 | return -1; 103 | } 104 | memcpy(_tm, x, sizeof(*_tm)); 105 | 106 | return 0; 107 | } 108 | #endif 109 | 110 | 111 | /* 112 | * I don't understand why Microsoft says this function is unsafe, so 113 | * do it anyway 114 | */ 115 | const char *strerror_x(int x) 116 | { 117 | #ifdef _MSC_VER 118 | #pragma warning(disable: 4996) 119 | #endif 120 | #undef strerror 121 | return strerror(x); 122 | } 123 | -------------------------------------------------------------------------------- /src/readfiles.c: -------------------------------------------------------------------------------- 1 | #include "packetdump.h" 2 | #include "readfiles.h" 3 | #include "lz4/lz4frame.h" 4 | #include 5 | 6 | void 7 | read_file(const struct PacketDump *conf, const char *filename) 8 | { 9 | FILE *fp; 10 | LZ4F_errorCode_t err; 11 | LZ4F_dctx *ctx = NULL; 12 | char buf[65536]; 13 | size_t bytes_read; 14 | size_t offset; 15 | LZ4F_frameInfo_t frame_info; 16 | 17 | fp = fopen(filename, "rb"); 18 | if (fp == NULL) { 19 | perror(filename); 20 | return; 21 | } 22 | 23 | /* 24 | * read the first chunk 25 | */ 26 | bytes_read = fread(buf, 1, sizeof(buf), fp); 27 | if (bytes_read == 0) { 28 | if (feof(fp)) { 29 | fprintf(stderr, "%s: empty file\n", filename); 30 | } else 31 | perror(filename); 32 | goto closefiles; 33 | } 34 | 35 | 36 | /* 37 | * Create decompression engine 38 | */ 39 | err = LZ4F_createDecompressionContext(&ctx, LZ4F_getVersion()); 40 | if (LZ4F_isError(err)) { 41 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(err)); 42 | goto closefiles; 43 | } 44 | 45 | /* 46 | * Read the initial parameters from the start of the first chunk 47 | */ 48 | offset = bytes_read; 49 | err = LZ4F_getFrameInfo(ctx, &frame_info, buf, &offset); 50 | if (LZ4F_isError(err)) { 51 | fprintf(stderr, "lz4:getframeinfo: %s\n", LZ4F_getErrorName(offset)); 52 | goto closefiles; 53 | } 54 | 55 | for (;;) { 56 | char dst[65536]; 57 | size_t sizeof_dst = sizeof(dst); 58 | size_t bytes_remaining = bytes_read - offset; 59 | size_t bytes_decompressed = bytes_remaining; 60 | 61 | err = LZ4F_decompress(ctx, 62 | dst, &sizeof_dst, 63 | buf+offset, &bytes_decompressed, 64 | NULL); 65 | if (LZ4F_isError(err)) { 66 | fprintf(stderr, "lz4:decompress: %s\n", LZ4F_getErrorName(offset)); 67 | goto closefiles; 68 | } 69 | 70 | 71 | } 72 | 73 | 74 | 75 | 76 | closefiles: 77 | if (ctx != NULL) { 78 | err = LZ4F_freeDecompressionContext(ctx); 79 | if (LZ4F_isError(err)) { 80 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(err)); 81 | } 82 | } 83 | } 84 | void 85 | read_files(const struct PacketDump *conf) 86 | { 87 | size_t i; 88 | const char **file_list = conf->readfiles; 89 | 90 | if (file_list == NULL) 91 | return; 92 | 93 | for (i=0; file_list[i]; i++) { 94 | const char *filename; 95 | 96 | filename = file_list[i]; 97 | 98 | read_file(conf, filename); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/string_s.h: -------------------------------------------------------------------------------- 1 | /* 2 | safe "string" functions, like Microsoft's 3 | 4 | This is for the "safe" clib functions, where things like "strcpy()" is 5 | replaced with a safer version of the function, like "strcpy_s()". Since 6 | these things are non-standard, compilers deal with them differently. 7 | 8 | Reference: 9 | http://msdn.microsoft.com/en-us/library/bb288454.aspx 10 | */ 11 | #ifndef STRCPY_S 12 | #define STRCPY_S 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #undef strcpy 21 | #define strcpy STRCPY_FUNCTION_IS_BAD 22 | 23 | #undef strncpy 24 | #define strncpy STRNCPY_FUNCTION_IS_BAD 25 | 26 | #undef strcat 27 | #define strcat STRCAT_FUNCTION_IS_BAD 28 | 29 | #undef strncat 30 | #define strncat STRNCAT_FUNCTION_IS_BAD 31 | 32 | #undef sprintf 33 | #define sprintf SPRINTF_FUNCTION_IS_BAD 34 | 35 | #undef vsprintf 36 | #define vsprintf VSPRINTF_FUNCTION_IS_BAD 37 | 38 | #undef strtok 39 | #define strtok STRTOK_FUNCTION_IS_BAD 40 | 41 | #undef gets 42 | #define gets GETS_FUNCTION_IS_BAD 43 | 44 | #undef scanf 45 | #define scanf SCANF_FUNCTION_IS_BAD 46 | 47 | #undef sscanf 48 | #define sscanf SSCANF_FUNCTION_IS_BAD 49 | 50 | #undef itoa 51 | #define itoa ITOA_FUNCTION_IS_BAD 52 | 53 | #undef strerror 54 | #define strerror STRERROR_FUNCTION_IS_BAD 55 | 56 | const char *strerror_x(int x); 57 | 58 | #if defined(_MSC_VER) && (_MSC_VER == 1900) 59 | /*Visual Studio 2015*/ 60 | # include 61 | # include 62 | # define strcasecmp _stricmp 63 | # define memcasecmp _memicmp 64 | # ifndef PRIu64 65 | # define PRIu64 "llu" 66 | # define PRId64 "lld" 67 | # define PRIx64 "llx" 68 | # endif 69 | 70 | #elif defined(_MSC_VER) && (_MSC_VER == 1600) 71 | /*Visual Studio 2010*/ 72 | # include 73 | # include 74 | # define strcasecmp _stricmp 75 | # define memcasecmp _memicmp 76 | # ifndef PRIu64 77 | # define PRIu64 "llu" 78 | # define PRId64 "lld" 79 | # define PRIx64 "llx" 80 | # endif 81 | 82 | 83 | #elif defined(_MSC_VER) && (_MSC_VER == 1200) 84 | /* Visual Studio 6.0 */ 85 | # define sprintf_s _snprintf 86 | # define strcasecmp _stricmp 87 | # define memcasecmp _memicmp 88 | # define vsprintf_s _vsnprintf 89 | typedef int errno_t; 90 | errno_t fopen_s(FILE **fp, const char *filename, const char *mode); 91 | 92 | #elif defined(__GNUC__) && (__GNUC__ >= 4) 93 | #include 94 | /* GCC 4 */ 95 | # define sprintf_s snprintf 96 | # define vsprintf_s vsnprintf 97 | int memcasecmp(const void *lhs, const void *rhs, int length); 98 | typedef int errno_t; 99 | errno_t fopen_s(FILE **fp, const char *filename, const char *mode); 100 | errno_t strcpy_s(char *dst, size_t sizeof_dst, const char *src); 101 | errno_t localtime_s(struct tm* _tm, const time_t *time); 102 | errno_t gmtime_s(struct tm* _tm, const time_t *time); 103 | #undef strerror 104 | 105 | #else 106 | # error unknown compiler 107 | #endif 108 | 109 | 110 | 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /lz4/lz4frame_static.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 auto-framing library 3 | Header File for static linking only 4 | Copyright (C) 2011-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #ifndef LZ4FRAME_STATIC_H_0398209384 37 | #define LZ4FRAME_STATIC_H_0398209384 38 | 39 | #if defined (__cplusplus) 40 | extern "C" { 41 | #endif 42 | 43 | /* lz4frame_static.h should be used solely in the context of static linking. 44 | * It contains definitions which are not stable and may change in the future. 45 | * Never use it in the context of DLL linking. 46 | */ 47 | 48 | 49 | /* --- Dependency --- */ 50 | #include "lz4frame.h" 51 | 52 | 53 | /* --- Error List --- */ 54 | #define LZ4F_LIST_ERRORS(ITEM) \ 55 | ITEM(OK_NoError) \ 56 | ITEM(ERROR_GENERIC) \ 57 | ITEM(ERROR_maxBlockSize_invalid) \ 58 | ITEM(ERROR_blockMode_invalid) \ 59 | ITEM(ERROR_contentChecksumFlag_invalid) \ 60 | ITEM(ERROR_compressionLevel_invalid) \ 61 | ITEM(ERROR_headerVersion_wrong) \ 62 | ITEM(ERROR_blockChecksum_unsupported) \ 63 | ITEM(ERROR_reservedFlag_set) \ 64 | ITEM(ERROR_allocation_failed) \ 65 | ITEM(ERROR_srcSize_tooLarge) \ 66 | ITEM(ERROR_dstMaxSize_tooSmall) \ 67 | ITEM(ERROR_frameHeader_incomplete) \ 68 | ITEM(ERROR_frameType_unknown) \ 69 | ITEM(ERROR_frameSize_wrong) \ 70 | ITEM(ERROR_srcPtr_wrong) \ 71 | ITEM(ERROR_decompressionFailed) \ 72 | ITEM(ERROR_headerChecksum_invalid) \ 73 | ITEM(ERROR_contentChecksum_invalid) \ 74 | ITEM(ERROR_frameDecoding_alreadyStarted) \ 75 | ITEM(ERROR_maxCode) 76 | 77 | #define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, 78 | 79 | /* enum list is exposed, to handle specific errors */ 80 | typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; 81 | 82 | LZ4F_errorCodes LZ4F_getErrorCode(size_t functionResult); 83 | 84 | 85 | #if defined (__cplusplus) 86 | } 87 | #endif 88 | 89 | #endif /* LZ4FRAME_STATIC_H_0398209384 */ 90 | -------------------------------------------------------------------------------- /src/rawsock-pcapfile.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2007 by Errata Security, All Rights Reserved 2 | * Programer(s): Robert David Graham [rdg] 3 | */ 4 | #ifndef __PCAPFILE_H 5 | #define __PCAPFILE_H 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | #include 10 | #include 11 | 12 | enum { 13 | PCAPFILE_NO_COMPRESSION=0, 14 | 15 | /* LZ4 default (fastest) compression */ 16 | PCAPFILE_LZ4, 17 | 18 | /* LZ4 maximum, and slowest, compress */ 19 | PCAPFILE_LZ4SLOW, 20 | }; 21 | struct PcapFile; 22 | 23 | 24 | unsigned pcapfile_datalink(struct PcapFile *handle); 25 | 26 | /** 27 | * Append a packet/frame to the output file. If the file was opened with 28 | * a compresison flag, this packet will be compressed. 29 | * @param capfile 30 | * A file handled opened with 'pcapfile_openwrite()' 31 | * @param buffer 32 | * The contents of the packet buffer. 33 | * @param buffer_size 34 | * The (possibly sliced) length of the packet buffer [caplen] 35 | * @param original_length 36 | * The original length of the packet [len] 37 | * @param time_sec 38 | * Number of seconds since 1970 (time_t) [ts.tv_sec] 39 | * @param time_usec 40 | * Number of microseconds since the start of the current 41 | * second [ts.tv_usec]. 42 | * @return 43 | * The number of bytes written, which may be smaller than the number 44 | * of bytes in the packet if compression is enabled. A negative 45 | * number is returned if there is an error. 46 | */ 47 | ssize_t pcapfile_writeframe( 48 | struct PcapFile *capfile, 49 | const void *buffer, 50 | unsigned buffer_size, 51 | unsigned original_length, 52 | long time_sec, 53 | long time_usec 54 | ); 55 | 56 | struct PcapFile *pcapfile_openread(const char *capfilename); 57 | 58 | 59 | /** 60 | * Opens a pcapfile for writing. 61 | * @param capfilename 62 | * Name of the file to open. If it exists, it'll be sillently overwritten 63 | * @param linktype 64 | * The type of data link layer, such as Ethernet or one fo the several 65 | * possible WiFi encapsulations. 66 | * @param compression_type 67 | * The type of compression supported, 0 for none, or PCAPFILE_LZ4 68 | * for the LZ4 algorithm. 69 | */ 70 | struct PcapFile *pcapfile_openwrite(const char *capfilename, unsigned linktype, int compression_type); 71 | 72 | struct PcapFile *pcapfile_openappend(const char *capfilename, unsigned linktype); 73 | 74 | unsigned pcapfile_percentdone(struct PcapFile *handle, uint64_t *r_bytes_read); 75 | 76 | void pcapfile_get_timestamps(struct PcapFile *handle, time_t *start, time_t *end); 77 | 78 | /** 79 | * Set a "maximum" size for a file. When the current file fills up with data, 80 | * it will close that file and open a new one, then continue to write 81 | * from that point on in the new file. 82 | */ 83 | void pcapfile_set_max(struct PcapFile *capfile, unsigned max_megabytes, unsigned max_files); 84 | 85 | /** 86 | * Read a single frame from the file. 87 | * Returns 0 if failed to read (from error or end of file), and 88 | * returns 1 if successful. 89 | */ 90 | int pcapfile_readframe( 91 | struct PcapFile *capfile, 92 | unsigned *r_time_secs, 93 | unsigned *r_time_usecs, 94 | unsigned *r_original_length, 95 | unsigned *r_captured_length, 96 | unsigned char *buf, 97 | unsigned sizeof_buf 98 | ); 99 | 100 | 101 | void pcapfile_close(struct PcapFile *handle); 102 | 103 | #ifdef __cplusplus 104 | } 105 | #endif 106 | #endif /*__PCAPFILE_H*/ 107 | -------------------------------------------------------------------------------- /src/test-compress.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "lz4/lz4frame.h" 3 | 4 | void compress_file(const char *infilename, const char *outfilename, int compression_level) 5 | { 6 | FILE *infp = NULL; 7 | FILE *outfp = NULL; 8 | char outbuf[65536 + 1024]; 9 | size_t bytes_compressed; 10 | size_t bytes_written; 11 | LZ4F_compressionContext_t ctx = 0; 12 | LZ4F_errorCode_t err; 13 | LZ4F_preferences_t prefs = {0}; 14 | 15 | /* 16 | * Open the files 17 | */ 18 | infp = fopen(infilename, "rb"); 19 | if (infp == NULL) { 20 | perror(infilename); 21 | goto closefiles; 22 | } 23 | outfp = fopen(outfilename, "wb"); 24 | if (outfp == NULL) { 25 | perror(outfilename); 26 | goto closefiles; 27 | } 28 | 29 | /* Set compression parameters */ 30 | prefs.autoFlush = 1; 31 | prefs.compressionLevel = compression_level; 32 | 33 | /* Create compression context */ 34 | err = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 35 | if (LZ4F_isError(err)) { 36 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(err)); 37 | goto closefiles; 38 | } 39 | 40 | /* Write the LZ4 magic header */ 41 | bytes_compressed = LZ4F_compressBegin(ctx, outbuf, sizeof(outbuf), &prefs); 42 | if (LZ4F_isError(bytes_compressed)) { 43 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(bytes_compressed)); 44 | goto closefiles; 45 | } 46 | bytes_written = fwrite(outbuf, 1, bytes_compressed, outfp); 47 | if (bytes_written != bytes_compressed) { 48 | perror(outfilename); 49 | goto closefiles; 50 | } 51 | 52 | /* 53 | * Now read all the blocks from the files 54 | */ 55 | for (;;) { 56 | char inbuf[65536]; 57 | size_t bytes_read; 58 | 59 | /* 60 | * Read the uncompressed block 61 | */ 62 | bytes_read = fread(inbuf, 1, sizeof(inbuf), infp); 63 | if (bytes_read == 0) { 64 | if (feof(infp)) { 65 | fprintf(stderr, "%s: end of file\n", infilename); 66 | break; 67 | } else if (ferror(infp)) 68 | perror(infilename); 69 | else 70 | fprintf(stderr, "%s: unknown error\n", infilename); 71 | goto closefiles; 72 | } 73 | 74 | /* 75 | * compress the block 76 | */ 77 | bytes_compressed = LZ4F_compressUpdate(ctx, outbuf, sizeof(outbuf), inbuf, bytes_read, NULL); 78 | if (LZ4F_isError(bytes_compressed)) { 79 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(bytes_compressed)); 80 | goto closefiles; 81 | } 82 | 83 | /* 84 | * write the compressed data 85 | */ 86 | bytes_written = fwrite(outbuf, 1, bytes_compressed, outfp); 87 | if (bytes_written != bytes_compressed) { 88 | perror(outfilename); 89 | goto closefiles; 90 | } 91 | 92 | } 93 | 94 | 95 | /* 96 | * Now properly close the file 97 | */ 98 | bytes_compressed = LZ4F_compressEnd(ctx, outbuf, sizeof(outbuf), NULL); 99 | if (LZ4F_isError(bytes_compressed)) { 100 | fprintf(stderr, "lz4: %s\n", LZ4F_getErrorName(bytes_compressed)); 101 | goto closefiles; 102 | } 103 | bytes_written = fwrite(outbuf, 1, bytes_compressed, outfp); 104 | if (bytes_written != bytes_compressed) { 105 | perror(outfilename); 106 | goto closefiles; 107 | } 108 | 109 | 110 | 111 | closefiles: 112 | if (ctx) 113 | LZ4F_freeCompressionContext(ctx); 114 | if (infp) 115 | fclose(infp); 116 | if (outfp) 117 | fclose(outfp); 118 | } 119 | 120 | -------------------------------------------------------------------------------- /src/pixie-threads.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | #include "pixie-threads.h" 3 | 4 | #if defined(WIN32) 5 | #include 6 | #include 7 | #endif 8 | #if defined(__GNUC__) 9 | #include 10 | #include 11 | #include 12 | #include 13 | #endif 14 | 15 | #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) 16 | #include 17 | #include 18 | #endif 19 | 20 | #ifndef UNUSEDPARM 21 | #ifdef _MSC_VER 22 | #define UNUSEDPARM(x) x 23 | #else 24 | #define UNUSEDPARM(x) 25 | #endif 26 | #endif 27 | 28 | /**************************************************************************** 29 | ****************************************************************************/ 30 | void 31 | pixie_cpu_raise_priority(void) 32 | { 33 | #if defined WIN32 34 | DWORD_PTR result; 35 | result = SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); 36 | if (result == 0) { 37 | fprintf(stderr, "set_priority: returned error win32:%u\n", (unsigned)GetLastError()); 38 | } 39 | #elif defined(__linux__) && defined(__GNUC__) 40 | pthread_t thread = pthread_self(); 41 | pthread_attr_t thAttr; 42 | int policy = 0; 43 | int max_prio_for_policy = 0; 44 | 45 | pthread_attr_init(&thAttr); 46 | pthread_attr_getschedpolicy(&thAttr, &policy); 47 | max_prio_for_policy = sched_get_priority_max(policy); 48 | 49 | 50 | pthread_setschedprio(thread, max_prio_for_policy); 51 | pthread_attr_destroy(&thAttr); 52 | return; 53 | 54 | #endif 55 | } 56 | 57 | /**************************************************************************** 58 | * Set the current thread (implicit) to run exclusively on the explicit 59 | * process. 60 | * http://en.wikipedia.org/wiki/Processor_affinity 61 | ****************************************************************************/ 62 | void 63 | pixie_cpu_set_affinity(unsigned processor) 64 | { 65 | #if defined WIN32 66 | DWORD_PTR mask; 67 | DWORD_PTR result; 68 | if (processor > 0) 69 | processor--; 70 | mask = ((size_t)1)<>= 1; 120 | } 121 | if (count == 0) 122 | return 1; 123 | else 124 | return count; 125 | #elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) 126 | /* BSD - use sysctl() function */ 127 | int x; 128 | int mib[2]; 129 | size_t ncpu_length; 130 | int ncpu = 1; 131 | 132 | mib[0] = CTL_HW; 133 | mib[1] = HW_NCPU; 134 | ncpu_length = sizeof(ncpu); 135 | x = sysctl(mib, 2, &ncpu, &ncpu_length, NULL, 0); 136 | if (x == -1) { 137 | perror("sysctl(HW_NCPU) failed"); 138 | return 1; 139 | } else 140 | return (unsigned)ncpu; 141 | #elif defined linux 142 | /* http://linux.die.net/man/2/sched_getaffinity */ 143 | { 144 | pid_t pid; 145 | cpu_set_t mask; 146 | int err; 147 | 148 | /* Gegret our process ID */ 149 | pid = getpid(); 150 | 151 | /* Get list of available CPUs for our system */ 152 | err = sched_getaffinity(pid, sizeof(mask), &mask); 153 | if (err) { 154 | perror("sched_getaffinity"); 155 | return 1; 156 | } else { 157 | #ifndef CPU_COUNT 158 | return 1; 159 | #else 160 | return CPU_COUNT(&mask); 161 | #endif 162 | } 163 | } 164 | #else 165 | #error need to find CPU count 166 | /* UNKNOWN - Well, we don't know the type of system which means we won't 167 | * be able to start multiple threads anyway, so just return '1' */ 168 | return 1; 169 | #endif 170 | } 171 | 172 | /**************************************************************************** 173 | ****************************************************************************/ 174 | size_t 175 | pixie_begin_thread( 176 | void (*worker_thread)(void*), 177 | unsigned flags, 178 | void *worker_data) 179 | { 180 | #if defined(WIN32) 181 | UNUSEDPARM(flags); 182 | return _beginthread(worker_thread, 0, worker_data); 183 | #elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__kFreeBSD__) || defined(__OpenBSD__) 184 | 185 | typedef void *(*PTHREADFUNC)(void*); 186 | pthread_t thread_id = 0; 187 | pthread_create( 188 | &thread_id, 189 | NULL, 190 | (PTHREADFUNC)worker_thread, 191 | worker_data); 192 | return (size_t)thread_id; 193 | #else 194 | #error pixie_begin_thread undefined 195 | #endif 196 | } 197 | 198 | /**************************************************************************** 199 | ****************************************************************************/ 200 | void pixie_thread_join(size_t thread_handle) 201 | { 202 | #if defined(WIN32) 203 | WaitForSingleObject((HANDLE)thread_handle, INFINITE); 204 | #else 205 | void *p; 206 | 207 | pthread_join((pthread_t)thread_handle, &p); 208 | #endif 209 | } 210 | -------------------------------------------------------------------------------- /src/pixie-timer.c: -------------------------------------------------------------------------------- 1 | /* 2 | portability: time 3 | 4 | Since this program runs on both Linux and Windows, I need a portable 5 | way to get a high-resolution timer. 6 | 7 | NOTE: The time I'm looking for is "elapsed time" not "wall clock" 8 | time. In other words, if you put the system to sleep and wake it 9 | up a day later, this function should see no change, since time 10 | wasn't elapsing while the system was asleep. 11 | 12 | Reference: 13 | http://www.python.org/dev/peps/pep-0418/#monotonic-clocks 14 | http://www.brain-dump.org/blog/entry/107 15 | 16 | */ 17 | #include "pixie-timer.h" 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | #if defined(WIN32) 26 | #include 27 | 28 | LARGE_INTEGER 29 | getFILETIMEoffset(void) 30 | { 31 | SYSTEMTIME s; 32 | FILETIME f; 33 | LARGE_INTEGER t; 34 | 35 | s.wYear = 1970; 36 | s.wMonth = 1; 37 | s.wDay = 1; 38 | s.wHour = 0; 39 | s.wMinute = 0; 40 | s.wSecond = 0; 41 | s.wMilliseconds = 0; 42 | SystemTimeToFileTime(&s, &f); 43 | t.QuadPart = f.dwHighDateTime; 44 | t.QuadPart <<= 32; 45 | t.QuadPart |= f.dwLowDateTime; 46 | return (t); 47 | } 48 | 49 | int 50 | clock_gettime(int X, struct timeval *tv) 51 | { 52 | LARGE_INTEGER t; 53 | FILETIME f; 54 | double microseconds; 55 | static LARGE_INTEGER offset; 56 | static double frequencyToMicroseconds; 57 | static int initialized = 0; 58 | static BOOL usePerformanceCounter = 0; 59 | 60 | X=X; 61 | 62 | if (!initialized) { 63 | LARGE_INTEGER performanceFrequency; 64 | initialized = 1; 65 | usePerformanceCounter = QueryPerformanceFrequency(&performanceFrequency); 66 | if (usePerformanceCounter) { 67 | QueryPerformanceCounter(&offset); 68 | frequencyToMicroseconds = (double)performanceFrequency.QuadPart / 1000000.; 69 | } else { 70 | offset = getFILETIMEoffset(); 71 | frequencyToMicroseconds = 10.; 72 | } 73 | } 74 | if (usePerformanceCounter) QueryPerformanceCounter(&t); 75 | else { 76 | GetSystemTimeAsFileTime(&f); 77 | t.QuadPart = f.dwHighDateTime; 78 | t.QuadPart <<= 32; 79 | t.QuadPart |= f.dwLowDateTime; 80 | } 81 | 82 | t.QuadPart -= offset.QuadPart; 83 | microseconds = (double)t.QuadPart / frequencyToMicroseconds; 84 | t.QuadPart = (LONGLONG)microseconds; 85 | tv->tv_sec = (long)(t.QuadPart / 1000000); 86 | tv->tv_usec = t.QuadPart % 1000000; 87 | return (0); 88 | } 89 | 90 | 91 | uint64_t 92 | pixie_gettime(void) 93 | { 94 | //struct timeval tv; 95 | //clock_gettime(0, &tv); 96 | 97 | uint64_t time1 = 0, freq = 0; 98 | double seconds; 99 | 100 | QueryPerformanceCounter((LARGE_INTEGER *) &time1); 101 | QueryPerformanceFrequency((LARGE_INTEGER *)&freq); 102 | 103 | seconds = (double)time1/(double)freq; 104 | 105 | return (uint64_t)(seconds * 1000000.0); 106 | 107 | //return (uint64_t)tv.tv_sec * 1000000UL + tv.tv_usec; 108 | } 109 | uint64_t 110 | pixie_nanotime(void) 111 | { 112 | uint64_t time1 = 0, freq = 0; 113 | double seconds; 114 | QueryPerformanceCounter((LARGE_INTEGER *) &time1); 115 | QueryPerformanceFrequency((LARGE_INTEGER *)&freq); 116 | seconds = (double)time1/(double)freq; 117 | return (uint64_t)(seconds * 1000000000.0); 118 | } 119 | 120 | void 121 | pixie_mssleep(unsigned waitTime) 122 | { 123 | Sleep(waitTime); 124 | } 125 | 126 | void 127 | pixie_usleep(uint64_t waitTime) 128 | { 129 | /* 130 | uint64_t time1 = 0, time2 = 0, freq = 0; 131 | 132 | QueryPerformanceCounter((LARGE_INTEGER *) &time1); 133 | QueryPerformanceFrequency((LARGE_INTEGER *)&freq); 134 | 135 | do { 136 | QueryPerformanceCounter((LARGE_INTEGER *) &time2); 137 | } while((time2-time1) < waitTime); 138 | */ 139 | 140 | uint64_t start; 141 | 142 | start = pixie_gettime(); 143 | 144 | if (waitTime > 1000) 145 | Sleep((DWORD)(waitTime/1000)); 146 | 147 | while (pixie_gettime() - start < waitTime) 148 | ; 149 | } 150 | #elif defined(CLOCK_MONOTONIC) 151 | #include 152 | 153 | void 154 | pixie_mssleep(unsigned milliseconds) 155 | { 156 | pixie_usleep(milliseconds * 1000ULL); 157 | } 158 | 159 | void 160 | pixie_usleep(uint64_t microseconds) 161 | { 162 | struct timespec ts; 163 | struct timespec remaining; 164 | int err; 165 | 166 | ts.tv_sec = microseconds/1000000; 167 | ts.tv_nsec = (microseconds%1000000) * 1000; 168 | 169 | again: 170 | err = nanosleep(&ts, &remaining); 171 | if (err == -1 && errno == EINTR) { 172 | memcpy(&ts, &remaining, sizeof(ts)); 173 | goto again; 174 | } 175 | 176 | //usleep(microseconds); 177 | } 178 | uint64_t 179 | pixie_gettime(void) 180 | { 181 | int x; 182 | struct timespec tv; 183 | 184 | #ifdef CLOCK_MONOTONIC_RAW 185 | x = clock_gettime(CLOCK_MONOTONIC_RAW, &tv); 186 | #else 187 | x = clock_gettime(CLOCK_MONOTONIC, &tv); 188 | #endif 189 | if (x != 0) { 190 | printf("clock_gettime() err %d\n", errno); 191 | } 192 | 193 | return tv.tv_sec * 1000000 + tv.tv_nsec/1000; 194 | } 195 | uint64_t 196 | pixie_nanotime(void) 197 | { 198 | int x; 199 | struct timespec tv; 200 | 201 | #ifdef CLOCK_MONOTONIC_RAW 202 | x = clock_gettime(CLOCK_MONOTONIC_RAW, &tv); 203 | #else 204 | x = clock_gettime(CLOCK_MONOTONIC, &tv); 205 | #endif 206 | if (x != 0) { 207 | printf("clock_gettime() err %d\n", errno); 208 | } 209 | 210 | return tv.tv_sec * 1000000000 + tv.tv_nsec; 211 | } 212 | #elif defined(__MACH__) || defined(__FreeBSD__) /* works for Apple */ 213 | #include 214 | #include 215 | 216 | void pixie_usleep(uint64_t microseconds) 217 | { 218 | struct timespec t; 219 | t.tv_nsec = microseconds * 1000; 220 | if (microseconds > 1000000) 221 | t.tv_sec = microseconds/1000000; 222 | else { 223 | t.tv_sec = 0; 224 | } 225 | 226 | nanosleep(&t, 0); 227 | //usleep(microseconds); 228 | } 229 | void 230 | pixie_mssleep(unsigned milliseconds) 231 | { 232 | pixie_usleep(milliseconds * 1000ULL); 233 | } 234 | uint64_t 235 | pixie_gettime(void) 236 | { 237 | return mach_absolute_time()/1000; 238 | } 239 | uint64_t 240 | pixie_nanotime(void) 241 | { 242 | return mach_absolute_time(); 243 | } 244 | #endif 245 | 246 | int pixie_time_selftest(void) 247 | { 248 | static const uint64_t duration = 123456; 249 | uint64_t start, stop, elapsed; 250 | 251 | 252 | start = pixie_gettime(); 253 | pixie_usleep(duration); 254 | stop = pixie_gettime(); 255 | elapsed = stop - start; 256 | 257 | if (elapsed < 0.9*duration || 1.1*duration < elapsed) { 258 | /* I wonder how often this will fail just because the process 259 | * gets swapped out, but I'm leaving it in to see if people notice */ 260 | fprintf(stderr, "timing error, long delay\n"); 261 | return 1; 262 | } 263 | 264 | return 0; 265 | } 266 | -------------------------------------------------------------------------------- /src/rawsock-pcap.h: -------------------------------------------------------------------------------- 1 | /* 2 | Dynamically load libpcap at runtime 3 | 4 | This library optionally loads the 'libpcap' library at runtime, rather 5 | than statically linked at compile time. The advantage of this is that 6 | the user can build this project with no dependencies -- although they 7 | may require this dependency in order to run the program. 8 | 9 | As of 2017, libpcap shared libraries are standard on major Linux 10 | distributions (Debian, Readhat), FreeBSD, OpenBSD, and macOS. On 11 | Windows, "winpcap" must be downloaded. 12 | */ 13 | #ifndef RAWSOCK_PCAP_H 14 | #define RAWSOCK_PCAP_H 15 | #include 16 | 17 | #ifdef STATICPCAP 18 | #include 19 | #endif 20 | 21 | #ifndef STATICPCAP 22 | enum { 23 | DLT_EN10MB = 1, 24 | DLT_RAW = 101, 25 | }; 26 | 27 | /* Including the right ".h" file to define "timeval" is difficult, so instead 28 | * so instead we are simply going to define our own structure. This should 29 | * match the binary definition within the operating system 30 | */ 31 | struct pcap_timeval { 32 | long tv_sec; /* seconds */ 33 | long tv_usec; /* and microseconds */ 34 | }; 35 | 36 | /* Forward reference of opaque 'pcap_t' structure */ 37 | struct pcap; 38 | typedef struct pcap pcap_t; 39 | 40 | /* Forward reference of opaque 'pcap_if_t' structure */ 41 | struct pcap_if; 42 | typedef struct pcap_if pcap_if_t; 43 | 44 | /* How many bytes to reserve for error messages. This is the number specified 45 | * in libpcap, smaller numbers can crash */ 46 | enum { 47 | PCAP_ERRBUF_SIZE=256, 48 | }; 49 | 50 | /* used in pcap_setdirection() */ 51 | typedef enum { 52 | PCAP_D_INOUT = 0, 53 | PCAP_D_IN = 1, 54 | PCAP_D_OUT = 2, 55 | } pcap_direction_t; 56 | 57 | /* The packet header for capturing packets. Apple macOS inexplicably adds 58 | * an extra comment-field onto the end of this, so the definition needs 59 | * to be careful to match the real definition */ 60 | struct pcap_pkthdr { 61 | struct pcap_timeval ts; 62 | unsigned caplen; 63 | unsigned len; 64 | #ifdef __APPLE__ 65 | char comment[256]; 66 | #endif 67 | }; 68 | 69 | struct pcap_stat { 70 | unsigned ps_recv; /* number of packets received */ 71 | unsigned ps_drop; /* number of packets dropped */ 72 | unsigned ps_ifdrop; /* drops by interface -- only supported on some platforms */ 73 | #ifdef WIN32 74 | unsigned ps_capt; /* number of packets that reach the application */ 75 | unsigned ps_sent; /* number of packets sent by the server on the network */ 76 | unsigned ps_netdrop; /* number of packets lost on the network */ 77 | #endif /* _WIN32 */ 78 | }; 79 | 80 | #endif /*STATICPCAP*/ 81 | 82 | /* 83 | * This block is for function declarations. Consult the libpcap 84 | * documentation for what these functions really mean 85 | */ 86 | typedef void (*PCAP_HANDLE_PACKET)(unsigned char *v_seap, const struct pcap_pkthdr *framehdr, const unsigned char *buf); 87 | 88 | typedef void (*PCAP_close)(pcap_t *hPcap); 89 | typedef int (*PCAP_datalink)(pcap_t *hPcap); 90 | typedef const char *(*PCAP_datalink_val_to_name)(int dlt); 91 | typedef int (*PCAP_dispatch)(pcap_t *hPcap, int how_many_packets, PCAP_HANDLE_PACKET handler, unsigned char *handle_data); 92 | typedef int (*PCAP_findalldevs)(pcap_if_t **alldevs, char *errbuf); 93 | typedef void (*PCAP_freealldevs)(pcap_if_t *alldevs); 94 | typedef const char *(*PCAP_lib_version)(void); 95 | typedef char * (*PCAP_lookupdev)(char *errbuf); 96 | typedef int (*PCAP_major_version)(pcap_t *p); 97 | typedef int (*PCAP_minor_version)(pcap_t *p); 98 | typedef const unsigned char *(*PCAP_next)(pcap_t *p, struct pcap_pkthdr *h); 99 | typedef int (*PCAP_next_ex)(pcap_t *p, struct pcap_pkthdr **h, const unsigned char **buf); 100 | typedef pcap_t * (*PCAP_open_live)(const char *, int, int, int, char *); 101 | typedef pcap_t * (*PCAP_open_offline)(const char *fname, char *errbuf); 102 | typedef void (*PCAP_perror)(pcap_t *p, const char *prefix); 103 | typedef int (*PCAP_sendpacket)(pcap_t *p, const unsigned char *buf, int size); 104 | typedef int (*PCAP_setdirection)(pcap_t *, pcap_direction_t); 105 | typedef int (*PCAP_stats)(pcap_t *p, struct pcap_stat *ps); 106 | 107 | /* 108 | * New PCAP 109 | */ 110 | typedef pcap_t *(*PCAP_create)(const char *, char *); 111 | typedef int (*PCAP_set_snaplen)(pcap_t *, int); 112 | typedef int (*PCAP_set_promisc)(pcap_t *, int); 113 | typedef int (*PCAP_can_set_rfmon)(pcap_t *); 114 | typedef int (*PCAP_set_rfmon)(pcap_t *, int); 115 | typedef int (*PCAP_set_timeout)(pcap_t *, int); 116 | typedef int (*PCAP_set_buffer_size)(pcap_t *, int); 117 | typedef int (*PCAP_activate)(pcap_t *); 118 | 119 | typedef const char *(*PCAP_dev_name)(const pcap_if_t *dev); 120 | typedef const char *(*PCAP_dev_description)(const pcap_if_t *dev); 121 | typedef const pcap_if_t *(*PCAP_dev_next)(const pcap_if_t *dev); 122 | 123 | 124 | /* 125 | * PORTABILITY: Windows supports the "sendq" feature, and is really slow 126 | * without this feature. It's not needed on Linux, so we just create 127 | * equivelent functions that do nothing 128 | */ 129 | struct pcap_send_queue; 130 | typedef struct pcap_send_queue pcap_send_queue; 131 | 132 | typedef pcap_send_queue *(*PCAP_sendqueue_alloc)(size_t size); 133 | typedef unsigned (*PCAP_sendqueue_transmit)(pcap_t *p, pcap_send_queue *queue, int sync); 134 | typedef void (*PCAP_sendqueue_destroy)(pcap_send_queue *queue); 135 | typedef int (*PCAP_sendqueue_queue)(pcap_send_queue *queue, const struct pcap_pkthdr *pkt_header, const unsigned char *pkt_data); 136 | 137 | 138 | 139 | 140 | 141 | struct PcapFunctions { 142 | unsigned func_err:1; 143 | unsigned is_available:1; 144 | unsigned is_printing_debug:1; 145 | unsigned status; 146 | unsigned errcode; 147 | 148 | PCAP_close close; 149 | PCAP_datalink datalink; 150 | PCAP_datalink_val_to_name datalink_val_to_name; 151 | PCAP_dispatch dispatch; 152 | PCAP_findalldevs findalldevs; 153 | PCAP_freealldevs freealldevs; 154 | PCAP_lookupdev lookupdev; 155 | PCAP_lib_version lib_version; 156 | PCAP_major_version major_version; 157 | PCAP_minor_version minor_version; 158 | PCAP_next next; 159 | PCAP_next_ex next_ex; 160 | PCAP_open_live open_live; 161 | PCAP_open_offline open_offline; 162 | PCAP_perror perror; 163 | PCAP_sendpacket sendpacket; 164 | PCAP_setdirection setdirection; 165 | PCAP_stats stats; 166 | 167 | /* New PCAP */ 168 | PCAP_create create; 169 | PCAP_set_snaplen set_snaplen; 170 | PCAP_set_promisc set_promisc; 171 | PCAP_can_set_rfmon can_set_rfmon; 172 | PCAP_set_rfmon set_rfmon; 173 | PCAP_set_timeout set_timeout; 174 | PCAP_set_buffer_size set_buffer_size; 175 | PCAP_activate activate; 176 | 177 | /* Accessor functions for opaque data structure, don't really 178 | * exist in libpcap */ 179 | PCAP_dev_name dev_name; 180 | PCAP_dev_description dev_description; 181 | PCAP_dev_next dev_next; 182 | 183 | /* Windows-only functions */ 184 | PCAP_sendqueue_alloc sendqueue_alloc; 185 | PCAP_sendqueue_transmit sendqueue_transmit; 186 | PCAP_sendqueue_destroy sendqueue_destroy; 187 | PCAP_sendqueue_queue sendqueue_queue; 188 | 189 | }; 190 | 191 | /** 192 | * This is global structure containing all the libpcap function pointers. 193 | * use in the form "PCAP.functionname()" rather than "pcap_functioname()". 194 | */ 195 | extern struct PcapFunctions PCAP; 196 | 197 | /** 198 | * Dynamically loads the shared library (libpcap.so, libpcap.dynlib, 199 | * or libpcap.dll. Call this during program startup like main() in order 200 | * to load the libraries. Not thread safe, so call from the startup 201 | * thread, but not within threads. 202 | * @return 203 | * 0 on success or 204 | * -1 on failure 205 | */ 206 | int pcap_init(void); 207 | 208 | 209 | #endif 210 | -------------------------------------------------------------------------------- /src/rawsock-pcap.c: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2007 by Errata Security, All Rights Reserved 2 | * Copyright (c) 2017 by Robert David Graham 3 | * Programer(s): Robert David Graham [rdg] 4 | */ 5 | /* 6 | LIBPCAP INTERFACE 7 | 8 | This VERY MESSY code is a hack to load the 'libpcap' library 9 | at runtime rather than compile time. 10 | 11 | This reason for this mess is that it gets rid of a dependency 12 | when compiling this project. Otherwise, developers would have 13 | to download the 'libpcap-dev' dependency in order to build 14 | this project. 15 | 16 | Almost every platform these days (OpenBSD, FreeBSD, macOS, 17 | Debian, RedHat) comes with a "libpcap.so" library already 18 | installed by default with a known BINARY interface. Thus, 19 | we can include the data structures definitions directly 20 | in this project, then load the library dynamically. 21 | 22 | For those systems without libpcap.so already installed, the 23 | user can either install those on the system, or compile 24 | this project in "STATIC" mode, which will link to the 25 | static libpcap.a library. 26 | 27 | */ 28 | #include "logger.h" 29 | 30 | 31 | #if _MSC_VER==1200 32 | #pragma warning(disable:4115 4201) 33 | #include 34 | #endif 35 | #include "rawsock-pcap.h" 36 | 37 | #ifdef WIN32 38 | #include 39 | #else 40 | #include 41 | #endif 42 | 43 | #include 44 | #include 45 | #include 46 | 47 | #ifndef UNUSEDPARM 48 | #define UNUSEDPARM(x) x=(x) 49 | #endif 50 | 51 | 52 | #ifndef STATICPCAP 53 | struct pcap_if { 54 | struct pcap_if *next; 55 | char *name; /* name to hand to "pcap_open_live()" */ 56 | char *description; /* textual description of interface, or NULL */ 57 | void *addresses; 58 | unsigned flags; /* PCAP_IF_ interface flags */ 59 | }; 60 | #endif 61 | 62 | 63 | static void *my_null(int x, ...) 64 | { 65 | UNUSEDPARM(x); 66 | return 0; 67 | } 68 | 69 | static void seterr(char *errbuf, const char *msg) 70 | { 71 | size_t length = strlen(msg); 72 | 73 | if (length > PCAP_ERRBUF_SIZE-1) 74 | length = PCAP_ERRBUF_SIZE-1; 75 | memcpy(errbuf, msg, length); 76 | errbuf[length] = '\0'; 77 | } 78 | 79 | #ifdef STATICPCAP 80 | #define DECLARESTUB(n) static PCAP_##n stub_##n = pcap_##n; 81 | #else 82 | #define DECLARESTUB(n) static PCAP_##n stub_##n = (PCAP_##n)my_null; 83 | #endif 84 | 85 | DECLARESTUB(close); 86 | DECLARESTUB(datalink); 87 | DECLARESTUB(dispatch); 88 | DECLARESTUB(findalldevs); 89 | DECLARESTUB(freealldevs); 90 | DECLARESTUB(lookupdev); 91 | DECLARESTUB(open_live); 92 | DECLARESTUB(major_version); 93 | DECLARESTUB(minor_version); 94 | DECLARESTUB(lib_version); 95 | DECLARESTUB(open_offline); 96 | DECLARESTUB(sendpacket); 97 | DECLARESTUB(next); 98 | DECLARESTUB(next_ex); 99 | DECLARESTUB(setdirection); 100 | DECLARESTUB(datalink_val_to_name); 101 | DECLARESTUB(perror); 102 | DECLARESTUB(stats); 103 | 104 | 105 | 106 | static const char *stub_dev_name(const pcap_if_t *dev) 107 | { 108 | return dev->name; 109 | } 110 | static const char *stub_dev_description(const pcap_if_t *dev) 111 | { 112 | return dev->description; 113 | } 114 | static const pcap_if_t *stub_dev_next(const pcap_if_t *dev) 115 | { 116 | return dev->next; 117 | } 118 | 119 | static pcap_send_queue *stub_sendqueue_alloc(size_t size) 120 | { 121 | UNUSEDPARM(size); 122 | return 0; 123 | } 124 | static unsigned stub_sendqueue_transmit(pcap_t *p, pcap_send_queue *queue, int sync) 125 | { 126 | my_null(3, p, queue, sync); 127 | return 0; 128 | } 129 | static void stub_sendqueue_destroy(pcap_send_queue *queue) 130 | { 131 | my_null(1, queue); 132 | UNUSEDPARM(queue); 133 | } 134 | static int stub_sendqueue_queue(pcap_send_queue *queue, 135 | const struct pcap_pkthdr *pkt_header, 136 | const unsigned char *pkt_data) 137 | { 138 | my_null(4, queue, pkt_header, pkt_data); 139 | return 0; 140 | } 141 | 142 | /* 143 | * New API 144 | */ 145 | DECLARESTUB(create); 146 | DECLARESTUB(set_snaplen); 147 | DECLARESTUB(set_promisc); 148 | DECLARESTUB(can_set_rfmon); 149 | DECLARESTUB(set_rfmon); 150 | DECLARESTUB(set_timeout); 151 | DECLARESTUB(set_buffer_size); 152 | DECLARESTUB(activate); 153 | 154 | 155 | /**************************************************************************** 156 | *****************************************************************************/ 157 | struct PcapFunctions PCAP = {0}; 158 | 159 | 160 | /** 161 | * Runtime-load the libpcap shared-object or the winpcap DLL. We 162 | * load at runtime rather than loadtime to allow this program to 163 | * be used to process offline content, and to provide more helpful 164 | * messages to people who don't realize they need to install PCAP. 165 | */ 166 | int pcap_init(void) 167 | { 168 | struct PcapFunctions *pl = &PCAP; 169 | #ifdef WIN32 170 | void * hPacket; 171 | void * hLibpcap; 172 | void * hAirpcap; 173 | 174 | pl->is_available = 0; 175 | pl->is_printing_debug = 1; 176 | 177 | /* Look for the Packet.dll */ 178 | hPacket = LoadLibraryA("Packet.dll"); 179 | if (hPacket == NULL) { 180 | if (pl->is_printing_debug) 181 | switch (GetLastError()) { 182 | case ERROR_MOD_NOT_FOUND: 183 | fprintf(stderr, "%s: not found\n", "Packet.dll"); 184 | return; 185 | default: 186 | fprintf(stderr, "%s: couldn't load %d\n", "Packet.dll", (int)GetLastError()); 187 | return; 188 | } 189 | } 190 | 191 | /* Look for the Packet.dll */ 192 | hLibpcap = LoadLibraryA("wpcap.dll"); 193 | if (hLibpcap == NULL) { 194 | if (pl->is_printing_debug) 195 | fprintf(stderr, "%s: couldn't load %d\n", "wpcap.dll", (int)GetLastError()); 196 | return; 197 | } 198 | 199 | /* Look for the Packet.dll */ 200 | hAirpcap = LoadLibraryA("airpcap.dll"); 201 | if (hLibpcap == NULL) { 202 | if (pl->is_printing_debug) 203 | fprintf(stderr, "%s: couldn't load %d\n", "airpcap.dll", (int)GetLastError()); 204 | return; 205 | } 206 | 207 | #define DOLINK(PCAP_DATALINK, datalink) \ 208 | pl->datalink = (PCAP_DATALINK)GetProcAddress(hLibpcap, "pcap_"#datalink); \ 209 | if (pl->datalink == NULL) pl->func_err=1, pl->datalink = null_##PCAP_DATALINK; 210 | #endif 211 | 212 | 213 | #ifndef WIN32 214 | #ifndef STATICPCAP 215 | void *hLibpcap = 0; 216 | 217 | pl->is_available = 0; 218 | pl->is_printing_debug = 1; 219 | 220 | { 221 | static const char *possible_names[] = { 222 | "libpcap.so", 223 | "libpcap.A.dylib", 224 | "libpcap.dylib", 225 | "libpcap.so.0.9.5", 226 | "libpcap.so.0.9.4", 227 | "libpcap.so.0.8", 228 | 0 229 | }; 230 | unsigned i; 231 | for (i=0; possible_names[i]; i++) { 232 | hLibpcap = dlopen(possible_names[i], RTLD_LAZY); 233 | if (hLibpcap) { 234 | LOG(1, "pcap: found library: %s\n", possible_names[i]); 235 | break; 236 | } else { 237 | LOG(2, "pcap: failed to load: %s\n", possible_names[i]); 238 | } 239 | } 240 | 241 | if (hLibpcap == NULL) { 242 | fprintf(stderr, "pcap: failed to load libpcap shared library\n"); 243 | fprintf(stderr, " HINT: you must install libpcap or WinPcap\n"); 244 | } 245 | } 246 | 247 | #define DOLINK(DATALINK, datalink) \ 248 | pl->datalink = (PCAP_##DATALINK)dlsym(hLibpcap, "pcap_"#datalink); \ 249 | if (pl->datalink == NULL) LOG(1, "pcap: pcap_%s: failed\n", #datalink); \ 250 | if (pl->datalink == NULL) pl->func_err=1, pl->datalink = stub_##DATALINK; 251 | #define DYNLINK(datalink) \ 252 | pl->datalink = (PCAP_##datalink)dlsym(hLibpcap, "pcap_"#datalink); \ 253 | if (pl->datalink == NULL) LOG(1, "pcap: pcap_%s: failed\n", #datalink); \ 254 | if (pl->datalink == NULL) pl->func_err=1, pl->datalink = stub_##datalink; 255 | #else 256 | #define DOLINK(PCAP_DATALINK, datalink) \ 257 | pl->func_err=0, pl->datalink = null_##PCAP_DATALINK; 258 | #endif 259 | #endif 260 | 261 | #ifdef WIN32 262 | DOLINK(PCAP_GET_AIRPCAP_HANDLE, get_airpcap_handle); 263 | if (pl->func_err) { 264 | pl->func_err = 0; 265 | } 266 | if (hAirpcap) { 267 | pl->airpcap_set_device_channel = (AIRPCAP_SET_DEVICE_CHANNEL)GetProcAddress(hAirpcap, "AirpcapSetDeviceChannel"); 268 | if (pl->airpcap_set_device_channel == NULL) 269 | pl->airpcap_set_device_channel = null_AIRPCAP_SET_DEVICE_CHANNEL; 270 | } 271 | #endif 272 | 273 | #ifdef STATICPCAP 274 | #define DYNLINK(n) PCAP.n = stub_##n 275 | #endif 276 | 277 | DYNLINK(close); 278 | DYNLINK(datalink); 279 | DYNLINK(datalink_val_to_name); 280 | DYNLINK(dispatch); 281 | DYNLINK(findalldevs); 282 | DYNLINK(freealldevs); 283 | DYNLINK(lib_version); 284 | DYNLINK(lookupdev); 285 | DYNLINK(major_version); 286 | DYNLINK(minor_version); 287 | DYNLINK(next); 288 | DYNLINK(next_ex); 289 | DYNLINK(open_live); 290 | DYNLINK(open_offline); 291 | DYNLINK(perror); 292 | DYNLINK(sendpacket); 293 | DYNLINK(setdirection); 294 | DYNLINK(stats); 295 | 296 | DYNLINK(create); 297 | DYNLINK(set_snaplen); 298 | DYNLINK(set_promisc); 299 | DYNLINK(can_set_rfmon); 300 | DYNLINK(set_rfmon); 301 | DYNLINK(set_timeout); 302 | DYNLINK(set_buffer_size); 303 | DYNLINK(activate); 304 | 305 | DYNLINK(dev_name); 306 | DYNLINK(dev_description); 307 | DYNLINK(dev_next); 308 | 309 | DYNLINK(sendqueue_alloc); 310 | DYNLINK(sendqueue_transmit); 311 | DYNLINK(sendqueue_destroy); 312 | DYNLINK(sendqueue_queue); 313 | 314 | 315 | if (!pl->func_err) 316 | pl->is_available = 1; 317 | else 318 | pl->is_available = 0; 319 | 320 | return 0; 321 | } 322 | 323 | -------------------------------------------------------------------------------- /lz4/xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | xxHash - Extremely Fast Hash algorithm 3 | Header File 4 | Copyright (C) 2012-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* Notice extracted from xxHash homepage : 36 | 37 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 38 | It also successfully passes all tests from the SMHasher suite. 39 | 40 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) 41 | 42 | Name Speed Q.Score Author 43 | xxHash 5.4 GB/s 10 44 | CrapWow 3.2 GB/s 2 Andrew 45 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 46 | SpookyHash 2.0 GB/s 10 Bob Jenkins 47 | SBox 1.4 GB/s 9 Bret Mulvey 48 | Lookup3 1.2 GB/s 9 Bob Jenkins 49 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 50 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 51 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 52 | CRC32 0.43 GB/s 9 53 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 54 | SHA1-32 0.28 GB/s 10 55 | 56 | Q.Score is a measure of quality of the hash function. 57 | It depends on successfully passing SMHasher test set. 58 | 10 is a perfect score. 59 | 60 | A 64-bits version, named XXH64, is available since r35. 61 | It offers much better speed, but for 64-bits applications only. 62 | Name Speed on 64 bits Speed on 32 bits 63 | XXH64 13.8 GB/s 1.9 GB/s 64 | XXH32 6.8 GB/s 6.0 GB/s 65 | */ 66 | 67 | #ifndef XXHASH_H_5627135585666179 68 | #define XXHASH_H_5627135585666179 1 69 | 70 | #if defined (__cplusplus) 71 | extern "C" { 72 | #endif 73 | 74 | 75 | /* **************************** 76 | * Definitions 77 | ******************************/ 78 | #include /* size_t */ 79 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; 80 | 81 | 82 | /* **************************** 83 | * API modifier 84 | ******************************/ 85 | /** XXH_PRIVATE_API 86 | * This is useful to include xxhash functions in `static` mode 87 | * in order to inline them, and remove their symbol from the public list. 88 | * Methodology : 89 | * #define XXH_PRIVATE_API 90 | * #include "xxhash.h" 91 | * `xxhash.c` is automatically included. 92 | * It's not useful to compile and link it as a separate module. 93 | */ 94 | #ifdef XXH_PRIVATE_API 95 | # ifndef XXH_STATIC_LINKING_ONLY 96 | # define XXH_STATIC_LINKING_ONLY 97 | # endif 98 | # if defined(__GNUC__) 99 | # define XXH_PUBLIC_API static __inline __attribute__((unused)) 100 | # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 101 | # define XXH_PUBLIC_API static inline 102 | # elif defined(_MSC_VER) 103 | # define XXH_PUBLIC_API static __inline 104 | # else 105 | # define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ 106 | # endif 107 | #else 108 | # define XXH_PUBLIC_API /* do nothing */ 109 | #endif /* XXH_PRIVATE_API */ 110 | 111 | /*!XXH_NAMESPACE, aka Namespace Emulation : 112 | 113 | If you want to include _and expose_ xxHash functions from within your own library, 114 | but also want to avoid symbol collisions with other libraries which may also include xxHash, 115 | 116 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library 117 | with the value of XXH_NAMESPACE (therefore, avoid NULL and numeric values). 118 | 119 | Note that no change is required within the calling program as long as it includes `xxhash.h` : 120 | regular symbol name will be automatically translated by this header. 121 | */ 122 | #ifdef XXH_NAMESPACE 123 | # define XXH_CAT(A,B) A##B 124 | # define XXH_NAME2(A,B) XXH_CAT(A,B) 125 | # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) 126 | # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) 127 | # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) 128 | # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) 129 | # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) 130 | # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) 131 | # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) 132 | # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) 133 | # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) 134 | # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) 135 | # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) 136 | # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) 137 | # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) 138 | # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) 139 | # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) 140 | # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) 141 | # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) 142 | # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) 143 | # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) 144 | #endif 145 | 146 | 147 | /* ************************************* 148 | * Version 149 | ***************************************/ 150 | #define XXH_VERSION_MAJOR 0 151 | #define XXH_VERSION_MINOR 6 152 | #define XXH_VERSION_RELEASE 2 153 | #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 154 | XXH_PUBLIC_API unsigned XXH_versionNumber (void); 155 | 156 | 157 | /*-********************************************************************** 158 | * 32-bits hash 159 | ************************************************************************/ 160 | typedef unsigned int XXH32_hash_t; 161 | 162 | /*! XXH32() : 163 | Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". 164 | The memory between input & input+length must be valid (allocated and read-accessible). 165 | "seed" can be used to alter the result predictably. 166 | Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s */ 167 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); 168 | 169 | /*====== Streaming ======*/ 170 | typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ 171 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); 172 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 173 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); 174 | 175 | XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); 176 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 177 | XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 178 | 179 | /* 180 | These functions generate the xxHash of an input provided in multiple segments. 181 | Note that, for small input, they are slower than single-call functions, due to state management. 182 | For small input, prefer `XXH32()` and `XXH64()` . 183 | 184 | XXH state must first be allocated, using XXH*_createState() . 185 | 186 | Start a new hash by initializing state with a seed, using XXH*_reset(). 187 | 188 | Then, feed the hash state by calling XXH*_update() as many times as necessary. 189 | Obviously, input must be allocated and read accessible. 190 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. 191 | 192 | Finally, a hash value can be produced anytime, by using XXH*_digest(). 193 | This function returns the nn-bits hash as an int or long long. 194 | 195 | It's still possible to continue inserting input into the hash state after a digest, 196 | and generate some new hashes later on, by calling again XXH*_digest(). 197 | 198 | When done, free XXH state space if it was allocated dynamically. 199 | */ 200 | 201 | /*====== Canonical representation ======*/ 202 | 203 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t; 204 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 205 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 206 | 207 | /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. 208 | * The canonical representation uses human-readable write convention, aka big-endian (large digits first). 209 | * These functions allow transformation of hash result into and from its canonical format. 210 | * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. 211 | */ 212 | 213 | 214 | #ifndef XXH_NO_LONG_LONG 215 | /*-********************************************************************** 216 | * 64-bits hash 217 | ************************************************************************/ 218 | typedef unsigned long long XXH64_hash_t; 219 | 220 | /*! XXH64() : 221 | Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". 222 | "seed" can be used to alter the result predictably. 223 | This function runs faster on 64-bits systems, but slower on 32-bits systems (see benchmark). 224 | */ 225 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); 226 | 227 | /*====== Streaming ======*/ 228 | typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 229 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); 230 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 231 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); 232 | 233 | XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); 234 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); 235 | XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); 236 | 237 | /*====== Canonical representation ======*/ 238 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t; 239 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); 240 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); 241 | #endif /* XXH_NO_LONG_LONG */ 242 | 243 | 244 | #ifdef XXH_STATIC_LINKING_ONLY 245 | 246 | /* ================================================================================================ 247 | This section contains definitions which are not guaranteed to remain stable. 248 | They may change in future versions, becoming incompatible with a different version of the library. 249 | They shall only be used with static linking. 250 | Never use these definitions in association with dynamic linking ! 251 | =================================================================================================== */ 252 | 253 | /* These definitions are only meant to allow allocation of XXH state 254 | statically, on stack, or in a struct for example. 255 | Do not use members directly. */ 256 | 257 | struct XXH32_state_s { 258 | unsigned total_len_32; 259 | unsigned large_len; 260 | unsigned v1; 261 | unsigned v2; 262 | unsigned v3; 263 | unsigned v4; 264 | unsigned mem32[4]; /* buffer defined as U32 for alignment */ 265 | unsigned memsize; 266 | unsigned reserved; /* never read nor write, will be removed in a future version */ 267 | }; /* typedef'd to XXH32_state_t */ 268 | 269 | #ifndef XXH_NO_LONG_LONG 270 | struct XXH64_state_s { 271 | unsigned long long total_len; 272 | unsigned long long v1; 273 | unsigned long long v2; 274 | unsigned long long v3; 275 | unsigned long long v4; 276 | unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ 277 | unsigned memsize; 278 | unsigned reserved[2]; /* never read nor write, will be removed in a future version */ 279 | }; /* typedef'd to XXH64_state_t */ 280 | #endif 281 | 282 | # ifdef XXH_PRIVATE_API 283 | # include "xxhash.c" /* include xxhash function bodies as `static`, for inlining */ 284 | # endif 285 | 286 | #endif /* XXH_STATIC_LINKING_ONLY */ 287 | 288 | 289 | #if defined (__cplusplus) 290 | } 291 | #endif 292 | 293 | #endif /* XXHASH_H_5627135585666179 */ 294 | -------------------------------------------------------------------------------- /lz4/lz4hc.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 HC - High Compression Mode of LZ4 3 | Header File 4 | Copyright (C) 2011-2017, Yann Collet. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | #ifndef LZ4_HC_H_19834876238432 35 | #define LZ4_HC_H_19834876238432 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | /* --- Dependency --- */ 42 | /* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ 43 | #include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ 44 | 45 | 46 | /* --- Useful constants --- */ 47 | #define LZ4HC_CLEVEL_MIN 3 48 | #define LZ4HC_CLEVEL_DEFAULT 9 49 | #define LZ4HC_CLEVEL_OPT_MIN 11 50 | #define LZ4HC_CLEVEL_MAX 12 51 | 52 | 53 | /*-************************************ 54 | * Block Compression 55 | **************************************/ 56 | /*! LZ4_compress_HC() : 57 | * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. 58 | * `dst` must be already allocated. 59 | * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") 60 | * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") 61 | * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_CLEVEL_MAX will work. 62 | * Values >LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX. 63 | * @return : the number of bytes written into 'dst' 64 | * or 0 if compression fails. 65 | */ 66 | LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); 67 | 68 | 69 | /* Note : 70 | * Decompression functions are provided within "lz4.h" (BSD license) 71 | */ 72 | 73 | 74 | /*! LZ4_compress_HC_extStateHC() : 75 | * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. 76 | * `state` size is provided by LZ4_sizeofStateHC(). 77 | * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). 78 | */ 79 | LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); 80 | LZ4LIB_API int LZ4_sizeofStateHC(void); 81 | 82 | 83 | /*-************************************ 84 | * Streaming Compression 85 | * Bufferless synchronous API 86 | **************************************/ 87 | typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ 88 | 89 | /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : 90 | * These functions create and release memory for LZ4 HC streaming state. 91 | * Newly created states are automatically initialized. 92 | * Existing states can be re-used several times, using LZ4_resetStreamHC(). 93 | * These methods are API and ABI stable, they can be used in combination with a DLL. 94 | */ 95 | LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); 96 | LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); 97 | 98 | LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); 99 | LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); 100 | 101 | LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); 102 | 103 | LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); 104 | 105 | /* 106 | These functions compress data in successive blocks of any size, using previous blocks as dictionary. 107 | One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. 108 | There is an exception for ring buffers, which can be smaller than 64 KB. 109 | Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). 110 | 111 | Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). 112 | A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). 113 | 114 | Then, use LZ4_compress_HC_continue() to compress each successive block. 115 | Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. 116 | 'dst' buffer should be sized to handle worst case scenarios (see LZ4_compressBound()), to ensure operation success. 117 | Because in case of failure, the API does not guarantee context recovery, and context will have to be reset. 118 | If `dst` buffer budget cannot be >= LZ4_compressBound(), consider using LZ4_compress_HC_continue_destSize() instead. 119 | 120 | If, for any reason, previous data block can't be preserved unmodified in memory for next compression block, 121 | you can save it to a more stable memory space, using LZ4_saveDictHC(). 122 | Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. 123 | */ 124 | 125 | 126 | /*-************************************* 127 | * PRIVATE DEFINITIONS : 128 | * Do not use these definitions. 129 | * They are exposed to allow static allocation of `LZ4_streamHC_t`. 130 | * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 131 | **************************************/ 132 | #define LZ4HC_DICTIONARY_LOGSIZE 17 /* because of btopt, hc would only need 16 */ 133 | #define LZ4HC_MAXD (1<= 199901L) /* C99 */) 142 | #include 143 | 144 | typedef struct 145 | { 146 | uint32_t hashTable[LZ4HC_HASHTABLESIZE]; 147 | uint16_t chainTable[LZ4HC_MAXD]; 148 | const uint8_t* end; /* next block here to continue on current prefix */ 149 | const uint8_t* base; /* All index relative to this position */ 150 | const uint8_t* dictBase; /* alternate base for extDict */ 151 | uint8_t* inputBuffer; /* deprecated */ 152 | uint32_t dictLimit; /* below that point, need extDict */ 153 | uint32_t lowLimit; /* below that point, no more dict */ 154 | uint32_t nextToUpdate; /* index from which to continue dictionary update */ 155 | uint32_t searchNum; /* only for optimal parser */ 156 | uint32_t compressionLevel; 157 | } LZ4HC_CCtx_internal; 158 | 159 | #else 160 | 161 | typedef struct 162 | { 163 | unsigned int hashTable[LZ4HC_HASHTABLESIZE]; 164 | unsigned short chainTable[LZ4HC_MAXD]; 165 | const unsigned char* end; /* next block here to continue on current prefix */ 166 | const unsigned char* base; /* All index relative to this position */ 167 | const unsigned char* dictBase; /* alternate base for extDict */ 168 | unsigned char* inputBuffer; /* deprecated */ 169 | unsigned int dictLimit; /* below that point, need extDict */ 170 | unsigned int lowLimit; /* below that point, no more dict */ 171 | unsigned int nextToUpdate; /* index from which to continue dictionary update */ 172 | unsigned int searchNum; /* only for optimal parser */ 173 | int compressionLevel; 174 | } LZ4HC_CCtx_internal; 175 | 176 | #endif 177 | 178 | #define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ 179 | #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) 180 | union LZ4_streamHC_u { 181 | size_t table[LZ4_STREAMHCSIZE_SIZET]; 182 | LZ4HC_CCtx_internal internal_donotuse; 183 | }; /* previously typedef'd to LZ4_streamHC_t */ 184 | /* 185 | LZ4_streamHC_t : 186 | This structure allows static allocation of LZ4 HC streaming state. 187 | State must be initialized using LZ4_resetStreamHC() before first use. 188 | 189 | Static allocation shall only be used in combination with static linking. 190 | When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. 191 | */ 192 | 193 | 194 | /*-************************************ 195 | * Deprecated Functions 196 | **************************************/ 197 | /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ 198 | 199 | /* deprecated compression functions */ 200 | /* these functions will trigger warning messages in future releases */ 201 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); 202 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); 203 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); 204 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 205 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); 206 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 207 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); 208 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 209 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); 210 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 211 | 212 | /* Deprecated Streaming functions using older model; should no longer be used */ 213 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); 214 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); 215 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); 216 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); 217 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 218 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); 219 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); 220 | 221 | 222 | #if defined (__cplusplus) 223 | } 224 | #endif 225 | 226 | #endif /* LZ4_HC_H_19834876238432 */ 227 | 228 | /*-************************************************ 229 | * !!!!! STATIC LINKING ONLY !!!!! 230 | * Following definitions are considered experimental. 231 | * They should not be linked from DLL, 232 | * as there is no guarantee of API stability yet. 233 | * Prototypes will be promoted to "stable" status 234 | * after successfull usage in real-life scenarios. 235 | *************************************************/ 236 | #ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */ 237 | #ifndef LZ4_HC_SLO_098092834 238 | #define LZ4_HC_SLO_098092834 239 | 240 | /*! LZ4_compress_HC_destSize() : 241 | * Will try to compress as much data from `src` as possible 242 | * that can fit in `targetDstSize` budget. 243 | * Result is provided in 2 parts : 244 | * @return : the number of bytes written into 'dst' 245 | * or 0 if compression fails. 246 | * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` 247 | */ 248 | LZ4LIB_API int LZ4_compress_HC_destSize(void* LZ4HC_Data, 249 | const char* src, char* dst, 250 | int* srcSizePtr, int targetDstSize, 251 | int compressionLevel); 252 | 253 | /*! LZ4_compress_HC_continue_destSize() : 254 | * Similar as LZ4_compress_HC_continue(), 255 | * but will read a variable nb of bytes from `src` 256 | * to fit into `targetDstSize` budget. 257 | * Result is provided in 2 parts : 258 | * @return : the number of bytes written into 'dst' 259 | * or 0 if compression fails. 260 | * `srcSizePtr` : value will be updated to indicate how much bytes were read from `src` 261 | * Important : due to limitations, this prototype only works well up to cLevel < LZ4HC_CLEVEL_OPT_MIN 262 | * beyond that level, compression performance will be much reduced due to internal incompatibilities 263 | */ 264 | LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr, 265 | const char* src, char* dst, 266 | int* srcSizePtr, int targetDstSize); 267 | 268 | #endif /* LZ4_HC_SLO_098092834 */ 269 | #endif /* LZ4_HC_STATIC_LINKING_ONLY */ 270 | -------------------------------------------------------------------------------- /lz4/lz4opt.h: -------------------------------------------------------------------------------- 1 | /* 2 | lz4opt.h - Optimal Mode of LZ4 3 | Copyright (C) 2015-2017, Przemyslaw Skibinski 4 | Note : this file is intended to be included within lz4hc.c 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #define LZ4_OPT_NUM (1<<12) 37 | 38 | 39 | typedef struct { 40 | int off; 41 | int len; 42 | } LZ4HC_match_t; 43 | 44 | typedef struct { 45 | int price; 46 | int off; 47 | int mlen; 48 | int litlen; 49 | } LZ4HC_optimal_t; 50 | 51 | 52 | /* price in bytes */ 53 | FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) 54 | { 55 | size_t price = litlen; 56 | if (litlen >= (size_t)RUN_MASK) 57 | price += 1 + (litlen-RUN_MASK)/255; 58 | return price; 59 | } 60 | 61 | 62 | /* requires mlen >= MINMATCH */ 63 | FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) 64 | { 65 | size_t price = 2 + 1; /* 16-bit offset + token */ 66 | 67 | price += LZ4HC_literalsPrice(litlen); 68 | 69 | if (mlen >= (size_t)(ML_MASK+MINMATCH)) 70 | price+= 1 + (mlen-(ML_MASK+MINMATCH))/255; 71 | 72 | return price; 73 | } 74 | 75 | 76 | /*-************************************* 77 | * Binary Tree search 78 | ***************************************/ 79 | FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( 80 | LZ4HC_CCtx_internal* ctx, 81 | const BYTE* const ip, 82 | const BYTE* const iHighLimit, 83 | size_t best_mlen, 84 | LZ4HC_match_t* matches, 85 | int* matchNum) 86 | { 87 | U16* const chainTable = ctx->chainTable; 88 | U32* const HashTable = ctx->hashTable; 89 | const BYTE* const base = ctx->base; 90 | const U32 dictLimit = ctx->dictLimit; 91 | const U32 current = (U32)(ip - base); 92 | const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); 93 | const BYTE* const dictBase = ctx->dictBase; 94 | const BYTE* match; 95 | int nbAttempts = ctx->searchNum; 96 | int mnum = 0; 97 | U16 *ptr0, *ptr1, delta0, delta1; 98 | U32 matchIndex; 99 | size_t matchLength = 0; 100 | U32* HashPos; 101 | 102 | if (ip + MINMATCH > iHighLimit) return 1; 103 | 104 | /* HC4 match finder */ 105 | HashPos = &HashTable[LZ4HC_hashPtr(ip)]; 106 | matchIndex = *HashPos; 107 | *HashPos = current; 108 | 109 | ptr0 = &DELTANEXTMAXD(current*2+1); 110 | ptr1 = &DELTANEXTMAXD(current*2); 111 | delta0 = delta1 = (U16)(current - matchIndex); 112 | 113 | while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { 114 | nbAttempts--; 115 | if (matchIndex >= dictLimit) { 116 | match = base + matchIndex; 117 | matchLength = LZ4_count(ip, match, iHighLimit); 118 | } else { 119 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 120 | match = dictBase + matchIndex; 121 | if (vLimit > iHighLimit) vLimit = iHighLimit; 122 | matchLength = LZ4_count(ip, match, vLimit); 123 | if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) 124 | matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); 125 | } 126 | 127 | if (matchLength > best_mlen) { 128 | best_mlen = matchLength; 129 | if (matches) { 130 | if (matchIndex >= dictLimit) 131 | matches[mnum].off = (int)(ip - match); 132 | else 133 | matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ 134 | matches[mnum].len = (int)matchLength; 135 | mnum++; 136 | } 137 | if (best_mlen > LZ4_OPT_NUM) break; 138 | } 139 | 140 | if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ 141 | break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ 142 | 143 | if (*(ip+matchLength) < *(match+matchLength)) { 144 | *ptr0 = delta0; 145 | ptr0 = &DELTANEXTMAXD(matchIndex*2); 146 | if (*ptr0 == (U16)-1) break; 147 | delta0 = *ptr0; 148 | delta1 += delta0; 149 | matchIndex -= delta0; 150 | } else { 151 | *ptr1 = delta1; 152 | ptr1 = &DELTANEXTMAXD(matchIndex*2+1); 153 | if (*ptr1 == (U16)-1) break; 154 | delta1 = *ptr1; 155 | delta0 += delta1; 156 | matchIndex -= delta1; 157 | } 158 | } 159 | 160 | *ptr0 = (U16)-1; 161 | *ptr1 = (U16)-1; 162 | if (matchNum) *matchNum = mnum; 163 | /* if (best_mlen > 8) return best_mlen-8; */ 164 | if (!matchNum) return 1; 165 | return 1; 166 | } 167 | 168 | 169 | FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) 170 | { 171 | const BYTE* const base = ctx->base; 172 | const U32 target = (U32)(ip - base); 173 | U32 idx = ctx->nextToUpdate; 174 | while(idx < target) 175 | idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); 176 | } 177 | 178 | 179 | /** Tree updater, providing best match */ 180 | FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( 181 | LZ4HC_CCtx_internal* ctx, 182 | const BYTE* const ip, const BYTE* const iHighLimit, 183 | size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) 184 | { 185 | int mnum = 0; 186 | if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ 187 | if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); 188 | best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); 189 | ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); 190 | return mnum; 191 | } 192 | 193 | 194 | #define SET_PRICE(pos, ml, offset, ll, cost) \ 195 | { \ 196 | while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ 197 | opt[pos].mlen = (int)ml; \ 198 | opt[pos].off = (int)offset; \ 199 | opt[pos].litlen = (int)ll; \ 200 | opt[pos].price = (int)cost; \ 201 | } 202 | 203 | 204 | static int LZ4HC_compress_optimal ( 205 | LZ4HC_CCtx_internal* ctx, 206 | const char* const source, 207 | char* dest, 208 | int inputSize, 209 | int maxOutputSize, 210 | limitedOutput_directive limit, 211 | size_t sufficient_len, 212 | const int fullUpdate 213 | ) 214 | { 215 | LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; /* this uses a bit too much stack memory to my taste ... */ 216 | LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; 217 | 218 | const BYTE* ip = (const BYTE*) source; 219 | const BYTE* anchor = ip; 220 | const BYTE* const iend = ip + inputSize; 221 | const BYTE* const mflimit = iend - MFLIMIT; 222 | const BYTE* const matchlimit = (iend - LASTLITERALS); 223 | BYTE* op = (BYTE*) dest; 224 | BYTE* const oend = op + maxOutputSize; 225 | 226 | /* init */ 227 | if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1; 228 | ctx->end += inputSize; 229 | ip++; 230 | 231 | /* Main Loop */ 232 | while (ip < mflimit) { 233 | size_t const llen = ip - anchor; 234 | size_t last_pos = 0; 235 | size_t match_num, cur, best_mlen, best_off; 236 | memset(opt, 0, sizeof(LZ4HC_optimal_t)); /* memset only the first one */ 237 | 238 | match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); 239 | if (!match_num) { ip++; continue; } 240 | 241 | if ((size_t)matches[match_num-1].len > sufficient_len) { 242 | /* good enough solution : immediate encoding */ 243 | best_mlen = matches[match_num-1].len; 244 | best_off = matches[match_num-1].off; 245 | cur = 0; 246 | last_pos = 1; 247 | goto encode; 248 | } 249 | 250 | /* set prices using matches at position = 0 */ 251 | { size_t matchNb; 252 | for (matchNb = 0; matchNb < match_num; matchNb++) { 253 | size_t mlen = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; 254 | best_mlen = matches[matchNb].len; /* necessarily < sufficient_len < LZ4_OPT_NUM */ 255 | for ( ; mlen <= best_mlen ; mlen++) { 256 | size_t const cost = LZ4HC_sequencePrice(llen, mlen) - LZ4HC_literalsPrice(llen); 257 | SET_PRICE(mlen, mlen, matches[matchNb].off, 0, cost); /* updates last_pos and opt[pos] */ 258 | } } } 259 | 260 | if (last_pos < MINMATCH) { ip++; continue; } /* note : on clang at least, this test improves performance */ 261 | 262 | /* check further positions */ 263 | opt[0].mlen = opt[1].mlen = 1; 264 | for (cur = 1; cur <= last_pos; cur++) { 265 | const BYTE* const curPtr = ip + cur; 266 | 267 | /* establish baseline price if cur is literal */ 268 | { size_t price, litlen; 269 | if (opt[cur-1].mlen == 1) { 270 | /* no match at previous position */ 271 | litlen = opt[cur-1].litlen + 1; 272 | if (cur > litlen) { 273 | price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); 274 | } else { 275 | price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); 276 | } 277 | } else { 278 | litlen = 1; 279 | price = opt[cur - 1].price + LZ4HC_literalsPrice(1); 280 | } 281 | 282 | if (price < (size_t)opt[cur].price) 283 | SET_PRICE(cur, 1 /*mlen*/, 0 /*off*/, litlen, price); /* note : increases last_pos */ 284 | } 285 | 286 | if (cur == last_pos || curPtr >= mflimit) break; 287 | 288 | match_num = LZ4HC_BinTree_GetAllMatches(ctx, curPtr, matchlimit, MINMATCH-1, matches, fullUpdate); 289 | if ((match_num > 0) && (size_t)matches[match_num-1].len > sufficient_len) { 290 | /* immediate encoding */ 291 | best_mlen = matches[match_num-1].len; 292 | best_off = matches[match_num-1].off; 293 | last_pos = cur + 1; 294 | goto encode; 295 | } 296 | 297 | /* set prices using matches at position = cur */ 298 | { size_t matchNb; 299 | for (matchNb = 0; matchNb < match_num; matchNb++) { 300 | size_t ml = (matchNb>0) ? (size_t)matches[matchNb-1].len+1 : MINMATCH; 301 | best_mlen = (cur + matches[matchNb].len < LZ4_OPT_NUM) ? 302 | (size_t)matches[matchNb].len : LZ4_OPT_NUM - cur; 303 | 304 | for ( ; ml <= best_mlen ; ml++) { 305 | size_t ll, price; 306 | if (opt[cur].mlen == 1) { 307 | ll = opt[cur].litlen; 308 | if (cur > ll) 309 | price = opt[cur - ll].price + LZ4HC_sequencePrice(ll, ml); 310 | else 311 | price = LZ4HC_sequencePrice(llen + ll, ml) - LZ4HC_literalsPrice(llen); 312 | } else { 313 | ll = 0; 314 | price = opt[cur].price + LZ4HC_sequencePrice(0, ml); 315 | } 316 | 317 | if (cur + ml > last_pos || price < (size_t)opt[cur + ml].price) { 318 | SET_PRICE(cur + ml, ml, matches[matchNb].off, ll, price); 319 | } } } } 320 | } /* for (cur = 1; cur <= last_pos; cur++) */ 321 | 322 | best_mlen = opt[last_pos].mlen; 323 | best_off = opt[last_pos].off; 324 | cur = last_pos - best_mlen; 325 | 326 | encode: /* cur, last_pos, best_mlen, best_off must be set */ 327 | opt[0].mlen = 1; 328 | while (1) { /* from end to beginning */ 329 | size_t const ml = opt[cur].mlen; 330 | int const offset = opt[cur].off; 331 | opt[cur].mlen = (int)best_mlen; 332 | opt[cur].off = (int)best_off; 333 | best_mlen = ml; 334 | best_off = offset; 335 | if (ml > cur) break; /* can this happen ? */ 336 | cur -= ml; 337 | } 338 | 339 | /* encode all recorded sequences */ 340 | cur = 0; 341 | while (cur < last_pos) { 342 | int const ml = opt[cur].mlen; 343 | int const offset = opt[cur].off; 344 | if (ml == 1) { ip++; cur++; continue; } 345 | cur += ml; 346 | if ( LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ip - offset, limit, oend) ) return 0; 347 | } 348 | } /* while (ip < mflimit) */ 349 | 350 | /* Encode Last Literals */ 351 | { int lastRun = (int)(iend - anchor); 352 | if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ 353 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 354 | else *op++ = (BYTE)(lastRun< 3 | #include 4 | #include 5 | #include 6 | 7 | enum { 8 | CONF_NUM, 9 | CONF_BOOL, 10 | CONF_NOTBOOL, 11 | CONF_STR, 12 | CONF_FILES, 13 | }; 14 | enum { 15 | CONF_NOECHO = 1, 16 | CONF_COMMENT = 2, 17 | }; 18 | struct ConfigurationOptions { 19 | const char *name; 20 | int type; 21 | size_t offset; 22 | int flags; 23 | }; 24 | 25 | /*************************************************************************** 26 | * Macro to grab structure member given the offsetof() value 27 | ***************************************************************************/ 28 | #define VAR(name) offsetof(PacketDump,name) 29 | #define MEMBERAT(conf, n, t) (t*)&((char*)(conf))[n] 30 | 31 | /*************************************************************************** 32 | * The configured list of all possible configuration values 33 | ***************************************************************************/ 34 | static const struct ConfigurationOptions options[] = { 35 | {"filesize", CONF_NUM, VAR(rotate_size)}, 36 | {"filetime", CONF_NUM, VAR(rotate_seconds)}, 37 | {"maxfiles", CONF_NUM, VAR(rotate_filecount)}, 38 | {"interface", CONF_STR, VAR(ifname)}, 39 | {"writefile", CONF_STR, VAR(filename)}, 40 | {"bpf", CONF_STR, VAR(bpf_rule)}, 41 | {"bpf-file", CONF_STR, VAR(bpf_file)}, 42 | {"compress", CONF_BOOL, VAR(is_compression)}, 43 | {"gmt", CONF_BOOL, VAR(is_gmt)}, 44 | 45 | 46 | {"monitor-mode",CONF_BOOL, VAR(is_monitor_mode)}, 47 | {"no-promiscuous-mode", CONF_NOTBOOL,VAR(is_promiscuous_mode)}, 48 | {"relinquish-privileges", CONF_STR, VAR(drop_user)}, 49 | 50 | {"help", CONF_BOOL, VAR(is_help), CONF_NOECHO}, 51 | {"version", CONF_BOOL, VAR(is_version), CONF_NOECHO}, 52 | {"list-interfaces",CONF_BOOL,VAR(is_iflist), CONF_NOECHO}, 53 | {"echo", CONF_BOOL, VAR(is_echo), CONF_NOECHO}, 54 | 55 | {"readfile", CONF_FILES, VAR(readfiles)}, 56 | {0} 57 | }; 58 | 59 | /*************************************************************************** 60 | ***************************************************************************/ 61 | void 62 | echo_configuration(FILE *fp, struct PacketDump *conf) 63 | { 64 | size_t i; 65 | uint64_t *num; 66 | char *b; 67 | char **str; 68 | char ***files; 69 | 70 | for (i=0; options[i].name; i++) { 71 | if (options[i].flags & CONF_NOECHO) 72 | continue; 73 | 74 | fprintf(fp, "%s = ", options[i].name); 75 | switch (options[i].type) { 76 | case CONF_NUM: 77 | num = MEMBERAT(conf, options[i].offset, uint64_t); 78 | fprintf(fp, "%llu\n", *num); 79 | break; 80 | 81 | case CONF_BOOL: 82 | b = MEMBERAT(conf, options[i].offset, char); 83 | fprintf(fp, "%s\n", (*b)?"true":"false"); 84 | break; 85 | case CONF_NOTBOOL: 86 | b = MEMBERAT(conf, options[i].offset, char); 87 | fprintf(fp, "%s\n", (*b)?"false":"true"); 88 | break; 89 | case CONF_STR: 90 | str = MEMBERAT(conf, options[i].offset, char *); 91 | fprintf(fp, "%s\n", (*str)?(*str):""); 92 | break; 93 | case CONF_FILES: 94 | files = MEMBERAT(conf, options[i].offset, char **); 95 | if (*files) { 96 | size_t j; 97 | 98 | for (j=0; (*files)[j]; j++) { 99 | char *file = (*files)[j]; 100 | if (j) 101 | fprintf(fp, "%s = ", options[i].name); 102 | fprintf(fp, "%s\n", file); 103 | } 104 | 105 | } 106 | break; 107 | default: 108 | fprintf(fp, "\n"); 109 | break; 110 | } 111 | } 112 | } 113 | 114 | /*************************************************************************** 115 | * Parse the boolean value (true, false, t, f, 1, 0, etc.). If the value 116 | * string isn't a legal value, then this returns 0 and does not touch 117 | * the original value. 118 | * @return 1 if success, 0 is failure 119 | ***************************************************************************/ 120 | int 121 | parse_boolean(const char *value, char *result) 122 | { 123 | int is_match = 0; 124 | 125 | if (value == 0 || value[0] == '\0') { 126 | return 0; 127 | } 128 | 129 | is_match = !strcasecmp(value, "t") 130 | || !strcasecmp(value, "true") 131 | || !strcasecmp(value, "1") 132 | || !strcasecmp(value, "on"); 133 | if (is_match) { 134 | *result = 1; 135 | return 1; 136 | } 137 | 138 | is_match = !strcasecmp(value, "f") 139 | || !strcasecmp(value, "false") 140 | || !strcasecmp(value, "0") 141 | || !strcasecmp(value, "off"); 142 | if (is_match) { 143 | *result = 0; 144 | return 1; 145 | } 146 | 147 | /* parse failure */ 148 | return 0; 149 | } 150 | 151 | /*************************************************************************** 152 | * Whether the named option is of a "boolean" type, according to our list 153 | * of options. Note that this isn't parsing the value to see if it's true 154 | * for false, but parsing the name, and looking up in our list of 155 | * possible options. 156 | ***************************************************************************/ 157 | static int 158 | is_option_boolean(const char *name) 159 | { 160 | size_t i; 161 | for (i=0; options[i].name; i++) { 162 | if (strcmp(options[i].name, name)==0) { 163 | if (options[i].type == CONF_BOOL || options[i].type == CONF_NOTBOOL) 164 | return 1; 165 | else 166 | return 0; 167 | } 168 | } 169 | return 0; 170 | } 171 | static int 172 | is_option_filelist(const char *name) 173 | { 174 | size_t i; 175 | for (i=0; options[i].name; i++) { 176 | if (strcmp(options[i].name, name)==0) { 177 | if (options[i].type == CONF_FILES) 178 | return 1; 179 | else 180 | return 0; 181 | } 182 | } 183 | return 0; 184 | } 185 | 186 | 187 | /*************************************************************************** 188 | ***************************************************************************/ 189 | void 190 | parse_option(const char *name, const char *value, struct PacketDump *conf) 191 | { 192 | uint64_t *num; 193 | char *b; 194 | char **str; 195 | char ***files; 196 | int is_valid; 197 | size_t i; 198 | size_t count; 199 | 200 | /* 201 | * Find the template for the option 202 | */ 203 | for (i=0; options[i].name; i++) { 204 | if (strcmp(options[i].name, name) == 0) 205 | break; 206 | } 207 | if (options[i].name == NULL) { 208 | fprintf(stderr, "%s: unknown option\n", name); 209 | return; 210 | } 211 | 212 | /* 213 | * parse the option according to the template 214 | */ 215 | switch (options[i].type) { 216 | case CONF_NUM: 217 | num = MEMBERAT(conf, options[i].offset, uint64_t); 218 | *num = strtoull(value, 0, 0); 219 | break; 220 | 221 | case CONF_BOOL: 222 | case CONF_NOTBOOL: 223 | b = MEMBERAT(conf, options[i].offset, char); 224 | 225 | is_valid = parse_boolean(value, b); 226 | 227 | if (!is_valid) { 228 | fprintf(stderr, "%s: not a boolean value, expected 'true' or 'false'\n", name); 229 | } else if (options[i].type == CONF_NOTBOOL) { 230 | *b = !(*b); 231 | } 232 | break; 233 | case CONF_STR: 234 | str = MEMBERAT(conf, options[i].offset, char*); 235 | if (*str) 236 | free(*str); 237 | if (value == NULL || value[0] == '\0') 238 | *str = NULL; 239 | else 240 | *str = strdup(value); 241 | break; 242 | case CONF_FILES: 243 | files = MEMBERAT(conf, options[i].offset, char **); 244 | if (*files == NULL) { 245 | *files = malloc(sizeof(char*) * 1); 246 | (*files)[0] = NULL; 247 | } 248 | for (count=0; (*files)[count]; count++) 249 | ; 250 | *files = realloc(*files, sizeof(char*) * (count + 2)); 251 | (*files)[count++] = strdup(value); 252 | (*files)[count] = NULL; 253 | break; 254 | default: 255 | fprintf(stderr, "internal error: option type unknown\n"); 256 | exit(1); 257 | } 258 | } 259 | 260 | /*************************************************************************** 261 | ***************************************************************************/ 262 | static int 263 | is_ending(const char *lhs, const char *rhs) 264 | { 265 | size_t left_len = strlen(lhs); 266 | size_t right_len = strlen(rhs); 267 | 268 | if (left_len < right_len) 269 | return 0; 270 | 271 | return memcmp(lhs + left_len - right_len, rhs, right_len) == 0; 272 | } 273 | 274 | 275 | /*************************************************************************** 276 | ***************************************************************************/ 277 | void 278 | read_configuration(int argc, char **argv, struct PacketDump *conf) 279 | { 280 | int i; 281 | 282 | for (i=1; i= argc || argv[i+1][0] == '-') { 303 | if (is_option_boolean(name)) 304 | value = "true"; 305 | else { 306 | fprintf(stderr, "%s: missing value\n", name); 307 | continue; 308 | } 309 | } else 310 | value = argv[++i]; 311 | } 312 | /* 313 | * Now parse the name=value 314 | */ 315 | parse_option(name, value, conf); 316 | 317 | /* Do special processing for file-lists, such as when we 318 | * read in a list of files on input. Because of shell filename 319 | * wildcard expansion, we many ahve a long list of filenames 320 | * after the option name */ 321 | if (is_option_filelist(name)) { 322 | while (i < argc && argv[i+1][0] != '-') 323 | parse_option(name, argv[++i], conf); 324 | } 325 | 326 | free(name); 327 | } else if (argv[i][0] == '-') { 328 | const char *name; 329 | switch (argv[i][1]) { 330 | case 'C': name = "filesize"; break; 331 | case 'D': name = "list-interfaces"; break; 332 | case 'F': name = "bpf-file"; break; 333 | case 'G': name = "filetime"; break; 334 | case 'h': name = "help"; break; 335 | case '?': name = "help"; break; 336 | case 'i': name = "interface"; break; 337 | case 'I': name = "monitor-mode"; break; 338 | case 'p': name = "no-promiscuous-mode"; break; 339 | case 'r': name = "readfile"; break; 340 | case 'w': name = "writefile"; break; 341 | case 'W': name = "maxfiles"; break; 342 | case 'z': name = "compress"; break; 343 | case 'Z': name = "relinquish-privileges"; break; 344 | default: 345 | fprintf(stderr, "%s: bad option\n", argv[i]); 346 | continue; 347 | } 348 | if (argv[i][2] == '=') { 349 | value = argv[i]+3; 350 | } else if (argv[i][2]) { 351 | value = argv[i]+2; 352 | } else if (i+1 >= argc || argv[i+1][0] == '-') { 353 | if (is_option_boolean(name)) 354 | value = "true"; 355 | else { 356 | fprintf(stderr, "%s: missing value", name); 357 | continue; 358 | } 359 | } else { 360 | value = argv[++i]; 361 | } 362 | 363 | /* 364 | * Now parse the name=value 365 | */ 366 | parse_option(name, value, conf); 367 | 368 | /* Do special processing for file-lists, such as when we 369 | * read in a list of files on input. Because of shell filename 370 | * wildcard expansion, we many ahve a long list of filenames 371 | * after the option name */ 372 | if (is_option_filelist(name)) { 373 | while (i < argc && argv[i+1][0] != '-') 374 | parse_option(name, argv[++i], conf); 375 | } 376 | } 377 | } 378 | 379 | /* 380 | * if the file ends in the name of a compression algorithm, then 381 | * enable the compression flag. 382 | */ 383 | if (conf->filename && is_ending(conf->filename, ".lz4")) 384 | conf->is_compression = 1; 385 | } 386 | 387 | /*************************************************************************** 388 | ***************************************************************************/ 389 | void 390 | configuration_help(void) 391 | { 392 | printf( 393 | "usage: packetdump -i -w [options]\n" 394 | "options:\n" 395 | " -C \n" 396 | " Maximum size of file before it rotates.\n" 397 | " -D\n" 398 | " --list-interfaces\n" 399 | " Prints list of possible packet capture interfaces.\n" 400 | " -F \n" 401 | " Read BPF filter rules from this file.\n" 402 | " -G \n" 403 | " Rotate file after this number of seconds.\n" 404 | " -i \n" 405 | " --interface=\n" 406 | " Sniff on this network interface\n" 407 | " -I\n" 408 | " --monitor-mode\n" 409 | " On WiFi interfaces, sets rfmon mode\n" 410 | " -p\n" 411 | " --no-promiscuous-mode\n" 412 | " Do NOT put the adapter into promiscuous mode.\n" 413 | " -r \n" 414 | " Read packets from a file.\n" 415 | " -w \n" 416 | " Write packets to a file.\n" 417 | " -W \n" 418 | " Creates ring-buffer with this number of files.\n" 419 | " --version\n" 420 | " Print version info.\n" 421 | " -z [compression type]\n" 422 | " Enable compression. Not needed if file suffix indicates compression.\n" 423 | " -Z \n" 424 | " --relinquish-privileges=user\n" 425 | " Drops root privileges to those of this user\n" 426 | "\n"); 427 | } 428 | 429 | 430 | -------------------------------------------------------------------------------- /src/main.c: -------------------------------------------------------------------------------- 1 | #include "packetdump.h" 2 | #include "config.h" 3 | #include "logger.h" 4 | #include "lz4/lz4.h" 5 | #include "pixie-threads.h" 6 | #include "pixie-timer.h" 7 | #include "rawsock-pcap.h" /* dynamicly load pcap library */ 8 | #include "rawsock-pcapfile.h" /* write capture files */ 9 | #include "readfiles.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | /* 17 | * All the globals in this project 18 | */ 19 | unsigned control_c_pressed = 0; 20 | unsigned control_c_pressed_again = 0; 21 | 22 | /*************************************************************************** 23 | ***************************************************************************/ 24 | static char * 25 | morph_filename(const struct PacketDump *conf, time_t now, size_t filecount) 26 | { 27 | struct tm *tm; 28 | char *newfilename; 29 | const char *oldfilename = conf->filename; 30 | size_t i, j=0; 31 | 32 | if (conf->is_gmt) { 33 | tm = gmtime(&now); 34 | } else { 35 | tm = localtime(&now); 36 | } 37 | 38 | newfilename = malloc(1024 + strlen(oldfilename)); 39 | 40 | for (i=0; oldfilename[i]; i++) { 41 | if (oldfilename[i] != '%') 42 | newfilename[j++] = oldfilename[i]; 43 | else { 44 | int code = oldfilename[++i]; 45 | switch (code) { 46 | case 'y': 47 | j += sprintf(newfilename+j, "%02u", tm->tm_year % 100); 48 | break; 49 | case 'Y': 50 | j += sprintf(newfilename+j, "%02u", tm->tm_year + 1900); 51 | break; 52 | case 'm': 53 | j += sprintf(newfilename+j, "%02u", tm->tm_mon + 1); 54 | break; 55 | case 'd': 56 | j += sprintf(newfilename+j, "%02u", tm->tm_mday); 57 | break; 58 | case 'H': 59 | j += sprintf(newfilename+j, "%02u", tm->tm_hour); 60 | break; 61 | case 'M': 62 | j += sprintf(newfilename+j, "%02u", tm->tm_min); 63 | break; 64 | case 'S': 65 | j += sprintf(newfilename+j, "%02u", tm->tm_sec); 66 | break; 67 | default: 68 | newfilename[j++] = (char)code; 69 | } 70 | } 71 | } 72 | 73 | newfilename[j] = '\0'; 74 | 75 | 76 | return newfilename; 77 | } 78 | 79 | /*************************************************************************** 80 | ***************************************************************************/ 81 | static time_t 82 | next_rotate_time(time_t last_rotate, unsigned period, unsigned offset) 83 | { 84 | time_t next; 85 | 86 | if (period == 0) 87 | next = INT_MAX; 88 | else 89 | next = last_rotate - (last_rotate % period) + period + offset; 90 | 91 | return next; 92 | } 93 | 94 | /*************************************************************************** 95 | ***************************************************************************/ 96 | struct WriteContext 97 | { 98 | /** 99 | * The configuration information that tells us how we should be writing 100 | * packets. 101 | */ 102 | const struct PacketDump *conf; 103 | 104 | /** 105 | * Handle to the file where we are writing packets. This changes while we 106 | * write packets whenever we need to rotate the file to a new one 107 | */ 108 | struct PcapFile *fp; 109 | 110 | /** 111 | * The current filename, which is based on morphing the configured 112 | * filename, such as adding data/timestamp information 113 | */ 114 | char *filename; 115 | 116 | /** 117 | * The total number of files that we have processed 118 | */ 119 | size_t total_file_count; 120 | 121 | /** 122 | * The timestamp when we should next rotate the output file. 123 | */ 124 | time_t rotate_time; 125 | 126 | /** 127 | * The libpcap data-link value (Ethernet, WiFi, etc.) 128 | */ 129 | int data_link; 130 | 131 | size_t file_bytes_written; 132 | size_t file_packets_written; 133 | 134 | }; 135 | 136 | /*************************************************************************** 137 | * Write a single packet to the output file. 138 | * 139 | * Note that most of the logic in this function is about rotating the 140 | * file when it gets too big, or when it exceeds a timestamp. Indeed, 141 | * because of rotation issues, we don't even open the file for the first 142 | * time until we are ready to write the first frame. 143 | ***************************************************************************/ 144 | static int 145 | handle_packet(struct WriteContext *ctx, const struct pcap_pkthdr *hdr, const void *buf) 146 | { 147 | const struct PacketDump *conf = ctx->conf; 148 | ssize_t bytes_written; 149 | 150 | /* 151 | * open the output file 152 | */ 153 | again: 154 | if (ctx->fp == NULL) { 155 | 156 | /* Create a new filename based on timestamp and filecount information */ 157 | ctx->filename = morph_filename(conf, hdr->ts.tv_sec, ctx->total_file_count); 158 | LOG(0, "%s: opening new file\n", ctx->filename); 159 | 160 | /* Open the file */ 161 | ctx->fp = pcapfile_openwrite(ctx->filename, ctx->data_link, PCAPFILE_LZ4); 162 | if (ctx->fp == NULL) { 163 | /* This is bad. I don't know how to recover at this point */ 164 | fprintf(stderr, "%s: couldn't open file\n", ctx->filename); 165 | return -1; 166 | } 167 | 168 | /* Calculate the timestamp when the file should next be rotated. 169 | * Note that his is aligned, so that if "hourly" rotation is desired, 170 | * it'll rotate every hour on the hour */ 171 | ctx->rotate_time = next_rotate_time(hdr->ts.tv_sec, 172 | (unsigned)conf->rotate_seconds, 173 | 0); 174 | ctx->file_bytes_written = 0; 175 | ctx->file_packets_written = 0; 176 | ctx->total_file_count++; 177 | } 178 | 179 | /* 180 | * Rotate the old capture file if necessary 181 | */ 182 | if ((conf->rotate_size && ctx->file_bytes_written >= conf->rotate_size) 183 | || (ctx->rotate_time && hdr->ts.tv_sec >= ctx->rotate_time)) { 184 | LOG(0, "%s: file#%llu, wrote %llu bytes, wrote %llu packets\n", 185 | ctx->filename, 186 | ctx->total_file_count, 187 | ctx->file_bytes_written, 188 | ctx->file_packets_written); 189 | pcapfile_close(ctx->fp); 190 | ctx->fp = NULL; 191 | free(ctx->filename); 192 | ctx->filename = NULL; 193 | goto again; 194 | } 195 | 196 | 197 | /* 198 | * write the frame 199 | */ 200 | bytes_written = pcapfile_writeframe(ctx->fp, 201 | buf, 202 | hdr->caplen, 203 | hdr->len, 204 | hdr->ts.tv_sec, 205 | hdr->ts.tv_usec 206 | ); 207 | if (bytes_written < 0) { 208 | fprintf(stderr, "packet write failure\n"); 209 | return -1; 210 | } 211 | 212 | ctx->file_bytes_written += bytes_written; 213 | ctx->file_packets_written++; 214 | 215 | return 0; 216 | } 217 | 218 | /*************************************************************************** 219 | ***************************************************************************/ 220 | void statistics_thread(void *userdata) 221 | { 222 | unsigned long long total_packets = 0; 223 | unsigned long long total_drops = 0; 224 | 225 | while (!control_c_pressed) { 226 | struct pcap_stat stats = {0}; 227 | size_t bytes_printed; 228 | size_t i; 229 | 230 | pixie_usleep(100000 ); 231 | 232 | PCAP.stats(userdata, &stats); 233 | 234 | total_packets = stats.ps_recv; 235 | total_drops = stats.ps_drop + stats.ps_ifdrop; 236 | 237 | bytes_printed = fprintf(stderr, "packets=%llu, drops=%llu ", 238 | total_packets, 239 | total_drops); 240 | for (i=0; iifname, /* network adapter to sniff from*/ 263 | 65536, /* snap length */ 264 | 1, /* promiscuous mode */ 265 | 10, /* read timeout in milliseconds */ 266 | errbuf /* error buffer */ 267 | ); 268 | if (p == NULL) { 269 | fprintf(stderr, "%s: %s\n", conf->ifname, errbuf); 270 | return; 271 | } else { 272 | //fprintf(stderr, "%s: buffsize = %d\n", conf->ifname, PCAP.bufsize(p)); 273 | fprintf(stderr, "%s: capture started\n", conf->ifname); 274 | } 275 | 276 | /* 277 | * Start a statistics thread 278 | */ 279 | t = pixie_begin_thread(statistics_thread, 0, p); 280 | 281 | /* 282 | * Setup the context 283 | */ 284 | ctx->conf = conf; 285 | ctx->data_link = PCAP.datalink(p); 286 | 287 | /* 288 | * now loop reading packets 289 | */ 290 | while (!control_c_pressed) { 291 | struct pcap_pkthdr *hdr; 292 | const unsigned char *buf; 293 | int x; 294 | 295 | /* 296 | * Read the next packet 297 | */ 298 | x = PCAP.next_ex(p, &hdr, &buf); 299 | if (x == 0) 300 | continue; /* timeout expired */ 301 | else if (x < 0) { 302 | PCAP.perror(p, conf->ifname); 303 | break; 304 | } 305 | 306 | x = handle_packet(ctx, hdr, buf); 307 | if (x < 0) 308 | break; 309 | } 310 | 311 | pixie_thread_join(t); 312 | fprintf(stderr, "read %u packets\n", (unsigned)total_packets_written); 313 | 314 | if (ctx->fp) 315 | pcapfile_close(ctx->fp); 316 | if (ctx->filename) 317 | free(ctx->filename); 318 | if (p) 319 | PCAP.close(p); 320 | } 321 | 322 | /*************************************************************************** 323 | * We trap the so that instead of exiting immediately, we sit in 324 | * a loop for a few seconds waiting for any late response. But, the user 325 | * can press a second time to exit that waiting. 326 | ***************************************************************************/ 327 | static void 328 | control_c_handler(int x) 329 | { 330 | if (control_c_pressed == 0) { 331 | fprintf(stderr, 332 | "waiting several seconds to exit..." 333 | " \n" 334 | ); 335 | fflush(stderr); 336 | control_c_pressed = 1+x; 337 | } else { 338 | control_c_pressed_again = 1; 339 | } 340 | } 341 | 342 | /*************************************************************************** 343 | * This function prints to the command line a list of all the network 344 | * intefaces/devices. 345 | ***************************************************************************/ 346 | static void 347 | rawsock_list_adapters(void) 348 | { 349 | pcap_if_t *alldevs; 350 | char errbuf[PCAP_ERRBUF_SIZE]; 351 | int x; 352 | int index; 353 | const pcap_if_t *d; 354 | 355 | /* 356 | * Get the list of network adapters 357 | */ 358 | x = PCAP.findalldevs(&alldevs, errbuf); 359 | if (x < 0) { 360 | fprintf(stderr, "%s\n", errbuf); 361 | return; 362 | } 363 | if (alldevs == NULL) { 364 | fprintf(stderr, "ERR:libpcap: no adapters found, are you sure you are root?\n"); 365 | return; 366 | } 367 | 368 | /* 369 | * Print the list, with a numeric index 370 | */ 371 | index = 0; 372 | for(d=alldevs; d; d=PCAP.dev_next(d)) { 373 | fprintf(stderr, " %d %s \t", index++, PCAP.dev_name(d)); 374 | if (PCAP.dev_description(d)) 375 | fprintf(stderr, "(%s)\n", PCAP.dev_description(d)); 376 | else 377 | fprintf(stderr, "(No description available)\n"); 378 | } 379 | fprintf(stderr,"\n"); 380 | 381 | 382 | /* 383 | * Free the memory. Not really necessary, since we are going to exit 384 | * immediately anyway. 385 | */ 386 | PCAP.freealldevs(alldevs); 387 | } 388 | 389 | /*************************************************************************** 390 | ***************************************************************************/ 391 | int 392 | main(int argc, char *argv[]) 393 | { 394 | struct PacketDump conf[1] = {0}; 395 | unsigned statuscount = 0; 396 | char errbuf[PCAP_ERRBUF_SIZE]; 397 | 398 | fprintf(stderr, "--- packetdump/1.0 by Robert Graham ---\n"); 399 | 400 | /* Dynamically load the 'libcap' library */ 401 | pcap_init(); 402 | 403 | /* Read in the configuration from the command-line, and if the --conf 404 | * option is used, configuration files from the disk */ 405 | if (argc == 1) { 406 | fprintf(stderr, "no options specified, use '-h' for help\n"); 407 | return 1; 408 | } 409 | read_configuration(argc, argv, conf); 410 | 411 | if (conf->ifname == 0) 412 | conf->ifname = PCAP.lookupdev(errbuf); 413 | 414 | /* 415 | * trap to pause 416 | */ 417 | signal(SIGINT, control_c_handler); 418 | 419 | 420 | /* 421 | * Print info items that user might have requested. If the user specifies 422 | * these, then we won't start capture 423 | */ 424 | if (conf->is_help) { 425 | configuration_help(); 426 | statuscount++; 427 | } 428 | if (conf->is_echo) { 429 | echo_configuration(stdout, conf); 430 | statuscount++; 431 | } 432 | if (conf->is_version) { 433 | fprintf(stderr, "packetdump version = 1.0\n"); 434 | fprintf(stderr, "%s\n", PCAP.lib_version()); 435 | fprintf(stderr, "LZ4 version = %s\n", LZ4_versionString()); 436 | statuscount++; 437 | } 438 | if (conf->is_iflist) { 439 | rawsock_list_adapters(); 440 | statuscount++; 441 | } 442 | if (statuscount) 443 | return 1; 444 | 445 | if (conf->readfiles) { 446 | read_files(conf); 447 | return 0; 448 | } 449 | 450 | /* 451 | * Make sure we have a capture interface and a file to write 452 | * to 453 | */ 454 | if (conf->ifname == NULL || conf->ifname[0] == '\0') { 455 | fprintf(stderr, "FAIL: no interface specified\n"); 456 | fprintf(stderr, " hint: use the '-i' option to specify an interface\n"); 457 | return 1; 458 | } 459 | if (conf->filename == NULL || conf->filename[0] == '\0') { 460 | fprintf(stderr, "FAIL: no output files specified\n"); 461 | fprintf(stderr, " hint: use the '-w' option to specify a file to write to\n"); 462 | return 1; 463 | } 464 | 465 | /* 466 | * Start the capture thread 467 | */ 468 | capture_thread(conf); 469 | 470 | return 0; 471 | } 472 | -------------------------------------------------------------------------------- /lz4/lz4frame.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 auto-framing library 3 | Header File 4 | Copyright (C) 2011-2017, Yann Collet. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | 35 | /* LZ4F is a stand-alone API to create LZ4-compressed frames 36 | * conformant with specification v1.5.1. 37 | * It also offers streaming capabilities. 38 | * lz4.h is not required when using lz4frame.h. 39 | * */ 40 | 41 | #ifndef LZ4F_H_09782039843 42 | #define LZ4F_H_09782039843 43 | 44 | #if defined (__cplusplus) 45 | extern "C" { 46 | #endif 47 | 48 | /* --- Dependency --- */ 49 | #include /* size_t */ 50 | 51 | 52 | /** 53 | Introduction 54 | 55 | lz4frame.h implements LZ4 frame specification (doc/lz4_Frame_format.md). 56 | lz4frame.h provides frame compression functions that take care 57 | of encoding standard metadata alongside LZ4-compressed blocks. 58 | */ 59 | 60 | /*-*************************************************************** 61 | * Compiler specifics 62 | *****************************************************************/ 63 | /* LZ4_DLL_EXPORT : 64 | * Enable exporting of functions when building a Windows DLL 65 | * LZ4FLIB_API : 66 | * Control library symbols visibility. 67 | */ 68 | #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) 69 | # define LZ4FLIB_API __declspec(dllexport) 70 | #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) 71 | # define LZ4FLIB_API __declspec(dllimport) 72 | #elif defined(__GNUC__) && (__GNUC__ >= 4) 73 | # define LZ4FLIB_API __attribute__ ((__visibility__ ("default"))) 74 | #else 75 | # define LZ4FLIB_API 76 | #endif 77 | 78 | #ifdef LZ4F_DISABLE_DEPRECATE_WARNINGS 79 | # define LZ4F_DEPRECATE(x) x 80 | #else 81 | # if defined(_MSC_VER) 82 | # define LZ4F_DEPRECATE(x) x /* __declspec(deprecated) x - only works with C++ */ 83 | # elif defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 6)) 84 | # define LZ4F_DEPRECATE(x) x __attribute__((deprecated)) 85 | # else 86 | # define LZ4F_DEPRECATE(x) x /* no deprecation warning for this compiler */ 87 | # endif 88 | #endif 89 | 90 | 91 | /*-************************************ 92 | * Error management 93 | **************************************/ 94 | typedef size_t LZ4F_errorCode_t; 95 | 96 | LZ4FLIB_API unsigned LZ4F_isError(LZ4F_errorCode_t code); /**< tells if a `LZ4F_errorCode_t` function result is an error code */ 97 | LZ4FLIB_API const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /**< return error code string; useful for debugging */ 98 | 99 | 100 | /*-************************************ 101 | * Frame compression types 102 | **************************************/ 103 | /* #define LZ4F_DISABLE_OBSOLETE_ENUMS */ /* uncomment to disable obsolete enums */ 104 | #ifndef LZ4F_DISABLE_OBSOLETE_ENUMS 105 | # define LZ4F_OBSOLETE_ENUM(x) , LZ4F_DEPRECATE(x) = LZ4F_##x 106 | #else 107 | # define LZ4F_OBSOLETE_ENUM(x) 108 | #endif 109 | 110 | /* The larger the block size, the (slightly) better the compression ratio, 111 | * though there are diminishing returns. 112 | * Larger blocks also increase memory usage on both compression and decompression sides. */ 113 | typedef enum { 114 | LZ4F_default=0, 115 | LZ4F_max64KB=4, 116 | LZ4F_max256KB=5, 117 | LZ4F_max1MB=6, 118 | LZ4F_max4MB=7 119 | LZ4F_OBSOLETE_ENUM(max64KB) 120 | LZ4F_OBSOLETE_ENUM(max256KB) 121 | LZ4F_OBSOLETE_ENUM(max1MB) 122 | LZ4F_OBSOLETE_ENUM(max4MB) 123 | } LZ4F_blockSizeID_t; 124 | 125 | /* Linked blocks sharply reduce inefficiencies when using small blocks, 126 | * they compress better. 127 | * However, some LZ4 decoders are only compatible with independent blocks */ 128 | typedef enum { 129 | LZ4F_blockLinked=0, 130 | LZ4F_blockIndependent 131 | LZ4F_OBSOLETE_ENUM(blockLinked) 132 | LZ4F_OBSOLETE_ENUM(blockIndependent) 133 | } LZ4F_blockMode_t; 134 | 135 | typedef enum { 136 | LZ4F_noContentChecksum=0, 137 | LZ4F_contentChecksumEnabled 138 | LZ4F_OBSOLETE_ENUM(noContentChecksum) 139 | LZ4F_OBSOLETE_ENUM(contentChecksumEnabled) 140 | } LZ4F_contentChecksum_t; 141 | 142 | typedef enum { 143 | LZ4F_frame=0, 144 | LZ4F_skippableFrame 145 | LZ4F_OBSOLETE_ENUM(skippableFrame) 146 | } LZ4F_frameType_t; 147 | 148 | #ifndef LZ4F_DISABLE_OBSOLETE_ENUMS 149 | typedef LZ4F_blockSizeID_t blockSizeID_t; 150 | typedef LZ4F_blockMode_t blockMode_t; 151 | typedef LZ4F_frameType_t frameType_t; 152 | typedef LZ4F_contentChecksum_t contentChecksum_t; 153 | #endif 154 | 155 | /*! LZ4F_frameInfo_t : 156 | * makes it possible to supply detailed frame parameters to the stream interface. 157 | * It's not required to set all fields, as long as the structure was initially memset() to zero. 158 | * All reserved fields must be set to zero. */ 159 | typedef struct { 160 | LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ 161 | LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ 162 | LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ 163 | LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ 164 | unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ 165 | unsigned reserved[2]; /* must be zero for forward compatibility */ 166 | } LZ4F_frameInfo_t; 167 | 168 | /*! LZ4F_preferences_t : 169 | * makes it possible to supply detailed compression parameters to the stream interface. 170 | * It's not required to set all fields, as long as the structure was initially memset() to zero. 171 | * All reserved fields must be set to zero. */ 172 | typedef struct { 173 | LZ4F_frameInfo_t frameInfo; 174 | int compressionLevel; /* 0 == default (fast mode); values above LZ4HC_CLEVEL_MAX count as LZ4HC_CLEVEL_MAX; values below 0 trigger "fast acceleration", proportional to value */ 175 | unsigned autoFlush; /* 1 == always flush (reduce usage of tmp buffer) */ 176 | unsigned reserved[4]; /* must be zero for forward compatibility */ 177 | } LZ4F_preferences_t; 178 | 179 | 180 | /*-********************************* 181 | * Simple compression function 182 | ***********************************/ 183 | /*!LZ4F_compressFrameBound() : 184 | * Returns the maximum possible size of a frame compressed with LZ4F_compressFrame() given srcSize content and preferences. 185 | * Note : this result is only usable with LZ4F_compressFrame(), not with multi-segments compression. 186 | */ 187 | LZ4FLIB_API size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); 188 | 189 | /*!LZ4F_compressFrame() : 190 | * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 191 | * An important rule is that dstBuffer MUST be large enough (dstCapacity) to store the result in worst case situation. 192 | * This value is supplied by LZ4F_compressFrameBound(). 193 | * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode). 194 | * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. 195 | * @return : number of bytes written into dstBuffer. 196 | * or an error code if it fails (can be tested using LZ4F_isError()) 197 | */ 198 | LZ4FLIB_API size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); 199 | 200 | 201 | 202 | /*-*********************************** 203 | * Advanced compression functions 204 | *************************************/ 205 | typedef struct LZ4F_cctx_s LZ4F_cctx; /* incomplete type */ 206 | typedef LZ4F_cctx* LZ4F_compressionContext_t; /* for compatibility with previous API version */ 207 | 208 | typedef struct { 209 | unsigned stableSrc; /* 1 == src content will remain present on future calls to LZ4F_compress(); skip copying src content within tmp buffer */ 210 | unsigned reserved[3]; 211 | } LZ4F_compressOptions_t; 212 | 213 | /*--- Resource Management ---*/ 214 | 215 | #define LZ4F_VERSION 100 216 | LZ4FLIB_API unsigned LZ4F_getVersion(void); 217 | /*! LZ4F_createCompressionContext() : 218 | * The first thing to do is to create a compressionContext object, which will be used in all compression operations. 219 | * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. 220 | * The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. 221 | * The function will provide a pointer to a fully allocated LZ4F_cctx object. 222 | * If @return != zero, there was an error during context creation. 223 | * Object can release its memory using LZ4F_freeCompressionContext(); 224 | */ 225 | LZ4FLIB_API LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_cctx** cctxPtr, unsigned version); 226 | LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx); 227 | 228 | 229 | /*---- Compression ----*/ 230 | 231 | #define LZ4F_HEADER_SIZE_MAX 15 232 | /*! LZ4F_compressBegin() : 233 | * will write the frame header into dstBuffer. 234 | * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes. 235 | * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. 236 | * @return : number of bytes written into dstBuffer for the header 237 | * or an error code (which can be tested using LZ4F_isError()) 238 | */ 239 | LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); 240 | 241 | /*! LZ4F_compressBound() : 242 | * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations. 243 | * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario. 244 | * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers. 245 | * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations. 246 | */ 247 | LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); 248 | 249 | /*! LZ4F_compressUpdate() : 250 | * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. 251 | * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations. 252 | * This value is provided by LZ4F_compressBound(). 253 | * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). 254 | * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. 255 | * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default. 256 | * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). 257 | * or an error code if it fails (which can be tested using LZ4F_isError()) 258 | */ 259 | LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); 260 | 261 | /*! LZ4F_flush() : 262 | * When data must be generated and sent immediately, without waiting for a block to be completely filled, 263 | * it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. 264 | * `dstCapacity` must be large enough to ensure the operation will be successful. 265 | * `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. 266 | * @return : number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) 267 | * or an error code if it fails (which can be tested using LZ4F_isError()) 268 | */ 269 | LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); 270 | 271 | /*! LZ4F_compressEnd() : 272 | * To properly finish an LZ4 frame, invoke LZ4F_compressEnd(). 273 | * It will flush whatever data remained within `cctx` (like LZ4_flush()) 274 | * and properly finalize the frame, with an endMark and a checksum. 275 | * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default. 276 | * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) 277 | * or an error code if it fails (which can be tested using LZ4F_isError()) 278 | * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. 279 | */ 280 | LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr); 281 | 282 | 283 | /*-********************************* 284 | * Decompression functions 285 | ***********************************/ 286 | typedef struct LZ4F_dctx_s LZ4F_dctx; /* incomplete type */ 287 | typedef LZ4F_dctx* LZ4F_decompressionContext_t; /* compatibility with previous API versions */ 288 | 289 | typedef struct { 290 | unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ 291 | unsigned reserved[3]; 292 | } LZ4F_decompressOptions_t; 293 | 294 | 295 | /* Resource management */ 296 | 297 | /*!LZ4F_createDecompressionContext() : 298 | * Create an LZ4F_decompressionContext_t object, which will be used to track all decompression operations. 299 | * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions. 300 | * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. 301 | * The result is an errorCode, which can be tested using LZ4F_isError(). 302 | * dctx memory can be released using LZ4F_freeDecompressionContext(); 303 | * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. 304 | * That is, it should be == 0 if decompression has been completed fully and correctly. 305 | */ 306 | LZ4FLIB_API LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_dctx** dctxPtr, unsigned version); 307 | LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_dctx* dctx); 308 | 309 | 310 | /*-*********************************** 311 | * Streaming decompression functions 312 | *************************************/ 313 | 314 | /*! LZ4F_getFrameInfo() : 315 | * This function extracts frame parameters (such as max blockSize, frame checksum, etc.). 316 | * Its usage is optional. Extracted information can be useful for allocation purposes, typically. 317 | * This function works in 2 situations : 318 | * - At the beginning of a new frame, in which case it will decode this information from `srcBuffer`, and start the decoding process. 319 | * Input size must be large enough to successfully decode the entire frame header. 320 | * Frame header size is variable, but is guaranteed to be <= LZ4F_HEADER_SIZE_MAX bytes. 321 | * It's allowed to provide more input data than this minimum. 322 | * - After decoding has been started. 323 | * In which case, no input is read, frame parameters are extracted from dctx. 324 | * If decoding has just started, but not yet extracted information from header, LZ4F_getFrameInfo() will fail. 325 | * The number of bytes consumed from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). 326 | * Decompression must resume from (srcBuffer + *srcSizePtr). 327 | * @return : an hint about how many srcSize bytes LZ4F_decompress() expects for next call, 328 | * or an error code which can be tested using LZ4F_isError() 329 | * note 1 : in case of error, dctx is not modified. Decoding operations can resume from where they stopped. 330 | * note 2 : frame parameters are *copied into* an already allocated LZ4F_frameInfo_t structure. 331 | */ 332 | LZ4FLIB_API size_t LZ4F_getFrameInfo(LZ4F_dctx* dctx, 333 | LZ4F_frameInfo_t* frameInfoPtr, 334 | const void* srcBuffer, size_t* srcSizePtr); 335 | 336 | /*! LZ4F_decompress() : 337 | * Call this function repetitively to regenerate data compressed within `srcBuffer`. 338 | * The function will attempt to decode up to *srcSizePtr bytes from srcBuffer, into dstBuffer of capacity *dstSizePtr. 339 | * 340 | * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). 341 | * 342 | * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). 343 | * Number of bytes read can be < number of bytes provided, meaning there is some more data to decode. 344 | * It typically happens when dstBuffer is not large enough to contain all decoded data. 345 | * Remaining data will have to be presented again in a subsequent invocation. 346 | * 347 | * `dstBuffer` content is expected to be flushed between each invocation, as its content will be overwritten. 348 | * `dstBuffer` can be changed at will between each consecutive function invocation. 349 | * 350 | * @return is an hint of how many `srcSize` bytes LZ4F_decompress() expects for next call. 351 | * Schematically, it's the size of the current (or remaining) compressed block + header of next block. 352 | * Respecting the hint provides some small speed benefit, because it skips intermediate buffers. 353 | * This is just a hint though, it's always possible to provide any srcSize. 354 | * When a frame is fully decoded, @return will be 0 (no more data expected). 355 | * If decompression failed, @return is an error code, which can be tested using LZ4F_isError(). 356 | * 357 | * After a frame is fully decoded, dctx can be used again to decompress another frame. 358 | * After a decompression error, use LZ4F_resetDecompressionContext() before re-using dctx, to return to clean state. 359 | */ 360 | LZ4FLIB_API size_t LZ4F_decompress(LZ4F_dctx* dctx, 361 | void* dstBuffer, size_t* dstSizePtr, 362 | const void* srcBuffer, size_t* srcSizePtr, 363 | const LZ4F_decompressOptions_t* dOptPtr); 364 | 365 | 366 | /*! LZ4F_resetDecompressionContext() : v1.8.0 367 | * In case of an error, the context is left in "undefined" state. 368 | * In which case, it's necessary to reset it, before re-using it. 369 | * This method can also be used to abruptly stop an unfinished decompression, 370 | * and start a new with the same context. */ 371 | LZ4FLIB_API void LZ4F_resetDecompressionContext(LZ4F_dctx* dctx); /* always successful */ 372 | 373 | 374 | 375 | #if defined (__cplusplus) 376 | } 377 | #endif 378 | 379 | #endif /* LZ4F_H_09782039843 */ 380 | -------------------------------------------------------------------------------- /lz4/lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | * LZ4 - Fast LZ compression algorithm 3 | * Header File 4 | * Copyright (C) 2011-2017, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 homepage : http://www.lz4.org 33 | - LZ4 source repository : https://github.com/lz4/lz4 34 | */ 35 | #if defined (__cplusplus) 36 | extern "C" { 37 | #endif 38 | 39 | #ifndef LZ4_H_2983827168210 40 | #define LZ4_H_2983827168210 41 | 42 | /* --- Dependency --- */ 43 | #include /* size_t */ 44 | 45 | 46 | /** 47 | Introduction 48 | 49 | LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, 50 | scalable with multi-cores CPU. It features an extremely fast decoder, with speed in 51 | multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. 52 | 53 | The LZ4 compression library provides in-memory compression and decompression functions. 54 | Compression can be done in: 55 | - a single step (described as Simple Functions) 56 | - a single step, reusing a context (described in Advanced Functions) 57 | - unbounded multiple steps (described as Streaming compression) 58 | 59 | lz4.h provides block compression functions. It gives full buffer control to user. 60 | Decompressing an lz4-compressed block also requires metadata (such as compressed size). 61 | Each application is free to encode such metadata in whichever way it wants. 62 | 63 | An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), 64 | take care of encoding standard metadata alongside LZ4-compressed blocks. 65 | If your application requires interoperability, it's recommended to use it. 66 | A library is provided to take care of it, see lz4frame.h. 67 | */ 68 | 69 | /*^*************************************************************** 70 | * Export parameters 71 | *****************************************************************/ 72 | /* 73 | * LZ4_DLL_EXPORT : 74 | * Enable exporting of functions when building a Windows DLL 75 | * LZ4LIB_API : 76 | * Control library symbols visibility. 77 | */ 78 | #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) 79 | # define LZ4LIB_API __declspec(dllexport) 80 | #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) 81 | # define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 82 | #elif defined(__GNUC__) && (__GNUC__ >= 4) 83 | # define LZ4LIB_API __attribute__ ((__visibility__ ("default"))) 84 | #else 85 | # define LZ4LIB_API 86 | #endif 87 | 88 | 89 | /*------ Version ------*/ 90 | #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 91 | #define LZ4_VERSION_MINOR 8 /* for new (non-breaking) interface capabilities */ 92 | #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ 93 | 94 | #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) 95 | 96 | #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE 97 | #define LZ4_QUOTE(str) #str 98 | #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) 99 | #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) 100 | 101 | LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; to be used when checking dll version */ 102 | LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; to be used when checking dll version */ 103 | 104 | 105 | /*-************************************ 106 | * Tuning parameter 107 | **************************************/ 108 | /*! 109 | * LZ4_MEMORY_USAGE : 110 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 111 | * Increasing memory usage improves compression ratio 112 | * Reduced memory usage can improve speed, due to cache effect 113 | * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache 114 | */ 115 | #ifndef LZ4_MEMORY_USAGE 116 | # define LZ4_MEMORY_USAGE 14 117 | #endif 118 | 119 | /*-************************************ 120 | * Simple Functions 121 | **************************************/ 122 | /*! LZ4_compress_default() : 123 | Compresses 'sourceSize' bytes from buffer 'source' 124 | into already allocated 'dest' buffer of size 'maxDestSize'. 125 | Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). 126 | It also runs faster, so it's a recommended setting. 127 | If the function cannot compress 'source' into a more limited 'dest' budget, 128 | compression stops *immediately*, and the function result is zero. 129 | As a consequence, 'dest' content is not valid. 130 | This function never writes outside 'dest' buffer, nor read outside 'source' buffer. 131 | sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE 132 | maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) 133 | return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) 134 | or 0 if compression fails */ 135 | LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); 136 | 137 | /*! LZ4_decompress_safe() : 138 | compressedSize : is the precise full size of the compressed block. 139 | maxDecompressedSize : is the size of destination buffer, which must be already allocated. 140 | return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) 141 | If destination buffer is not large enough, decoding will stop and output an error code (<0). 142 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 143 | This function is protected against buffer overflow exploits, including malicious data packets. 144 | It never writes outside output buffer, nor reads outside input buffer. 145 | */ 146 | LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); 147 | 148 | 149 | /*-************************************ 150 | * Advanced Functions 151 | **************************************/ 152 | #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ 153 | #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 154 | 155 | /*! 156 | LZ4_compressBound() : 157 | Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) 158 | This function is primarily useful for memory allocation purposes (destination buffer size). 159 | Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). 160 | Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) 161 | inputSize : max supported value is LZ4_MAX_INPUT_SIZE 162 | return : maximum output size in a "worst case" scenario 163 | or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) 164 | */ 165 | LZ4LIB_API int LZ4_compressBound(int inputSize); 166 | 167 | /*! 168 | LZ4_compress_fast() : 169 | Same as LZ4_compress_default(), but allows to select an "acceleration" factor. 170 | The larger the acceleration value, the faster the algorithm, but also the lesser the compression. 171 | It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. 172 | An acceleration value of "1" is the same as regular LZ4_compress_default() 173 | Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. 174 | */ 175 | LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); 176 | 177 | 178 | /*! 179 | LZ4_compress_fast_extState() : 180 | Same compression function, just using an externally allocated memory space to store compression state. 181 | Use LZ4_sizeofState() to know how much memory must be allocated, 182 | and allocate it on 8-bytes boundaries (using malloc() typically). 183 | Then, provide it as 'void* state' to compression function. 184 | */ 185 | LZ4LIB_API int LZ4_sizeofState(void); 186 | LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); 187 | 188 | 189 | /*! 190 | LZ4_compress_destSize() : 191 | Reverse the logic, by compressing as much data as possible from 'source' buffer 192 | into already allocated buffer 'dest' of size 'targetDestSize'. 193 | This function either compresses the entire 'source' content into 'dest' if it's large enough, 194 | or fill 'dest' buffer completely with as much data as possible from 'source'. 195 | *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. 196 | New value is necessarily <= old value. 197 | return : Nb bytes written into 'dest' (necessarily <= targetDestSize) 198 | or 0 if compression fails 199 | */ 200 | LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); 201 | 202 | 203 | /*! 204 | LZ4_decompress_fast() : 205 | originalSize : is the original and therefore uncompressed size 206 | return : the number of bytes read from the source buffer (in other words, the compressed size) 207 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 208 | Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. 209 | note : This function fully respect memory boundaries for properly formed compressed data. 210 | It is a bit faster than LZ4_decompress_safe(). 211 | However, it does not provide any protection against intentionally modified data stream (malicious input). 212 | Use this function in trusted environment only (data to decode comes from a trusted source). 213 | */ 214 | LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); 215 | 216 | /*! 217 | LZ4_decompress_safe_partial() : 218 | This function decompress a compressed block of size 'compressedSize' at position 'source' 219 | into destination buffer 'dest' of size 'maxDecompressedSize'. 220 | The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, 221 | reducing decompression time. 222 | return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) 223 | Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. 224 | Always control how many bytes were decoded. 225 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 226 | This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets 227 | */ 228 | LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); 229 | 230 | 231 | /*-********************************************* 232 | * Streaming Compression Functions 233 | ***********************************************/ 234 | typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ 235 | 236 | /*! LZ4_createStream() and LZ4_freeStream() : 237 | * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. 238 | * LZ4_freeStream() releases its memory. 239 | */ 240 | LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); 241 | LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); 242 | 243 | /*! LZ4_resetStream() : 244 | * An LZ4_stream_t structure can be allocated once and re-used multiple times. 245 | * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. 246 | */ 247 | LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); 248 | 249 | /*! LZ4_loadDict() : 250 | * Use this function to load a static dictionary into LZ4_stream. 251 | * Any previous data will be forgotten, only 'dictionary' will remain in memory. 252 | * Loading a size of 0 is allowed. 253 | * Return : dictionary size, in bytes (necessarily <= 64 KB) 254 | */ 255 | LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); 256 | 257 | /*! LZ4_compress_fast_continue() : 258 | * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. 259 | * Important : Previous data blocks are assumed to remain present and unmodified ! 260 | * 'dst' buffer must be already allocated. 261 | * If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. 262 | * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. 263 | * After an error, the stream status is invalid, and it can only be reset or freed. 264 | */ 265 | LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration); 266 | 267 | /*! LZ4_saveDict() : 268 | * If previously compressed data block is not guaranteed to remain available at its current memory location, 269 | * save it into a safer place (char* safeBuffer). 270 | * Note : it's not necessary to call LZ4_loadDict() after LZ4_saveDict(), dictionary is immediately usable. 271 | * @return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. 272 | */ 273 | LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); 274 | 275 | 276 | /*-********************************************** 277 | * Streaming Decompression Functions 278 | * Bufferless synchronous API 279 | ************************************************/ 280 | typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ 281 | 282 | /*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() : 283 | * creation / destruction of streaming decompression tracking structure */ 284 | LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); 285 | LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); 286 | 287 | /*! LZ4_setStreamDecode() : 288 | * Use this function to instruct where to find the dictionary. 289 | * Setting a size of 0 is allowed (same effect as reset). 290 | * @return : 1 if OK, 0 if error 291 | */ 292 | LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); 293 | 294 | /*! 295 | LZ4_decompress_*_continue() : 296 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 297 | Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) 298 | In the case of a ring buffers, decoding buffer must be either : 299 | - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) 300 | In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). 301 | - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. 302 | maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. 303 | In which case, encoding and decoding buffers do not need to be synchronized, 304 | and encoding ring buffer can have any size, including small ones ( < 64 KB). 305 | - _At least_ 64 KB + 8 bytes + maxBlockSize. 306 | In which case, encoding and decoding buffers do not need to be synchronized, 307 | and encoding ring buffer can have any size, including larger than decoding buffer. 308 | Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, 309 | and indicate where it is saved using LZ4_setStreamDecode() 310 | */ 311 | LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); 312 | LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); 313 | 314 | 315 | /*! LZ4_decompress_*_usingDict() : 316 | * These decoding functions work the same as 317 | * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() 318 | * They are stand-alone, and don't need an LZ4_streamDecode_t structure. 319 | */ 320 | LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); 321 | LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); 322 | 323 | 324 | /*^********************************************** 325 | * !!!!!! STATIC LINKING ONLY !!!!!! 326 | ***********************************************/ 327 | /*-************************************ 328 | * Private definitions 329 | ************************************** 330 | * Do not use these definitions. 331 | * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. 332 | * Using these definitions will expose code to API and/or ABI break in future versions of the library. 333 | **************************************/ 334 | #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) 335 | #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) 336 | #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ 337 | 338 | #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 339 | #include 340 | 341 | typedef struct { 342 | uint32_t hashTable[LZ4_HASH_SIZE_U32]; 343 | uint32_t currentOffset; 344 | uint32_t initCheck; 345 | const uint8_t* dictionary; 346 | uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ 347 | uint32_t dictSize; 348 | } LZ4_stream_t_internal; 349 | 350 | typedef struct { 351 | const uint8_t* externalDict; 352 | size_t extDictSize; 353 | const uint8_t* prefixEnd; 354 | size_t prefixSize; 355 | } LZ4_streamDecode_t_internal; 356 | 357 | #else 358 | 359 | typedef struct { 360 | unsigned int hashTable[LZ4_HASH_SIZE_U32]; 361 | unsigned int currentOffset; 362 | unsigned int initCheck; 363 | const unsigned char* dictionary; 364 | unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ 365 | unsigned int dictSize; 366 | } LZ4_stream_t_internal; 367 | 368 | typedef struct { 369 | const unsigned char* externalDict; 370 | size_t extDictSize; 371 | const unsigned char* prefixEnd; 372 | size_t prefixSize; 373 | } LZ4_streamDecode_t_internal; 374 | 375 | #endif 376 | 377 | /*! 378 | * LZ4_stream_t : 379 | * information structure to track an LZ4 stream. 380 | * init this structure before first use. 381 | * note : only use in association with static linking ! 382 | * this definition is not API/ABI safe, 383 | * and may change in a future version ! 384 | */ 385 | #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) 386 | #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) 387 | union LZ4_stream_u { 388 | unsigned long long table[LZ4_STREAMSIZE_U64]; 389 | LZ4_stream_t_internal internal_donotuse; 390 | } ; /* previously typedef'd to LZ4_stream_t */ 391 | 392 | 393 | /*! 394 | * LZ4_streamDecode_t : 395 | * information structure to track an LZ4 stream during decompression. 396 | * init this structure using LZ4_setStreamDecode (or memset()) before first use 397 | * note : only use in association with static linking ! 398 | * this definition is not API/ABI safe, 399 | * and may change in a future version ! 400 | */ 401 | #define LZ4_STREAMDECODESIZE_U64 4 402 | #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) 403 | union LZ4_streamDecode_u { 404 | unsigned long long table[LZ4_STREAMDECODESIZE_U64]; 405 | LZ4_streamDecode_t_internal internal_donotuse; 406 | } ; /* previously typedef'd to LZ4_streamDecode_t */ 407 | 408 | 409 | /*-************************************ 410 | * Obsolete Functions 411 | **************************************/ 412 | 413 | /*! Deprecation warnings 414 | Should deprecation warnings be a problem, 415 | it is generally possible to disable them, 416 | typically with -Wno-deprecated-declarations for gcc 417 | or _CRT_SECURE_NO_WARNINGS in Visual. 418 | Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ 419 | #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS 420 | # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ 421 | #else 422 | # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 423 | # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ 424 | # define LZ4_DEPRECATED(message) [[deprecated(message)]] 425 | # elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) 426 | # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) 427 | # elif (LZ4_GCC_VERSION >= 301) 428 | # define LZ4_DEPRECATED(message) __attribute__((deprecated)) 429 | # elif defined(_MSC_VER) 430 | # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) 431 | # else 432 | # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") 433 | # define LZ4_DEPRECATED(message) 434 | # endif 435 | #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ 436 | 437 | /* Obsolete compression functions */ 438 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); 439 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 440 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 441 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 442 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 443 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 444 | 445 | /* Obsolete decompression functions */ 446 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast() instead") int LZ4_uncompress (const char* source, char* dest, int outputSize); 447 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe() instead") int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); 448 | 449 | /* Obsolete streaming functions; use new streaming interface whenever possible */ 450 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); 451 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); 452 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); 453 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); 454 | 455 | /* Obsolete streaming decoding functions */ 456 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); 457 | LZ4LIB_API LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); 458 | 459 | #endif /* LZ4_H_2983827168210 */ 460 | 461 | #if defined (__cplusplus) 462 | } 463 | #endif 464 | -------------------------------------------------------------------------------- /lz4/xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Fast Hash algorithm 3 | * Copyright (C) 2012-2016, Yann Collet 4 | * 5 | * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are 9 | * met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following disclaimer 15 | * in the documentation and/or other materials provided with the 16 | * distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | * 30 | * You can contact the author at : 31 | * - xxHash homepage: http://www.xxhash.com 32 | * - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | 36 | /* ************************************* 37 | * Tuning parameters 38 | ***************************************/ 39 | /*!XXH_FORCE_MEMORY_ACCESS : 40 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 41 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 42 | * The below switch allow to select different access method for improved performance. 43 | * Method 0 (default) : use `memcpy()`. Safe and portable. 44 | * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). 45 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 46 | * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. 47 | * It can generate buggy code on targets which do not support unaligned memory accesses. 48 | * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) 49 | * See http://stackoverflow.com/a/32095106/646947 for details. 50 | * Prefer these methods in priority order (0 > 1 > 2) 51 | */ 52 | #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 53 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 54 | # define XXH_FORCE_MEMORY_ACCESS 2 55 | # elif defined(__INTEL_COMPILER) || \ 56 | (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) 57 | # define XXH_FORCE_MEMORY_ACCESS 1 58 | # endif 59 | #endif 60 | 61 | /*!XXH_ACCEPT_NULL_INPUT_POINTER : 62 | * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. 63 | * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. 64 | * By default, this option is disabled. To enable it, uncomment below define : 65 | */ 66 | /* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ 67 | 68 | /*!XXH_FORCE_NATIVE_FORMAT : 69 | * By default, xxHash library provides endian-independent Hash values, based on little-endian convention. 70 | * Results are therefore identical for little-endian and big-endian CPU. 71 | * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. 72 | * Should endian-independence be of no importance for your application, you may set the #define below to 1, 73 | * to improve speed for Big-endian CPU. 74 | * This option has no impact on Little_Endian CPU. 75 | */ 76 | #ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ 77 | # define XXH_FORCE_NATIVE_FORMAT 0 78 | #endif 79 | 80 | /*!XXH_FORCE_ALIGN_CHECK : 81 | * This is a minor performance trick, only useful with lots of very small keys. 82 | * It means : check for aligned/unaligned input. 83 | * The check costs one initial branch per hash; set to 0 when the input data 84 | * is guaranteed to be aligned. 85 | */ 86 | #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ 87 | # if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) 88 | # define XXH_FORCE_ALIGN_CHECK 0 89 | # else 90 | # define XXH_FORCE_ALIGN_CHECK 1 91 | # endif 92 | #endif 93 | 94 | 95 | /* ************************************* 96 | * Includes & Memory related functions 97 | ***************************************/ 98 | /*! Modify the local functions below should you wish to use some other memory routines 99 | * for malloc(), free() */ 100 | #include 101 | static void* XXH_malloc(size_t s) { return malloc(s); } 102 | static void XXH_free (void* p) { free(p); } 103 | /*! and for memcpy() */ 104 | #include 105 | static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } 106 | 107 | #define XXH_STATIC_LINKING_ONLY 108 | #include "xxhash.h" 109 | 110 | 111 | /* ************************************* 112 | * Compiler Specific Options 113 | ***************************************/ 114 | #ifdef _MSC_VER /* Visual Studio */ 115 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 116 | # define FORCE_INLINE static __forceinline 117 | #else 118 | # if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 119 | # ifdef __GNUC__ 120 | # define FORCE_INLINE static inline __attribute__((always_inline)) 121 | # else 122 | # define FORCE_INLINE static inline 123 | # endif 124 | # else 125 | # define FORCE_INLINE static 126 | # endif /* __STDC_VERSION__ */ 127 | #endif 128 | 129 | 130 | /* ************************************* 131 | * Basic Types 132 | ***************************************/ 133 | #ifndef MEM_MODULE 134 | # if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 135 | # include 136 | typedef uint8_t BYTE; 137 | typedef uint16_t U16; 138 | typedef uint32_t U32; 139 | typedef int32_t S32; 140 | # else 141 | typedef unsigned char BYTE; 142 | typedef unsigned short U16; 143 | typedef unsigned int U32; 144 | typedef signed int S32; 145 | # endif 146 | #endif 147 | 148 | #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) 149 | 150 | /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ 151 | static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } 152 | 153 | #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) 154 | 155 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 156 | /* currently only defined for gcc and icc */ 157 | typedef union { U32 u32; } __attribute__((packed)) unalign; 158 | static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } 159 | 160 | #else 161 | 162 | /* portable and safe solution. Generally efficient. 163 | * see : http://stackoverflow.com/a/32095106/646947 164 | */ 165 | static U32 XXH_read32(const void* memPtr) 166 | { 167 | U32 val; 168 | memcpy(&val, memPtr, sizeof(val)); 169 | return val; 170 | } 171 | 172 | #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ 173 | 174 | 175 | /* **************************************** 176 | * Compiler-specific Functions and Macros 177 | ******************************************/ 178 | #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 179 | 180 | /* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ 181 | #if defined(_MSC_VER) 182 | # define XXH_rotl32(x,r) _rotl(x,r) 183 | # define XXH_rotl64(x,r) _rotl64(x,r) 184 | #else 185 | # define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) 186 | # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) 187 | #endif 188 | 189 | #if defined(_MSC_VER) /* Visual Studio */ 190 | # define XXH_swap32 _byteswap_ulong 191 | #elif XXH_GCC_VERSION >= 403 192 | # define XXH_swap32 __builtin_bswap32 193 | #else 194 | static U32 XXH_swap32 (U32 x) 195 | { 196 | return ((x << 24) & 0xff000000 ) | 197 | ((x << 8) & 0x00ff0000 ) | 198 | ((x >> 8) & 0x0000ff00 ) | 199 | ((x >> 24) & 0x000000ff ); 200 | } 201 | #endif 202 | 203 | 204 | /* ************************************* 205 | * Architecture Macros 206 | ***************************************/ 207 | typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; 208 | 209 | /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ 210 | #ifndef XXH_CPU_LITTLE_ENDIAN 211 | static const int g_one = 1; 212 | # define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) 213 | #endif 214 | 215 | 216 | /* *************************** 217 | * Memory reads 218 | *****************************/ 219 | typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; 220 | 221 | FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) 222 | { 223 | if (align==XXH_unaligned) 224 | return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); 225 | else 226 | return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); 227 | } 228 | 229 | FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) 230 | { 231 | return XXH_readLE32_align(ptr, endian, XXH_unaligned); 232 | } 233 | 234 | static U32 XXH_readBE32(const void* ptr) 235 | { 236 | return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); 237 | } 238 | 239 | 240 | /* ************************************* 241 | * Macros 242 | ***************************************/ 243 | #define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ 244 | XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } 245 | 246 | 247 | /* ******************************************************************* 248 | * 32-bits hash functions 249 | *********************************************************************/ 250 | static const U32 PRIME32_1 = 2654435761U; 251 | static const U32 PRIME32_2 = 2246822519U; 252 | static const U32 PRIME32_3 = 3266489917U; 253 | static const U32 PRIME32_4 = 668265263U; 254 | static const U32 PRIME32_5 = 374761393U; 255 | 256 | static U32 XXH32_round(U32 seed, U32 input) 257 | { 258 | seed += input * PRIME32_2; 259 | seed = XXH_rotl32(seed, 13); 260 | seed *= PRIME32_1; 261 | return seed; 262 | } 263 | 264 | FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) 265 | { 266 | const BYTE* p = (const BYTE*)input; 267 | const BYTE* bEnd = p + len; 268 | U32 h32; 269 | #define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) 270 | 271 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 272 | if (p==NULL) { 273 | len=0; 274 | bEnd=p=(const BYTE*)(size_t)16; 275 | } 276 | #endif 277 | 278 | if (len>=16) { 279 | const BYTE* const limit = bEnd - 16; 280 | U32 v1 = seed + PRIME32_1 + PRIME32_2; 281 | U32 v2 = seed + PRIME32_2; 282 | U32 v3 = seed + 0; 283 | U32 v4 = seed - PRIME32_1; 284 | 285 | do { 286 | v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; 287 | v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; 288 | v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; 289 | v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; 290 | } while (p<=limit); 291 | 292 | h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); 293 | } else { 294 | h32 = seed + PRIME32_5; 295 | } 296 | 297 | h32 += (U32) len; 298 | 299 | while (p+4<=bEnd) { 300 | h32 += XXH_get32bits(p) * PRIME32_3; 301 | h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; 302 | p+=4; 303 | } 304 | 305 | while (p> 15; 312 | h32 *= PRIME32_2; 313 | h32 ^= h32 >> 13; 314 | h32 *= PRIME32_3; 315 | h32 ^= h32 >> 16; 316 | 317 | return h32; 318 | } 319 | 320 | 321 | XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) 322 | { 323 | #if 0 324 | /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 325 | XXH32_state_t state; 326 | XXH32_reset(&state, seed); 327 | XXH32_update(&state, input, len); 328 | return XXH32_digest(&state); 329 | #else 330 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 331 | 332 | if (XXH_FORCE_ALIGN_CHECK) { 333 | if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ 334 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 335 | return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); 336 | else 337 | return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); 338 | } } 339 | 340 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 341 | return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); 342 | else 343 | return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); 344 | #endif 345 | } 346 | 347 | 348 | 349 | /*====== Hash streaming ======*/ 350 | 351 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) 352 | { 353 | return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); 354 | } 355 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) 356 | { 357 | XXH_free(statePtr); 358 | return XXH_OK; 359 | } 360 | 361 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) 362 | { 363 | memcpy(dstState, srcState, sizeof(*dstState)); 364 | } 365 | 366 | XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) 367 | { 368 | XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ 369 | memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ 370 | state.v1 = seed + PRIME32_1 + PRIME32_2; 371 | state.v2 = seed + PRIME32_2; 372 | state.v3 = seed + 0; 373 | state.v4 = seed - PRIME32_1; 374 | memcpy(statePtr, &state, sizeof(state)); 375 | return XXH_OK; 376 | } 377 | 378 | 379 | FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) 380 | { 381 | const BYTE* p = (const BYTE*)input; 382 | const BYTE* const bEnd = p + len; 383 | 384 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 385 | if (input==NULL) return XXH_ERROR; 386 | #endif 387 | 388 | state->total_len_32 += (unsigned)len; 389 | state->large_len |= (len>=16) | (state->total_len_32>=16); 390 | 391 | if (state->memsize + len < 16) { /* fill in tmp buffer */ 392 | XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); 393 | state->memsize += (unsigned)len; 394 | return XXH_OK; 395 | } 396 | 397 | if (state->memsize) { /* some data left from previous update */ 398 | XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); 399 | { const U32* p32 = state->mem32; 400 | state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; 401 | state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; 402 | state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; 403 | state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; 404 | } 405 | p += 16-state->memsize; 406 | state->memsize = 0; 407 | } 408 | 409 | if (p <= bEnd-16) { 410 | const BYTE* const limit = bEnd - 16; 411 | U32 v1 = state->v1; 412 | U32 v2 = state->v2; 413 | U32 v3 = state->v3; 414 | U32 v4 = state->v4; 415 | 416 | do { 417 | v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; 418 | v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; 419 | v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; 420 | v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; 421 | } while (p<=limit); 422 | 423 | state->v1 = v1; 424 | state->v2 = v2; 425 | state->v3 = v3; 426 | state->v4 = v4; 427 | } 428 | 429 | if (p < bEnd) { 430 | XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); 431 | state->memsize = (unsigned)(bEnd-p); 432 | } 433 | 434 | return XXH_OK; 435 | } 436 | 437 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) 438 | { 439 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 440 | 441 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 442 | return XXH32_update_endian(state_in, input, len, XXH_littleEndian); 443 | else 444 | return XXH32_update_endian(state_in, input, len, XXH_bigEndian); 445 | } 446 | 447 | 448 | 449 | FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) 450 | { 451 | const BYTE * p = (const BYTE*)state->mem32; 452 | const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; 453 | U32 h32; 454 | 455 | if (state->large_len) { 456 | h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); 457 | } else { 458 | h32 = state->v3 /* == seed */ + PRIME32_5; 459 | } 460 | 461 | h32 += state->total_len_32; 462 | 463 | while (p+4<=bEnd) { 464 | h32 += XXH_readLE32(p, endian) * PRIME32_3; 465 | h32 = XXH_rotl32(h32, 17) * PRIME32_4; 466 | p+=4; 467 | } 468 | 469 | while (p> 15; 476 | h32 *= PRIME32_2; 477 | h32 ^= h32 >> 13; 478 | h32 *= PRIME32_3; 479 | h32 ^= h32 >> 16; 480 | 481 | return h32; 482 | } 483 | 484 | 485 | XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) 486 | { 487 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 488 | 489 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 490 | return XXH32_digest_endian(state_in, XXH_littleEndian); 491 | else 492 | return XXH32_digest_endian(state_in, XXH_bigEndian); 493 | } 494 | 495 | 496 | /*====== Canonical representation ======*/ 497 | 498 | /*! Default XXH result types are basic unsigned 32 and 64 bits. 499 | * The canonical representation follows human-readable write convention, aka big-endian (large digits first). 500 | * These functions allow transformation of hash result into and from its canonical format. 501 | * This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. 502 | */ 503 | 504 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) 505 | { 506 | XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); 507 | if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); 508 | memcpy(dst, &hash, sizeof(*dst)); 509 | } 510 | 511 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) 512 | { 513 | return XXH_readBE32(src); 514 | } 515 | 516 | 517 | #ifndef XXH_NO_LONG_LONG 518 | 519 | /* ******************************************************************* 520 | * 64-bits hash functions 521 | *********************************************************************/ 522 | 523 | /*====== Memory access ======*/ 524 | 525 | #ifndef MEM_MODULE 526 | # define MEM_MODULE 527 | # if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 528 | # include 529 | typedef uint64_t U64; 530 | # else 531 | typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ 532 | # endif 533 | #endif 534 | 535 | 536 | #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) 537 | 538 | /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ 539 | static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } 540 | 541 | #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) 542 | 543 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 544 | /* currently only defined for gcc and icc */ 545 | typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign64; 546 | static U64 XXH_read64(const void* ptr) { return ((const unalign64*)ptr)->u64; } 547 | 548 | #else 549 | 550 | /* portable and safe solution. Generally efficient. 551 | * see : http://stackoverflow.com/a/32095106/646947 552 | */ 553 | 554 | static U64 XXH_read64(const void* memPtr) 555 | { 556 | U64 val; 557 | memcpy(&val, memPtr, sizeof(val)); 558 | return val; 559 | } 560 | 561 | #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ 562 | 563 | #if defined(_MSC_VER) /* Visual Studio */ 564 | # define XXH_swap64 _byteswap_uint64 565 | #elif XXH_GCC_VERSION >= 403 566 | # define XXH_swap64 __builtin_bswap64 567 | #else 568 | static U64 XXH_swap64 (U64 x) 569 | { 570 | return ((x << 56) & 0xff00000000000000ULL) | 571 | ((x << 40) & 0x00ff000000000000ULL) | 572 | ((x << 24) & 0x0000ff0000000000ULL) | 573 | ((x << 8) & 0x000000ff00000000ULL) | 574 | ((x >> 8) & 0x00000000ff000000ULL) | 575 | ((x >> 24) & 0x0000000000ff0000ULL) | 576 | ((x >> 40) & 0x000000000000ff00ULL) | 577 | ((x >> 56) & 0x00000000000000ffULL); 578 | } 579 | #endif 580 | 581 | FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) 582 | { 583 | if (align==XXH_unaligned) 584 | return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); 585 | else 586 | return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); 587 | } 588 | 589 | FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) 590 | { 591 | return XXH_readLE64_align(ptr, endian, XXH_unaligned); 592 | } 593 | 594 | static U64 XXH_readBE64(const void* ptr) 595 | { 596 | return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); 597 | } 598 | 599 | 600 | /*====== xxh64 ======*/ 601 | 602 | static const U64 PRIME64_1 = 11400714785074694791ULL; 603 | static const U64 PRIME64_2 = 14029467366897019727ULL; 604 | static const U64 PRIME64_3 = 1609587929392839161ULL; 605 | static const U64 PRIME64_4 = 9650029242287828579ULL; 606 | static const U64 PRIME64_5 = 2870177450012600261ULL; 607 | 608 | static U64 XXH64_round(U64 acc, U64 input) 609 | { 610 | acc += input * PRIME64_2; 611 | acc = XXH_rotl64(acc, 31); 612 | acc *= PRIME64_1; 613 | return acc; 614 | } 615 | 616 | static U64 XXH64_mergeRound(U64 acc, U64 val) 617 | { 618 | val = XXH64_round(0, val); 619 | acc ^= val; 620 | acc = acc * PRIME64_1 + PRIME64_4; 621 | return acc; 622 | } 623 | 624 | FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) 625 | { 626 | const BYTE* p = (const BYTE*)input; 627 | const BYTE* const bEnd = p + len; 628 | U64 h64; 629 | #define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) 630 | 631 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 632 | if (p==NULL) { 633 | len=0; 634 | bEnd=p=(const BYTE*)(size_t)32; 635 | } 636 | #endif 637 | 638 | if (len>=32) { 639 | const BYTE* const limit = bEnd - 32; 640 | U64 v1 = seed + PRIME64_1 + PRIME64_2; 641 | U64 v2 = seed + PRIME64_2; 642 | U64 v3 = seed + 0; 643 | U64 v4 = seed - PRIME64_1; 644 | 645 | do { 646 | v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; 647 | v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; 648 | v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; 649 | v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; 650 | } while (p<=limit); 651 | 652 | h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); 653 | h64 = XXH64_mergeRound(h64, v1); 654 | h64 = XXH64_mergeRound(h64, v2); 655 | h64 = XXH64_mergeRound(h64, v3); 656 | h64 = XXH64_mergeRound(h64, v4); 657 | 658 | } else { 659 | h64 = seed + PRIME64_5; 660 | } 661 | 662 | h64 += (U64) len; 663 | 664 | while (p+8<=bEnd) { 665 | U64 const k1 = XXH64_round(0, XXH_get64bits(p)); 666 | h64 ^= k1; 667 | h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; 668 | p+=8; 669 | } 670 | 671 | if (p+4<=bEnd) { 672 | h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; 673 | h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; 674 | p+=4; 675 | } 676 | 677 | while (p> 33; 684 | h64 *= PRIME64_2; 685 | h64 ^= h64 >> 29; 686 | h64 *= PRIME64_3; 687 | h64 ^= h64 >> 32; 688 | 689 | return h64; 690 | } 691 | 692 | 693 | XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) 694 | { 695 | #if 0 696 | /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ 697 | XXH64_state_t state; 698 | XXH64_reset(&state, seed); 699 | XXH64_update(&state, input, len); 700 | return XXH64_digest(&state); 701 | #else 702 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 703 | 704 | if (XXH_FORCE_ALIGN_CHECK) { 705 | if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ 706 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 707 | return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); 708 | else 709 | return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); 710 | } } 711 | 712 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 713 | return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); 714 | else 715 | return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); 716 | #endif 717 | } 718 | 719 | /*====== Hash Streaming ======*/ 720 | 721 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) 722 | { 723 | return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); 724 | } 725 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) 726 | { 727 | XXH_free(statePtr); 728 | return XXH_OK; 729 | } 730 | 731 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dstState, const XXH64_state_t* srcState) 732 | { 733 | memcpy(dstState, srcState, sizeof(*dstState)); 734 | } 735 | 736 | XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) 737 | { 738 | XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ 739 | memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ 740 | state.v1 = seed + PRIME64_1 + PRIME64_2; 741 | state.v2 = seed + PRIME64_2; 742 | state.v3 = seed + 0; 743 | state.v4 = seed - PRIME64_1; 744 | memcpy(statePtr, &state, sizeof(state)); 745 | return XXH_OK; 746 | } 747 | 748 | FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) 749 | { 750 | const BYTE* p = (const BYTE*)input; 751 | const BYTE* const bEnd = p + len; 752 | 753 | #ifdef XXH_ACCEPT_NULL_INPUT_POINTER 754 | if (input==NULL) return XXH_ERROR; 755 | #endif 756 | 757 | state->total_len += len; 758 | 759 | if (state->memsize + len < 32) { /* fill in tmp buffer */ 760 | XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); 761 | state->memsize += (U32)len; 762 | return XXH_OK; 763 | } 764 | 765 | if (state->memsize) { /* tmp buffer is full */ 766 | XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); 767 | state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); 768 | state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); 769 | state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); 770 | state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); 771 | p += 32-state->memsize; 772 | state->memsize = 0; 773 | } 774 | 775 | if (p+32 <= bEnd) { 776 | const BYTE* const limit = bEnd - 32; 777 | U64 v1 = state->v1; 778 | U64 v2 = state->v2; 779 | U64 v3 = state->v3; 780 | U64 v4 = state->v4; 781 | 782 | do { 783 | v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; 784 | v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; 785 | v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; 786 | v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; 787 | } while (p<=limit); 788 | 789 | state->v1 = v1; 790 | state->v2 = v2; 791 | state->v3 = v3; 792 | state->v4 = v4; 793 | } 794 | 795 | if (p < bEnd) { 796 | XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); 797 | state->memsize = (unsigned)(bEnd-p); 798 | } 799 | 800 | return XXH_OK; 801 | } 802 | 803 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) 804 | { 805 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 806 | 807 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 808 | return XXH64_update_endian(state_in, input, len, XXH_littleEndian); 809 | else 810 | return XXH64_update_endian(state_in, input, len, XXH_bigEndian); 811 | } 812 | 813 | FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) 814 | { 815 | const BYTE * p = (const BYTE*)state->mem64; 816 | const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; 817 | U64 h64; 818 | 819 | if (state->total_len >= 32) { 820 | U64 const v1 = state->v1; 821 | U64 const v2 = state->v2; 822 | U64 const v3 = state->v3; 823 | U64 const v4 = state->v4; 824 | 825 | h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); 826 | h64 = XXH64_mergeRound(h64, v1); 827 | h64 = XXH64_mergeRound(h64, v2); 828 | h64 = XXH64_mergeRound(h64, v3); 829 | h64 = XXH64_mergeRound(h64, v4); 830 | } else { 831 | h64 = state->v3 + PRIME64_5; 832 | } 833 | 834 | h64 += (U64) state->total_len; 835 | 836 | while (p+8<=bEnd) { 837 | U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); 838 | h64 ^= k1; 839 | h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; 840 | p+=8; 841 | } 842 | 843 | if (p+4<=bEnd) { 844 | h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; 845 | h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; 846 | p+=4; 847 | } 848 | 849 | while (p> 33; 856 | h64 *= PRIME64_2; 857 | h64 ^= h64 >> 29; 858 | h64 *= PRIME64_3; 859 | h64 ^= h64 >> 32; 860 | 861 | return h64; 862 | } 863 | 864 | XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) 865 | { 866 | XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; 867 | 868 | if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) 869 | return XXH64_digest_endian(state_in, XXH_littleEndian); 870 | else 871 | return XXH64_digest_endian(state_in, XXH_bigEndian); 872 | } 873 | 874 | 875 | /*====== Canonical representation ======*/ 876 | 877 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) 878 | { 879 | XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); 880 | if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); 881 | memcpy(dst, &hash, sizeof(*dst)); 882 | } 883 | 884 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) 885 | { 886 | return XXH_readBE64(src); 887 | } 888 | 889 | #endif /* XXH_NO_LONG_LONG */ 890 | --------------------------------------------------------------------------------