├── .gitignore ├── COPYING ├── GPL-3 ├── README ├── build.gradle ├── build.xml ├── buildWithClean.gradle ├── docs ├── Makefile └── state-machine.dot ├── jcpp ├── include │ ├── lzo │ │ ├── lzo1x.h │ │ ├── lzo1y.h │ │ ├── lzo1z.h │ │ └── lzoconf.h │ └── lzojava.h ├── org │ └── anarres │ │ └── lzo │ │ ├── LzoCompressor1x_1.jcpp │ │ ├── LzoCompressor1y_1.jcpp │ │ ├── LzoDecompressor1x.jcpp │ │ ├── LzoDecompressor1x_safe.jcpp │ │ ├── LzoDecompressor1y.jcpp │ │ ├── LzoDecompressor1y_safe.jcpp │ │ ├── LzoDecompressor1z.jcpp │ │ ├── LzoDecompressor1z_safe.jcpp │ │ └── LzoVersion.jcpp └── src │ ├── compr1b.h │ ├── compr1c.h │ ├── config1.h │ ├── config1a.h │ ├── config1b.h │ ├── config1c.h │ ├── config1f.h │ ├── config1x.h │ ├── config1y.h │ ├── config1z.h │ ├── config2a.h │ ├── lzo1_cm.ch │ ├── lzo1_d.ch │ ├── lzo1a_cm.ch │ ├── lzo1a_cr.ch │ ├── lzo1a_de.h │ ├── lzo1b_c.ch │ ├── lzo1b_cc.h │ ├── lzo1b_cm.ch │ ├── lzo1b_cr.ch │ ├── lzo1b_d.ch │ ├── lzo1b_de.h │ ├── lzo1b_r.ch │ ├── lzo1b_sm.ch │ ├── lzo1b_tm.ch │ ├── lzo1c_cc.h │ ├── lzo1f_d.ch │ ├── lzo1x_9x.c │ ├── lzo1x_c.ch │ ├── lzo1x_d.ch │ ├── lzo1x_oo.ch │ ├── lzo2a_d.ch │ ├── lzo_conf.h │ ├── lzo_dict.h │ ├── lzo_dll.ch │ ├── lzo_func.ch │ ├── lzo_mchw.ch │ ├── lzo_ptr.h │ ├── lzo_swd.ch │ ├── miniacc.h │ ├── stats1a.h │ ├── stats1b.h │ └── stats1c.h ├── lib ├── ant │ └── ant-contrib.jar ├── jalopy │ ├── aelfred-1.2.jar │ ├── jalopy-1.0b11.jar │ ├── jalopy-ant-0.6.2.jar │ ├── jaxp-1.2.jar │ ├── jdom-1.0b8.jar │ ├── log4j-1.2.6.jar │ ├── oro-2.0.6.jar │ └── sax-2.0.1.jar ├── jcpp │ ├── anarres-cpp.jar │ └── gnu.getopt.jar ├── junit │ ├── hamcrest-core-1.1.jar │ └── junit4-4.8.2.jar └── runtime │ ├── commons-io-1.4-src.jar │ ├── commons-io-1.4.jar │ ├── commons-lang-2.5-src.jar │ ├── commons-lang-2.5.jar │ ├── commons-logging-1.1.1.jar │ ├── hadoop-core-0.21.0-dev.jar │ └── hadoop-mapred-0.21.0-dev.jar ├── license-header.txt ├── nbproject ├── build-impl.xml ├── genfiles.properties ├── private │ ├── config.properties │ ├── private.properties │ └── private.xml ├── project.properties └── project.xml ├── src ├── com │ └── hadoop │ │ ├── compression │ │ └── lzo │ │ │ ├── DistributedLzoIndexer.java │ │ │ ├── LzoCodec.java │ │ │ ├── LzoIndex.java │ │ │ ├── LzoIndexer.java │ │ │ └── LzopCodec.java │ │ ├── mapred │ │ ├── DeprecatedLzoLineRecordReader.java │ │ └── DeprecatedLzoTextInputFormat.java │ │ └── mapreduce │ │ ├── LzoIndexOutputFormat.java │ │ ├── LzoIndexRecordWriter.java │ │ ├── LzoLineRecordReader.java │ │ ├── LzoSplitInputFormat.java │ │ ├── LzoSplitRecordReader.java │ │ └── LzoTextInputFormat.java └── org │ ├── anarres │ └── lzo │ │ ├── AbstractLzo1Compressor.java │ │ ├── AbstractLzo2Compressor.java │ │ ├── AbstractLzoCompressor.java │ │ ├── AbstractLzoDecompressor.java │ │ ├── AbstractLzoTransformer.java │ │ ├── LzoAlgorithm.java │ │ ├── LzoCompressor.java │ │ ├── LzoConstraint.java │ │ ├── LzoDecompressor.java │ │ ├── LzoInputStream.java │ │ ├── LzoLibrary.java │ │ ├── LzoOutputStream.java │ │ ├── LzoTransformer.java │ │ ├── LzopConstants.java │ │ ├── LzopInputStream.java │ │ ├── LzopOutputStream.java │ │ ├── hadoop │ │ └── codec │ │ │ ├── LzoCodec.java │ │ │ ├── LzoCompressor.java │ │ │ ├── LzoDecompressor.java │ │ │ ├── LzopCodec.java │ │ │ ├── LzopInputStream.java │ │ │ └── LzopOutputStream.java │ │ └── lzo_uintp.java │ └── apache │ └── hadoop │ └── io │ └── compress │ ├── LzoCodec.java │ └── LzopCodec.java └── test ├── data ├── README ├── compressed.txt ├── compressed.txt.lzo.adler32 ├── compressed.txt.lzo.adler32d ├── compressed.txt.lzo.crc32 ├── compressed.txt.lzo.crc32d ├── temp.txt └── temp.txt.lzo └── org └── anarres └── lzo ├── BinaryTest.java ├── BlockCompressorStreamTest.java ├── HadoopLzopFileTest.java ├── LzoAlgorithmTest.java ├── LzoStreamTest.java ├── LzopFileTest.java ├── LzopStreamTest.java ├── PerformanceTest.java ├── RandomInputStream.java └── ZlibCompressorTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | *.out 2 | *.lzo 3 | docs/*.png 4 | build 5 | dist 6 | 7 | test/org/anarres/lzo/CustomerDataTest.java 8 | test/data/*.lzo 9 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | This package is licensed under the GPL, version 3. A copy of that 2 | license is included in the file GPL-3. 3 | 4 | In addition, some files contain headers which grant additional rights, 5 | however those additional rights apply only to the files in which they 6 | are granted, and not to the library as a whole. 7 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | There is no version of LZO in pure Java. The obvious solution is to 2 | take the C source code, and feed it to the Java compiler, modifying 3 | the Java compiler as necessary to make it compile. 4 | 5 | This package is an implementation of that obvious solution, for which 6 | I can only apologise to the world. 7 | 8 | It turns out, however, that the compression performance on a single 9 | 2.4GHz laptop CPU is in excess of 500Mb/sec, and decompression runs 10 | at 815Mb/sec, which seems to be more than adequate. Run 11 | PerformanceTest on an appropriate file to reproduce these figures. 12 | 13 | 14 | Notes on BlockCompressionStream, as of Hadoop 0.21.x: 15 | 16 | * If you write 1 byte, then a large block, BlockCompressorStream will 17 | flush the single-byte block before compressing the large block. This 18 | is inefficient. 19 | 20 | * If you write a large block to a fresh stream, BlockCompressorStream 21 | will flush existing data, which will write a zero uncompressed 22 | length to the file, but follow it with no blocks, thus breaking the 23 | ulen-clen-data format. This is wrong. There is no contract for the 24 | finished() method to avoid this, since it must return false at the 25 | top of write(), then must (with no other mutator calls) return true 26 | in BlockCompressorStream.finish() in order to avoid the empty block; 27 | having returned true there, compress() must be able to return a 28 | nonempty block, even though we have no data. This is wrong. 29 | 30 | * Large blocks are written (ulen (clen data)*) not (ulen clen data)* 31 | due to the loop in compress(). This is not the same as the format for 32 | lzop, thus a data file written using LzopCodec cannot be read by lzop. 33 | See lzop-1.03/src/p_lzo.c method lzo_compress, which contains a 34 | single very simple loop, which is how Hadoop's BlockCompressorStream 35 | should be written. This is both inefficient and wrong. 36 | 37 | * If the LZO compressor needs to use its holdover field (or, 38 | equivalently in other people's code, setInputFromSavedData()), 39 | then the ulen-clen-data format is broken because getBytesRead() 40 | MUST return the full number of bytes passed to setInput(), not 41 | just the number of bytes actually compressed so far; then if there 42 | is holdover data, there is nowhere for it to go but into the 43 | returned data from a second call to compress(), at which point the 44 | API has forced us to break ulen-clen-data, as per lzop's file 45 | format. This is wrong, and badly designed. 46 | 47 | * The number of uncompressed bytes is written to the stream in lzop. 48 | There is therefore no excuse for a "Buffer too small" error in 49 | decompression. However, this value is NOT used to resize the 50 | decompressor's output buffer, and so the error occurs. One cannot, 51 | as a rule, know the size of output buffer required to decompress a 52 | given file, so Hadoop must be configured by trial and error. This 53 | is badly designed, and harder to use. 54 | 55 | 56 | Shevek 57 | 58 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'maven' 3 | 4 | apply plugin: 'idea' 5 | 6 | defaultTasks 'deploy' 7 | 8 | // Including Ant build from a separate Gradle file to avoid clashes with 9 | // existing targets e.g. clean, etc. from java plugin 10 | task importTasksRunInOrder(type: GradleBuild) { GradleBuild antBuild -> 11 | antBuild.buildFile ='buildWithClean.gradle' 12 | antBuild.tasks = ['jar'] 13 | } 14 | 15 | version = '1.0-SNAPSHOT' 16 | group = 'org.annares.lzo' 17 | 18 | repositories { 19 | mavenLocal() 20 | } 21 | 22 | task rename (type: Copy) { 23 | from 'dist' 24 | into project.libsDir 25 | rename { String fileName -> 26 | fileName.replaceAll('lzo', project.name + "-" + version) 27 | } 28 | rename '(lzo)(.jar)', '$2' 29 | } 30 | 31 | jar.onlyIf {"Ant build provided jar will be moved in place".length() == 0} 32 | 33 | task deploy (dependsOn: [importTasksRunInOrder, rename, uploadArchives]) << { 34 | println 'Buidling and deploying' 35 | } 36 | 37 | uploadArchives { 38 | repositories { 39 | mavenDeployer { 40 | repository(url: "http://archiva:8290/archiva/repository/snapshots/") { 41 | // Make sure to have correctly configured user/password in 42 | // ~/.m2/settings.xml to access the repository above 43 | } 44 | pom.project { 45 | inceptionYear '2011' 46 | licenses { 47 | license { 48 | name 'GNU GENERAL PUBLIC LICENSE Version 3' 49 | distribution 'repo' 50 | } 51 | } 52 | } 53 | } 54 | } 55 | } 56 | 57 | -------------------------------------------------------------------------------- /buildWithClean.gradle: -------------------------------------------------------------------------------- 1 | ant.importBuild('build.xml') 2 | 3 | jar << { 4 | println "Ant jar task is complete." 5 | println "File should be available from " + ant.properties['dist.jar'] 6 | } 7 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | TARGETS= 2 | 3 | all : ${TARGETS} 4 | 5 | .PRECIOUS : %.png 6 | 7 | %.png : %.dot 8 | dot -Tpng -o $*.png $*.dot 9 | 10 | %.map : %.dot 11 | dot -Tcmapx -o $*.map $*.dot 12 | 13 | %.eog : %.png 14 | eog $*.png 15 | 16 | clean: 17 | rm -f ${TARGETS} 18 | -------------------------------------------------------------------------------- /docs/state-machine.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | node [shape=box]; 3 | init -> a_literal_run [label="0-15"]; 4 | init -> match [label="16,17"]; 5 | init -> match_next [label="18,19,20"]; 6 | init -> init_copy [label="21-255"]; 7 | 8 | init_copy [label="Initial copy\n*ip - 17 bytes follow"]; 9 | init_copy -> first_literal_run; 10 | 11 | a_literal_run [label="A literal run\nlength=3 + RLE(15)"]; 12 | 13 | a_literal_run -> first_literal_run; 14 | 15 | first_literal_run -> match [label="16-255"]; 16 | 17 | first_literal_run -> first_literal_run_do [label="0-15 (4 bits)"]; 18 | 19 | first_literal_run_do [label="First literal run\n0000ppxx qqqqqqqq\nop - M2_MAX_OFFSET - 1 - qqqqqqqqpp\n3 bytes\n"]; 20 | 21 | first_literal_run_do -> match_done; 22 | 23 | match -> match_m2 [label=">= 64 / x1......"]; 24 | match -> match_m3 [label=">= 32 / 001....."]; 25 | match -> match_m4 [label=">= 16 / 0001...."]; 26 | match -> match_m1 [label="0000...."]; 27 | 28 | match_m2 [label="M2 match\nlllpppxx qqqqqqqq\nm_pos = op - 1 - qqqqqqqqppp\nt(len) = lll - 1"]; 29 | match_m2 -> copy_match; 30 | 31 | match_m3 [label="M3 match\n001lllll RLE(31) ppppppxx qqqqqqqq\nm_pos = op - 1 - qqqqqqqqpppppp\nt(len) = RLE(31)"]; 32 | match_m3 -> pre_copy_match; 33 | 34 | match_m4 [label="M4 match\n0001plll RLE(7) qqqqqqxx rrrrrrrr\nm_pos = op - 1prrrrrrrrqqqqqq\nt(len) = RLE(7)"]; 35 | match_m4 -> pre_copy_match; 36 | 37 | match_m1 [label="M1 match\n0000ppxx qqqqqqqq\nm_pos = op - 1 - qqqqqqqqpp\nt(len) = 2"]; 38 | match_m1 -> match_done; 39 | 40 | pre_copy_match -> copy_match; 41 | 42 | copy_match [label="\N\ncopy t + 2 bytes from m_pos"]; 43 | copy_match -> match_done; 44 | 45 | match_done [label="\N\nt(len) = ip[-2] & 3"]; 46 | match_done -> eof; 47 | match_done -> match_next; 48 | match_done -> init [label="xx = 0",constraint=false]; 49 | 50 | match_next [label="\N\nt = ______xx @ ip[-2]\ncopy xx from ip"]; 51 | match_next -> match [constraint=false]; 52 | 53 | eof; 54 | 55 | } 56 | -------------------------------------------------------------------------------- /jcpp/include/lzo/lzo1x.h: -------------------------------------------------------------------------------- 1 | /* Memory required for the wrkmem parameter. 2 | * When the required size is 0, you can also pass a NULL pointer. 3 | */ 4 | 5 | #define LZO1X_MEM_COMPRESS LZO1X_1_MEM_COMPRESS 6 | #define LZO1X_MEM_DECOMPRESS (0) 7 | #define LZO1X_MEM_OPTIMIZE (0) 8 | 9 | -------------------------------------------------------------------------------- /jcpp/include/lzo/lzo1y.h: -------------------------------------------------------------------------------- 1 | /* Memory required for the wrkmem parameter. 2 | * When the required size is 0, you can also pass a NULL pointer. 3 | */ 4 | 5 | #define LZO1Y_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t)) 6 | #define LZO1Y_MEM_DECOMPRESS (0) 7 | #define LZO1Y_MEM_OPTIMIZE (0) 8 | 9 | -------------------------------------------------------------------------------- /jcpp/include/lzo/lzo1z.h: -------------------------------------------------------------------------------- 1 | /* Memory required for the wrkmem parameter. 2 | * When the required size is 0, you can also pass a NULL pointer. 3 | */ 4 | 5 | #define LZO1Z_MEM_COMPRESS ((lzo_uint32) (16384L * lzo_sizeof_dict_t)) 6 | #define LZO1Z_MEM_DECOMPRESS (0) 7 | #define LZO1Z_MEM_OPTIMIZE (0) 8 | 9 | -------------------------------------------------------------------------------- /jcpp/include/lzo/lzoconf.h: -------------------------------------------------------------------------------- 1 | #ifndef __LZOCONF_H_INCLUDED 2 | #define __LZOCONF_H_INCLUDED 1 3 | 4 | #define LZO_VERSION 0x2050 5 | #define LZO_VERSION_STRING "2.05-Java" 6 | #define LZO_VERSION_DATE "Jul 8 2011" 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /jcpp/include/lzojava.h: -------------------------------------------------------------------------------- 1 | #define LZO_CFG_FREESTANDING 1 2 | #define LZO_PUBLIC(x) public static x 3 | #define LZO_UNUSED(x) 4 | #define LZO_DEFINE_UNINITIALIZED_VAR(t,n,v) t n = v 5 | 6 | #define const 7 | #define register 8 | #define unsigned 9 | #define char byte 10 | #define lzo_bytep byte[] 11 | #define lzo_uint int 12 | #define lzo_uint32 long 13 | #define lzo_xint long // ? 14 | #define lzo_voidp Object 15 | 16 | #define GOTO_INIT() int state = init; // Java-goto 17 | #define GOTO_BEGIN(label) label: for (;;) { switch (state) { case init: // Java-goto 18 | #define GOTO_END() break; default: throw new IllegalStateException("Illegal state " + state); } break; } 19 | #define GOTO_END_UNREACHABLE() default: throw new IllegalStateException("Illegal state " + state); } } 20 | 21 | #define __lzo_unlikely(x) (x) 22 | 23 | #define LZO_CPP_STRINGIZE(x) #x 24 | #define LZO_CPP_MACRO_EXPAND(x) LZO_CPP_STRINGIZE(x) 25 | #define LZO_CPP_CONCAT2(a,b) a ## b 26 | #define LZO_CPP_CONCAT3(a,b,c) a ## b ## c 27 | #define LZO_CPP_CONCAT4(a,b,c,d) a ## b ## c ## d 28 | #define LZO_CPP_CONCAT5(a,b,c,d,e) a ## b ## c ## d ## e 29 | #define LZO_CPP_ECONCAT2(a,b) LZO_CPP_CONCAT2(a,b) 30 | #define LZO_CPP_ECONCAT3(a,b,c) LZO_CPP_CONCAT3(a,b,c) 31 | #define LZO_CPP_ECONCAT4(a,b,c,d) LZO_CPP_CONCAT4(a,b,c,d) 32 | #define LZO_CPP_ECONCAT5(a,b,c,d,e) LZO_CPP_CONCAT5(a,b,c,d,e) 33 | 34 | // In Java, all of these are array indices. 35 | #define PTR(x) (x) 36 | #define PTR_LT(a,b) (PTR(a) < PTR(b)) 37 | #define PTR_GE(a,b) (PTR(a) >= PTR(b)) 38 | #define PTR_DIFF(a, b) (PTR(a) - PTR(b)) 39 | 40 | #define pd(a,b) PTR_DIFF(a,b) 41 | 42 | #if 1 //Java && defined(UA_GET32) 43 | #undef LZO_DICT_USE_PTR 44 | #define LZO_DICT_USE_PTR 0 45 | #endif 46 | 47 | // for lzo1y.h and lzo1z.h 48 | #define lzo_sizeof_dict_t 4 49 | 50 | // Unfortunately clobbered by config1x.h etc 51 | // #define LZO_DETERMINISTIC (1) 52 | 53 | #define U(x) ((x) & 0xff) 54 | #define PRINT(x) // System.out.println(x) 55 | 56 | // NOT a macro because liblzo2 assumes that if UA_GET32 is a macro, 57 | // then it is faster than byte-array accesses, which it is not - 58 | // or, if it is, hotspot will deal with it. 59 | private static int UA_GET32(byte[] in, int in_ptr) { 60 | return (U(in[in_ptr]) << 24) | (U(in[in_ptr + 1]) << 16) | (U(in[in_ptr + 2]) << 8) | U(in[in_ptr + 3]); 61 | } 62 | 63 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoCompressor1x_1.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoCompressor1x_1 extends AbstractLzo1Compressor { 6 | 7 | public LzoCompressor1x_1() { 8 | super(LzoAlgorithm.LZO1X); 9 | } 10 | 11 | #include "lzojava.h" 12 | 13 | #define LZO_NEED_DICT_H 1 14 | #ifndef D_BITS 15 | #define D_BITS 14 16 | #endif 17 | #define D_INDEX1(d,p) d = DM(DMUL(0x21,DX3(p,5,5,6)) >> 5) 18 | #define D_INDEX2(d,p) d = (d & (D_MASK & 0x7ff)) ^ (D_HIGH | 0x1f) 19 | #if 1 20 | #define DINDEX(dv,p) DM(((DMUL(0x1824429d,dv)) >> (32-D_BITS))) 21 | #else 22 | #define DINDEX(dv,p) DM((dv) + ((dv) >> (32-D_BITS))) 23 | #endif 24 | #include "config1x.h" 25 | #define LZO_DETERMINISTIC (1) 26 | 27 | #ifndef DO_COMPRESS 28 | #define DO_COMPRESS compress 29 | #endif 30 | 31 | #include "lzo1x_c.ch" 32 | 33 | private final lzo_dict_p dictionary = new lzo_dict_t[1 << D_BITS]; 34 | 35 | public int compress(byte[] in, int in_base, int in_len, 36 | byte[] out, int out_base, lzo_uintp out_len) { 37 | return compress(in, in_base, in_len, 38 | out, out_base, out_len, 39 | dictionary); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoCompressor1y_1.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoCompressor1y_1 extends AbstractLzo1Compressor { 6 | 7 | public LzoCompressor1y_1() { 8 | super(LzoAlgorithm.LZO1Y); 9 | } 10 | 11 | #include "lzojava.h" 12 | 13 | #define LZO_NEED_DICT_H 1 14 | #ifndef D_BITS 15 | #define D_BITS 14 16 | #endif 17 | #define D_INDEX1(d,p) d = DX3(p,5,5,6); d += d >> 5; d = DM(d) 18 | #define D_INDEX2(d,p) d = (d & (D_MASK & 0x7ff)) ^ (D_HIGH | 0x1f) 19 | #if 1 20 | #define DINDEX(dv,p) DM(((DMUL(0x1824429d,dv)) >> (32-D_BITS))) 21 | #else 22 | #define DINDEX(dv,p) DM((dv) + ((dv) >> (32-D_BITS))) 23 | #endif 24 | #include "config1y.h" 25 | #define LZO_DETERMINISTIC (1) 26 | 27 | #ifndef DO_COMPRESS 28 | #define DO_COMPRESS compress 29 | #endif 30 | 31 | #include "lzo1x_c.ch" 32 | 33 | private final lzo_dict_p dictionary = new lzo_dict_t[16384]; 34 | 35 | public int compress(byte[] in, int in_base, int in_len, 36 | byte[] out, int out_base, lzo_uintp out_len) { 37 | return compress(in, in_base, in_len, 38 | out, out_base, out_len, 39 | dictionary); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1x.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1x extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1x() { 8 | super(LzoAlgorithm.LZO1X); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1x.h" 13 | 14 | #undef LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1x_safe.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1x_safe extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1x_safe() { 8 | super(LzoAlgorithm.LZO1X, LzoConstraint.SAFETY); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1x.h" 13 | 14 | #define LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1y.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1y extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1y() { 8 | super(LzoAlgorithm.LZO1Y); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1y.h" 13 | 14 | #undef LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1y_safe.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1y_safe extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1y_safe() { 8 | super(LzoAlgorithm.LZO1Y, LzoConstraint.SAFETY); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1y.h" 13 | 14 | #define LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1z.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1z extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1z() { 8 | super(LzoAlgorithm.LZO1Z); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1z.h" 13 | 14 | #undef LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoDecompressor1z_safe.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | import java.util.Arrays; 4 | 5 | public class LzoDecompressor1z_safe extends AbstractLzoDecompressor { 6 | 7 | public LzoDecompressor1z_safe() { 8 | super(LzoAlgorithm.LZO1Z, LzoConstraint.SAFETY); 9 | } 10 | 11 | #include "lzojava.h" 12 | #include "config1z.h" 13 | 14 | #define LZO_TEST_OVERRUN 15 | #define DO_DECOMPRESS decompress 16 | 17 | #include "lzo1x_d.ch" 18 | 19 | @Override 20 | public int decompress(byte[] in, int in_base, int in_len, 21 | byte[] out, int out_base, lzo_uintp out_len) { 22 | return decompress(in, in_base, in_len, 23 | out, out_base, out_len, null); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /jcpp/org/anarres/lzo/LzoVersion.jcpp: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo; 2 | 3 | #include "lzo/lzoconf.h" 4 | 5 | public class LzoVersion { 6 | public static final short LZO_LIBRARY_VERSION = LZO_VERSION; 7 | public static final String LZO_LIBRARY_VERSION_STRING = LZO_VERSION_STRING; 8 | } 9 | -------------------------------------------------------------------------------- /jcpp/src/compr1b.h: -------------------------------------------------------------------------------- 1 | /* compr1b.h -- 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | #define LZO_NEED_DICT_H 1 45 | #include "config1b.h" 46 | 47 | 48 | #if !defined(COMPRESS_ID) 49 | #define COMPRESS_ID LZO_CPP_ECONCAT2(DD_BITS,CLEVEL) 50 | #endif 51 | 52 | 53 | #include "lzo1b_c.ch" 54 | 55 | 56 | /*********************************************************************** 57 | // 58 | ************************************************************************/ 59 | 60 | #define LZO_COMPRESS \ 61 | LZO_CPP_ECONCAT3(lzo1b_,COMPRESS_ID,_compress) 62 | 63 | #define LZO_COMPRESS_FUNC \ 64 | LZO_CPP_ECONCAT3(_lzo1b_,COMPRESS_ID,_compress_func) 65 | 66 | 67 | 68 | /*********************************************************************** 69 | // 70 | ************************************************************************/ 71 | 72 | const lzo_compress_t LZO_COMPRESS_FUNC = do_compress; 73 | 74 | LZO_PUBLIC(int) 75 | LZO_COMPRESS ( const lzo_bytep in, lzo_uint in_len, 76 | lzo_bytep out, lzo_uintp out_len, 77 | lzo_voidp wrkmem ) 78 | { 79 | return _lzo1b_do_compress(in,in_len,out,out_len,wrkmem,do_compress); 80 | } 81 | 82 | /* 83 | vi:ts=4:et 84 | */ 85 | -------------------------------------------------------------------------------- /jcpp/src/compr1c.h: -------------------------------------------------------------------------------- 1 | /* compr1c.h -- 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | #define LZO_NEED_DICT_H 1 45 | #include "config1c.h" 46 | 47 | 48 | #if !defined(COMPRESS_ID) 49 | #define COMPRESS_ID LZO_CPP_ECONCAT2(DD_BITS,CLEVEL) 50 | #endif 51 | 52 | 53 | #include "lzo1b_c.ch" 54 | 55 | 56 | /*********************************************************************** 57 | // 58 | ************************************************************************/ 59 | 60 | #define LZO_COMPRESS \ 61 | LZO_CPP_ECONCAT3(lzo1c_,COMPRESS_ID,_compress) 62 | 63 | #define LZO_COMPRESS_FUNC \ 64 | LZO_CPP_ECONCAT3(_lzo1c_,COMPRESS_ID,_compress_func) 65 | 66 | 67 | 68 | /*********************************************************************** 69 | // 70 | ************************************************************************/ 71 | 72 | const lzo_compress_t LZO_COMPRESS_FUNC = do_compress; 73 | 74 | LZO_PUBLIC(int) 75 | LZO_COMPRESS ( const lzo_bytep in, lzo_uint in_len, 76 | lzo_bytep out, lzo_uintp out_len, 77 | lzo_voidp wrkmem ) 78 | { 79 | return _lzo1c_do_compress(in,in_len,out,out_len,wrkmem,do_compress); 80 | } 81 | 82 | /* 83 | vi:ts=4:et 84 | */ 85 | -------------------------------------------------------------------------------- /jcpp/src/config1.h: -------------------------------------------------------------------------------- 1 | /* config1.h -- configuration for the LZO1 algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | #ifndef __LZO_CONFIG1_H 50 | #define __LZO_CONFIG1_H 1 51 | 52 | #include "lzo_conf.h" 53 | #include "lzo/lzo1.h" 54 | 55 | #define LZO_NO_R1 1 56 | #include "config1a.h" 57 | 58 | #endif /* already included */ 59 | 60 | 61 | /* 62 | vi:ts=4:et 63 | */ 64 | 65 | -------------------------------------------------------------------------------- /jcpp/src/config1b.h: -------------------------------------------------------------------------------- 1 | /* config1b.h -- configuration for the LZO1B algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1B_H 51 | #define __LZO_CONFIG1B_H 1 52 | 53 | #include "lzo_conf.h" 54 | #include "lzo/lzo1b.h" 55 | 56 | 57 | /*********************************************************************** 58 | // algorithm configuration 59 | ************************************************************************/ 60 | 61 | /* run bits (4 - 5) - the compressor and the decompressor 62 | * must use the same value. */ 63 | #if !defined(RBITS) 64 | # define RBITS 5 65 | #endif 66 | 67 | /* dictionary depth (0 - 6) - this only affects the compressor. 68 | * 0 is fastest, 6 is best compression ratio */ 69 | #if !defined(DDBITS) 70 | # define DDBITS 0 71 | #endif 72 | 73 | /* compression level (1 - 9) - this only affects the compressor. 74 | * 1 is fastest, 9 is best compression ratio */ 75 | #if !defined(CLEVEL) 76 | # define CLEVEL 1 /* fastest by default */ 77 | #endif 78 | 79 | 80 | /* check configuration */ 81 | #if (RBITS < 4 || RBITS > 5) 82 | # error "invalid RBITS" 83 | #endif 84 | #if (DDBITS < 0 || DDBITS > 6) 85 | # error "invalid DDBITS" 86 | #endif 87 | #if (CLEVEL < 1 || CLEVEL > 9) 88 | # error "invalid CLEVEL" 89 | #endif 90 | 91 | 92 | /*********************************************************************** 93 | // internal configuration 94 | ************************************************************************/ 95 | 96 | /* add a special code so that the decompressor can detect the 97 | * end of the compressed data block (overhead is 3 bytes per block) */ 98 | #define LZO_EOF_CODE 1 99 | 100 | 101 | /*********************************************************************** 102 | // algorithm internal configuration 103 | ************************************************************************/ 104 | 105 | /* choose the hashing strategy */ 106 | #ifndef LZO_HASH 107 | #define LZO_HASH LZO_HASH_LZO_INCREMENTAL_A 108 | #endif 109 | 110 | /* config */ 111 | #define R_BITS RBITS 112 | #define DD_BITS DDBITS 113 | #ifndef D_BITS 114 | #define D_BITS 14 115 | #endif 116 | 117 | 118 | /*********************************************************************** 119 | // optimization and debugging 120 | ************************************************************************/ 121 | 122 | /* Collect statistics */ 123 | #if 0 && !defined(LZO_COLLECT_STATS) 124 | # define LZO_COLLECT_STATS 1 125 | #endif 126 | 127 | 128 | /*********************************************************************** 129 | // 130 | ************************************************************************/ 131 | 132 | #include "lzo1b_de.h" 133 | #include "stats1b.h" 134 | 135 | #include "lzo1b_cc.h" 136 | 137 | 138 | #endif /* already included */ 139 | 140 | /* 141 | vi:ts=4:et 142 | */ 143 | 144 | -------------------------------------------------------------------------------- /jcpp/src/config1c.h: -------------------------------------------------------------------------------- 1 | /* config1c.h -- configuration for the LZO1C algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1C_H 51 | #define __LZO_CONFIG1C_H 1 52 | 53 | #include "lzo_conf.h" 54 | #include "lzo/lzo1c.h" 55 | 56 | 57 | /*********************************************************************** 58 | // algorithm configuration 59 | ************************************************************************/ 60 | 61 | /* run bits (4 - 5) - the compressor and the decompressor 62 | * must use the same value. */ 63 | #if !defined(RBITS) 64 | # define RBITS 5 65 | #endif 66 | 67 | /* dictionary depth (0 - 6) - this only affects the compressor. 68 | * 0 is fastest, 6 is best compression ratio */ 69 | #if !defined(DDBITS) 70 | # define DDBITS 0 71 | #endif 72 | 73 | /* compression level (1 - 9) - this only affects the compressor. 74 | * 1 is fastest, 9 is best compression ratio */ 75 | #if !defined(CLEVEL) 76 | # define CLEVEL 1 /* fastest by default */ 77 | #endif 78 | 79 | 80 | /* check configuration */ 81 | #if (RBITS < 4 || RBITS > 5) 82 | # error "invalid RBITS" 83 | #endif 84 | #if (DDBITS < 0 || DDBITS > 6) 85 | # error "invalid DDBITS" 86 | #endif 87 | #if (CLEVEL < 1 || CLEVEL > 9) 88 | # error "invalid CLEVEL" 89 | #endif 90 | 91 | 92 | /*********************************************************************** 93 | // internal configuration 94 | ************************************************************************/ 95 | 96 | /* add a special code so that the decompressor can detect the 97 | * end of the compressed data block (overhead is 3 bytes per block) */ 98 | #define LZO_EOF_CODE 1 99 | 100 | 101 | /*********************************************************************** 102 | // algorithm internal configuration 103 | ************************************************************************/ 104 | 105 | /* choose the hashing strategy */ 106 | #ifndef LZO_HASH 107 | #define LZO_HASH LZO_HASH_LZO_INCREMENTAL_A 108 | #endif 109 | 110 | /* config */ 111 | #define R_BITS RBITS 112 | #define DD_BITS DDBITS 113 | #ifndef D_BITS 114 | #define D_BITS 14 115 | #endif 116 | 117 | 118 | /*********************************************************************** 119 | // optimization and debugging 120 | ************************************************************************/ 121 | 122 | /* Collect statistics */ 123 | #if 0 && !defined(LZO_COLLECT_STATS) 124 | # define LZO_COLLECT_STATS 1 125 | #endif 126 | 127 | 128 | /*********************************************************************** 129 | // 130 | ************************************************************************/ 131 | 132 | /* good parameters when using a blocksize of 8kB */ 133 | #define M3O_BITS 6 134 | #undef LZO_DETERMINISTIC 135 | 136 | 137 | #include "lzo1b_de.h" 138 | #include "stats1c.h" 139 | 140 | #include "lzo1c_cc.h" 141 | 142 | 143 | #endif /* already included */ 144 | 145 | /* 146 | vi:ts=4:et 147 | */ 148 | 149 | -------------------------------------------------------------------------------- /jcpp/src/config1f.h: -------------------------------------------------------------------------------- 1 | /* config1f.h -- configuration for the LZO1F algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1F_H 51 | #define __LZO_CONFIG1F_H 1 52 | 53 | #include "lzo_conf.h" 54 | #include "lzo/lzo1f.h" 55 | 56 | 57 | /*********************************************************************** 58 | // 59 | ************************************************************************/ 60 | 61 | #define LZO_EOF_CODE 1 62 | #undef LZO_DETERMINISTIC 63 | 64 | #define M2_MAX_OFFSET 0x0800 65 | #define M3_MAX_OFFSET 0x3fff 66 | 67 | #define M2_MIN_LEN 3 68 | #define M2_MAX_LEN 8 69 | #define M3_MIN_LEN 3 70 | #define M3_MAX_LEN 33 71 | 72 | #define M3_MARKER 224 73 | 74 | 75 | /*********************************************************************** 76 | // 77 | ************************************************************************/ 78 | 79 | #ifndef MIN_LOOKAHEAD 80 | #define MIN_LOOKAHEAD (M2_MAX_LEN + 1) 81 | #endif 82 | 83 | #if defined(LZO_NEED_DICT_H) 84 | 85 | #ifndef LZO_HASH 86 | #define LZO_HASH LZO_HASH_LZO_INCREMENTAL_A 87 | #endif 88 | #define DL_MIN_LEN M2_MIN_LEN 89 | #include "lzo_dict.h" 90 | 91 | #endif 92 | 93 | 94 | 95 | #endif /* already included */ 96 | 97 | /* 98 | vi:ts=4:et 99 | */ 100 | 101 | -------------------------------------------------------------------------------- /jcpp/src/config1x.h: -------------------------------------------------------------------------------- 1 | /* config1x.h -- configuration for the LZO1X algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1X_H 51 | #define __LZO_CONFIG1X_H 1 52 | 53 | #if !defined(LZO1X) && !defined(LZO1Y) && !defined(LZO1Z) 54 | # define LZO1X 1 55 | #endif 56 | 57 | #include "lzo_conf.h" 58 | #if !defined(__LZO_IN_MINILZO) 59 | #include "lzo/lzo1x.h" 60 | #endif 61 | 62 | 63 | /*********************************************************************** 64 | // 65 | ************************************************************************/ 66 | 67 | #ifndef LZO_EOF_CODE 68 | #define LZO_EOF_CODE 1 69 | #endif 70 | #undef LZO_DETERMINISTIC 71 | 72 | #define M1_MAX_OFFSET 0x0400 73 | #ifndef M2_MAX_OFFSET 74 | #define M2_MAX_OFFSET 0x0800 75 | #endif 76 | #define M3_MAX_OFFSET 0x4000 77 | #define M4_MAX_OFFSET 0xbfff 78 | 79 | #define MX_MAX_OFFSET (M1_MAX_OFFSET + M2_MAX_OFFSET) 80 | 81 | #define M1_MIN_LEN 2 82 | #define M1_MAX_LEN 2 83 | #define M2_MIN_LEN 3 84 | #ifndef M2_MAX_LEN 85 | #define M2_MAX_LEN 8 86 | #endif 87 | #define M3_MIN_LEN 3 88 | #define M3_MAX_LEN 33 89 | #define M4_MIN_LEN 3 90 | #define M4_MAX_LEN 9 91 | 92 | #define M1_MARKER 0 93 | #define M2_MARKER 64 94 | #define M3_MARKER 32 95 | #define M4_MARKER 16 96 | 97 | 98 | /*********************************************************************** 99 | // 100 | ************************************************************************/ 101 | 102 | #ifndef MIN_LOOKAHEAD 103 | #define MIN_LOOKAHEAD (M2_MAX_LEN + 1) 104 | #endif 105 | 106 | #if defined(LZO_NEED_DICT_H) 107 | 108 | #ifndef LZO_HASH 109 | #define LZO_HASH LZO_HASH_LZO_INCREMENTAL_B 110 | #endif 111 | #define DL_MIN_LEN M2_MIN_LEN 112 | #include "lzo_dict.h" 113 | 114 | #endif 115 | 116 | 117 | 118 | #endif /* already included */ 119 | 120 | /* 121 | vi:ts=4:et 122 | */ 123 | 124 | -------------------------------------------------------------------------------- /jcpp/src/config1y.h: -------------------------------------------------------------------------------- 1 | /* config1y.h -- configuration for the LZO1Y algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1Y_H 51 | #define __LZO_CONFIG1Y_H 1 52 | 53 | #if !defined(LZO1Y) 54 | # define LZO1Y 1 55 | #endif 56 | 57 | #include "lzo_conf.h" 58 | #include "lzo/lzo1y.h" 59 | 60 | #define M2_MAX_LEN 14 61 | #define M2_MAX_OFFSET 0x0400 62 | #include "config1x.h" 63 | 64 | #endif /* already included */ 65 | 66 | /* 67 | vi:ts=4:et 68 | */ 69 | 70 | -------------------------------------------------------------------------------- /jcpp/src/config1z.h: -------------------------------------------------------------------------------- 1 | /* config1z.h -- configuration for the LZO1Z algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG1Z_H 51 | #define __LZO_CONFIG1Z_H 1 52 | 53 | #if !defined(LZO1Z) 54 | # define LZO1Z 1 55 | #endif 56 | 57 | #include "lzo_conf.h" 58 | #include "lzo/lzo1z.h" 59 | 60 | #define M2_MAX_OFFSET 0x0700 61 | #include "config1x.h" 62 | 63 | #endif /* already included */ 64 | 65 | /* 66 | vi:ts=4:et 67 | */ 68 | 69 | -------------------------------------------------------------------------------- /jcpp/src/config2a.h: -------------------------------------------------------------------------------- 1 | /* config2a.h -- configuration for the LZO2A algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_CONFIG2A_H 51 | #define __LZO_CONFIG2A_H 1 52 | 53 | #include "lzo_conf.h" 54 | #include "lzo/lzo2a.h" 55 | 56 | 57 | /*********************************************************************** 58 | // algorithm configuration 59 | ************************************************************************/ 60 | 61 | /* dictionary depth (0 - 6) - this only affects the compressor. 62 | * 0 is fastest, 6 is best compression ratio */ 63 | #if !defined(DDBITS) 64 | # define DDBITS 0 65 | #endif 66 | 67 | /* compression level (1 - 9) - this only affects the compressor. 68 | * 1 is fastest, 9 is best compression ratio */ 69 | #if !defined(CLEVEL) 70 | # define CLEVEL 1 /* fastest by default */ 71 | #endif 72 | 73 | 74 | /* check configuration */ 75 | #if (DDBITS < 0 || DDBITS > 6) 76 | # error "invalid DDBITS" 77 | #endif 78 | #if (CLEVEL < 1 || CLEVEL > 9) 79 | # error "invalid CLEVEL" 80 | #endif 81 | 82 | 83 | /*********************************************************************** 84 | // internal configuration 85 | ************************************************************************/ 86 | 87 | #if 1 88 | #define SWD_N 8191 /* size of ring buffer */ 89 | #else 90 | #define SWD_N 16383 /* size of ring buffer */ 91 | #endif 92 | 93 | #define M1_MIN_LEN 2 94 | #define M1_MAX_LEN 5 95 | #define M2_MIN_LEN 3 96 | #define M3_MIN_LEN 3 97 | 98 | 99 | /* add a special code so that the decompressor can detect the 100 | * end of the compressed data block (overhead is 3 bytes per block) */ 101 | #define LZO_EOF_CODE 1 102 | 103 | #undef LZO_DETERMINISTIC 104 | 105 | 106 | /*********************************************************************** 107 | // algorithm internal configuration 108 | ************************************************************************/ 109 | 110 | /* choose the hashing strategy */ 111 | #ifndef LZO_HASH 112 | #define LZO_HASH LZO_HASH_LZO_INCREMENTAL_A 113 | #endif 114 | 115 | /* config */ 116 | #define DD_BITS DDBITS 117 | #ifndef D_BITS 118 | #define D_BITS 14 119 | #endif 120 | 121 | 122 | 123 | /*********************************************************************** 124 | // optimization and debugging 125 | ************************************************************************/ 126 | 127 | /* Collect statistics */ 128 | #if 0 && !defined(LZO_COLLECT_STATS) 129 | # define LZO_COLLECT_STATS 1 130 | #endif 131 | 132 | 133 | /*********************************************************************** 134 | // 135 | ************************************************************************/ 136 | 137 | /* get bits */ 138 | #define _NEEDBITS \ 139 | { _NEEDBYTE; b |= ((lzo_uint32) _NEXTBYTE) << k; k += 8; assert(k <= 32); } 140 | #define NEEDBITS(j) { assert((j) < 8); if (k < (j)) _NEEDBITS } 141 | 142 | /* set bits */ 143 | #define SETBITS(j,x) { b |= (x) << k; k += (j); assert(k <= 32); } 144 | 145 | /* access bits */ 146 | #define MASKBITS(j) (b & ((((lzo_uint32)1 << (j)) - 1))) 147 | 148 | /* drop bits */ 149 | #define DUMPBITS(j) { assert(k >= j); b >>= (j); k -= (j); } 150 | 151 | 152 | 153 | #endif /* already included */ 154 | 155 | /* 156 | vi:ts=4:et 157 | */ 158 | 159 | -------------------------------------------------------------------------------- /jcpp/src/lzo1_cm.ch: -------------------------------------------------------------------------------- 1 | /* lzo1_cm.ch -- implementation of the LZO1 compression algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #include "lzo1a_cm.ch" 51 | 52 | 53 | /* 54 | vi:ts=4:et 55 | */ 56 | -------------------------------------------------------------------------------- /jcpp/src/lzo1a_cr.ch: -------------------------------------------------------------------------------- 1 | /* lzo1a_cr.ch -- literal run handling for the the LZO1A algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the LZO package and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO1A_CR_H 51 | #define __LZO1A_CR_H 1 52 | 53 | 54 | /*********************************************************************** 55 | // code a literal run 56 | ************************************************************************/ 57 | 58 | static lzo_bytep 59 | store_run(lzo_bytep const oo, const lzo_bytep const ii, lzo_uint r_len) 60 | { 61 | register lzo_bytep op; 62 | register const lzo_bytep ip; 63 | register lzo_uint t; 64 | 65 | op = oo; 66 | ip = ii; 67 | assert(r_len > 0); 68 | 69 | /* code a long R0 run */ 70 | if (r_len >= 512) 71 | { 72 | unsigned r_bits = 6; /* 256 << 6 == 16384 */ 73 | lzo_uint tt = 32768u; 74 | 75 | while (r_len >= (t = tt)) 76 | { 77 | r_len -= t; 78 | *op++ = 0; *op++ = (R0MAX - R0MIN); 79 | MEMCPY8_DS(op, ip, t); 80 | LZO_STATS(lzo_stats->r0long_runs++); 81 | } 82 | tt >>= 1; 83 | do { 84 | if (r_len >= (t = tt)) 85 | { 86 | r_len -= t; 87 | *op++ = 0; *op++ = LZO_BYTE((R0FAST - R0MIN) + r_bits); 88 | MEMCPY8_DS(op, ip, t); 89 | LZO_STATS(lzo_stats->r0long_runs++); 90 | } 91 | tt >>= 1; 92 | } while (--r_bits > 0); 93 | } 94 | assert(r_len < 512); 95 | 96 | while (r_len >= (t = R0FAST)) 97 | { 98 | r_len -= t; 99 | *op++ = 0; *op++ = (R0FAST - R0MIN); 100 | MEMCPY8_DS(op, ip, t); 101 | LZO_STATS(lzo_stats->r0fast_runs++); 102 | } 103 | 104 | t = r_len; 105 | if (t >= R0MIN) 106 | { 107 | /* code a short R0 run */ 108 | *op++ = 0; *op++ = LZO_BYTE(t - R0MIN); 109 | MEMCPY_DS(op, ip, t); 110 | LZO_STATS(lzo_stats->r0short_runs++); 111 | } 112 | else if (t > 0) 113 | { 114 | /* code a short literal run */ 115 | LZO_STATS(lzo_stats->lit_runs++); 116 | LZO_STATS(lzo_stats->lit_run[t]++); 117 | *op++ = LZO_BYTE(t); 118 | MEMCPY_DS(op, ip, t); 119 | } 120 | 121 | return op; 122 | } 123 | 124 | 125 | 126 | #endif /* already included */ 127 | 128 | /* 129 | vi:ts=4:et 130 | */ 131 | 132 | -------------------------------------------------------------------------------- /jcpp/src/lzo1b_cc.h: -------------------------------------------------------------------------------- 1 | /* lzo1b_cc.h -- definitions for the the LZO1B compression driver 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO1B_CC_H 51 | #define __LZO1B_CC_H 1 52 | 53 | 54 | /*********************************************************************** 55 | // 56 | ************************************************************************/ 57 | 58 | extern const lzo_compress_t _lzo1b_1_compress_func; 59 | extern const lzo_compress_t _lzo1b_2_compress_func; 60 | extern const lzo_compress_t _lzo1b_3_compress_func; 61 | extern const lzo_compress_t _lzo1b_4_compress_func; 62 | extern const lzo_compress_t _lzo1b_5_compress_func; 63 | extern const lzo_compress_t _lzo1b_6_compress_func; 64 | extern const lzo_compress_t _lzo1b_7_compress_func; 65 | extern const lzo_compress_t _lzo1b_8_compress_func; 66 | extern const lzo_compress_t _lzo1b_9_compress_func; 67 | 68 | extern const lzo_compress_t _lzo1b_99_compress_func; 69 | 70 | 71 | /*********************************************************************** 72 | // 73 | ************************************************************************/ 74 | 75 | LZO_EXTERN(lzo_bytep ) 76 | _lzo1b_store_run ( lzo_bytep const oo, const lzo_bytep const ii, 77 | lzo_uint r_len); 78 | 79 | #define STORE_RUN _lzo1b_store_run 80 | 81 | 82 | lzo_compress_t _lzo1b_get_compress_func(int clevel); 83 | 84 | int _lzo1b_do_compress ( const lzo_bytep in, lzo_uint in_len, 85 | lzo_bytep out, lzo_uintp out_len, 86 | lzo_voidp wrkmem, 87 | lzo_compress_t func ); 88 | 89 | 90 | #endif /* already included */ 91 | 92 | /* 93 | vi:ts=4:et 94 | */ 95 | 96 | 97 | -------------------------------------------------------------------------------- /jcpp/src/lzo1b_cr.ch: -------------------------------------------------------------------------------- 1 | /* lzo1b_cr.ch -- implementation of the LZO1B compression algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | 51 | /*********************************************************************** 52 | // store the current literal run 53 | ************************************************************************/ 54 | 55 | assert(ip < ip_end); 56 | if (pd(ip,ii) > 0) 57 | { 58 | lzo_uint t = pd(ip,ii); 59 | 60 | #if defined(LZO_HAVE_R1) 61 | if (ip == r1) 62 | { 63 | /* Code a context sensitive R1 match. */ 64 | LZO_STATS(lzo_stats->literals += t); 65 | LZO_STATS(lzo_stats->r1_matches++); 66 | assert(t == 1); 67 | /* modify marker byte */ 68 | assert((op[-2] >> M2O_BITS) == (M2_MARKER >> M2O_BITS)); 69 | op[-2] &= M2O_MASK; 70 | assert((op[-2] >> M2O_BITS) == 0); 71 | /* copy 1 literal */ 72 | *op++ = *ii++; 73 | r1 = ip + (M2_MIN_LEN + 1); /* set new R1 pointer */ 74 | } 75 | else 76 | #endif 77 | if (t < R0MIN) 78 | { 79 | /* inline the copying of a short run */ 80 | LZO_STATS(lzo_stats->literals += t); 81 | LZO_STATS(lzo_stats->lit_runs++); 82 | LZO_STATS(lzo_stats->lit_run[t]++); 83 | #if defined(LZO_HAVE_M3) 84 | if (t < LZO_SIZE(8-M3O_BITS) && op == m3) 85 | { 86 | /* Code a very short literal run into the low offset bits 87 | * of the previous M3/M4 match. 88 | */ 89 | LZO_STATS(lzo_stats->lit_runs_after_m3_match++); 90 | LZO_STATS(lzo_stats->lit_run_after_m3_match[t]++); 91 | assert((m3[-2] >> M3O_BITS) == 0); 92 | m3[-2] |= LZO_BYTE(t << M3O_BITS); 93 | } 94 | else 95 | #endif 96 | { 97 | *op++ = LZO_BYTE(t); 98 | } 99 | MEMCPY_DS(op, ii, t); 100 | #if defined(LZO_HAVE_R1) 101 | r1 = ip + (M2_MIN_LEN + 1); /* set new R1 pointer */ 102 | #endif 103 | } 104 | else if (t < R0FAST) 105 | { 106 | /* inline the copying of a short R0 run */ 107 | LZO_STATS(lzo_stats->literals += t); 108 | LZO_STATS(lzo_stats->r0short_runs++); 109 | *op++ = 0; *op++ = LZO_BYTE(t - R0MIN); 110 | MEMCPY_DS(op, ii, t); 111 | #if defined(LZO_HAVE_R1) 112 | r1 = ip + (M2_MIN_LEN + 1); /* set new R1 pointer */ 113 | #endif 114 | } 115 | else 116 | { 117 | op = STORE_RUN(op,ii,t); 118 | ii = ip; 119 | } 120 | } 121 | 122 | 123 | /* ii now points to the start of the current match */ 124 | assert(ii == ip); 125 | 126 | 127 | /* 128 | vi:ts=4:et 129 | */ 130 | -------------------------------------------------------------------------------- /jcpp/src/lzo1b_r.ch: -------------------------------------------------------------------------------- 1 | /* lzo1b_r.ch -- literal run handling for the the LZO1B/LZO1C algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /*********************************************************************** 45 | // store a literal run (internal) 46 | ************************************************************************/ 47 | 48 | LZO_PUBLIC(lzo_bytep ) 49 | STORE_RUN ( lzo_bytep const oo, const lzo_bytep const ii, lzo_uint r_len) 50 | { 51 | register lzo_bytep op; 52 | register const lzo_bytep ip; 53 | register lzo_uint t; 54 | 55 | LZO_STATS(lzo_stats->literals += r_len); 56 | 57 | op = oo; 58 | ip = ii; 59 | assert(r_len > 0); 60 | 61 | /* code a long R0 run */ 62 | if (r_len >= 512) 63 | { 64 | unsigned r_bits = 6; /* 256 << 6 == 16384 */ 65 | lzo_uint tt = 32768u; 66 | 67 | while (r_len >= (t = tt)) 68 | { 69 | r_len -= t; 70 | *op++ = 0; *op++ = (R0FAST - R0MIN) + 7; 71 | MEMCPY8_DS(op, ip, t); 72 | LZO_STATS(lzo_stats->r0long_runs++); 73 | } 74 | tt >>= 1; 75 | do { 76 | if (r_len >= (t = tt)) 77 | { 78 | r_len -= t; 79 | *op++ = 0; *op++ = LZO_BYTE((R0FAST - R0MIN) + r_bits); 80 | MEMCPY8_DS(op, ip, t); 81 | LZO_STATS(lzo_stats->r0long_runs++); 82 | } 83 | tt >>= 1; 84 | } while (--r_bits > 0); 85 | } 86 | assert(r_len < 512); 87 | 88 | while (r_len >= (t = R0FAST)) 89 | { 90 | r_len -= t; 91 | *op++ = 0; *op++ = (R0FAST - R0MIN); 92 | MEMCPY8_DS(op, ip, t); 93 | LZO_STATS(lzo_stats->r0fast_runs++); 94 | } 95 | 96 | t = r_len; 97 | if (t >= R0MIN) 98 | { 99 | /* code a short R0 run */ 100 | *op++ = 0; *op++ = LZO_BYTE(t - R0MIN); 101 | MEMCPY_DS(op, ip, t); 102 | LZO_STATS(lzo_stats->r0short_runs++); 103 | } 104 | else if (t > 0) 105 | { 106 | /* code a short literal run */ 107 | LZO_STATS(lzo_stats->lit_runs++); 108 | LZO_STATS(lzo_stats->lit_run[t]++); 109 | *op++ = LZO_BYTE(t); 110 | MEMCPY_DS(op, ip, t); 111 | } 112 | 113 | return op; 114 | } 115 | 116 | 117 | /* 118 | vi:ts=4:et 119 | */ 120 | 121 | -------------------------------------------------------------------------------- /jcpp/src/lzo1b_tm.ch: -------------------------------------------------------------------------------- 1 | /* lzo1b_tm.ch -- implementation of the LZO1B compression algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | 51 | /*********************************************************************** 52 | // test for a potential match 53 | ************************************************************************/ 54 | 55 | 56 | #if (DD_BITS == 0) 57 | 58 | try_match: 59 | #if !defined(NDEBUG) && (LZO_DICT_USE_PTR) 60 | #if (LZO_DETERMINISTIC) 61 | assert(m_pos == NULL || m_pos >= in); 62 | assert(m_pos == NULL || m_pos < ip); 63 | #endif 64 | m_pos_sav = m_pos; 65 | #endif 66 | if (m_pos[0] == ip[0] && m_pos[1] == ip[1] && m_pos[2] == ip[2]) 67 | { 68 | m_pos += 3; 69 | goto match; 70 | } 71 | 72 | 73 | #else /* (DD_BITS == 0) */ 74 | 75 | 76 | /* test potential match */ 77 | 78 | if (m_len > M2_MIN_LEN) 79 | goto match; 80 | if (m_len == M2_MIN_LEN) 81 | { 82 | #if (_MAX_OFFSET == _M2_MAX_OFFSET) 83 | goto match; 84 | #else 85 | if (m_off <= M2_MAX_OFFSET) 86 | goto match; 87 | #if 0 && (M3_MIN_LEN == M2_MIN_LEN) 88 | if (ip == ii) 89 | goto match; 90 | #endif 91 | #endif 92 | } 93 | goto literal; 94 | 95 | 96 | #endif /* (DD_BITS == 0) */ 97 | 98 | 99 | 100 | /* 101 | vi:ts=4:et 102 | */ 103 | -------------------------------------------------------------------------------- /jcpp/src/lzo1c_cc.h: -------------------------------------------------------------------------------- 1 | /* lzo1c_cc.h -- definitions for the the LZO1C compression driver 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO1C_CC_H 51 | #define __LZO1C_CC_H 1 52 | 53 | 54 | /*********************************************************************** 55 | // 56 | ************************************************************************/ 57 | 58 | extern const lzo_compress_t _lzo1c_1_compress_func; 59 | extern const lzo_compress_t _lzo1c_2_compress_func; 60 | extern const lzo_compress_t _lzo1c_3_compress_func; 61 | extern const lzo_compress_t _lzo1c_4_compress_func; 62 | extern const lzo_compress_t _lzo1c_5_compress_func; 63 | extern const lzo_compress_t _lzo1c_6_compress_func; 64 | extern const lzo_compress_t _lzo1c_7_compress_func; 65 | extern const lzo_compress_t _lzo1c_8_compress_func; 66 | extern const lzo_compress_t _lzo1c_9_compress_func; 67 | 68 | extern const lzo_compress_t _lzo1c_99_compress_func; 69 | 70 | 71 | /*********************************************************************** 72 | // 73 | ************************************************************************/ 74 | 75 | LZO_EXTERN(lzo_bytep ) 76 | _lzo1c_store_run ( lzo_bytep const oo, const lzo_bytep const ii, 77 | lzo_uint r_len); 78 | 79 | #define STORE_RUN _lzo1c_store_run 80 | 81 | 82 | lzo_compress_t _lzo1c_get_compress_func(int clevel); 83 | 84 | int _lzo1c_do_compress ( const lzo_bytep in, lzo_uint in_len, 85 | lzo_bytep out, lzo_uintp out_len, 86 | lzo_voidp wrkmem, 87 | lzo_compress_t func ); 88 | 89 | 90 | #endif /* already included */ 91 | 92 | /* 93 | vi:ts=4:et 94 | */ 95 | 96 | 97 | -------------------------------------------------------------------------------- /jcpp/src/lzo_dll.ch: -------------------------------------------------------------------------------- 1 | /* lzo_dll.ch -- DLL initialization of the LZO library 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /*********************************************************************** 45 | // Windows 16 bit + Watcom C + DLL 46 | ************************************************************************/ 47 | 48 | #if (LZO_OS_WIN16 && LZO_CC_WATCOMC) && defined(__SW_BD) 49 | 50 | /* don't pull in - we don't need it */ 51 | #if 0 52 | BOOL FAR PASCAL LibMain ( HANDLE hInstance, WORD wDataSegment, 53 | WORD wHeapSize, LPSTR lpszCmdLine ) 54 | #else 55 | int __far __pascal LibMain ( int a, short b, short c, long d ) 56 | #endif 57 | { 58 | LZO_UNUSED(a); LZO_UNUSED(b); LZO_UNUSED(c); LZO_UNUSED(d); 59 | return 1; 60 | } 61 | 62 | #endif 63 | 64 | 65 | /* 66 | vi:ts=4:et 67 | */ 68 | -------------------------------------------------------------------------------- /jcpp/src/lzo_func.ch: -------------------------------------------------------------------------------- 1 | /* lzo_func.ch -- functions 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | /*********************************************************************** 51 | // bitops 52 | ************************************************************************/ 53 | 54 | #if (defined(_WIN32) || defined(_WIN64)) && ((LZO_CC_INTELC && (__INTEL_COMPILER >= 1000)) || (LZO_CC_MSC && (_MSC_VER >= 1400))) 55 | #include 56 | #if !defined(lzo_bitops_clz32) && defined(WANT_lzo_bitops_clz32) && 0 57 | #pragma intrinsic(_BitScanReverse) 58 | static __lzo_inline unsigned lzo_bitops_clz32(lzo_uint32 v) 59 | { 60 | unsigned long r; 61 | (void) _BitScanReverse(&r, v); 62 | return (unsigned) r; 63 | } 64 | #define lzo_bitops_clz32 lzo_bitops_clz32 65 | #endif 66 | #if !defined(lzo_bitops_clz64) && defined(WANT_lzo_bitops_clz64) && defined(LZO_UINT64_MAX) && 0 67 | #pragma intrinsic(_BitScanReverse64) 68 | static __lzo_inline unsigned lzo_bitops_clz64(lzo_uint64 v) 69 | { 70 | unsigned long r; 71 | (void) _BitScanReverse64(&r, v); 72 | return (unsigned) r; 73 | } 74 | #define lzo_bitops_clz64 lzo_bitops_clz64 75 | #endif 76 | #if !defined(lzo_bitops_ctz32) && defined(WANT_lzo_bitops_ctz32) 77 | #pragma intrinsic(_BitScanForward) 78 | static __lzo_inline unsigned lzo_bitops_ctz32(lzo_uint32 v) 79 | { 80 | unsigned long r; 81 | (void) _BitScanForward(&r, v); 82 | return (unsigned) r; 83 | } 84 | #define lzo_bitops_ctz32 lzo_bitops_ctz32 85 | #endif 86 | #if !defined(lzo_bitops_ctz64) && defined(WANT_lzo_bitops_ctz64) && defined(LZO_UINT64_MAX) 87 | #pragma intrinsic(_BitScanForward64) 88 | static __lzo_inline unsigned lzo_bitops_ctz64(lzo_uint64 v) 89 | { 90 | unsigned long r; 91 | (void) _BitScanForward64(&r, v); 92 | return (unsigned) r; 93 | } 94 | #define lzo_bitops_ctz64 lzo_bitops_ctz64 95 | #endif 96 | 97 | #elif (LZO_CC_CLANG || (LZO_CC_GNUC >= 0x030400ul) || (LZO_CC_INTELC && (__INTEL_COMPILER >= 1000)) || LZO_CC_LLVM) 98 | #if !defined(lzo_bitops_clz32) && defined(WANT_lzo_bitops_clz32) 99 | #define lzo_bitops_clz32(v) ((unsigned) __builtin_clz(v)) 100 | #endif 101 | #if !defined(lzo_bitops_clz64) && defined(WANT_lzo_bitops_clz64) && defined(LZO_UINT64_MAX) 102 | #define lzo_bitops_clz64(v) ((unsigned) __builtin_clzll(v)) 103 | #endif 104 | #if !defined(lzo_bitops_ctz32) && defined(WANT_lzo_bitops_ctz32) 105 | #define lzo_bitops_ctz32(v) ((unsigned) __builtin_ctz(v)) 106 | #endif 107 | #if !defined(lzo_bitops_ctz64) && defined(WANT_lzo_bitops_ctz64) && defined(LZO_UINT64_MAX) 108 | #define lzo_bitops_ctz64(v) ((unsigned) __builtin_ctzll(v)) 109 | #endif 110 | #if !defined(lzo_bitops_popcount32) && defined(WANT_lzo_bitops_popcount32) 111 | #define lzo_bitops_popcount32(v) ((unsigned) __builtin_popcount(v)) 112 | #endif 113 | #if !defined(lzo_bitops_popcount32) && defined(WANT_lzo_bitops_popcount64) && defined(LZO_UINT64_MAX) 114 | #define lzo_bitops_popcount64(v) ((unsigned) __builtin_popcountll(v)) 115 | #endif 116 | #endif 117 | 118 | 119 | /* 120 | vi:ts=4:et 121 | */ 122 | 123 | -------------------------------------------------------------------------------- /jcpp/src/stats1a.h: -------------------------------------------------------------------------------- 1 | /* stats1a.h -- statistics for the the LZO1A algorithm 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the LZO package and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_STATS1A_H 51 | #define __LZO_STATS1A_H 1 52 | 53 | #ifdef __cplusplus 54 | extern "C" { 55 | #endif 56 | 57 | 58 | 59 | /*********************************************************************** 60 | // collect statistical information when compressing 61 | // used for finetuning, view with a debugger 62 | ************************************************************************/ 63 | 64 | #if (LZO_COLLECT_STATS) 65 | # define LZO_STATS(expr) expr 66 | #else 67 | # define LZO_STATS(expr) ((void) 0) 68 | #endif 69 | 70 | 71 | /*********************************************************************** 72 | // 73 | ************************************************************************/ 74 | 75 | typedef struct { 76 | 77 | /* configuration */ 78 | unsigned rbits; 79 | unsigned clevel; 80 | 81 | /* internal configuration */ 82 | unsigned dbits; 83 | unsigned lbits; 84 | 85 | /* constants */ 86 | unsigned min_match_short; 87 | unsigned max_match_short; 88 | unsigned min_match_long; 89 | unsigned max_match_long; 90 | unsigned min_offset; 91 | unsigned max_offset; 92 | unsigned r0min; 93 | unsigned r0fast; 94 | unsigned r0max; 95 | 96 | /* counts */ 97 | long short_matches; 98 | long long_matches; 99 | long r1_matches; 100 | long lit_runs; 101 | long lit_runs_after_long_match; 102 | long r0short_runs; 103 | long r0fast_runs; 104 | long r0long_runs; 105 | 106 | /* */ 107 | long lit_run[RSIZE]; 108 | long lit_run_after_long_match[RSIZE]; 109 | long short_match[MAX_MATCH_SHORT + 1]; 110 | long long_match[MAX_MATCH_LONG + 1]; 111 | long marker[256]; 112 | 113 | /* these could prove useful for further optimizations */ 114 | long short_match_offset_osize[MAX_MATCH_SHORT + 1]; 115 | long short_match_offset_256[MAX_MATCH_SHORT + 1]; 116 | long short_match_offset_1024[MAX_MATCH_SHORT + 1]; 117 | long matches_out_of_range; 118 | long matches_out_of_range_2; 119 | long matches_out_of_range_4; 120 | long match_out_of_range[MAX_MATCH_SHORT + 1]; 121 | 122 | /* */ 123 | long in_len; 124 | long out_len; 125 | } 126 | lzo1a_stats_t; 127 | 128 | extern lzo1a_stats_t *lzo1a_stats; 129 | 130 | 131 | 132 | #ifdef __cplusplus 133 | } /* extern "C" */ 134 | #endif 135 | 136 | #endif /* already included */ 137 | 138 | /* 139 | vi:ts=4:et 140 | */ 141 | -------------------------------------------------------------------------------- /jcpp/src/stats1b.h: -------------------------------------------------------------------------------- 1 | /* stats1b.h -- statistics for the the LZO library 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_STATS1B_H 51 | #define __LZO_STATS1B_H 1 52 | 53 | #ifdef __cplusplus 54 | extern "C" { 55 | #endif 56 | 57 | 58 | /*********************************************************************** 59 | // Collect statistical information when compressing. 60 | // Useful for finetuning the compression algorithm. 61 | // Examine the symbol 'lzo1b_stats' with a debugger. 62 | ************************************************************************/ 63 | 64 | #if (LZO_COLLECT_STATS) 65 | # define LZO_STATS(expr) expr 66 | #else 67 | # define LZO_STATS(expr) ((void) 0) 68 | #endif 69 | 70 | 71 | #if (LZO_COLLECT_STATS) 72 | 73 | typedef struct 74 | { 75 | /* algorithm configuration */ 76 | unsigned r_bits; 77 | unsigned m3o_bits; 78 | unsigned dd_bits; 79 | unsigned clevel; 80 | 81 | /* internal configuration */ 82 | unsigned d_bits; 83 | long min_lookahead; 84 | long max_lookbehind; 85 | const char *compress_id; 86 | 87 | /* counts */ 88 | long lit_runs; 89 | long r0short_runs; 90 | long r0fast_runs; 91 | long r0long_runs; 92 | long m1_matches; 93 | long m2_matches; 94 | long m3_matches; 95 | long m4_matches; 96 | long r1_matches; 97 | 98 | /* */ 99 | long lit_run[R0MIN]; 100 | long m2_match[M2_MAX_LEN + 1]; 101 | long m3_match[M3_MAX_LEN + 1]; 102 | #if (M3O_BITS < 8) 103 | long lit_runs_after_m3_match; 104 | long lit_run_after_m3_match[LZO_SIZE(8-M3O_BITS)]; 105 | #endif 106 | 107 | /* */ 108 | long matches; 109 | long match_bytes; 110 | long literals; 111 | long literal_overhead; 112 | long literal_bytes; 113 | double literal_overhead_percent; 114 | 115 | /* */ 116 | long unused_dict_entries; 117 | double unused_dict_entries_percent; 118 | 119 | /* */ 120 | long in_len; 121 | long out_len; 122 | } 123 | lzo1b_stats_t; 124 | 125 | 126 | void _lzo1b_stats_init(lzo1b_stats_t *lzo_stats); 127 | void _lzo1b_stats_calc(lzo1b_stats_t *lzo_stats); 128 | 129 | extern lzo1b_stats_t * const lzo1b_stats; 130 | 131 | #define lzo_stats_t lzo1b_stats_t 132 | #define lzo_stats lzo1b_stats 133 | 134 | #endif 135 | 136 | 137 | #ifdef __cplusplus 138 | } /* extern "C" */ 139 | #endif 140 | 141 | #endif /* already included */ 142 | 143 | /* 144 | vi:ts=4:et 145 | */ 146 | -------------------------------------------------------------------------------- /jcpp/src/stats1c.h: -------------------------------------------------------------------------------- 1 | /* stats1c.h -- statistics for the the LZO library 2 | 3 | This file is part of the LZO real-time data compression library. 4 | 5 | Copyright (C) 2011 Markus Franz Xaver Johannes Oberhumer 6 | Copyright (C) 2010 Markus Franz Xaver Johannes Oberhumer 7 | Copyright (C) 2009 Markus Franz Xaver Johannes Oberhumer 8 | Copyright (C) 2008 Markus Franz Xaver Johannes Oberhumer 9 | Copyright (C) 2007 Markus Franz Xaver Johannes Oberhumer 10 | Copyright (C) 2006 Markus Franz Xaver Johannes Oberhumer 11 | Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer 12 | Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer 13 | Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer 14 | Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer 15 | Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer 16 | Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer 17 | Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer 18 | Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer 19 | Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer 20 | Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer 21 | All Rights Reserved. 22 | 23 | The LZO library is free software; you can redistribute it and/or 24 | modify it under the terms of the GNU General Public License as 25 | published by the Free Software Foundation; either version 2 of 26 | the License, or (at your option) any later version. 27 | 28 | The LZO library is distributed in the hope that it will be useful, 29 | but WITHOUT ANY WARRANTY; without even the implied warranty of 30 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 31 | GNU General Public License for more details. 32 | 33 | You should have received a copy of the GNU General Public License 34 | along with the LZO library; see the file COPYING. 35 | If not, write to the Free Software Foundation, Inc., 36 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 37 | 38 | Markus F.X.J. Oberhumer 39 | 40 | http://www.oberhumer.com/opensource/lzo/ 41 | */ 42 | 43 | 44 | /* WARNING: this file should *not* be used by applications. It is 45 | part of the implementation of the library and is subject 46 | to change. 47 | */ 48 | 49 | 50 | #ifndef __LZO_STATS1C_H 51 | #define __LZO_STATS1C_H 1 52 | 53 | #define lzo1b_stats_t lzo1c_stats_t 54 | #define lzo1b_stats lzo1c_stats 55 | #define _lzo1b_stats_init _lzo1c_stats_init 56 | #define _lzo1b_stats_calc _lzo1c_stats_calc 57 | 58 | #include "stats1b.h" 59 | 60 | #endif /* already included */ 61 | 62 | /* 63 | vi:ts=4:et 64 | */ 65 | -------------------------------------------------------------------------------- /lib/ant/ant-contrib.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/ant/ant-contrib.jar -------------------------------------------------------------------------------- /lib/jalopy/aelfred-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/aelfred-1.2.jar -------------------------------------------------------------------------------- /lib/jalopy/jalopy-1.0b11.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/jalopy-1.0b11.jar -------------------------------------------------------------------------------- /lib/jalopy/jalopy-ant-0.6.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/jalopy-ant-0.6.2.jar -------------------------------------------------------------------------------- /lib/jalopy/jaxp-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/jaxp-1.2.jar -------------------------------------------------------------------------------- /lib/jalopy/jdom-1.0b8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/jdom-1.0b8.jar -------------------------------------------------------------------------------- /lib/jalopy/log4j-1.2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/log4j-1.2.6.jar -------------------------------------------------------------------------------- /lib/jalopy/oro-2.0.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/oro-2.0.6.jar -------------------------------------------------------------------------------- /lib/jalopy/sax-2.0.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jalopy/sax-2.0.1.jar -------------------------------------------------------------------------------- /lib/jcpp/anarres-cpp.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jcpp/anarres-cpp.jar -------------------------------------------------------------------------------- /lib/jcpp/gnu.getopt.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/jcpp/gnu.getopt.jar -------------------------------------------------------------------------------- /lib/junit/hamcrest-core-1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/junit/hamcrest-core-1.1.jar -------------------------------------------------------------------------------- /lib/junit/junit4-4.8.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/junit/junit4-4.8.2.jar -------------------------------------------------------------------------------- /lib/runtime/commons-io-1.4-src.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/commons-io-1.4-src.jar -------------------------------------------------------------------------------- /lib/runtime/commons-io-1.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/commons-io-1.4.jar -------------------------------------------------------------------------------- /lib/runtime/commons-lang-2.5-src.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/commons-lang-2.5-src.jar -------------------------------------------------------------------------------- /lib/runtime/commons-lang-2.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/commons-lang-2.5.jar -------------------------------------------------------------------------------- /lib/runtime/commons-logging-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/commons-logging-1.1.1.jar -------------------------------------------------------------------------------- /lib/runtime/hadoop-core-0.21.0-dev.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/hadoop-core-0.21.0-dev.jar -------------------------------------------------------------------------------- /lib/runtime/hadoop-mapred-0.21.0-dev.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/lib/runtime/hadoop-mapred-0.21.0-dev.jar -------------------------------------------------------------------------------- /license-header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | -------------------------------------------------------------------------------- /nbproject/genfiles.properties: -------------------------------------------------------------------------------- 1 | build.xml.data.CRC32=24ac336f 2 | build.xml.script.CRC32=11f77e30 3 | build.xml.stylesheet.CRC32=28e38971@1.38.2.45 4 | # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. 5 | # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. 6 | nbproject/build-impl.xml.data.CRC32=24ac336f 7 | nbproject/build-impl.xml.script.CRC32=a32a6e94 8 | nbproject/build-impl.xml.stylesheet.CRC32=f33e10ff@1.38.2.45 9 | -------------------------------------------------------------------------------- /nbproject/private/config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/nbproject/private/config.properties -------------------------------------------------------------------------------- /nbproject/private/private.properties: -------------------------------------------------------------------------------- 1 | compile.on.save=true 2 | do.depend=false 3 | do.jar=true 4 | javac.debug=true 5 | javadoc.preview=true 6 | user.properties.file=/home/shevek/.netbeans/6.9/build.properties 7 | -------------------------------------------------------------------------------- /nbproject/private/private.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /nbproject/project.properties: -------------------------------------------------------------------------------- 1 | annotation.processing.enabled=true 2 | annotation.processing.enabled.in.editor=false 3 | annotation.processing.run.all.processors=true 4 | annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output 5 | application.desc=LZO Compression and Decompression for Java 6 | application.title=LZO 7 | application.vendor=shevek 8 | build.classes.dir=${build.dir}/classes 9 | build.classes.excludes=**/*.java,**/*.form,**/*.c,**/*.ch,**/*.h,**/*.jcpp 10 | # This directory is removed when the project is cleaned: 11 | build.dir=build 12 | build.generated.dir=${build.dir}/generated 13 | build.generated.sources.dir=${build.dir}/generated-sources 14 | # Only compile against the classpath explicitly listed here: 15 | build.sysclasspath=ignore 16 | build.test.classes.dir=${build.dir}/test/classes 17 | build.test.results.dir=${build.dir}/test/results 18 | # Uncomment to specify the preferred debugger connection transport: 19 | #debug.transport=dt_socket 20 | debug.classpath=\ 21 | ${run.classpath} 22 | debug.test.classpath=\ 23 | ${run.test.classpath} 24 | # This directory is removed when the project is cleaned: 25 | dist.dir=dist 26 | dist.jar=${dist.dir}/lzo.jar 27 | dist.javadoc.dir=${dist.dir}/javadoc 28 | endorsed.classpath= 29 | excludes= 30 | file.reference.commons-io-1.4-src.jar=lib/runtime/commons-io-1.4-src.jar 31 | file.reference.commons-io-1.4.jar=lib/runtime/commons-io-1.4.jar 32 | file.reference.commons-lang-2.5-src.jar=lib/runtime/commons-lang-2.5-src.jar 33 | file.reference.commons-lang-2.5.jar=lib/runtime/commons-lang-2.5.jar 34 | file.reference.commons-logging-1.1.1.jar=lib/runtime/commons-logging-1.1.1.jar 35 | file.reference.hadoop-core-0.21.0-dev.jar=lib/runtime/hadoop-core-0.21.0-dev.jar 36 | file.reference.hadoop-mapred-0.21.0-dev.jar=lib/runtime/hadoop-mapred-0.21.0-dev.jar 37 | file.reference.hamcrest-core-1.1.jar=lib/junit/hamcrest-core-1.1.jar 38 | file.reference.junit4-4.8.2.jar=lib/junit/junit4-4.8.2.jar 39 | includes=** 40 | jar.compress=false 41 | javac.classpath=\ 42 | ${file.reference.hadoop-core-0.21.0-dev.jar}:\ 43 | ${file.reference.hadoop-mapred-0.21.0-dev.jar}:\ 44 | ${file.reference.commons-logging-1.1.1.jar} 45 | # Space-separated list of extra javac options 46 | javac.compilerargs= 47 | javac.deprecation=false 48 | javac.processorpath=\ 49 | ${javac.classpath} 50 | javac.source=1.5 51 | javac.target=1.5 52 | javac.test.classpath=\ 53 | ${javac.classpath}:\ 54 | ${build.classes.dir}:\ 55 | ${file.reference.commons-io-1.4-src.jar}:\ 56 | ${file.reference.commons-io-1.4.jar}:\ 57 | ${file.reference.commons-lang-2.5-src.jar}:\ 58 | ${file.reference.commons-lang-2.5.jar}:\ 59 | ${file.reference.junit4-4.8.2.jar}:\ 60 | ${file.reference.hamcrest-core-1.1.jar} 61 | javac.test.processorpath=\ 62 | ${javac.test.classpath} 63 | javadoc.additionalparam= 64 | javadoc.author=false 65 | javadoc.encoding=${source.encoding} 66 | javadoc.noindex=false 67 | javadoc.nonavbar=false 68 | javadoc.notree=false 69 | javadoc.private=false 70 | javadoc.splitindex=true 71 | javadoc.use=true 72 | javadoc.version=false 73 | javadoc.windowtitle= 74 | meta.inf.dir=${src.dir}/META-INF 75 | platform.active=default_platform 76 | run.classpath=\ 77 | ${javac.classpath}:\ 78 | ${build.classes.dir} 79 | # Space-separated list of JVM arguments used when running the project 80 | # (you may also define separate properties like run-sys-prop.name=value instead of -Dname=value 81 | # or test-sys-prop.name=value to set system properties for unit tests): 82 | run.jvmargs=-Xms128m -Xmx2400m -server -ea 83 | run.test.classpath=\ 84 | ${javac.test.classpath}:\ 85 | ${build.test.classes.dir} 86 | source.encoding=UTF-8 87 | src.dir=src 88 | src.jcpp.dir=jcpp 89 | test.src.dir=test 90 | 91 | test-sys-prop.test.data.dir=${basedir}/test/data 92 | 93 | -------------------------------------------------------------------------------- /nbproject/project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | org.netbeans.modules.java.j2seproject 4 | 5 | 6 | lzo 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/com/hadoop/compression/lzo/DistributedLzoIndexer.java: -------------------------------------------------------------------------------- 1 | package com.hadoop.compression.lzo; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | 8 | import com.hadoop.mapreduce.LzoIndexOutputFormat; 9 | import com.hadoop.mapreduce.LzoSplitInputFormat; 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | import org.apache.hadoop.conf.Configurable; 13 | import org.apache.hadoop.conf.Configuration; 14 | import org.apache.hadoop.conf.Configured; 15 | import org.apache.hadoop.fs.FileStatus; 16 | import org.apache.hadoop.fs.FileSystem; 17 | import org.apache.hadoop.fs.Path; 18 | import org.apache.hadoop.fs.PathFilter; 19 | import org.apache.hadoop.io.LongWritable; 20 | import org.apache.hadoop.io.compress.CompressionCodec; 21 | import org.apache.hadoop.io.compress.CompressionCodecFactory; 22 | import org.apache.hadoop.mapreduce.Job; 23 | import org.apache.hadoop.mapreduce.Mapper; 24 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 25 | import org.apache.hadoop.util.Tool; 26 | import org.apache.hadoop.util.ToolRunner; 27 | 28 | public class DistributedLzoIndexer extends Configured implements Tool { 29 | private static final Log LOG = LogFactory.getLog(DistributedLzoIndexer.class); 30 | private final String LZO_EXTENSION = new LzopCodec().getDefaultExtension(); 31 | 32 | private final PathFilter nonTemporaryFilter = new PathFilter() { 33 | @Override 34 | public boolean accept(Path path) { 35 | return !path.toString().endsWith("/_temporary"); 36 | } 37 | }; 38 | 39 | private void walkPath(Path path, PathFilter pathFilter, List accumulator) { 40 | try { 41 | FileSystem fs = path.getFileSystem(getConf()); 42 | FileStatus fileStatus = fs.getFileStatus(path); 43 | 44 | if (fileStatus.isDir()) { 45 | FileStatus[] children = fs.listStatus(path, pathFilter); 46 | for (FileStatus childStatus : children) { 47 | walkPath(childStatus.getPath(), pathFilter, accumulator); 48 | } 49 | } else if (path.toString().endsWith(LZO_EXTENSION)) { 50 | Path lzoIndexPath = path.suffix(LzoIndex.LZO_INDEX_SUFFIX); 51 | if (fs.exists(lzoIndexPath)) { 52 | // If the index exists and is of nonzero size, we're already done. 53 | // We re-index a file with a zero-length index, because every file has at least one block. 54 | if (fs.getFileStatus(lzoIndexPath).getLen() > 0) { 55 | LOG.info("[SKIP] LZO index file already exists for " + path); 56 | return; 57 | } else { 58 | LOG.info("Adding LZO file " + path + " to indexing list (index file exists but is zero length)"); 59 | accumulator.add(path); 60 | } 61 | } else { 62 | // If no index exists, we need to index the file. 63 | LOG.info("Adding LZO file " + path + " to indexing list (no index currently exists)"); 64 | accumulator.add(path); 65 | } 66 | } 67 | } catch (IOException ioe) { 68 | LOG.warn("Error walking path: " + path, ioe); 69 | } 70 | } 71 | 72 | @Override 73 | public int run(String[] args) throws Exception { 74 | if (args.length == 0 || (args.length == 1 && "--help".equals(args[0]))) { 75 | printUsage(); 76 | ToolRunner.printGenericCommandUsage(System.err); 77 | return -1; 78 | } 79 | 80 | List inputPaths = new ArrayList(); 81 | for (String strPath: args) { 82 | walkPath(new Path(strPath), nonTemporaryFilter, inputPaths); 83 | } 84 | 85 | if (inputPaths.isEmpty()) { 86 | System.err.println("No input paths found - perhaps all " + 87 | ".lzo files have already been indexed."); 88 | return 0; 89 | } 90 | 91 | Configuration conf = new Configuration(); 92 | Job job = new Job(conf); 93 | job.setJobName("Distributed Lzo Indexer " + Arrays.toString(args)); 94 | 95 | job.setOutputKeyClass(Path.class); 96 | job.setOutputValueClass(LongWritable.class); 97 | 98 | // The LzoIndexOutputFormat doesn't currently work with speculative execution. 99 | // Patches welcome. 100 | job.getConfiguration().setBoolean( 101 | "mapred.map.tasks.speculative.execution", false); 102 | 103 | job.setJarByClass(DistributedLzoIndexer.class); 104 | job.setInputFormatClass(LzoSplitInputFormat.class); 105 | job.setOutputFormatClass(LzoIndexOutputFormat.class); 106 | job.setNumReduceTasks(0); 107 | job.setMapperClass(Mapper.class); 108 | 109 | for (Path p : inputPaths) { 110 | FileInputFormat.addInputPath(job, p); 111 | } 112 | 113 | return job.waitForCompletion(true) ? 0 : 1; 114 | } 115 | 116 | public static void main(String[] args) throws Exception { 117 | int exitCode = ToolRunner.run(new DistributedLzoIndexer(), args); 118 | System.exit(exitCode); 119 | } 120 | 121 | public static void printUsage() { 122 | System.err.println("Usage: hadoop jar /path/to/this/jar com.hadoop.compression.lzo.DistributedLzoIndexer [file2.lzo directory3 ...]"); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/com/hadoop/compression/lzo/LzoCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | package com.hadoop.compression.lzo; 19 | 20 | 21 | import org.apache.hadoop.io.compress.CompressionCodec; 22 | 23 | /** 24 | * A {@link org.apache.hadoop.io.compress.CompressionCodec} for a streaming 25 | * lzo compression/decompression pair. 26 | * http://www.oberhumer.com/opensource/lzo/ 27 | * 28 | */ 29 | public class LzoCodec extends org.anarres.lzo.hadoop.codec.LzoCodec { 30 | } 31 | -------------------------------------------------------------------------------- /src/com/hadoop/compression/lzo/LzopCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | package com.hadoop.compression.lzo; 19 | 20 | 21 | import org.apache.hadoop.io.compress.CompressionCodec; 22 | 23 | /** 24 | * A {@link CompressionCodec} for a streaming 25 | * lzo compression/decompression pair compatible with lzop. 26 | * http://www.lzop.org/ 27 | */ 28 | public class LzopCodec extends org.anarres.lzo.hadoop.codec.LzopCodec { 29 | } 30 | -------------------------------------------------------------------------------- /src/com/hadoop/mapred/DeprecatedLzoLineRecordReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | 19 | package com.hadoop.mapred; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.fs.FSDataInputStream; 25 | import org.apache.hadoop.fs.FileSystem; 26 | import org.apache.hadoop.fs.Path; 27 | import org.apache.hadoop.io.LongWritable; 28 | import org.apache.hadoop.io.Text; 29 | import org.apache.hadoop.io.compress.CompressionCodec; 30 | import org.apache.hadoop.io.compress.CompressionCodecFactory; 31 | import org.apache.hadoop.mapred.FileSplit; 32 | import org.apache.hadoop.mapred.RecordReader; 33 | import org.apache.hadoop.util.LineReader; 34 | 35 | @SuppressWarnings("deprecation") 36 | public class DeprecatedLzoLineRecordReader implements RecordReader { 37 | private CompressionCodecFactory codecFactory = null; 38 | private long start; 39 | private long pos; 40 | private final long end; 41 | private final LineReader in; 42 | private final FSDataInputStream fileIn; 43 | 44 | DeprecatedLzoLineRecordReader(Configuration conf, FileSplit split) throws IOException { 45 | start = split.getStart(); 46 | end = start + split.getLength(); 47 | final Path file = split.getPath(); 48 | 49 | FileSystem fs = file.getFileSystem(conf); 50 | codecFactory = new CompressionCodecFactory(conf); 51 | final CompressionCodec codec = codecFactory.getCodec(file); 52 | if (codec == null) { 53 | throw new IOException("No LZO codec found, cannot run."); 54 | } 55 | 56 | // Open the file and seek to the next split. 57 | fileIn = fs.open(file); 58 | // Create input stream and read the file header. 59 | in = new LineReader(codec.createInputStream(fileIn), conf); 60 | if (start != 0) { 61 | fileIn.seek(start); 62 | 63 | // Read and ignore the first line. 64 | in.readLine(new Text()); 65 | start = fileIn.getPos(); 66 | } 67 | 68 | pos = start; 69 | } 70 | 71 | public LongWritable createKey() { 72 | return new LongWritable(); 73 | } 74 | 75 | public Text createValue() { 76 | return new Text(); 77 | } 78 | 79 | public boolean next(LongWritable key, Text value) throws IOException { 80 | // Since the LZOP codec reads everything in LZO blocks, we can't stop if pos == end. 81 | // Instead, wait for the next block to be read in when pos will be > end. 82 | while (pos <= end) { 83 | key.set(pos); 84 | 85 | int newSize = in.readLine(value); 86 | if (newSize == 0) { 87 | return false; 88 | } 89 | pos = fileIn.getPos(); 90 | return true; 91 | } 92 | return false; 93 | } 94 | 95 | public float getProgress() throws IOException { 96 | if (start == end) { 97 | return 0.0f; 98 | } else { 99 | return Math.min(1.0f, (pos - start)/ (float)(end - start)); 100 | } 101 | } 102 | 103 | public synchronized long getPos() throws IOException { 104 | return pos; 105 | } 106 | 107 | public synchronized void close() throws IOException { 108 | if (in != null) { 109 | in.close(); 110 | } 111 | } 112 | } -------------------------------------------------------------------------------- /src/com/hadoop/mapred/DeprecatedLzoTextInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | 19 | package com.hadoop.mapred; 20 | 21 | import java.io.IOException; 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.HashMap; 25 | import java.util.Iterator; 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | import org.apache.hadoop.fs.FileStatus; 30 | import org.apache.hadoop.fs.FileSystem; 31 | import org.apache.hadoop.fs.Path; 32 | import org.apache.hadoop.io.LongWritable; 33 | import org.apache.hadoop.io.Text; 34 | import org.apache.hadoop.mapred.FileInputFormat; 35 | import org.apache.hadoop.mapred.FileSplit; 36 | import org.apache.hadoop.mapred.InputSplit; 37 | import org.apache.hadoop.mapred.JobConf; 38 | import org.apache.hadoop.mapred.RecordReader; 39 | import org.apache.hadoop.mapred.Reporter; 40 | 41 | import com.hadoop.compression.lzo.LzoIndex; 42 | import com.hadoop.compression.lzo.LzopCodec; 43 | 44 | /** 45 | * This class conforms to the old (org.apache.hadoop.mapred.*) hadoop API style 46 | * which is deprecated but still required in places. Streaming, for example, 47 | * does a check that the given input format is a descendant of 48 | * org.apache.hadoop.mapred.InputFormat, which any InputFormat-derived class 49 | * from the new API fails. In order for streaming to work, you must use 50 | * com.hadoop.mapred.DeprecatedLzoTextInputFormat, not 51 | * com.hadoop.mapreduce.LzoTextInputFormat. The classes attempt to be alike in 52 | * every other respect. 53 | */ 54 | 55 | @SuppressWarnings("deprecation") 56 | public class DeprecatedLzoTextInputFormat extends FileInputFormat { 57 | public static final String LZO_INDEX_SUFFIX = ".index"; 58 | private final Map indexes = new HashMap(); 59 | 60 | @Override 61 | protected FileStatus[] listStatus(JobConf conf) throws IOException { 62 | List files = new ArrayList(Arrays.asList(super.listStatus(conf))); 63 | 64 | String fileExtension = new LzopCodec().getDefaultExtension(); 65 | 66 | Iterator it = files.iterator(); 67 | while (it.hasNext()) { 68 | FileStatus fileStatus = it.next(); 69 | Path file = fileStatus.getPath(); 70 | 71 | if (!file.toString().endsWith(fileExtension)) { 72 | // Get rid of non-LZO files. 73 | it.remove(); 74 | } else { 75 | FileSystem fs = file.getFileSystem(conf); 76 | LzoIndex index = LzoIndex.readIndex(fs, file); 77 | indexes.put(file, index); 78 | } 79 | } 80 | 81 | return files.toArray(new FileStatus[] {}); 82 | } 83 | 84 | @Override 85 | protected boolean isSplitable(FileSystem fs, Path filename) { 86 | LzoIndex index = indexes.get(filename); 87 | return !index.isEmpty(); 88 | } 89 | 90 | @Override 91 | public InputSplit[] getSplits(JobConf conf, int numSplits) throws IOException { 92 | FileSplit[] splits = (FileSplit[])super.getSplits(conf, numSplits); 93 | // Find new starts/ends of the filesplit that align with the LZO blocks. 94 | 95 | List result = new ArrayList(); 96 | 97 | for (FileSplit fileSplit: splits) { 98 | Path file = fileSplit.getPath(); 99 | FileSystem fs = file.getFileSystem(conf); 100 | LzoIndex index = indexes.get(file); 101 | if (index == null) { 102 | throw new IOException("Index not found for " + file); 103 | } 104 | if (index.isEmpty()) { 105 | // Empty index, keep it as is. 106 | result.add(fileSplit); 107 | continue; 108 | } 109 | 110 | long start = fileSplit.getStart(); 111 | long end = start + fileSplit.getLength(); 112 | 113 | long lzoStart = index.alignSliceStartToIndex(start, end); 114 | long lzoEnd = index.alignSliceEndToIndex(end, fs.getFileStatus(file).getLen()); 115 | 116 | if (lzoStart != LzoIndex.NOT_FOUND && lzoEnd != LzoIndex.NOT_FOUND) { 117 | result.add(new FileSplit(file, lzoStart, lzoEnd - lzoStart, fileSplit.getLocations())); 118 | } 119 | } 120 | 121 | return result.toArray(new FileSplit[result.size()]); 122 | } 123 | 124 | @Override 125 | public RecordReader getRecordReader(InputSplit split, 126 | JobConf conf, Reporter reporter) throws IOException { 127 | reporter.setStatus(split.toString()); 128 | return new DeprecatedLzoLineRecordReader(conf, (FileSplit)split); 129 | } 130 | 131 | } 132 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoIndexOutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.hadoop.mapreduce; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapred.FileAlreadyExistsException; 8 | import org.apache.hadoop.mapreduce.JobContext; 9 | import org.apache.hadoop.mapreduce.OutputCommitter; 10 | import org.apache.hadoop.mapreduce.OutputFormat; 11 | import org.apache.hadoop.mapreduce.RecordWriter; 12 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 13 | 14 | public class LzoIndexOutputFormat extends OutputFormat { 15 | @Override 16 | public RecordWriter getRecordWriter(TaskAttemptContext taskAttemptContext) 17 | throws IOException, InterruptedException { 18 | return new LzoIndexRecordWriter(taskAttemptContext); 19 | } 20 | 21 | @Override 22 | public void checkOutputSpecs(JobContext job) throws FileAlreadyExistsException, IOException {} 23 | 24 | // A totally no-op output committer, because the LzoIndexRecordWriter opens a file on the side 25 | // and writes to that instead. 26 | @Override 27 | public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) 28 | throws IOException, InterruptedException { 29 | return new OutputCommitter() { 30 | @Override public void setupJob(JobContext jobContext) throws IOException {} 31 | @Override public void cleanupJob(JobContext jobContext) throws IOException {} 32 | @Override public void setupTask(TaskAttemptContext taskAttemptContext) throws IOException {} 33 | @Override public void commitTask(TaskAttemptContext taskAttemptContext) throws IOException {} 34 | @Override public void abortTask(TaskAttemptContext taskAttemptContext) throws IOException {} 35 | @Override public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) throws IOException { 36 | return false; 37 | } 38 | }; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoIndexRecordWriter.java: -------------------------------------------------------------------------------- 1 | package com.hadoop.mapreduce; 2 | 3 | import java.io.IOException; 4 | 5 | import com.hadoop.compression.lzo.LzoIndex; 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.hadoop.fs.FSDataOutputStream; 9 | import org.apache.hadoop.fs.FileSystem; 10 | import org.apache.hadoop.fs.Path; 11 | import org.apache.hadoop.io.LongWritable; 12 | import org.apache.hadoop.mapreduce.RecordWriter; 13 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 14 | 15 | public class LzoIndexRecordWriter extends RecordWriter { 16 | private static final Log LOG = LogFactory.getLog(LzoIndexRecordWriter.class); 17 | 18 | private FSDataOutputStream outputStream; 19 | private final TaskAttemptContext context; 20 | 21 | private FileSystem fs; 22 | private Path inputPath; 23 | private Path tmpIndexPath; 24 | private Path realIndexPath; 25 | 26 | public LzoIndexRecordWriter(TaskAttemptContext taskAttemptContext) { 27 | context = taskAttemptContext; 28 | } 29 | 30 | @Override 31 | public void write(Path path, LongWritable offset) throws IOException, InterruptedException { 32 | if (outputStream == null) { 33 | // Set up the output file on the first record. 34 | LOG.info("Setting up output stream to write index file for " + path); 35 | outputStream = setupOutputFile(path); 36 | } 37 | offset.write(outputStream); 38 | } 39 | 40 | @Override 41 | public void close(TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { 42 | if (outputStream != null) { 43 | // Close the output stream so that the tmp file is synced, then move it. 44 | outputStream.close(); 45 | 46 | LOG.info("In close, now renaming " + tmpIndexPath + " to final location " + realIndexPath); 47 | // Rename, indexing completed. 48 | fs.rename(tmpIndexPath, realIndexPath); 49 | } 50 | } 51 | 52 | private FSDataOutputStream setupOutputFile(Path path) throws IOException { 53 | fs = path.getFileSystem(context.getConfiguration()); 54 | inputPath = path; 55 | 56 | // For /a/b/c.lzo, tmpIndexPath = /a/b/c.lzo.index.tmp, 57 | // and it is moved to realIndexPath = /a/b/c.lzo.index upon completion. 58 | tmpIndexPath = path.suffix(LzoIndex.LZO_TMP_INDEX_SUFFIX); 59 | realIndexPath = path.suffix(LzoIndex.LZO_INDEX_SUFFIX); 60 | 61 | // Delete the old index files if they exist. 62 | fs.delete(tmpIndexPath, false); 63 | fs.delete(realIndexPath, false); 64 | 65 | return fs.create(tmpIndexPath, false); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoLineRecordReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | 19 | package com.hadoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.fs.FSDataInputStream; 25 | import org.apache.hadoop.fs.FileSystem; 26 | import org.apache.hadoop.fs.Path; 27 | import org.apache.hadoop.io.LongWritable; 28 | import org.apache.hadoop.io.Text; 29 | import org.apache.hadoop.io.compress.CompressionCodec; 30 | import org.apache.hadoop.io.compress.CompressionCodecFactory; 31 | import org.apache.hadoop.mapreduce.InputSplit; 32 | import org.apache.hadoop.mapreduce.RecordReader; 33 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 34 | import org.apache.hadoop.mapreduce.lib.input.FileSplit; 35 | import org.apache.hadoop.util.LineReader; 36 | 37 | /** 38 | * Reads line from an lzo compressed text file. Treats keys as offset in file 39 | * and value as line. 40 | */ 41 | public class LzoLineRecordReader extends RecordReader { 42 | 43 | private long start; 44 | private long pos; 45 | private long end; 46 | private LineReader in; 47 | private FSDataInputStream fileIn; 48 | 49 | private final LongWritable key = new LongWritable(); 50 | private final Text value = new Text(); 51 | 52 | /** 53 | * Get the progress within the split. 54 | */ 55 | @Override 56 | public float getProgress() { 57 | if (start == end) { 58 | return 0.0f; 59 | } else { 60 | return Math.min(1.0f, (pos - start) / (float) (end - start)); 61 | } 62 | } 63 | 64 | public synchronized long getPos() throws IOException { 65 | return pos; 66 | } 67 | 68 | @Override 69 | public synchronized void close() throws IOException { 70 | if (in != null) { 71 | in.close(); 72 | } 73 | } 74 | 75 | @Override 76 | public LongWritable getCurrentKey() throws IOException, InterruptedException { 77 | return key; 78 | } 79 | 80 | @Override 81 | public Text getCurrentValue() throws IOException, InterruptedException { 82 | return value; 83 | } 84 | 85 | @Override 86 | public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException { 87 | FileSplit split = (FileSplit) genericSplit; 88 | start = split.getStart(); 89 | end = start + split.getLength(); 90 | final Path file = split.getPath(); 91 | Configuration job = context.getConfiguration(); 92 | 93 | FileSystem fs = file.getFileSystem(job); 94 | CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(job); 95 | final CompressionCodec codec = compressionCodecs.getCodec(file); 96 | if (codec == null) { 97 | throw new IOException("Codec for file " + file + " not found, cannot run"); 98 | } 99 | 100 | // open the file and seek to the start of the split 101 | fileIn = fs.open(split.getPath()); 102 | 103 | // creates input stream and also reads the file header 104 | in = new LineReader(codec.createInputStream(fileIn), job); 105 | 106 | if (start != 0) { 107 | fileIn.seek(start); 108 | 109 | // read and ignore the first line 110 | in.readLine(new Text()); 111 | start = fileIn.getPos(); 112 | } 113 | 114 | this.pos = start; 115 | } 116 | 117 | @Override 118 | public boolean nextKeyValue() throws IOException, InterruptedException { 119 | //since the lzop codec reads everything in lzo blocks 120 | //we can't stop if the pos == end 121 | //instead we wait for the next block to be read in when 122 | //pos will be > end 123 | while (pos <= end) { 124 | key.set(pos); 125 | 126 | int newSize = in.readLine(value); 127 | if (newSize == 0) { 128 | return false; 129 | } 130 | pos = fileIn.getPos(); 131 | 132 | return true; 133 | } 134 | 135 | return false; 136 | } 137 | } 138 | 139 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoSplitInputFormat.java: -------------------------------------------------------------------------------- 1 | package com.hadoop.mapreduce; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.fs.Path; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.InputSplit; 8 | import org.apache.hadoop.mapreduce.JobContext; 9 | import org.apache.hadoop.mapreduce.RecordReader; 10 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | 13 | public class LzoSplitInputFormat extends FileInputFormat { 14 | 15 | @Override 16 | public RecordReader createRecordReader(InputSplit inputSplit, 17 | TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { 18 | return new LzoSplitRecordReader(); 19 | } 20 | 21 | @Override 22 | protected boolean isSplitable(JobContext context, Path filename) { 23 | // Force the files to be unsplittable, because indexing requires seeing all the 24 | // compressed blocks in succession. 25 | return false; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoSplitRecordReader.java: -------------------------------------------------------------------------------- 1 | package com.hadoop.mapreduce; 2 | 3 | import java.io.EOFException; 4 | import java.io.IOException; 5 | 6 | import org.anarres.lzo.LzopInputStream; 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | import org.apache.hadoop.conf.Configurable; 10 | import org.apache.hadoop.conf.Configuration; 11 | import org.apache.hadoop.fs.FSDataInputStream; 12 | import org.apache.hadoop.fs.FileSystem; 13 | import org.apache.hadoop.fs.Path; 14 | import org.apache.hadoop.io.LongWritable; 15 | import org.apache.hadoop.io.compress.CompressionCodec; 16 | import org.apache.hadoop.io.compress.CompressionCodecFactory; 17 | import org.apache.hadoop.mapreduce.InputSplit; 18 | import org.apache.hadoop.mapreduce.RecordReader; 19 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 20 | import org.apache.hadoop.mapreduce.lib.input.FileSplit; 21 | 22 | public class LzoSplitRecordReader extends RecordReader { 23 | private static final Log LOG = LogFactory.getLog(LzoSplitRecordReader.class); 24 | 25 | private final int LOG_EVERY_N_BLOCKS = 1000; 26 | 27 | private final LongWritable curValue = new LongWritable(-1); 28 | private FSDataInputStream rawInputStream; 29 | private TaskAttemptContext context; 30 | 31 | private int numBlocksRead = 0; 32 | private int numDecompressedChecksums = -1; 33 | private int numCompressedChecksums = -1; 34 | private long totalFileSize = 0; 35 | private Path lzoFile; 36 | 37 | @Override 38 | public void initialize(InputSplit genericSplit, TaskAttemptContext taskAttemptContext) throws IOException { 39 | context = taskAttemptContext; 40 | FileSplit fileSplit = (FileSplit)genericSplit; 41 | lzoFile = fileSplit.getPath(); 42 | // The LzoSplitInputFormat is not splittable, so the split length is the whole file. 43 | totalFileSize = fileSplit.getLength(); 44 | 45 | // Jump through some hoops to create the lzo codec. 46 | Configuration conf = context.getConfiguration(); 47 | CompressionCodecFactory factory = new CompressionCodecFactory(conf); 48 | CompressionCodec codec = factory.getCodec(lzoFile); 49 | ((Configurable)codec).setConf(conf); 50 | 51 | FileSystem fs = lzoFile.getFileSystem(conf); 52 | rawInputStream = fs.open(lzoFile); 53 | 54 | // Creating the LzopInputStream here just reads the lzo header for us, nothing more. 55 | // We do the rest of our input off of the raw stream is. 56 | LzopInputStream lzis = new LzopInputStream(rawInputStream); 57 | 58 | // This must be called AFTER createInputStream is called, because createInputStream 59 | // is what reads the header, which has the checksum information. Otherwise getChecksumsCount 60 | // erroneously returns zero, and all block offsets will be wrong. 61 | numCompressedChecksums = lzis.getCompressedChecksumCount(); 62 | numDecompressedChecksums = lzis.getUncompressedChecksumCount(); 63 | } 64 | 65 | @Override 66 | public boolean nextKeyValue() throws IOException { 67 | int uncompressedBlockSize = rawInputStream.readInt(); 68 | if (uncompressedBlockSize == 0) { 69 | // An uncompressed block size of zero means end of file. 70 | return false; 71 | } else if (uncompressedBlockSize < 0) { 72 | throw new EOFException("Could not read uncompressed block size at position " + 73 | rawInputStream.getPos() + " in file " + lzoFile); 74 | } 75 | 76 | int compressedBlockSize = rawInputStream.readInt(); 77 | if (compressedBlockSize <= 0) { 78 | throw new EOFException("Could not read compressed block size at position " + 79 | rawInputStream.getPos() + " in file " + lzoFile); 80 | } 81 | 82 | // See LzopInputStream.getCompressedData 83 | boolean isUncompressedBlock = (uncompressedBlockSize == compressedBlockSize); 84 | int numChecksumsToSkip = isUncompressedBlock ? 85 | numDecompressedChecksums : numDecompressedChecksums + numCompressedChecksums; 86 | 87 | // Get the current position. Since we've read two ints, the current block started 8 bytes ago. 88 | long pos = rawInputStream.getPos(); 89 | curValue.set(pos - 8); 90 | // Seek beyond the checksums and beyond the block data to the beginning of the next block. 91 | rawInputStream.seek(pos + compressedBlockSize + (4 * numChecksumsToSkip)); 92 | ++numBlocksRead; 93 | 94 | // Log some progress every so often. 95 | if (numBlocksRead % LOG_EVERY_N_BLOCKS == 0) { 96 | LOG.info("Reading block " + numBlocksRead + " at pos " + pos + " of " + totalFileSize + ". Read is " + 97 | (100.0 * getProgress()) + "% done. "); 98 | } 99 | 100 | return true; 101 | } 102 | 103 | @Override 104 | public Path getCurrentKey() { 105 | return lzoFile; 106 | } 107 | 108 | @Override 109 | public LongWritable getCurrentValue() { 110 | return curValue; 111 | } 112 | 113 | @Override 114 | public float getProgress() throws IOException { 115 | if (totalFileSize == 0) { 116 | return 0.0f; 117 | } else { 118 | return (float)rawInputStream.getPos() / totalFileSize; 119 | } 120 | } 121 | 122 | @Override 123 | public void close() throws IOException { 124 | LOG.info("Closing input stream after reading " + numBlocksRead + " blocks from " + lzoFile); 125 | rawInputStream.close(); 126 | } 127 | } 128 | 129 | -------------------------------------------------------------------------------- /src/com/hadoop/mapreduce/LzoTextInputFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it 5 | * and/or modify it under the terms of the GNU General Public License 6 | * as published by the Free Software Foundation, either version 3 of 7 | * the License, or (at your option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be 10 | * useful, but WITHOUT ANY WARRANTY; without even the implied warranty 11 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Hadoop-Gpl-Compression. If not, see 16 | * . 17 | */ 18 | 19 | package com.hadoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | import java.util.ArrayList; 23 | import java.util.HashMap; 24 | import java.util.Iterator; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | import org.apache.hadoop.conf.Configuration; 29 | import org.apache.hadoop.fs.FileStatus; 30 | import org.apache.hadoop.fs.FileSystem; 31 | import org.apache.hadoop.fs.Path; 32 | import org.apache.hadoop.io.LongWritable; 33 | import org.apache.hadoop.io.Text; 34 | import org.apache.hadoop.mapreduce.InputFormat; 35 | import org.apache.hadoop.mapreduce.InputSplit; 36 | import org.apache.hadoop.mapreduce.JobContext; 37 | import org.apache.hadoop.mapreduce.RecordReader; 38 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 39 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 40 | import org.apache.hadoop.mapreduce.lib.input.FileSplit; 41 | 42 | import com.hadoop.compression.lzo.LzoIndex; 43 | import com.hadoop.compression.lzo.LzopCodec; 44 | 45 | /** 46 | * An {@link InputFormat} for lzop compressed text files. Files are broken into 47 | * lines. Either linefeed or carriage-return are used to signal end of line. 48 | * Keys are the position in the file, and values are the line of text. 49 | */ 50 | public class LzoTextInputFormat extends FileInputFormat { 51 | 52 | private final Map indexes = new HashMap(); 53 | 54 | @Override 55 | protected List listStatus(JobContext job) throws IOException { 56 | List files = super.listStatus(job); 57 | 58 | String fileExtension = new LzopCodec().getDefaultExtension(); 59 | Configuration conf = job.getConfiguration(); 60 | 61 | for (Iterator iterator = files.iterator(); iterator.hasNext();) { 62 | FileStatus fileStatus = iterator.next(); 63 | Path file = fileStatus.getPath(); 64 | FileSystem fs = file.getFileSystem(conf); 65 | 66 | if (!file.toString().endsWith(fileExtension)) { 67 | //get rid of non lzo files 68 | iterator.remove(); 69 | } else { 70 | //read the index file 71 | LzoIndex index = LzoIndex.readIndex(fs, file); 72 | indexes.put(file, index); 73 | } 74 | } 75 | 76 | return files; 77 | } 78 | 79 | @Override 80 | protected boolean isSplitable(JobContext context, Path filename) { 81 | LzoIndex index = indexes.get(filename); 82 | return !index.isEmpty(); 83 | } 84 | 85 | @Override 86 | public List getSplits(JobContext job) throws IOException { 87 | List splits = super.getSplits(job); 88 | Configuration conf = job.getConfiguration(); 89 | // find new start/ends of the filesplit that aligns 90 | // with the lzo blocks 91 | 92 | List result = new ArrayList(); 93 | 94 | for (InputSplit genericSplit : splits) { 95 | // load the index 96 | FileSplit fileSplit = (FileSplit) genericSplit; 97 | Path file = fileSplit.getPath(); 98 | FileSystem fs = file.getFileSystem(conf); 99 | LzoIndex index = indexes.get(file); 100 | if (index == null) { 101 | throw new IOException("Index not found for " + file); 102 | } 103 | 104 | if (index.isEmpty()) { 105 | // empty index, keep as is 106 | result.add(fileSplit); 107 | continue; 108 | } 109 | 110 | long start = fileSplit.getStart(); 111 | long end = start + fileSplit.getLength(); 112 | 113 | long lzoStart = index.alignSliceStartToIndex(start, end); 114 | long lzoEnd = index.alignSliceEndToIndex(end, fs.getFileStatus(file).getLen()); 115 | 116 | if (lzoStart != LzoIndex.NOT_FOUND && lzoEnd != LzoIndex.NOT_FOUND) { 117 | result.add(new FileSplit(file, lzoStart, lzoEnd - lzoStart, fileSplit.getLocations())); 118 | } 119 | } 120 | 121 | return result; 122 | } 123 | 124 | @Override 125 | public RecordReader createRecordReader(InputSplit split, 126 | TaskAttemptContext taskAttempt) throws IOException, InterruptedException { 127 | 128 | return new LzoLineRecordReader(); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/AbstractLzo1Compressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public abstract class AbstractLzo1Compressor extends AbstractLzoCompressor { 49 | 50 | public AbstractLzo1Compressor(LzoAlgorithm algorithm, LzoConstraint... constraints) { 51 | super(algorithm, constraints); 52 | } 53 | 54 | @Override 55 | public int getCompressionOverhead(int inputBufferSize) { 56 | return (inputBufferSize >> 4) + 64 + 3; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/AbstractLzo2Compressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public abstract class AbstractLzo2Compressor extends AbstractLzoCompressor { 49 | 50 | public AbstractLzo2Compressor(LzoAlgorithm algorithm, LzoConstraint... constraints) { 51 | super(algorithm, constraints); 52 | } 53 | 54 | @Override 55 | public int getCompressionOverhead(int inputBufferSize) { 56 | return (inputBufferSize >> 3) + 128 + 3; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/AbstractLzoCompressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public abstract class AbstractLzoCompressor extends AbstractLzoTransformer implements LzoCompressor { 49 | 50 | public AbstractLzoCompressor(LzoAlgorithm algorithm, LzoConstraint... constraints) { 51 | super(algorithm, constraints); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/AbstractLzoDecompressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public abstract class AbstractLzoDecompressor extends AbstractLzoTransformer implements LzoDecompressor { 49 | 50 | public AbstractLzoDecompressor(LzoAlgorithm algorithm, LzoConstraint... constraints) { 51 | super(algorithm, constraints); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/AbstractLzoTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public class AbstractLzoTransformer implements LzoTransformer { 49 | 50 | private final LzoAlgorithm algorithm; 51 | private final LzoConstraint[] constraints; 52 | 53 | public AbstractLzoTransformer(LzoAlgorithm algorithm, LzoConstraint... constraints) { 54 | this.algorithm = algorithm; 55 | this.constraints = constraints; 56 | } 57 | 58 | @Override 59 | public LzoAlgorithm getAlgorithm() { 60 | return algorithm; 61 | } 62 | 63 | @Override 64 | public LzoConstraint[] getConstraints() { 65 | return constraints; 66 | } 67 | 68 | @Override 69 | public String toErrorString(int code) { 70 | switch (code) { 71 | case LZO_E_OK: 72 | return "OK"; 73 | case LZO_E_ERROR: 74 | return "Error"; 75 | case LZO_E_OUT_OF_MEMORY: 76 | return "Out of memory"; 77 | case LZO_E_NOT_COMPRESSIBLE: 78 | return "Not compressible"; 79 | case LZO_E_INPUT_OVERRUN: 80 | return "Input overrun"; 81 | case LZO_E_OUTPUT_OVERRUN: 82 | return "Output overrun"; 83 | case LZO_E_LOOKBEHIND_OVERRUN: 84 | return "Lookbehind overrun"; 85 | case LZO_E_EOF_NOT_FOUND: 86 | return "EOF not found"; 87 | case LZO_E_INPUT_NOT_CONSUMED: 88 | return "Input not consumed"; 89 | default: 90 | return "Unknown-" + code; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoAlgorithm.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public enum LzoAlgorithm { 49 | 50 | LZO1, 51 | LZO1A, 52 | LZO1B, 53 | LZO1C, 54 | LZO1F, 55 | LZO1X, 56 | LZO1Y, 57 | LZO1Z, 58 | LZO2A; 59 | } 60 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoCompressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public interface LzoCompressor extends LzoTransformer { 49 | 50 | public int getCompressionOverhead(int inputBufferSize); 51 | 52 | public int compress(byte[] in, int in_base, int in_len, 53 | byte[] out, int out_base, lzo_uintp out_len); 54 | } 55 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoConstraint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public enum LzoConstraint { 49 | 50 | SPEED, MEMORY, COMPRESSION, SAFETY; 51 | } 52 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoDecompressor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public interface LzoDecompressor extends LzoTransformer { 49 | 50 | public int decompress(byte[] in, int in_base, int in_len, 51 | byte[] out, int out_base, lzo_uintp out_len); 52 | } 53 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoLibrary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public class LzoLibrary { 49 | 50 | private static class Inner { 51 | 52 | private static final LzoLibrary INSTANCE = new LzoLibrary(); 53 | } 54 | 55 | public static LzoLibrary getInstance() { 56 | return Inner.INSTANCE; 57 | } 58 | 59 | /** 60 | * Returns a new compressor for the given algorithm. 61 | */ 62 | public LzoCompressor newCompressor(LzoAlgorithm algorithm, LzoConstraint constraint) { 63 | if (algorithm == null) 64 | return new LzoCompressor1x_1(); 65 | switch (algorithm) { 66 | case LZO1X: 67 | return new LzoCompressor1x_1(); 68 | case LZO1Y: 69 | return new LzoCompressor1y_1(); 70 | default: 71 | throw new UnsupportedOperationException("Unsupported algorithm " + algorithm); 72 | } 73 | } 74 | 75 | /** 76 | * Returns a new decompressor for the given algorithm. 77 | * The only constraint which makes sense is {@link LzoConstraint#SAFETY}. 78 | */ 79 | public LzoDecompressor newDecompressor(LzoAlgorithm algorithm, LzoConstraint constraint) { 80 | if (algorithm == null) 81 | throw new NullPointerException("No algorithm specified."); 82 | switch (algorithm) { 83 | 84 | case LZO1X: 85 | if (constraint == LzoConstraint.SAFETY) 86 | return new LzoDecompressor1x_safe(); 87 | else 88 | return new LzoDecompressor1x(); 89 | 90 | case LZO1Y: 91 | if (constraint == LzoConstraint.SAFETY) 92 | return new LzoDecompressor1y_safe(); 93 | else 94 | return new LzoDecompressor1y(); 95 | 96 | case LZO1Z: 97 | if (constraint == LzoConstraint.SAFETY) 98 | return new LzoDecompressor1z_safe(); 99 | else 100 | return new LzoDecompressor1z(); 101 | 102 | default: 103 | throw new UnsupportedOperationException("Unsupported algorithm " + algorithm); 104 | 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/LzoTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public interface LzoTransformer { 49 | 50 | public static final int LZO_E_OK = 0; 51 | public static final int LZO_E_ERROR = -1; 52 | public static final int LZO_E_OUT_OF_MEMORY = -2; 53 | public static final int LZO_E_NOT_COMPRESSIBLE = -3; 54 | public static final int LZO_E_INPUT_OVERRUN = -4; 55 | public static final int LZO_E_OUTPUT_OVERRUN = -5; 56 | public static final int LZO_E_LOOKBEHIND_OVERRUN = -6; 57 | public static final int LZO_E_EOF_NOT_FOUND = -7; 58 | public static final int LZO_E_INPUT_NOT_CONSUMED = -8; 59 | 60 | public LzoAlgorithm getAlgorithm(); 61 | 62 | public LzoConstraint[] getConstraints(); 63 | 64 | public String toErrorString(int code); 65 | } 66 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/hadoop/codec/LzopCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * This file is based on a file from hadoop-gpl-compression. 10 | * 11 | * This library is free software; you can redistribute it and/or 12 | * modify it under the terms of the GNU General Public License 13 | * as published by the Free Software Foundation; either version 14 | * 2 of the License, or (at your option) any later version. 15 | * 16 | * This library is distributed in the hope that it will be useful, 17 | * but WITHOUT ANY WARRANTY; without even the implied warranty 18 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 19 | * See the GNU General Public License for more details. 20 | * 21 | * You should have received a copy of the GNU General Public 22 | * License along with the LZO library; see the file COPYING. 23 | * If not, see or write to the 24 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 25 | * Floor, Boston, MA 02110-1301, USA. 26 | */ 27 | package org.anarres.lzo.hadoop.codec; 28 | 29 | import java.io.IOException; 30 | import java.io.InputStream; 31 | import java.io.OutputStream; 32 | 33 | import org.apache.hadoop.io.compress.CompressionInputStream; 34 | import org.apache.hadoop.io.compress.CompressionOutputStream; 35 | import org.apache.hadoop.io.compress.Compressor; 36 | import org.apache.hadoop.io.compress.Decompressor; 37 | 38 | import org.apache.commons.logging.Log; 39 | import org.apache.commons.logging.LogFactory; 40 | import org.apache.hadoop.conf.Configuration; 41 | import org.apache.hadoop.io.compress.CompressionCodec; 42 | 43 | /** 44 | * A {@link CompressionCodec} for a streaming 45 | * lzo compression/decompression pair compatible with lzop. 46 | * http://www.lzop.org/ 47 | */ 48 | public class LzopCodec extends LzoCodec { 49 | 50 | @Override 51 | public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { 52 | Configuration conf = getConf(); 53 | LzoCompressor.CompressionStrategy strategy = LzoCodec.getCompressionStrategy(conf); 54 | int bufferSize = LzoCodec.getBufferSize(conf); 55 | return new LzopOutputStream(out, strategy, bufferSize); 56 | } 57 | 58 | @Override 59 | public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { 60 | // Ensure native-lzo library is loaded & initialized 61 | return new LzopInputStream(in); 62 | } 63 | 64 | @Override 65 | public Decompressor createDecompressor() { 66 | return null; 67 | } 68 | 69 | @Override 70 | public String getDefaultExtension() { 71 | return ".lzo"; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/hadoop/codec/LzopInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo.hadoop.codec; 6 | 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import org.apache.hadoop.io.compress.CompressionInputStream; 10 | 11 | /** 12 | * 13 | * @author shevek 14 | */ 15 | public class LzopInputStream extends CompressionInputStream { 16 | 17 | public LzopInputStream(InputStream in) throws IOException { 18 | super(new org.anarres.lzo.LzopInputStream(in)); 19 | } 20 | 21 | @Override 22 | public int read(byte[] buf, int off, int len) throws IOException { 23 | return in.read(buf, off, len); 24 | } 25 | 26 | @Override 27 | public int read() throws IOException { 28 | return in.read(); 29 | } 30 | 31 | @Override 32 | public void resetState() throws IOException { 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/hadoop/codec/LzopOutputStream.java: -------------------------------------------------------------------------------- 1 | package org.anarres.lzo.hadoop.codec; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | import org.apache.hadoop.io.compress.CompressionOutputStream; 6 | 7 | public class LzopOutputStream extends CompressionOutputStream { 8 | 9 | public LzopOutputStream(OutputStream out, LzoCompressor.CompressionStrategy strategy, int bufferSize) throws IOException { 10 | super(new org.anarres.lzo.LzopOutputStream(out, strategy.newCompressor(), bufferSize)); 11 | } 12 | 13 | @Override 14 | public void write(byte[] buf, int off, int len) throws IOException { 15 | out.write(buf, off, len); 16 | } 17 | 18 | @Override 19 | public void write(int b) throws IOException { 20 | out.write(b); 21 | } 22 | 23 | @Override 24 | public void finish() throws IOException { 25 | } 26 | 27 | @Override 28 | public void resetState() throws IOException { 29 | throw new UnsupportedOperationException("Not supported yet."); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/org/anarres/lzo/lzo_uintp.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of lzo-java, an implementation of LZO in Java. 3 | * https://github.com/Karmasphere/lzo-java 4 | * 5 | * The Java portion of this library is: 6 | * Copyright (C) 2011 Shevek 7 | * All Rights Reserved. 8 | * 9 | * The preprocessed C portion of this library is: 10 | * Copyright (C) 2006-2011 Markus Franz Xaver Johannes Oberhumer 11 | * All Rights Reserved. 12 | * 13 | * This library is free software; you can redistribute it and/or 14 | * modify it under the terms of the GNU General Public License 15 | * as published by the Free Software Foundation; either version 16 | * 2 of the License, or (at your option) any later version. 17 | * 18 | * This library is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty 20 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 21 | * See the GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public 24 | * License along with the LZO library; see the file COPYING. 25 | * If not, see or write to the 26 | * Free Software Foundation, Inc., 51 Franklin Street, Fifth 27 | * Floor, Boston, MA 02110-1301, USA. 28 | 29 | * As a special exception, the copyright holders of this file 30 | * give you permission to link this file with independent 31 | * modules to produce an executable, regardless of the license 32 | * terms of these independent modules, and to copy and distribute 33 | * the resulting executable under terms of your choice, provided 34 | * that you also meet, for each linked independent module, 35 | * the terms and conditions of the license of that module. An 36 | * independent module is a module which is not derived from or 37 | * based on this library or file. If you modify this file, you may 38 | * extend this exception to your version of the file, but 39 | * you are not obligated to do so. If you do not wish to do so, 40 | * delete this exception statement from your version. 41 | */ 42 | package org.anarres.lzo; 43 | 44 | /** 45 | * 46 | * @author shevek 47 | */ 48 | public class lzo_uintp { 49 | 50 | public int value; 51 | 52 | public lzo_uintp() { 53 | } 54 | 55 | public lzo_uintp(int value) { 56 | this.value = value; 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return String.valueOf(value); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/org/apache/hadoop/io/compress/LzoCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Hadoop-Gpl-Compression. 3 | * 4 | * Hadoop-Gpl-Compression is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or (at your 7 | * option) any later version. 8 | * 9 | * Hadoop-Gpl-Compression is distributed in the hope that it will be useful, but 10 | * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 12 | * details. 13 | * 14 | * You should have received a copy of the GNU General Public License along with 15 | * Hadoop-Gpl-Compression. If not, see . 16 | */ 17 | 18 | package org.apache.hadoop.io.compress; 19 | 20 | import java.io.IOException; 21 | import java.io.OutputStream; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | 26 | /** 27 | * This is a bridging class whose sole purpose is to provide backward 28 | * compatibility of applications that historically depend on 29 | * {@link org.apache.hadoop.io.compress.LzoCodec} (such as SequenceFile). 30 | * 31 | * The class is marked as @deprecated and should not be used explicitly in the 32 | * future. 33 | * 34 | * A warning message will be generated when a user wants to use this class to 35 | * generate LZO compressed data. Not the case for decompression because this is 36 | * the legitimate backward compatibility usage. 37 | */ 38 | @Deprecated 39 | public class LzoCodec extends com.hadoop.compression.lzo.LzoCodec { 40 | private static final Log LOG = LogFactory.getLog(LzoCodec.class); 41 | 42 | static final String oahLzoCodec = LzoCodec.class.getName(); 43 | static final String chclLzoCodec = 44 | com.hadoop.compression.lzo.LzoCodec.class.getName(); 45 | static boolean warned = false; 46 | 47 | static { 48 | LOG.info("Bridging " + oahLzoCodec + " to " + chclLzoCodec + "."); 49 | } 50 | 51 | @Override 52 | public CompressionOutputStream createOutputStream(OutputStream out, 53 | Compressor compressor) throws IOException { 54 | if (!warned) { 55 | LOG.warn(oahLzoCodec + " is deprecated. You should use " + chclLzoCodec 56 | + " instead to generate LZO compressed data."); 57 | warned = true; 58 | } 59 | return super.createOutputStream(out, compressor); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/org/apache/hadoop/io/compress/LzopCodec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.apache.hadoop.io.compress; 6 | 7 | /** 8 | * 9 | * @author shevek 10 | */ 11 | public class LzopCodec extends com.hadoop.compression.lzo.LzopCodec { 12 | } 13 | -------------------------------------------------------------------------------- /test/data/README: -------------------------------------------------------------------------------- 1 | These files were created using the commandline version of lzop. 2 | The various flags included --crc32, -C -C, and so forth. 3 | -------------------------------------------------------------------------------- /test/data/compressed.txt: -------------------------------------------------------------------------------- 1 | foo a baz 1 2 | foo x baz 2 3 | foo y baz 3 4 | foo z baz 4 5 | foo b baz 5 6 | -------------------------------------------------------------------------------- /test/data/compressed.txt.lzo.adler32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/test/data/compressed.txt.lzo.adler32 -------------------------------------------------------------------------------- /test/data/compressed.txt.lzo.adler32d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/test/data/compressed.txt.lzo.adler32d -------------------------------------------------------------------------------- /test/data/compressed.txt.lzo.crc32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/test/data/compressed.txt.lzo.crc32 -------------------------------------------------------------------------------- /test/data/compressed.txt.lzo.crc32d: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/test/data/compressed.txt.lzo.crc32d -------------------------------------------------------------------------------- /test/data/temp.txt: -------------------------------------------------------------------------------- 1 | This is my test text file. 2 | This is my test text file. 3 | This is my test text file. 4 | This is my test text file. 5 | This is my test text file. 6 | This is my test text file. 7 | This is my test text file. 8 | -------------------------------------------------------------------------------- /test/data/temp.txt.lzo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Karmasphere/lzo-java/af259500dc851019ae62a0adfd9ce8883d2694f6/test/data/temp.txt.lzo -------------------------------------------------------------------------------- /test/org/anarres/lzo/BinaryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import org.junit.Test; 8 | import static org.junit.Assert.*; 9 | 10 | /** 11 | * 12 | * @author shevek 13 | */ 14 | public class BinaryTest { 15 | 16 | private static int U(byte b) { 17 | return b & 0xff; 18 | } 19 | 20 | // Reproduced here so that we can make the method in LZO private and static, for inlining. 21 | private static int UA_GET32(byte[] in, int in_ptr) { 22 | return (U(in[in_ptr]) << 24) | (U(in[in_ptr + 1]) << 16) | (U(in[in_ptr + 2]) << 8) | U(in[in_ptr + 3]); 23 | } 24 | 25 | @Test 26 | public void testShifts() { 27 | byte[] data = new byte[]{(byte) 0x81, 1, (byte) 0x82, 3}; 28 | int value = UA_GET32(data, 0); 29 | System.out.println("Byte is " + Integer.toHexString((data[0] & 0xff) << 24)); 30 | System.out.println("Byte is " + Integer.toHexString((data[1] & 0xff) << 16)); 31 | System.out.println("Byte is " + Integer.toHexString((data[2] & 0xff) << 8)); 32 | System.out.println("Byte is " + Integer.toHexString((data[3] & 0xff) << 0)); 33 | System.out.println("Value is " + Integer.toHexString(value)); 34 | assertEquals(0x81018203, value); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/BlockCompressorStreamTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.io.ByteArrayInputStream; 8 | import java.io.ByteArrayOutputStream; 9 | import java.io.DataInputStream; 10 | import java.io.File; 11 | import java.io.FileInputStream; 12 | import org.anarres.lzo.hadoop.codec.LzoCompressor; 13 | import org.anarres.lzo.hadoop.codec.LzoDecompressor; 14 | import org.apache.commons.io.FileUtils; 15 | import org.apache.commons.io.IOUtils; 16 | import org.apache.commons.logging.Log; 17 | import org.apache.commons.logging.LogFactory; 18 | import org.apache.hadoop.io.compress.BlockCompressorStream; 19 | import org.apache.hadoop.io.compress.BlockDecompressorStream; 20 | import org.junit.Test; 21 | 22 | /** 23 | * 24 | * @author shevek 25 | */ 26 | public class BlockCompressorStreamTest { 27 | 28 | private static final Log LOG = LogFactory.getLog(BlockCompressorStreamTest.class); 29 | private static final String PATH = "/home/shevek/vm/karmasphere-aws-vm/Karmasphere AWS VM-s003.vmdk"; 30 | 31 | @Test 32 | public void testBlockCompressorStream() throws Throwable { 33 | try { 34 | LOG.info("Total memory is " + Runtime.getRuntime().totalMemory()); 35 | LOG.info("Max memory is " + Runtime.getRuntime().maxMemory()); 36 | 37 | FileInputStream fi = new FileInputStream(new File(PATH)); 38 | DataInputStream di = new DataInputStream(fi); 39 | byte[] data = new byte[512 * 1024 /* * 1024 */]; 40 | di.readFully(data); 41 | LOG.info("Original data is " + data.length + " bytes."); 42 | 43 | for (int i = 0; i < 1; i++) { 44 | ByteArrayInputStream bi = new ByteArrayInputStream(data); 45 | ByteArrayOutputStream bo = new ByteArrayOutputStream(data.length); 46 | BlockCompressorStream co = new BlockCompressorStream(bo, new LzoCompressor(), 64 * 1024, 18); 47 | LOG.info("Starting."); 48 | long start = System.currentTimeMillis(); 49 | IOUtils.copy(bi, co); 50 | co.close(); 51 | long end = System.currentTimeMillis(); 52 | LOG.info("Compression took " + ((end - start) / 1000d) + " ms"); 53 | LOG.info("Compressed data is " + bo.size() + " bytes."); 54 | 55 | byte[] cb = bo.toByteArray(); 56 | FileUtils.writeByteArrayToFile(new File("compressed.out"), cb); 57 | 58 | bi = new ByteArrayInputStream(cb); 59 | BlockDecompressorStream ci = new BlockDecompressorStream(bi, new LzoDecompressor()); 60 | bo.reset(); 61 | start = System.currentTimeMillis(); 62 | IOUtils.copy(ci, bo); 63 | end = System.currentTimeMillis(); 64 | LOG.info("Uncompression took " + ((end - start) / 1000d) + " ms"); 65 | LOG.info("Uncompressed data is " + bo.size() + " bytes."); 66 | } 67 | } catch (Throwable t) { 68 | LOG.error(t, t); 69 | throw t; 70 | } finally { 71 | System.out.flush(); 72 | System.err.flush(); 73 | Thread.sleep(100); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/HadoopLzopFileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.io.ByteArrayOutputStream; 8 | import java.io.File; 9 | import java.io.FileInputStream; 10 | import org.anarres.lzo.hadoop.codec.LzopInputStream; 11 | import org.apache.commons.io.IOUtils; 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | import org.apache.hadoop.io.LongWritable; 15 | import org.apache.hadoop.io.Text; 16 | import org.apache.hadoop.mapred.LineRecordReader; 17 | import org.junit.Test; 18 | 19 | /** 20 | * 21 | * @author shevek 22 | */ 23 | public class HadoopLzopFileTest { 24 | 25 | private static final Log LOG = LogFactory.getLog(BlockCompressorStreamTest.class); 26 | 27 | @Test 28 | public void testInputFile() throws Exception { 29 | try { 30 | File dir = LzopFileTest.getDataDirectory(); 31 | File file = new File(dir, "input.txt.lzo"); 32 | FileInputStream fi = new FileInputStream(file); 33 | LzopInputStream ci = new LzopInputStream(fi); 34 | ByteArrayOutputStream bo = new ByteArrayOutputStream((int) (file.length() * 2)); 35 | bo.reset(); 36 | long start = System.currentTimeMillis(); 37 | IOUtils.copy(ci, bo); 38 | long end = System.currentTimeMillis(); 39 | LOG.info("Uncompression took " + ((end - start) / 1000d) + " ms"); 40 | LOG.info("Uncompressed data is " + bo.size() + " bytes."); 41 | LOG.info("Uncompressed data is\n" + bo); 42 | } finally { 43 | System.out.flush(); 44 | System.err.flush(); 45 | Thread.sleep(100); 46 | } 47 | } 48 | 49 | @Test 50 | public void testLineReader() throws Exception { 51 | try { 52 | File dir = LzopFileTest.getDataDirectory(); 53 | File file = new File(dir, "input.txt.lzo"); 54 | FileInputStream fi = new FileInputStream(file); 55 | LzopInputStream ci = new LzopInputStream(fi); 56 | LineRecordReader reader = new LineRecordReader(ci, 0L, 9999L, 9999); 57 | LongWritable key = new LongWritable(); 58 | Text value = new Text(); 59 | while (reader.next(key, value)) { 60 | LOG.info("key=" + key + "; value=" + value); 61 | } 62 | } catch (Exception e) { 63 | LOG.info("Test failed", e); 64 | throw e; 65 | } finally { 66 | System.out.flush(); 67 | System.err.flush(); 68 | Thread.sleep(100); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/LzoAlgorithmTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | import java.util.Formatter; 10 | import java.io.InputStream; 11 | import java.util.Random; 12 | import java.util.Arrays; 13 | import org.apache.commons.io.IOUtils; 14 | import org.junit.Test; 15 | import static org.junit.Assert.*; 16 | 17 | /** 18 | * 19 | * @author shevek 20 | */ 21 | public class LzoAlgorithmTest { 22 | 23 | private static final Log LOG = LogFactory.getLog(LzoAlgorithmTest.class); 24 | 25 | public void testAlgorithm(LzoAlgorithm algorithm, byte[] orig, String desc) { 26 | LzoCompressor compressor = LzoLibrary.getInstance().newCompressor(algorithm, null); 27 | LOG.info("\nCompressing " + orig.length + " " + desc + " bytes using " + algorithm); 28 | 29 | // LOG.info("Original: " + Arrays.toString(orig)); 30 | 31 | byte[] compressed = new byte[orig.length * 2]; 32 | lzo_uintp compressed_length = new lzo_uintp(compressed.length); 33 | int compressed_code = compressor.compress(orig, 0, orig.length, compressed, 0, compressed_length); 34 | 35 | LOG.info("Compressed: " + compressor.toErrorString(compressed_code) + "; length=" + compressed_length); 36 | // LOG.info("Compressed: " + Arrays.toString(Arrays.copyOf(compressed, compressed_length.value))); 37 | assertEquals(LzoTransformer.LZO_E_OK, compressed_code); 38 | 39 | LzoDecompressor decompressor = LzoLibrary.getInstance().newDecompressor(algorithm, null); 40 | byte[] uncompressed = new byte[orig.length]; 41 | lzo_uintp uncompressed_length = new lzo_uintp(uncompressed.length); 42 | int uncompressed_code = decompressor.decompress(compressed, 0, compressed_length.value, uncompressed, 0, uncompressed_length); 43 | 44 | LOG.info("Output: " + decompressor.toErrorString(uncompressed_code)); 45 | // LOG.info("Output: " + Arrays.toString(uncompressed)); 46 | 47 | assertEquals(LzoTransformer.LZO_E_OK, uncompressed_code); 48 | assertArrayEquals(orig, uncompressed); 49 | } 50 | 51 | // Totally RLE. 52 | @Test 53 | public void testBlank() throws Exception { 54 | byte[] orig = new byte[512 * 1024]; 55 | Arrays.fill(orig, (byte) 0); 56 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 57 | try { 58 | testAlgorithm(algorithm, orig, "blank"); 59 | } catch (UnsupportedOperationException e) { 60 | // LOG.info("Unsupported algorithm " + algorithm); 61 | } 62 | } 63 | } 64 | 65 | // Highly cyclic. 66 | @Test 67 | public void testSequence() throws Exception { 68 | byte[] orig = new byte[512 * 1024]; 69 | for (int i = 0; i < orig.length; i++) 70 | orig[i] = (byte) (i & 0xf); 71 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 72 | try { 73 | testAlgorithm(algorithm, orig, "sequential"); 74 | } catch (UnsupportedOperationException e) { 75 | // LOG.info("Unsupported algorithm " + algorithm); 76 | } 77 | } 78 | } 79 | 80 | // Essentially uncompressible. 81 | @Test 82 | public void testRandom() throws Exception { 83 | Random r = new Random(); 84 | for (int i = 0; i < 10; i++) { 85 | byte[] orig = new byte[256 * 1024]; 86 | r.nextBytes(orig); 87 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 88 | try { 89 | testAlgorithm(algorithm, orig, "random"); 90 | } catch (UnsupportedOperationException e) { 91 | // LOG.info("Unsupported algorithm " + algorithm); 92 | } 93 | } 94 | } 95 | } 96 | 97 | public void testClass(Class type) throws Exception { 98 | String name = type.getName(); 99 | name = name.replace('.', '/') + ".class"; 100 | LOG.info("Class is " + name); 101 | InputStream in = getClass().getClassLoader().getResourceAsStream(name); 102 | byte[] orig = IOUtils.toByteArray(in); 103 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 104 | try { 105 | testAlgorithm(algorithm, orig, "class-file"); 106 | } catch (UnsupportedOperationException e) { 107 | // LOG.info("Unsupported algorithm " + algorithm); 108 | } 109 | } 110 | } 111 | 112 | @Test 113 | public void testClass() throws Exception { 114 | testClass(getClass()); 115 | testClass(Integer.class); 116 | testClass(Formatter.class); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/LzoStreamTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | 6 | package org.anarres.lzo; 7 | 8 | import java.io.ByteArrayInputStream; 9 | import java.io.ByteArrayOutputStream; 10 | import java.io.DataInputStream; 11 | import java.io.IOException; 12 | import java.io.InputStream; 13 | import java.util.Arrays; 14 | import java.util.Formatter; 15 | import java.util.Random; 16 | import org.apache.commons.io.IOUtils; 17 | import org.apache.commons.logging.Log; 18 | import org.apache.commons.logging.LogFactory; 19 | import org.junit.Test; 20 | import static org.junit.Assert.*; 21 | 22 | /** 23 | * 24 | * @author shevek 25 | */ 26 | public class LzoStreamTest { 27 | 28 | private static final Log LOG = LogFactory.getLog(LzoStreamTest.class); 29 | 30 | public void testAlgorithm(LzoAlgorithm algorithm, byte[] orig) throws IOException { 31 | LzoCompressor compressor = LzoLibrary.getInstance().newCompressor(algorithm, null); 32 | LOG.info("\nCompressing " + orig.length + " bytes using " + algorithm); 33 | 34 | // LOG.info("Original: " + Arrays.toString(orig)); 35 | 36 | ByteArrayOutputStream os = new ByteArrayOutputStream(); 37 | LzoOutputStream cs = new LzoOutputStream(os, compressor, 256); 38 | cs.write(orig); 39 | cs.close(); 40 | 41 | LOG.info("Compressed: OK."); 42 | 43 | LzoDecompressor decompressor = LzoLibrary.getInstance().newDecompressor(algorithm, null); 44 | 45 | ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray()); 46 | LzoInputStream us = new LzoInputStream(is, decompressor); 47 | DataInputStream ds = new DataInputStream(us); 48 | byte[] uncompressed = new byte[orig.length]; 49 | ds.readFully(uncompressed); 50 | 51 | LOG.info("Output: OK."); 52 | // LOG.info("Output: " + Arrays.toString(uncompressed)); 53 | 54 | assertArrayEquals(orig, uncompressed); 55 | } 56 | 57 | // Totally RLE. 58 | @Test 59 | public void testBlank() throws Exception { 60 | byte[] orig = new byte[512 * 1024]; 61 | Arrays.fill(orig, (byte) 0); 62 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 63 | try { 64 | testAlgorithm(algorithm, orig); 65 | } catch (UnsupportedOperationException e) { 66 | // LOG.info("Unsupported algorithm " + algorithm); 67 | } 68 | } 69 | } 70 | 71 | // Highly cyclic. 72 | @Test 73 | public void testSequence() throws Exception { 74 | byte[] orig = new byte[512 * 1024]; 75 | for (int i = 0; i < orig.length; i++) 76 | orig[i] = (byte) (i & 0xf); 77 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 78 | try { 79 | testAlgorithm(algorithm, orig); 80 | } catch (UnsupportedOperationException e) { 81 | // LOG.info("Unsupported algorithm " + algorithm); 82 | } 83 | } 84 | } 85 | 86 | // Essentially uncompressible. 87 | @Test 88 | public void testRandom() throws Exception { 89 | Random r = new Random(); 90 | for (int i = 0; i < 10; i++) { 91 | byte[] orig = new byte[256 * 1024]; 92 | r.nextBytes(orig); 93 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 94 | try { 95 | testAlgorithm(algorithm, orig); 96 | } catch (UnsupportedOperationException e) { 97 | // LOG.info("Unsupported algorithm " + algorithm); 98 | } 99 | } 100 | } 101 | } 102 | 103 | public void testClass(Class type) throws Exception { 104 | String name = type.getName(); 105 | name = name.replace('.', '/') + ".class"; 106 | LOG.info("Class is " + name); 107 | InputStream in = getClass().getClassLoader().getResourceAsStream(name); 108 | byte[] orig = IOUtils.toByteArray(in); 109 | for (LzoAlgorithm algorithm : LzoAlgorithm.values()) { 110 | try { 111 | testAlgorithm(algorithm, orig); 112 | } catch (UnsupportedOperationException e) { 113 | // LOG.info("Unsupported algorithm " + algorithm); 114 | } 115 | } 116 | } 117 | 118 | @Test 119 | public void testClass() throws Exception { 120 | testClass(getClass()); 121 | testClass(Integer.class); 122 | testClass(Formatter.class); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/LzopFileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.io.File; 8 | import java.io.FileInputStream; 9 | import org.apache.commons.io.IOUtils; 10 | import org.junit.Test; 11 | import static org.junit.Assert.*; 12 | 13 | /** 14 | * 15 | * @author shevek 16 | */ 17 | public class LzopFileTest { 18 | 19 | public static File getDataDirectory() { 20 | String path = System.getProperty("test.data.dir"); 21 | File file = new File(path == null ? "test/data" : path); 22 | assertTrue(file + " is a directory.", file.isDirectory()); 23 | return file; 24 | } 25 | 26 | private void testLzopFile(String name, long flags) throws Exception { 27 | try { 28 | File dir = getDataDirectory(); 29 | File lzo = new File(dir, name); 30 | File raw = new File(dir, "compressed.txt"); 31 | 32 | FileInputStream lzo_fin = new FileInputStream(lzo); 33 | LzopInputStream lzo_zin = new LzopInputStream(lzo_fin); 34 | byte[] lzo_data = IOUtils.toByteArray(lzo_zin); 35 | 36 | FileInputStream raw_fin = new FileInputStream(raw); 37 | byte[] raw_data = IOUtils.toByteArray(raw_fin); 38 | 39 | long tflags = lzo_zin.getFlags() & flags; 40 | assertEquals("Missing flags in file", flags, tflags); 41 | 42 | assertArrayEquals("Failed to decompress properly.", raw_data, lzo_data); 43 | } finally { 44 | System.out.flush(); 45 | System.err.flush(); 46 | Thread.sleep(100); 47 | } 48 | } 49 | 50 | @Test 51 | public void testLzopFile() throws Exception { 52 | testLzopFile("compressed.txt.lzo.adler32d", LzopConstants.F_ADLER32_D); 53 | testLzopFile("compressed.txt.lzo.crc32d", LzopConstants.F_CRC32_D); 54 | testLzopFile("compressed.txt.lzo.adler32", LzopConstants.F_ADLER32_D | LzopConstants.F_ADLER32_C); 55 | testLzopFile("compressed.txt.lzo.crc32", LzopConstants.F_CRC32_D | LzopConstants.F_CRC32_C); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/PerformanceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.io.DataInputStream; 8 | import java.io.File; 9 | import java.io.FileInputStream; 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | import org.junit.Test; 13 | import static org.junit.Assert.*; 14 | import static org.junit.Assume.*; 15 | 16 | /** 17 | * 18 | * @author shevek 19 | */ 20 | public class PerformanceTest { 21 | 22 | private static final Log LOG = LogFactory.getLog(BlockCompressorStreamTest.class); 23 | // I am not going to commit a 512Mb file to git. You will have to choose your own. 24 | // This is a VMWare disk image, which counts as "real data". 25 | private static final String PATH = "/home/shevek/vm/karmasphere-aws-vm/Karmasphere AWS VM-s003.vmdk"; 26 | // private static final String PATH = "/home/shevek/thirdparty/archive/fp.log"; 27 | 28 | @Test 29 | public void testBlockCompress() throws Exception { 30 | try { 31 | LOG.info("Total memory is " + Runtime.getRuntime().totalMemory()); 32 | LOG.info("Max memory is " + Runtime.getRuntime().maxMemory()); 33 | 34 | File file = new File(PATH); 35 | assumeTrue(file.isFile()); 36 | FileInputStream fi = new FileInputStream(new File(PATH)); 37 | DataInputStream di = new DataInputStream(fi); 38 | int length = Math.min((int) file.length(), 512 * 1024 * 1024); 39 | byte[] data = new byte[length]; 40 | di.readFully(data); 41 | LOG.info("Original data is " + data.length + " bytes."); 42 | byte[] compressed = new byte[data.length]; 43 | 44 | LzoCompressor compressor = new LzoCompressor1x_1(); 45 | LzoDecompressor decompressor = new LzoDecompressor1x(); 46 | 47 | for (int i = 0; i < 8; i++) { 48 | lzo_uintp compressed_length = new lzo_uintp(compressed.length); 49 | LOG.info("Starting."); 50 | long start = System.currentTimeMillis(); 51 | compressor.compress(data, 0, data.length, compressed, 0, compressed_length); 52 | long end = System.currentTimeMillis(); 53 | LOG.info("Compression took " + ((end - start) / 1000d) + " ms " 54 | + "and output " + compressed_length + " bytes, " 55 | + "ratio=" + (data.length / (double) compressed_length.value)); 56 | 57 | lzo_uintp uncompressed_length = new lzo_uintp(data.length); 58 | start = System.currentTimeMillis(); 59 | decompressor.decompress(compressed, 0, compressed_length.value, data, 0, uncompressed_length); 60 | end = System.currentTimeMillis(); 61 | LOG.info("Uncompression took " + ((end - start) / 1000d) + " ms and output " + uncompressed_length + " bytes"); 62 | assertEquals(data.length, uncompressed_length.value); 63 | } 64 | } finally { 65 | System.out.flush(); 66 | System.err.flush(); 67 | Thread.sleep(100); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/RandomInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.util.Random; 8 | import org.apache.commons.io.input.NullInputStream; 9 | 10 | /** 11 | * 12 | * @author shevek 13 | */ 14 | public class RandomInputStream extends NullInputStream { 15 | 16 | private final Random r = new Random(); 17 | 18 | public RandomInputStream(long size) { 19 | super(size); 20 | } 21 | 22 | @Override 23 | protected int processByte() { 24 | return r.nextInt() & 0xff; 25 | } 26 | 27 | @Override 28 | protected void processBytes(byte[] bytes, int offset, int length) { 29 | r.nextBytes(bytes); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/org/anarres/lzo/ZlibCompressorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this template, choose Tools | Templates 3 | * and open the template in the editor. 4 | */ 5 | package org.anarres.lzo; 6 | 7 | import java.io.ByteArrayInputStream; 8 | import java.io.ByteArrayOutputStream; 9 | import java.io.DataInputStream; 10 | import java.io.File; 11 | import java.io.FileInputStream; 12 | import org.apache.commons.io.IOUtils; 13 | import org.apache.commons.logging.Log; 14 | import org.apache.commons.logging.LogFactory; 15 | import org.apache.hadoop.io.compress.BlockCompressorStream; 16 | import org.apache.hadoop.io.compress.BlockDecompressorStream; 17 | import org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater; 18 | import org.apache.hadoop.io.compress.zlib.BuiltInZlibInflater; 19 | import org.junit.Ignore; 20 | import org.junit.Test; 21 | 22 | /** 23 | * 24 | * @author shevek 25 | */ 26 | public class ZlibCompressorTest { 27 | 28 | private static final Log LOG = LogFactory.getLog(ZlibCompressorTest.class); 29 | private static final String PATH = "/home/shevek/vm/karmasphere-aws-vm/Karmasphere AWS VM-s003.vmdk"; 30 | 31 | @Test 32 | public void testEmpty() { 33 | LOG.info("Every test suite must have a case, or JUnit gets unhappy."); 34 | } 35 | 36 | @Ignore 37 | @Test 38 | public void testBlockCompressorStream() throws Exception { 39 | try { 40 | LOG.info("Total memory is " + Runtime.getRuntime().totalMemory()); 41 | LOG.info("Max memory is " + Runtime.getRuntime().maxMemory()); 42 | 43 | FileInputStream fi = new FileInputStream(new File(PATH)); 44 | DataInputStream di = new DataInputStream(fi); 45 | byte[] data = new byte[512 * 1024 * 1024]; 46 | di.readFully(data); 47 | LOG.info("Original data is " + data.length + " bytes."); 48 | 49 | ByteArrayInputStream bi = new ByteArrayInputStream(data); 50 | ByteArrayOutputStream bo = new ByteArrayOutputStream(data.length); 51 | BlockCompressorStream co = new BlockCompressorStream(bo, new BuiltInZlibDeflater()); 52 | LOG.info("Starting."); 53 | long start = System.currentTimeMillis(); 54 | IOUtils.copy(bi, co); 55 | co.close(); 56 | long end = System.currentTimeMillis(); 57 | LOG.info("Compression took " + ((end - start) / 1000d) + " ms"); 58 | LOG.info("Compressed data is " + bo.size() + " bytes."); 59 | 60 | if (true) 61 | return; 62 | 63 | bi = new ByteArrayInputStream(bo.toByteArray()); 64 | BlockDecompressorStream ci = new BlockDecompressorStream(bi, new BuiltInZlibInflater()); 65 | bo.reset(); 66 | start = System.currentTimeMillis(); 67 | IOUtils.copy(ci, bo); 68 | end = System.currentTimeMillis(); 69 | LOG.info("Uncompression took " + ((end - start) / 1000d) + " ms"); 70 | LOG.info("Uncompressed data is " + bo.size() + " bytes."); 71 | } finally { 72 | System.out.flush(); 73 | System.err.flush(); 74 | Thread.sleep(100); 75 | } 76 | } 77 | } 78 | --------------------------------------------------------------------------------