├── .document ├── .gitignore ├── CHANGELOG.rdoc ├── Gemfile ├── LICENSE.txt ├── README.rdoc ├── Rakefile ├── VERSION ├── benchmarking ├── bench.rb ├── bench_all.sh ├── compressor.rb ├── compressor_lz4.rb ├── compressor_lzo.rb └── compressor_snappy.rb ├── build_all.sh ├── ext └── lz4ruby │ ├── extconf.rb │ └── lz4ruby.c ├── lib ├── lz4-jruby.rb └── lz4-ruby.rb ├── pom.xml ├── setup_env.sh ├── spec ├── helper.rb ├── lz4_compressHC_spec.rb ├── lz4_compress_spec.rb ├── lz4_raw_compress_spec.rb ├── lz4_raw_decompress_spec.rb └── lz4_raw_spec.rb └── src └── main └── java └── com └── headius └── jruby └── lz4 ├── LZ4Internal.java └── LZ4Library.java /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # rcov generated 2 | coverage 3 | coverage.data 4 | 5 | # rdoc generated 6 | rdoc 7 | 8 | # yard generated 9 | doc 10 | .yardoc 11 | 12 | # bundler 13 | .bundle 14 | 15 | # jeweler generated 16 | pkg 17 | 18 | # JRuby ext build artifacts 19 | target 20 | dependency-reduced-pom.xml 21 | 22 | # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 23 | # 24 | # * Create a file at ~/.gitignore 25 | # * Include files you want ignored 26 | # * Run: git config --global core.excludesfile ~/.gitignore 27 | # 28 | # After doing this, these files will be ignored in all your git projects, 29 | # saving you from having to 'pollute' every project you touch with them 30 | # 31 | # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line) 32 | # 33 | # For MacOS: 34 | # 35 | #.DS_Store 36 | 37 | # For TextMate 38 | #*.tmproj 39 | #tmtags 40 | 41 | # For emacs: 42 | *~ 43 | \#* 44 | .\#* 45 | 46 | # For vim: 47 | #*.swp 48 | 49 | # For redcar: 50 | #.redcar 51 | 52 | # For rubinius: 53 | #*.rbc 54 | 55 | # LZ4 sources and Makefile 56 | ext/lz4ruby/Makefile 57 | ext/lz4ruby/lz4.? 58 | ext/lz4ruby/lz4hc.? 59 | 60 | # binaries 61 | *.o 62 | *.so 63 | *.jar 64 | 65 | # Development, test and temporaries 66 | Gemfile 67 | Gemfile.lock 68 | *.gemspec 69 | benchmarking/testdata/ 70 | tmp/ 71 | -------------------------------------------------------------------------------- /CHANGELOG.rdoc: -------------------------------------------------------------------------------- 1 | = ChangeLog 2 | 3 | == 0.3.3 4 | 5 | * Update to lz4 latest version (r119) #9 6 | 7 | == 0.3.2 8 | 9 | * Fix a bug #7 10 | 11 | == 0.3.1 12 | 13 | * Support raw data handling methods in JRuby 14 | 15 | == 0.3.0 16 | 17 | * Support raw data handling methods (but not worked in JRuby). 18 | * LZ4.Raw.compress() 19 | * LZ4.Raw.compressHC() 20 | * LZ4.Raw.decompress() 21 | * Rename LZ4.uncompress() to LZ4.decompress(). 22 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | # Add dependencies required to use your gem here. 3 | # Example: 4 | # gem "activesupport", ">= 2.3.5" 5 | 6 | # Add dependencies to develop your gem here. 7 | # Include everything needed to run rake, tests, features, etc. 8 | group :development do 9 | gem "rspec" 10 | gem "rdoc", "~> 3.12" 11 | gem "bundler" 12 | gem "jeweler", "~> 1.8.3" 13 | gem "rake-compiler", ">= 0" 14 | end 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 KOMIYA Atsushi 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = lz4-ruby 2 | 3 | Ruby bindings for {LZ4}[http://code.google.com/p/lz4/]. 4 | 5 | == Installation 6 | 7 | gem install lz4-ruby 8 | 9 | == Usage 10 | 11 | require 'rubygems' 12 | require 'lz4-ruby' 13 | 14 | # compress (fast) 15 | compressed = LZ4::compress("hello, world") 16 | 17 | # compress (high compression) 18 | compressed = LZ4::compressHC("hello, world") 19 | 20 | # uncompress 21 | uncompressed = LZ4::uncompress(compressed) 22 | 23 | == Benchmark 24 | 25 | Tested on VirtualBox VM : 2-core / 4GB RAM (Host : Core i5-2520M / 8GB RAM). 26 | 27 | === {enwik8}[http://mattmahoney.net/dc/enwik8.zip] 28 | method ratio(bpc) comp.time(ms) uncomp.time(ms) 29 | ------------------------------------------------------- 30 | lz4 4.559 519.25 182.67 31 | snappy 4.668 1050.73 257.94 32 | lzo 4.279 1000.13 574.77 33 | 34 | == TODO 35 | 36 | * Support streaming methods. 37 | * Support compression level (LZ4HC) 38 | * Write API documents. 39 | 40 | == Copyright 41 | 42 | Copyright (c) 2012 KOMIYA Atsushi. 43 | See LICENSE.txt for further details. 44 | 45 | == Contributors 46 | 47 | * {Charles Oliver Nutter}[https://github.com/headius] 48 | * dearblue[https://github.com/dearblue] 49 | 50 | == About LZ4 51 | 52 | This Rubygem includes LZ4 codes written by Yann Collet. 53 | 54 | LZ4 - Fast LZ compression algorithm 55 | Copyright (C) 2011-2012, Yann Collet. 56 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 57 | 58 | Redistribution and use in source and binary forms, with or without 59 | modification, are permitted provided that the following conditions are 60 | met: 61 | 62 | * Redistributions of source code must retain the above copyright 63 | notice, this list of conditions and the following disclaimer. 64 | * Redistributions in binary form must reproduce the above 65 | copyright notice, this list of conditions and the following disclaimer 66 | in the documentation and/or other materials provided with the 67 | distribution. 68 | 69 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 70 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 71 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 72 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 73 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 74 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 75 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 76 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 77 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 78 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 79 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 80 | 81 | You can contact the author at : 82 | - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html 83 | - LZ4 source repository : http://code.google.com/p/lz4/ 84 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # encoding: utf-8 3 | 4 | require 'rubygems' 5 | require 'bundler' 6 | begin 7 | Bundler.setup(:default, :development) 8 | rescue Bundler::BundlerError => e 9 | $stderr.puts e.message 10 | $stderr.puts "Run `bundle install` to install missing gems" 11 | exit e.status_code 12 | end 13 | require 'rake' 14 | 15 | require 'jeweler' 16 | jeweler_tasks = Jeweler::Tasks.new do |gem| 17 | # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options 18 | gem.name = "lz4-ruby" 19 | gem.homepage = "http://github.com/komiya-atsushi/lz4-ruby" 20 | gem.license = "MIT" 21 | gem.summary = %Q{Ruby bindings for LZ4 (Extremely Fast Compression algorithm).} 22 | gem.description = %Q{Ruby bindings for LZ4. LZ4 is a very fast lossless compression algorithm.} 23 | gem.email = "komiya.atsushi@gmail.com" 24 | gem.authors = ["KOMIYA Atsushi"] 25 | gem.extensions = ["ext/lz4ruby/extconf.rb"] 26 | 27 | gem.files.exclude("*.sh") 28 | 29 | gem.files.include("ext/lz4ruby/*.c") 30 | gem.files.include("ext/lz4ruby/*.h") 31 | 32 | gem.required_ruby_version = '>= 1.9' 33 | 34 | # dependencies defined in Gemfile 35 | end 36 | Jeweler::RubygemsDotOrgTasks.new 37 | 38 | $gemspec = jeweler_tasks.gemspec 39 | $gemspec.version = jeweler_tasks.jeweler.version 40 | 41 | require 'rspec/core/rake_task' 42 | RSpec::Core::RakeTask.new(:spec) 43 | 44 | task :default => :spec 45 | 46 | require 'rdoc/task' 47 | Rake::RDocTask.new do |rdoc| 48 | version = File.exist?('VERSION') ? File.read('VERSION') : "" 49 | 50 | rdoc.rdoc_dir = 'rdoc' 51 | rdoc.title = "lz4-ruby #{version}" 52 | rdoc.rdoc_files.include('README*') 53 | rdoc.rdoc_files.include('lib/**/*.rb') 54 | end 55 | 56 | require 'rake/extensiontask' 57 | Rake::ExtensionTask.new("lz4ruby", $gemspec) do |ext| 58 | ext.cross_compile = true 59 | ext.cross_platform = ["x86-mingw32"] 60 | end 61 | 62 | Rake::Task.tasks.each do |task_name| 63 | case task_name.to_s 64 | when /^native/ 65 | task_name.prerequisites.unshift('fix_rake_compiler_gemspec_dump') 66 | end 67 | end 68 | 69 | task :fix_rake_compiler_gemspec_dump do 70 | %w{files extra_rdoc_files test_files}.each do |accessor| 71 | $gemspec.send(accessor).instance_eval { 72 | @exclude_procs = Array.new 73 | } 74 | end 75 | end 76 | 77 | task :gems do 78 | sh "rake clean build:cross" 79 | sh "rake clean build" 80 | end 81 | 82 | task "build:cross" => [:modify_gemspec_for_windows, :build] do 83 | file = "pkg/lz4-ruby-#{get_version}.gem" 84 | end 85 | 86 | task "build:jruby" => [:modify_gemspec_for_jruby, :compile_jruby, :build] 87 | 88 | task :modify_gemspec_for_windows do 89 | $gemspec.extensions = [] 90 | $gemspec.files.include("lib/?.?/*.so") 91 | $gemspec.platform = "x86-mingw32" 92 | end 93 | 94 | task :modify_gemspec_for_jruby do 95 | $gemspec.extensions = [] 96 | $gemspec.files.include("lib/*.jar") 97 | $gemspec.platform = "java" 98 | end 99 | 100 | task :compile_jruby do 101 | system 'mvn package' 102 | end 103 | 104 | def get_version 105 | `cat VERSION`.chomp 106 | end 107 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.3.3 -------------------------------------------------------------------------------- /benchmarking/bench.rb: -------------------------------------------------------------------------------- 1 | require 'benchmark' 2 | require 'rubygems' 3 | 4 | CHUNK_SIZE = 8 << 20 5 | NUM_LOOP = 10 6 | 7 | class DevNullIO 8 | def write(arg) 9 | # Do nothing 10 | end 11 | end 12 | 13 | class StringIO 14 | def initialize(text) 15 | @text = text 16 | @len = text.length 17 | @pos = 0 18 | end 19 | 20 | def read(length) 21 | return nil if @pos == @len 22 | 23 | length = @len - @pos if @pos + length > @len 24 | 25 | result = @text[@pos, length] 26 | @pos += length 27 | 28 | return result 29 | end 30 | end 31 | 32 | class CompressorBenchmark 33 | USAGE = <#{@compressed_filename}` 73 | end 74 | 75 | def benchmark_compression 76 | data = nil 77 | File.open(@testdata_filename) do |file| 78 | data = file.read(File.size(@testdata_filename)) 79 | end 80 | 81 | devnull = DevNullIO.new 82 | 83 | # warm up 84 | @compressor.compress(StringIO.new(data), devnull) 85 | 86 | result = Benchmark.measure { 87 | NUM_LOOP.times { |t| @compressor.compress(StringIO.new(data), devnull) } 88 | } 89 | 90 | return result 91 | end 92 | 93 | def benchmark_uncompression 94 | data = nil 95 | File.open(@compressed_filename) do |file| 96 | data = file.read(File.size(@compressed_filename)) 97 | end 98 | 99 | devnull = DevNullIO.new 100 | 101 | # warm up 102 | @compressor.uncompress(StringIO.new(data), devnull) 103 | 104 | result = Benchmark.measure { 105 | NUM_LOOP.times { |t| @compressor.uncompress(StringIO.new(data), devnull) } 106 | } 107 | 108 | return result 109 | end 110 | 111 | def show_result(result_comp, result_uncomp) 112 | orig_size = File.size(@testdata_filename) 113 | comp_size = File.size(@compressed_filename) 114 | 115 | ratio = comp_size * 8.0 / orig_size 116 | comp_time = result_comp.real * 1000.0 / NUM_LOOP 117 | uncomp_time = result_uncomp.real * 1000.0 / NUM_LOOP 118 | 119 | puts "method\tratio(bpc)\tcomp.time(ms)\tuncomp.time(ms)" 120 | puts "-------------------------------------------------------" 121 | puts "%s\t%.3f\t\t%.2f\t\t%.2f" % [@signature, ratio, comp_time, uncomp_time] 122 | end 123 | 124 | def do_benchmark 125 | setup_compressed() 126 | 127 | result_comp = benchmark_compression() 128 | result_uncomp = benchmark_uncompression() 129 | 130 | show_result(result_comp, result_uncomp) 131 | end 132 | end 133 | 134 | if $0 == __FILE__ 135 | CompressorBenchmark.new(ARGV).do_benchmark 136 | end 137 | 138 | -------------------------------------------------------------------------------- /benchmarking/bench_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ruby bench.rb lz4 $1 4 | ruby bench.rb snappy $1 | tail -n 1 5 | ruby bench.rb lzo $1 | tail -n 1 6 | -------------------------------------------------------------------------------- /benchmarking/compressor.rb: -------------------------------------------------------------------------------- 1 | class Compressor 2 | DEFAULT_CHUNK_SIZE = 8 << 20 3 | 4 | def initialize(chunk_size) 5 | if chunk_size == nil 6 | @chunk_size = DEFAULT_CHUNK_SIZE 7 | else 8 | @chunk_size = chunk_size 9 | end 10 | 11 | require_libs() 12 | end 13 | 14 | def compress(infile, outfile) 15 | loop do 16 | text = infile.read(@chunk_size) 17 | break if text == nil || text.length == 0 18 | 19 | compressed = compress_text(text) 20 | comp_size = compressed.length 21 | 22 | outfile.write([comp_size].pack("L")) 23 | outfile.write(compressed) 24 | end 25 | end 26 | 27 | def uncompress(infile, outfile) 28 | loop do 29 | comp_size = infile.read(4) 30 | break if comp_size == nil || comp_size.length == 0 31 | 32 | comp_size = comp_size.unpack("L")[0] 33 | compressed = infile.read(comp_size) 34 | 35 | text = uncompress_text(compressed) 36 | 37 | outfile.write(text) 38 | end 39 | end 40 | 41 | def self.unit_driver() 42 | if !(ARGV.length == 1) && !(ARGV.length == 2 && ARGV[0] == 'c') 43 | puts <outfile 46 | 47 | Uncompress: 48 | ./#{$0} u outfile 49 | EOS 50 | exit 1 51 | end 52 | 53 | require 'rubygems' 54 | 55 | case ARGV[0] 56 | when 'c' 57 | chunk_size = nil 58 | chunk_size = ARGV[1].to_i if ARGV.length == 2 59 | compressor = create_compressor(chunk_size) 60 | compressor.compress($stdin, $stdout) 61 | 62 | when 'u' 63 | compressor = create_compressor(nil) 64 | compressor.uncompress($stdin, $stdout) 65 | 66 | else 67 | puts "Error: illegal argument '#{ARGV[0]}'" 68 | exit 1 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /benchmarking/compressor_lz4.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require './compressor.rb' 4 | 5 | class LZ4Compressor < Compressor 6 | def require_libs 7 | require 'lz4-ruby' 8 | end 9 | 10 | def compress_text(text) 11 | return LZ4::compress(text) 12 | end 13 | 14 | def uncompress_text(compressed) 15 | return LZ4::uncompress(compressed) 16 | end 17 | end 18 | 19 | def create_compressor(chunk_size) 20 | return LZ4Compressor.new(chunk_size) 21 | end 22 | 23 | if $0 == __FILE__ 24 | Compressor.unit_driver() { |chunk_size| LZ4Compressor.new(chunk_size) } 25 | end 26 | -------------------------------------------------------------------------------- /benchmarking/compressor_lzo.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require './compressor.rb' 4 | 5 | class LZOCompressor < Compressor 6 | def require_libs 7 | require 'lzoruby' 8 | end 9 | 10 | def compress_text(text) 11 | return LZO.compress(text) 12 | end 13 | 14 | def uncompress_text(compressed) 15 | return LZO.decompress(compressed) 16 | end 17 | end 18 | 19 | def create_compressor(chunk_size) 20 | return LZOCompressor.new(chunk_size) 21 | end 22 | 23 | if $0 == __FILE__ 24 | Compressor.unit_driver() { |chunk_size| LZOCompressor.new(chunk_size) } 25 | end 26 | -------------------------------------------------------------------------------- /benchmarking/compressor_snappy.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require './compressor.rb' 4 | 5 | class SnappyCompressor < Compressor 6 | def require_libs 7 | require 'snappy' 8 | end 9 | 10 | def compress_text(text) 11 | return Snappy.deflate(text) 12 | end 13 | 14 | def uncompress_text(compressed) 15 | return Snappy.inflate(compressed) 16 | end 17 | end 18 | 19 | def create_compressor(chunk_size) 20 | return SnappyCompressor.new(chunk_size) 21 | end 22 | 23 | if $0 == __FILE__ 24 | Compressor.unit_driver() { |chunk_size| SnappyCompressor.new(chunk_size) } 25 | end 26 | -------------------------------------------------------------------------------- /build_all.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | LZ4_REV_NO=119 4 | URL=http://lz4.googlecode.com/svn-history/r${LZ4_REV_NO}/trunk 5 | 6 | # get lz4 sources from web 7 | rm ext/lz4ruby/lz4.c; wget -P ext/lz4ruby/ $URL/lz4.c 8 | rm ext/lz4ruby/lz4.h; wget -P ext/lz4ruby/ $URL/lz4.h 9 | rm ext/lz4ruby/lz4hc.c; wget -P ext/lz4ruby/ $URL/lz4hc.c 10 | rm ext/lz4ruby/lz4hc.h; wget -P ext/lz4ruby/ $URL/lz4hc.h 11 | 12 | [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" 13 | 14 | rm -f ext/lz4ruby/*.o 15 | rm -f ext/lz4ruby/*.so 16 | rm -rf tmp/* 17 | rm -rf pkg/* 18 | rm -rf target/* 19 | 20 | rm -rf lib/1.8 lib/1.9 lib/*.jar 21 | 22 | # compile & build .jar 23 | rvm use jruby --default 24 | rvm gemset use lz4-ruby 25 | bundle exec rake build:jruby 26 | 27 | # compile 1.8.7 native extensions for MinGW 28 | rvm use 1.8.7 --default 29 | rvm gemset use lz4-ruby 30 | bundle exec rake cross compile RUBY_CC_VERSION=1.8.7 31 | 32 | # compile 1.9.3 native extensions for MinGW 33 | rvm use 1.9.3 --default 34 | rvm gemset use lz4-ruby 35 | bundle exec rake cross compile RUBY_CC_VERSION=1.9.3 36 | 37 | # copy native extensions -> lib/1.x 38 | rvm use 1.8.7 --default 39 | rvm gemset use lz4-ruby 40 | bundle exec rake cross compile RUBY_CC_VERSION=1.8.7:1.9.3 41 | 42 | rm lib/lz4ruby.so 43 | 44 | # build pre-compiled gem for MinGW 45 | bundle exec rake build:cross 46 | 47 | # build "Compile-It-Yourself" gem 48 | rm -rf lib/1.8 lib/1.9 49 | bundle exec rake build 50 | -------------------------------------------------------------------------------- /ext/lz4ruby/extconf.rb: -------------------------------------------------------------------------------- 1 | require 'mkmf' 2 | 3 | $CFLAGS += " -Wall " 4 | 5 | create_makefile('lz4ruby') 6 | 7 | -------------------------------------------------------------------------------- /ext/lz4ruby/lz4ruby.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "lz4.h" 3 | #include "lz4hc.h" 4 | 5 | typedef int (*CompressFunc)(const char *source, char *dest, int isize); 6 | typedef int (*CompressLimitedOutputFunc)(const char* source, char* dest, int inputSize, int maxOutputSize); 7 | 8 | static VALUE lz4internal; 9 | static VALUE lz4_error; 10 | 11 | /** 12 | * LZ4Internal functions. 13 | */ 14 | static VALUE compress_internal(CompressFunc compressor, VALUE header, VALUE input, VALUE in_size) { 15 | const char *src_p; 16 | int src_size; 17 | 18 | const char *header_p; 19 | int header_size; 20 | 21 | VALUE result; 22 | char *buf; 23 | int buf_size; 24 | 25 | int comp_size; 26 | 27 | Check_Type(input, T_STRING); 28 | src_p = RSTRING_PTR(input); 29 | src_size = NUM2INT(in_size); 30 | buf_size = LZ4_compressBound(src_size); 31 | 32 | Check_Type(header, T_STRING); 33 | header_p = RSTRING_PTR(header); 34 | header_size = RSTRING_LEN(header); 35 | 36 | result = rb_str_new(NULL, buf_size + header_size); 37 | buf = RSTRING_PTR(result); 38 | 39 | memcpy(buf, header_p, header_size); 40 | 41 | comp_size = compressor(src_p, buf + header_size, src_size); 42 | rb_str_resize(result, comp_size + header_size); 43 | 44 | return result; 45 | } 46 | 47 | static VALUE compress_raw_internal( 48 | CompressLimitedOutputFunc compressor, 49 | VALUE _input, 50 | VALUE _input_size, 51 | VALUE _output_buffer, 52 | VALUE _max_output_size) { 53 | 54 | const char *src_p; 55 | int src_size; 56 | 57 | int needs_resize; 58 | char *buf_p; 59 | 60 | int max_output_size; 61 | 62 | int comp_size; 63 | 64 | 65 | Check_Type(_input, T_STRING); 66 | src_p = RSTRING_PTR(_input); 67 | src_size = NUM2INT(_input_size); 68 | 69 | if (NIL_P(_output_buffer)) { 70 | needs_resize = 1; 71 | _output_buffer = rb_str_new(NULL, _max_output_size); 72 | 73 | } else { 74 | needs_resize = 0; 75 | } 76 | 77 | buf_p = RSTRING_PTR(_output_buffer); 78 | 79 | max_output_size = NUM2INT(_max_output_size); 80 | 81 | comp_size = compressor(src_p, buf_p, src_size, max_output_size); 82 | 83 | if (comp_size > 0 && needs_resize) { 84 | rb_str_resize(_output_buffer, comp_size); 85 | } 86 | 87 | return rb_ary_new3(2, _output_buffer, INT2NUM(comp_size)); 88 | } 89 | 90 | static VALUE lz4internal_compress_raw( 91 | VALUE self, 92 | VALUE _input, 93 | VALUE _input_size, 94 | VALUE _output_buffer, 95 | VALUE _max_output_size) 96 | { 97 | return compress_raw_internal( 98 | LZ4_compress_limitedOutput, 99 | _input, 100 | _input_size, 101 | _output_buffer, 102 | _max_output_size); 103 | } 104 | 105 | static VALUE lz4internal_compressHC_raw( 106 | VALUE self, 107 | VALUE _input, 108 | VALUE _input_size, 109 | VALUE _output_buffer, 110 | VALUE _max_output_size) 111 | { 112 | return compress_raw_internal( 113 | LZ4_compressHC_limitedOutput, 114 | _input, 115 | _input_size, 116 | _output_buffer, 117 | _max_output_size); 118 | } 119 | 120 | 121 | static VALUE lz4internal_compress(VALUE self, VALUE header, VALUE input, VALUE in_size) { 122 | return compress_internal(LZ4_compress, header, input, in_size); 123 | } 124 | 125 | static VALUE lz4internal_compressHC(VALUE self, VALUE header, VALUE input, VALUE in_size) { 126 | return compress_internal(LZ4_compressHC, header, input, in_size); 127 | } 128 | 129 | static VALUE lz4internal_uncompress(VALUE self, VALUE input, VALUE in_size, VALUE offset, VALUE out_size) { 130 | const char *src_p; 131 | int src_size; 132 | 133 | int header_size; 134 | 135 | VALUE result; 136 | char *buf; 137 | int buf_size; 138 | 139 | int read_bytes; 140 | 141 | Check_Type(input, T_STRING); 142 | src_p = RSTRING_PTR(input); 143 | src_size = NUM2INT(in_size); 144 | 145 | header_size = NUM2INT(offset); 146 | buf_size = NUM2INT(out_size); 147 | 148 | result = rb_str_new(NULL, buf_size); 149 | buf = RSTRING_PTR(result); 150 | 151 | read_bytes = LZ4_decompress_safe(src_p + header_size, buf, src_size - header_size, buf_size); 152 | if (read_bytes < 0) { 153 | rb_raise(lz4_error, "Compressed data is maybe corrupted."); 154 | } 155 | 156 | return result; 157 | } 158 | 159 | static VALUE lz4internal_decompress_raw( 160 | VALUE self, 161 | VALUE _input, 162 | VALUE _input_size, 163 | VALUE _output_buffer, 164 | VALUE _max_output_size) { 165 | 166 | const char *src_p; 167 | int src_size; 168 | 169 | int max_output_size; 170 | 171 | int needs_resize; 172 | char *buf_p; 173 | 174 | int decomp_size; 175 | 176 | Check_Type(_input, T_STRING); 177 | src_p = RSTRING_PTR(_input); 178 | src_size = NUM2INT(_input_size); 179 | 180 | max_output_size = NUM2INT(_max_output_size); 181 | 182 | if (NIL_P(_output_buffer)) { 183 | needs_resize = 1; 184 | _output_buffer = rb_str_new(NULL, max_output_size); 185 | 186 | } else { 187 | needs_resize = 0; 188 | } 189 | 190 | buf_p = RSTRING_PTR(_output_buffer); 191 | decomp_size = LZ4_decompress_safe(src_p, buf_p, src_size, max_output_size); 192 | 193 | if (decomp_size > 0 && needs_resize) { 194 | rb_str_resize(_output_buffer, decomp_size); 195 | } 196 | 197 | return rb_ary_new3(2, _output_buffer, INT2NUM(decomp_size)); 198 | } 199 | 200 | #if 0 201 | 202 | static inline void lz4internal_raw_compress_scanargs(int argc, VALUE *argv, VALUE *src, VALUE *dest, size_t *srcsize, size_t *maxsize) { 203 | switch (argc) { 204 | case 1: 205 | *src = argv[0]; 206 | Check_Type(*src, RUBY_T_STRING); 207 | *srcsize = RSTRING_LEN(*src); 208 | *dest = rb_str_buf_new(0); 209 | *maxsize = LZ4_compressBound(*srcsize); 210 | break; 211 | case 2: 212 | *src = argv[0]; 213 | Check_Type(*src, RUBY_T_STRING); 214 | *srcsize = RSTRING_LEN(*src); 215 | if (TYPE(argv[1]) == T_STRING) { 216 | *dest = argv[1]; 217 | *maxsize = LZ4_compressBound(*srcsize); 218 | } else { 219 | *dest = rb_str_buf_new(0); 220 | *maxsize = NUM2SIZET(argv[1]); 221 | } 222 | break; 223 | case 3: 224 | *src = argv[0]; 225 | Check_Type(*src, RUBY_T_STRING); 226 | *srcsize = RSTRING_LEN(*src); 227 | *dest = argv[1]; 228 | Check_Type(*dest, RUBY_T_STRING); 229 | *maxsize = NUM2SIZET(argv[2]); 230 | break; 231 | default: 232 | //rb_error_arity(argc, 1, 3); 233 | rb_scan_args(argc, argv, "12", NULL, NULL, NULL); 234 | // the following code is used to eliminate compiler warnings. 235 | *src = *dest = 0; 236 | *srcsize = *maxsize = 0; 237 | } 238 | } 239 | 240 | static inline VALUE lz4internal_raw_compress_common(int argc, VALUE *argv, VALUE lz4, CompressLimitedOutputFunc compressor) { 241 | VALUE src, dest; 242 | size_t srcsize; 243 | size_t maxsize; 244 | 245 | lz4internal_raw_compress_scanargs(argc, argv, &src, &dest, &srcsize, &maxsize); 246 | 247 | if (srcsize > LZ4_MAX_INPUT_SIZE) { 248 | rb_raise(lz4_error, 249 | "input size is too big for lz4 compress (max %u bytes)", 250 | LZ4_MAX_INPUT_SIZE); 251 | } 252 | rb_str_modify(dest); 253 | rb_str_resize(dest, maxsize); 254 | rb_str_set_len(dest, 0); 255 | 256 | int size = compressor(RSTRING_PTR(src), RSTRING_PTR(dest), srcsize, maxsize); 257 | if (size < 0) { 258 | rb_raise(lz4_error, "failed LZ4 raw compress"); 259 | } 260 | 261 | rb_str_resize(dest, size); 262 | rb_str_set_len(dest, size); 263 | 264 | return dest; 265 | } 266 | 267 | /* 268 | * call-seq: 269 | * (compressed string data) raw_compress(src) 270 | * (compressed string data) raw_compress(src, max_dest_size) 271 | * (dest with compressed string data) raw_compress(src, dest) 272 | * (dest with compressed string data) raw_compress(src, dest, max_dest_size) 273 | */ 274 | static VALUE lz4internal_raw_compress(int argc, VALUE *argv, VALUE lz4i) { 275 | return lz4internal_raw_compress_common(argc, argv, lz4i, LZ4_compress_limitedOutput); 276 | } 277 | 278 | /* 279 | * call-seq: 280 | * (compressed string data) raw_compressHC(src) 281 | * (compressed string data) raw_compressHC(src, max_dest_size) 282 | * (dest with compressed string data) raw_compressHC(src, dest) 283 | * (dest with compressed string data) raw_compressHC(src, dest, max_dest_size) 284 | */ 285 | static VALUE lz4internal_raw_compressHC(int argc, VALUE *argv, VALUE lz4i) { 286 | return lz4internal_raw_compress_common(argc, argv, lz4i, LZ4_compressHC_limitedOutput); 287 | } 288 | 289 | enum { 290 | LZ4RUBY_UNCOMPRESS_MAXSIZE = 1 << 24, // tentative value 291 | }; 292 | 293 | static inline void lz4internal_raw_uncompress_scanargs(int argc, VALUE *argv, VALUE *src, VALUE *dest, size_t *maxsize) { 294 | switch (argc) { 295 | case 1: 296 | *src = argv[0]; 297 | Check_Type(*src, RUBY_T_STRING); 298 | *dest = rb_str_buf_new(0); 299 | *maxsize = LZ4RUBY_UNCOMPRESS_MAXSIZE; 300 | break; 301 | case 2: 302 | *src = argv[0]; 303 | Check_Type(*src, RUBY_T_STRING); 304 | *dest = argv[1]; 305 | if (TYPE(*dest) == T_STRING) { 306 | *maxsize = LZ4RUBY_UNCOMPRESS_MAXSIZE; 307 | } else { 308 | *maxsize = NUM2SIZET(*dest); 309 | *dest = rb_str_buf_new(0); 310 | } 311 | break; 312 | case 3: 313 | *src = argv[0]; 314 | Check_Type(*src, RUBY_T_STRING); 315 | *dest = argv[1]; 316 | Check_Type(*dest, RUBY_T_STRING); 317 | *maxsize = NUM2SIZET(argv[2]); 318 | break; 319 | default: 320 | //rb_error_arity(argc, 2, 3); 321 | rb_scan_args(argc, argv, "21", NULL, NULL, NULL); 322 | // the following code is used to eliminate compiler warnings. 323 | *src = *dest = 0; 324 | *maxsize = 0; 325 | } 326 | } 327 | 328 | /* 329 | * call-seq: 330 | * (uncompressed string data) raw_uncompress(src, max_dest_size = 1 << 24) 331 | * (dest for uncompressed string data) raw_uncompress(src, dest, max_dest_size = 1 << 24) 332 | */ 333 | static VALUE lz4internal_raw_uncompress(int argc, VALUE *argv, VALUE lz4i) { 334 | VALUE src, dest; 335 | size_t maxsize; 336 | lz4internal_raw_uncompress_scanargs(argc, argv, &src, &dest, &maxsize); 337 | 338 | rb_str_modify(dest); 339 | rb_str_resize(dest, maxsize); 340 | rb_str_set_len(dest, 0); 341 | 342 | int size = LZ4_decompress_safe(RSTRING_PTR(src), RSTRING_PTR(dest), RSTRING_LEN(src), maxsize); 343 | if (size < 0) { 344 | rb_raise(lz4_error, "failed LZ4 raw uncompress at %d", -size); 345 | } 346 | 347 | rb_str_resize(dest, size); 348 | rb_str_set_len(dest, size); 349 | 350 | return dest; 351 | } 352 | 353 | #endif 354 | 355 | void Init_lz4ruby(void) { 356 | lz4internal = rb_define_module("LZ4Internal"); 357 | 358 | rb_define_module_function(lz4internal, "compress", lz4internal_compress, 3); 359 | rb_define_module_function(lz4internal, "compressHC", lz4internal_compressHC, 3); 360 | rb_define_module_function(lz4internal, "uncompress", lz4internal_uncompress, 4); 361 | 362 | //rb_define_module_function(lz4internal, "raw_compress", lz4internal_raw_compress, -1); 363 | //rb_define_module_function(lz4internal, "raw_compressHC", lz4internal_raw_compressHC, -1); 364 | //rb_define_module_function(lz4internal, "raw_uncompress", lz4internal_raw_uncompress, -1); 365 | 366 | rb_define_module_function(lz4internal, "compress_raw", lz4internal_compress_raw, 4); 367 | rb_define_module_function(lz4internal, "compressHC_raw", lz4internal_compressHC_raw, 4); 368 | rb_define_module_function(lz4internal, "decompress_raw", lz4internal_decompress_raw, 4); 369 | 370 | lz4_error = rb_define_class_under(lz4internal, "Error", rb_eStandardError); 371 | } 372 | -------------------------------------------------------------------------------- /lib/lz4-jruby.rb: -------------------------------------------------------------------------------- 1 | require 'lz4ruby.jar' 2 | 3 | com.headius.jruby.lz4.LZ4Library.new.load(JRuby.runtime, false) 4 | -------------------------------------------------------------------------------- /lib/lz4-ruby.rb: -------------------------------------------------------------------------------- 1 | if /(mswin|mingw)/ =~ RUBY_PLATFORM 2 | /(\d+\.\d+)/ =~ RUBY_VERSION 3 | ver = $1 4 | require "#{ver}/lz4ruby.so" 5 | elsif RUBY_PLATFORM == 'java' 6 | require 'lz4-jruby' 7 | else 8 | require 'lz4ruby' 9 | end 10 | 11 | class LZ4 12 | def self.compress(input, in_size = nil) 13 | return _compress(input, in_size, false) 14 | end 15 | 16 | def self.compressHC(input, in_size = nil) 17 | return _compress(input, in_size, true) 18 | end 19 | 20 | def self.decompress(input, in_size = nil, encoding = nil) 21 | in_size = input.bytesize if in_size == nil 22 | out_size, varbyte_len = decode_varbyte(input) 23 | 24 | if out_size < 0 || varbyte_len < 0 25 | raise "Compressed data is maybe corrupted" 26 | end 27 | 28 | result = LZ4Internal::uncompress(input, in_size, varbyte_len, out_size) 29 | result.force_encoding(encoding) if encoding != nil 30 | 31 | return result 32 | end 33 | 34 | # @deprecated Use {#decompress} and will be removed. 35 | def self.uncompress(input, in_size = nil) 36 | return decompress(input, in_size) 37 | end 38 | 39 | private 40 | def self._compress(input, in_size, high_compression) 41 | in_size = input.bytesize if in_size == nil 42 | header = encode_varbyte(in_size) 43 | 44 | if high_compression 45 | return LZ4Internal.compressHC(header, input, in_size) 46 | else 47 | return LZ4Internal.compress(header, input, in_size) 48 | end 49 | end 50 | 51 | private 52 | def self.encode_varbyte(val) 53 | varbytes = [] 54 | 55 | loop do 56 | byte = val & 0x7f 57 | val >>= 7 58 | 59 | if val == 0 60 | varbytes.push(byte) 61 | break 62 | else 63 | varbytes.push(byte | 0x80) 64 | end 65 | end 66 | 67 | return varbytes.pack("C*") 68 | end 69 | 70 | private 71 | def self.decode_varbyte(text) 72 | len = [text.bytesize, 5].min 73 | bytes = text[0, len].unpack("C*") 74 | 75 | varbyte_len = 0 76 | val = 0 77 | bytes.each do |b| 78 | val |= (b & 0x7f) << (7 * varbyte_len) 79 | varbyte_len += 1 80 | return val, varbyte_len if b & 0x80 == 0 81 | end 82 | 83 | return -1, -1 84 | end 85 | 86 | # Handles LZ4 native data stream (without any additional headers). 87 | class Raw 88 | # Compresses `source` string. 89 | # 90 | # @param [String] source string to be compressed 91 | # @param [Hash] options 92 | # @option options [Fixnum] :input_size byte size of source to compress (must be less than or equal to `source.bytesize`) 93 | # @option options [String] :dest output buffer which will receive compressed string 94 | # @option options [Fixnum] :max_output_size acceptable maximum output size 95 | # @return [String, Fixnum] compressed string and its length. 96 | def self.compress(source, options = {}) 97 | return _compress(source, false, options) 98 | end 99 | 100 | # Compresses `source` string using High Compress Mode. 101 | # 102 | # @param [String] source string to be compressed 103 | # @param [Hash] options 104 | # @option options [Fixnum] :input_size byte size of source to compress (must be less than or equal to `source.bytesize`) 105 | # @option options [String] :dest output buffer which will receive compressed string 106 | # @option options [Fixnum] :max_output_size acceptable maximum output size 107 | # @return [String, Fixnum] compressed string and its length. 108 | def self.compressHC(source, options = {}) 109 | return _compress(source, true, options) 110 | end 111 | 112 | private 113 | def self._compress(source, high_compression, options = {}) 114 | input_size = options[:input_size] 115 | if input_size == nil 116 | input_size = source.bytesize 117 | 118 | else 119 | if source.bytesize < input_size 120 | raise ArgumentError, "`:input_size` (#{input_size}) must be less than or equal `source.bytesize` (#{source.bytesize})" 121 | end 122 | end 123 | 124 | dest = options[:dest] 125 | 126 | max_output_size = options[:max_output_size] 127 | if max_output_size == nil 128 | if dest != nil 129 | max_output_size = dest.bytesize 130 | else 131 | max_output_size = input_size + (input_size / 255) + 16 if dest == nil 132 | end 133 | 134 | else 135 | if dest != nil && dest.bytesize < max_output_size 136 | raise ArgumentError, "`:dest` buffer size (#{dest.bytesize}) must be greater than or equal `:max_output_size` (#{max_output_size})" 137 | end 138 | end 139 | 140 | result = nil 141 | if high_compression 142 | result = LZ4Internal.compressHC_raw(source, input_size, dest, max_output_size) 143 | else 144 | result = LZ4Internal.compress_raw(source, input_size, dest, max_output_size) 145 | end 146 | 147 | if result[1] <= 0 148 | raise LZ4Error, "compression failed" 149 | end 150 | 151 | return result[0], result[1] 152 | end 153 | 154 | # Decompresses `source` compressed string. 155 | # 156 | # @param [String] source 157 | # @param [Fixnum] max_output_size 158 | # @param [Hash] options 159 | # @option options [Fixnum] :input_size byte size of source to decompress (must be less than or equal to `source.bytesize`) 160 | # @option options [String] :dest output buffer which will receive decompressed string 161 | # @return [String, Fixnum] decompressed string and its length. 162 | def self.decompress(source, max_output_size, options = {}) 163 | input_size = options[:input_size] 164 | if input_size == nil 165 | input_size = source.bytesize 166 | 167 | else 168 | if source.bytesize < input_size 169 | raise ArgumentError, "`:input_size` (#{input_size}) must be less than or equal `source.bytesize` (#{source.bytesize})" 170 | end 171 | end 172 | 173 | dest = options[:dest] 174 | 175 | if dest != nil && dest.bytesize < max_output_size 176 | raise ArgumentError, "`:dest` buffer size (#{dest.bytesize}) must be greater than or equal `max_output_size` (#{max_output_size})" 177 | end 178 | 179 | result = LZ4Internal.decompress_raw(source, input_size, dest, max_output_size) 180 | 181 | if result[1] <= 0 182 | return "", 0 if source == "\x00" 183 | raise LZ4Error, "decompression failed" 184 | end 185 | 186 | return result[0], result[1] 187 | end 188 | end 189 | end 190 | 191 | class LZ4Error < StandardError 192 | end 193 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.headius.jruby.lz4 6 | lz4ruby 7 | 0.2.0 8 | jar 9 | 10 | lz4ruby 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | junit 20 | junit 21 | 3.8.1 22 | test 23 | 24 | 25 | net.jpountz.lz4 26 | lz4 27 | 1.1.1 28 | 29 | 30 | org.jruby 31 | jruby 32 | 1.7.0 33 | provided 34 | 35 | 36 | 37 | 38 | 39 | 40 | maven-compiler-plugin 41 | 3.1 42 | 43 | 1.6 44 | 1.6 45 | 46 | 47 | 48 | org.apache.maven.plugins 49 | maven-shade-plugin 50 | 2.0 51 | 52 | 53 | 54 | net.jpountz 55 | com.headius.jruby.lz4.vendor.net.jpountz 56 | 57 | 58 | ${basedir}/lib/lz4ruby.jar 59 | 60 | 61 | 62 | package 63 | 64 | shade 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /setup_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Requirements: 4 | # mingw32 (# sudo aptitude install mingw32 ) 5 | # rvm (# curl -L get.rvm.io | bash -s stable ) 6 | 7 | [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" 8 | 9 | function setup_ruby_env() { 10 | VER=$1 11 | 12 | rvm install ${VER} 13 | rvm use ${VER} --default 14 | rvm gemset create lz4-ruby 15 | rvm gemset use lz4-ruby 16 | gem install bundler 17 | bundle install 18 | } 19 | 20 | VER_MRI_18=`rvm list known_strings | grep 1.8.7-p | sed -e s/ruby-//` 21 | VER_MRI_19=`rvm list known_strings | grep 1.9.3-p | sed -e s/ruby-//` 22 | VER_JRUBY=`rvm list known_strings | grep jruby-1.7.8` 23 | 24 | setup_ruby_env ${VER_MRI_18} 25 | bundle exec rake-compiler cross-ruby VERSION=${VER_MRI_18} EXTS=--without-extensions 26 | setup_ruby_env ${VER_MRI_19} 27 | bundle exec rake-compiler cross-ruby VERSION=${VER_MRI_19} EXTS=--without-extensions 28 | setup_ruby_env ${VER_JRUBY} 29 | -------------------------------------------------------------------------------- /spec/helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | begin 4 | Bundler.setup(:default, :development) 5 | rescue Bundler::BundlerError => e 6 | $stderr.puts e.message 7 | $stderr.puts "Run `bundle install` to install missing gems" 8 | exit e.status_code 9 | end 10 | require 'test/unit' 11 | 12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 13 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'ext/lz4ruby')) 14 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 15 | 16 | build_native = < input_size) 39 | 40 | aaa_ex = [0x30, 0x61, 0x61, 0x61] 41 | 42 | it "should be compressed to length #{aaa_ex.size}" do 43 | expect(size).to eql(aaa_ex.size) 44 | expect(compressed.size).to eql(aaa_ex.size) 45 | end 46 | 47 | it "should be compressed into #{aaa_ex}" do 48 | expect(compressed).to eql(aaa_ex.pack("C*")) 49 | end 50 | end 51 | 52 | context "(30) which is greater than input text length (#{text.length})" do 53 | input_size = 30 54 | it "should be thrown ArgumentError" do 55 | expect { 56 | compressed, size = LZ4::Raw.compress(text, :input_size => input_size) 57 | }.to raise_error(ArgumentError, "`:input_size` (30) must be less than or equal `source.bytesize` (20)") 58 | end 59 | end 60 | end 61 | 62 | context "with output buffer" do 63 | context "enough buffer size: 10" do 64 | out_buf_size = 10 65 | out_buf = " " * out_buf_size 66 | 67 | compressed, size = LZ4::Raw.compress(text, :dest => out_buf) 68 | 69 | it "should be compressed to length #{expected.size}" do 70 | expect(size).to eql(expected.size) 71 | end 72 | 73 | it "should be unchanged output buffer size" do 74 | expect(compressed.size).to eql(out_buf_size) 75 | end 76 | 77 | it "should be compressed into '#{expected}'" do 78 | expect(compressed[0, size]).to eql(expected.pack("C*")) 79 | end 80 | 81 | it "should be used output buffer" do 82 | expect(compressed).to eql(out_buf) 83 | end 84 | end 85 | 86 | context "poor buffer size: 3" do 87 | out_buf_size = 3 88 | out_buf = " " * out_buf_size 89 | 90 | it "shoud be thrown LZ4Error" do 91 | expect { 92 | compressed, size = LZ4::Raw.compress(text, :dest => out_buf) 93 | }.to raise_error(LZ4Error, "compression failed") 94 | end 95 | end 96 | end 97 | 98 | context "with max_output_size" do 99 | context "enough max_output_size: 10" do 100 | max_output_size = 10 101 | compressed, size = LZ4::Raw.compress(text, :max_output_size => max_output_size) 102 | 103 | it "should be compressed to length #{expected.size}" do 104 | expect(size).to eql(expected.size) 105 | end 106 | 107 | it "should be compressed into '#{expected}'" do 108 | expect(compressed[0, size]).to eql(expected.pack("C*")) 109 | end 110 | end 111 | 112 | context "poor max_output_size: 3" do 113 | max_output_size = 3 114 | it "shoud be thrown LZ4Error" do 115 | expect { 116 | compressed, size = LZ4::Raw.compress(text, :max_output_size => max_output_size) 117 | }.to raise_error(LZ4Error, "compression failed") 118 | end 119 | end 120 | end 121 | 122 | context "with output buffer and max_output_size" do 123 | context "when size of output buffer: 30, which is greater than max_output_size: 20" do 124 | out_buf_size = 30 125 | max_output_size = 20 126 | out_buf = " " * out_buf_size 127 | 128 | compressed, size = LZ4::Raw.compress(text, :dest => out_buf, :max_output_size => max_output_size) 129 | 130 | it "should be compressed to length #{expected.size}" do 131 | expect(size).to eql(expected.size) 132 | end 133 | 134 | it "should be compressed into '#{expected}'" do 135 | expect(compressed[0, size]).to eql(expected.pack("C*")) 136 | end 137 | end 138 | 139 | context "when size of output buffer: 10, which is less than max_output_size: 20" do 140 | out_buf_size = 10 141 | max_output_size = 20 142 | out_buf = " " * out_buf_size 143 | 144 | it "should be thrown ArgumentError" do 145 | expect { 146 | compressed, size = LZ4::Raw.compress(text, :dest => out_buf, :max_output_size => max_output_size) 147 | }.to raise_error(ArgumentError, "`:dest` buffer size (10) must be greater than or equal `:max_output_size` (20)") 148 | end 149 | end 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /spec/lz4_raw_decompress_spec.rb: -------------------------------------------------------------------------------- 1 | require './spec/helper' 2 | 3 | describe "LZ4::Raw.decompress" do 4 | context "give compressed empty text" do 5 | decompressed, size = LZ4::Raw.decompress([0x00].pack("C*"), 0) 6 | 7 | it "should be decompressed to length zero" do 8 | expect(size).to eql(0) 9 | expect(decompressed.size).to eql(0) 10 | expect(decompressed).to eql("") 11 | end 12 | end 13 | 14 | context "give compressed text '#{compressed = [0x1a, 0x61, 0x01, 0x00, 0x50, 0x61, 0x61, 0x61, 0x61, 0x61]}'" do 15 | expected = "aaaaaaaaaaaaaaaaaaaa" 16 | 17 | context "with enough max_output_size" do 18 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), 30) 19 | 20 | it "should be decompressed to length #{expected.size}" do 21 | expect(size).to eql(expected.size) 22 | expect(decompressed.size).to eql(expected.size) 23 | end 24 | 25 | it "should be decompressed into '#{expected}'" do 26 | expect(decompressed).to eql(expected) 27 | end 28 | end 29 | 30 | context "with poor max_output_size" do 31 | it "should be thrown LZ4Error" do 32 | expect { 33 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), 10) 34 | }.to raise_error LZ4Error, "decompression failed" 35 | end 36 | end 37 | 38 | context "with input_size" do 39 | context "#{compressed.length}, which is equal to compressed text length" do 40 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*") + " ", 30, :input_size => compressed.size) 41 | 42 | it "should be decompressed to length #{expected.size}" do 43 | expect(size).to eql(expected.size) 44 | expect(decompressed.size).to eql(expected.size) 45 | end 46 | 47 | it "should be decompressed into '#{expected}'" do 48 | expect(decompressed).to eql(expected) 49 | end 50 | end 51 | 52 | context "20 which is greater than compressed text length (#{compressed.length})" do 53 | input_size = 20 54 | 55 | it "should be thrown ArgumentError" do 56 | expect { 57 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), 30, :input_size => input_size) 58 | }.to raise_error ArgumentError, "`:input_size` (20) must be less than or equal `source.bytesize` (#{compressed.length})" 59 | end 60 | end 61 | end 62 | 63 | context "with output buffer" do 64 | context "enough buffer size: 30" do 65 | out_buf_size = 30 66 | out_buf = " " * out_buf_size 67 | 68 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), out_buf_size, :dest => out_buf) 69 | 70 | it "should be decompressed to length #{expected.size}" do 71 | expect(size).to eql(expected.size) 72 | end 73 | 74 | it "should be unchanged output buffer size" do 75 | expect(decompressed.size).to eql(out_buf_size) 76 | end 77 | 78 | it "should be decompressed into '#{expected}'" do 79 | expect(decompressed[0, size]).to eql(expected) 80 | end 81 | 82 | it "should be used output buffer" do 83 | expect(decompressed).to eql(out_buf) 84 | end 85 | end 86 | 87 | context "poor buffer size: 19" do 88 | out_buf_size = 19 89 | out_buf = " " * out_buf_size 90 | 91 | it "should be thrown LZ4Error" do 92 | expect { 93 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), out_buf_size, :dest => out_buf) 94 | }.to raise_error LZ4Error, "decompression failed" 95 | end 96 | end 97 | end 98 | 99 | context "with output buffer and max_output_size" do 100 | context "when size of output buffer: 30, which is greater than max_output_size: 20" do 101 | out_buf_size = 30 102 | max_output_size = 20 103 | out_buf = " " * out_buf_size 104 | 105 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), max_output_size, :dest => out_buf) 106 | 107 | it "should be decompressed to length #{expected.size}" do 108 | expect(size).to eql(expected.size) 109 | end 110 | 111 | it "should be decompressed into '#{expected}'" do 112 | expect(decompressed[0, size]).to eql(expected) 113 | end 114 | end 115 | 116 | context "when size of output buffer: 10, which is less than max_output_size: 20" do 117 | out_buf_size = 10 118 | max_output_size = 20 119 | out_buf = " " * out_buf_size 120 | 121 | it "should be thrown ArgumentError" do 122 | expect { 123 | decompressed, size = LZ4::Raw.decompress(compressed.pack("C*"), max_output_size, :dest => out_buf) 124 | }.to raise_error ArgumentError, "`:dest` buffer size (10) must be greater than or equal `max_output_size` (20)" 125 | end 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /spec/lz4_raw_spec.rb: -------------------------------------------------------------------------------- 1 | require './spec/helper' 2 | 3 | describe "LZ4::Raw compressibility" do 4 | context "give 'lz4-ruby.rb' file" do 5 | filename = "./lib/lz4-ruby.rb" 6 | text = IO.readlines(filename).join("\n") 7 | 8 | compressed, comp_size = LZ4::Raw.compress(text) 9 | compressedHC, compHC_size = LZ4::Raw.compressHC(text) 10 | 11 | decompressed, decomp_size = LZ4::Raw.decompress(compressed, text.length) 12 | decompressedHC, decompHC_size = LZ4::Raw.decompress(compressedHC, text.length) 13 | 14 | it "should be able to comprese smaller than original text" do 15 | expect(comp_size).to be < text.length 16 | end 17 | 18 | it "should be able to compressHC smaller than original text" do 19 | expect(compHC_size).to be < text.length 20 | end 21 | 22 | it "should be able to compressHC smaller than compress" do 23 | expect(compHC_size).to be < comp_size 24 | end 25 | 26 | it "should be able to decompress from compressed text" do 27 | expect(decompressed).to eql(text) 28 | end 29 | 30 | it "should be able to decompress from compressHCed text" do 31 | expect(decompressedHC).to eql(text) 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /src/main/java/com/headius/jruby/lz4/LZ4Internal.java: -------------------------------------------------------------------------------- 1 | package com.headius.jruby.lz4; 2 | 3 | import net.jpountz.lz4.LZ4Compressor; 4 | import net.jpountz.lz4.LZ4Exception; 5 | import net.jpountz.lz4.LZ4Factory; 6 | import org.jruby.RubyArray; 7 | import org.jruby.RubyInteger; 8 | import org.jruby.RubyString; 9 | import org.jruby.anno.JRubyMethod; 10 | import org.jruby.runtime.ThreadContext; 11 | import org.jruby.runtime.builtin.IRubyObject; 12 | import org.jruby.util.ByteList; 13 | 14 | import java.util.Arrays; 15 | 16 | @SuppressWarnings("UnusedParameters") 17 | public class LZ4Internal { 18 | private static final LZ4Factory FACTORY = LZ4Factory.fastestInstance(); 19 | 20 | private static RubyArray compressInternal( 21 | ThreadContext context, 22 | LZ4Compressor compressor, 23 | IRubyObject _header, 24 | IRubyObject _input, 25 | IRubyObject _inputSize, 26 | IRubyObject _outputBuffer, 27 | IRubyObject _maxOutputSize) { 28 | 29 | byte[] headerBytes = {}; 30 | int headerSize = 0; 31 | if (isNotNil(_header)) { 32 | ByteList header = _header.convertToString().getByteList(); 33 | headerBytes = header.getUnsafeBytes(); 34 | headerSize = header.getRealSize(); 35 | } 36 | 37 | int inputSize = intFrom(_inputSize); 38 | ByteOutput output; 39 | if (isNotNil(_outputBuffer)) { 40 | output = new ByteOutput(_outputBuffer, _maxOutputSize); 41 | 42 | } else if (isNotNil(_maxOutputSize)) { 43 | int maxOutputSize = intFrom(_maxOutputSize) + headerSize; 44 | output = new ByteOutput(maxOutputSize); 45 | 46 | } else { 47 | int maxOutputSize = compressor.maxCompressedLength(inputSize) + headerSize; 48 | output = new ByteOutput(maxOutputSize); 49 | } 50 | 51 | System.arraycopy( 52 | headerBytes, 0, 53 | output.unsafeBytes, 0, 54 | headerSize); 55 | 56 | ByteInput input = new ByteInput(_input); 57 | try { 58 | int compressedSize = compressor.compress( 59 | input.unsafeBytes, input.offset, inputSize, 60 | output.unsafeBytes, output.offset + headerSize, output.bufferSize - headerSize); 61 | 62 | return RubyArray.newArray(context.runtime, Arrays.asList( 63 | output.toRubyString(context, compressedSize + headerSize), 64 | RubyInteger.int2fix(context.runtime, compressedSize))); 65 | 66 | } catch (LZ4Exception ignore) { 67 | return RubyArray.newArray(context.runtime, Arrays.asList( 68 | null, 69 | RubyInteger.int2fix(context.runtime, -1))); 70 | } 71 | } 72 | 73 | private static RubyArray decompressInternal( 74 | ThreadContext context, 75 | IRubyObject _offset, 76 | IRubyObject _input, 77 | IRubyObject _inputSize, 78 | IRubyObject _outputBuffer, 79 | IRubyObject _maxOutputSize) { 80 | 81 | int offset = 0; 82 | if (isNotNil(_offset)) { 83 | offset = intFrom(_offset); 84 | } 85 | 86 | ByteOutput output; 87 | if (isNotNil(_outputBuffer)) { 88 | output = new ByteOutput(_outputBuffer, _maxOutputSize); 89 | 90 | } else { 91 | output = new ByteOutput(intFrom(_maxOutputSize)); 92 | } 93 | 94 | ByteInput input = new ByteInput(_input); 95 | try { 96 | int decompressedSize = FACTORY.unknwonSizeDecompressor() 97 | .decompress( 98 | input.unsafeBytes, input.offset + offset, intFrom(_inputSize) - offset, 99 | output.unsafeBytes, output.offset, output.bufferSize); 100 | 101 | return RubyArray.newArray(context.runtime, Arrays.asList( 102 | output.toRubyString(context, decompressedSize), 103 | RubyInteger.int2fix(context.runtime, decompressedSize))); 104 | 105 | } catch (LZ4Exception ignore) { 106 | return RubyArray.newArray(context.runtime, Arrays.asList( 107 | null, 108 | RubyInteger.int2fix(context.runtime, -1))); 109 | } 110 | } 111 | 112 | @JRubyMethod(module = true) 113 | public static IRubyObject compress(ThreadContext context, IRubyObject self, IRubyObject _header, IRubyObject _input, IRubyObject _in_size) { 114 | RubyArray array = compressInternal(context, 115 | FACTORY.fastCompressor(), 116 | _header, 117 | _input, 118 | _in_size, 119 | null, 120 | null); 121 | return array.first(); 122 | } 123 | 124 | @JRubyMethod(module = true) 125 | public static IRubyObject compressHC(ThreadContext context, IRubyObject self, IRubyObject _header, IRubyObject _input, IRubyObject _in_size) { 126 | RubyArray array = compressInternal(context, 127 | FACTORY.highCompressor(), 128 | _header, 129 | _input, 130 | _in_size, 131 | null, 132 | null); 133 | return array.first(); 134 | } 135 | 136 | @JRubyMethod(required = 4, module = true) 137 | public static IRubyObject uncompress(ThreadContext context, IRubyObject self, IRubyObject[] args) { 138 | RubyString input = args[0].convertToString(); 139 | RubyInteger in_size = args[1].convertToInteger(); 140 | RubyInteger header_size = args[2].convertToInteger(); 141 | RubyInteger buf_size = args[3].convertToInteger(); 142 | 143 | RubyArray array = decompressInternal(context, 144 | header_size, 145 | input, 146 | in_size, 147 | null, 148 | buf_size); 149 | return array.first(); 150 | } 151 | 152 | @JRubyMethod(required = 4, module = true) 153 | public static IRubyObject compress_raw( 154 | ThreadContext context, 155 | IRubyObject self, 156 | IRubyObject[] args) { 157 | 158 | IRubyObject _input = args[0]; 159 | IRubyObject _inputSize = args[1]; 160 | IRubyObject _outputBuffer = args[2]; 161 | IRubyObject _maxOutputSize = args[3]; 162 | 163 | return compressInternal(context, 164 | FACTORY.fastCompressor(), 165 | null, 166 | _input, 167 | _inputSize, 168 | _outputBuffer, 169 | _maxOutputSize); 170 | } 171 | 172 | @JRubyMethod(required = 4, module = true) 173 | public static IRubyObject compressHC_raw( 174 | ThreadContext context, 175 | IRubyObject self, 176 | IRubyObject[] args) { 177 | 178 | IRubyObject _input = args[0]; 179 | IRubyObject _inputSize = args[1]; 180 | IRubyObject _outputBuffer = args[2]; 181 | IRubyObject _maxOutputSize = args[3]; 182 | 183 | return compressInternal(context, 184 | FACTORY.highCompressor(), 185 | null, 186 | _input, 187 | _inputSize, 188 | _outputBuffer, 189 | _maxOutputSize); 190 | } 191 | 192 | @JRubyMethod(required = 4, module = true) 193 | public static IRubyObject decompress_raw( 194 | ThreadContext context, 195 | IRubyObject self, 196 | IRubyObject[] args) { 197 | 198 | IRubyObject _input = args[0]; 199 | IRubyObject _inputSize = args[1]; 200 | IRubyObject _outputBuffer = args[2]; 201 | IRubyObject _maxOutputSize = args[3]; 202 | 203 | return decompressInternal( 204 | context, 205 | null, 206 | _input, 207 | _inputSize, 208 | _outputBuffer, 209 | _maxOutputSize); 210 | } 211 | 212 | static class ByteInput { 213 | public final RubyString rubyString; 214 | public final ByteList byteList; 215 | public final byte[] unsafeBytes; 216 | public final int offset; 217 | public final int realSize; 218 | 219 | public ByteInput(IRubyObject buffer) { 220 | rubyString = buffer.convertToString(); 221 | byteList = rubyString.getByteList(); 222 | unsafeBytes = byteList.getUnsafeBytes(); 223 | offset = byteList.getBegin(); 224 | realSize = byteList.getRealSize(); 225 | } 226 | } 227 | 228 | static class ByteOutput { 229 | public final RubyString rubyString; 230 | public final ByteList byteList; 231 | public final byte[] unsafeBytes; 232 | public final int offset; 233 | public final int bufferSize; 234 | 235 | public ByteOutput(IRubyObject buffer, IRubyObject size) { 236 | bufferSize = intFrom(size); 237 | rubyString = buffer.convertToString(); 238 | byteList = rubyString.getByteList(); 239 | unsafeBytes = byteList.getUnsafeBytes(); 240 | offset = byteList.getBegin(); 241 | } 242 | 243 | public ByteOutput(int size) { 244 | bufferSize = size; 245 | rubyString = null; 246 | byteList = null; 247 | unsafeBytes = new byte[bufferSize]; 248 | offset = 0; 249 | } 250 | 251 | RubyString toRubyString(ThreadContext context, int length) { 252 | if (rubyString != null) { 253 | return rubyString; 254 | } 255 | 256 | return RubyString.newString(context.runtime, unsafeBytes, 0, length); 257 | } 258 | } 259 | 260 | /** 261 | * Test if specified IRubyObject is not null / nil. 262 | * 263 | * @param obj 264 | * @return 265 | */ 266 | private static boolean isNotNil(IRubyObject obj) { 267 | return obj != null && !obj.isNil(); 268 | } 269 | 270 | /** 271 | * Returns a integer value from specified IRubyObject. 272 | * 273 | * @param obj 274 | * @return 275 | */ 276 | private static int intFrom(IRubyObject obj) { 277 | RubyInteger rubyInt = obj.convertToInteger(); 278 | return (int) rubyInt.getLongValue(); 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /src/main/java/com/headius/jruby/lz4/LZ4Library.java: -------------------------------------------------------------------------------- 1 | package com.headius.jruby.lz4; 2 | 3 | import java.io.IOException; 4 | import org.jruby.Ruby; 5 | import org.jruby.RubyModule; 6 | import org.jruby.runtime.load.Library; 7 | 8 | public class LZ4Library implements Library { 9 | 10 | public void load(Ruby runtime, boolean wrap) throws IOException { 11 | RubyModule lz4Internal = runtime.defineModule("LZ4Internal"); 12 | 13 | lz4Internal.defineAnnotatedMethods(LZ4Internal.class); 14 | 15 | lz4Internal.defineClassUnder("Error", runtime.getStandardError(), runtime.getStandardError().getAllocator()); 16 | } 17 | 18 | } 19 | --------------------------------------------------------------------------------