├── VERSION ├── .document ├── Gemfile ├── .gitignore ├── LICENSE.txt ├── Rakefile ├── Gemfile.lock ├── em-files.gemspec ├── test.rb ├── README.md └── lib └── em-files.rb /VERSION: -------------------------------------------------------------------------------- 1 | 0.2.4 -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | # Add dependencies required to use your gem here. 3 | # Example: 4 | gem "eventmachine", ">= 0" 5 | 6 | # Add dependencies to develop your gem here. 7 | # Include everything needed to run rake, tests, features, etc. 8 | group :development do 9 | gem "bundler", ">= 1.0.0" 10 | gem "jeweler", ">= 1.5.2" 11 | gem "riot", ">= 0.12.1" 12 | end 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # rcov generated 2 | coverage 3 | 4 | # rdoc generated 5 | rdoc 6 | 7 | # yard generated 8 | doc 9 | .yardoc 10 | 11 | # bundler 12 | .bundle 13 | 14 | # jeweler generated 15 | pkg 16 | 17 | # aptana 18 | .project 19 | 20 | # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 21 | # 22 | # * Create a file at ~/.gitignore 23 | # * Include files you want ignored 24 | # * Run: git config --global core.excludesfile ~/.gitignore 25 | # 26 | # After doing this, these files will be ignored in all your git projects, 27 | # saving you from having to 'pollute' every project you touch with them 28 | # 29 | # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line) 30 | # 31 | # For MacOS: 32 | # 33 | #.DS_Store 34 | # 35 | # For TextMate 36 | #*.tmproj 37 | #tmtags 38 | # 39 | # For emacs: 40 | #*~ 41 | #\#* 42 | #.\#* 43 | # 44 | # For vim: 45 | #*.swp 46 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 - 2015 Martin Poljak (martin@poljak.cz) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require 'rubygems' 3 | require 'bundler' 4 | 5 | begin 6 | Bundler.setup(:default, :development) 7 | rescue Bundler::BundlerError => e 8 | $stderr.puts e.message 9 | $stderr.puts "Run `bundle install` to install missing gems" 10 | exit e.status_code 11 | end 12 | 13 | require 'rake' 14 | require 'jeweler' 15 | 16 | Jeweler::Tasks.new do |gem| 17 | # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options 18 | gem.name = "em-files" 19 | gem.homepage = "https://github.com/martinkozak/em-files" 20 | gem.license = "MIT" 21 | gem.summary = "Sequenced file reader and writer through EventMachine. Solves problem of blocking disk IO when operating with large files." 22 | gem.email = "martin@poljak.cz" 23 | gem.authors = ["Martin Poljak"] 24 | # Include your dependencies below. Runtime dependencies are required when using your gem, 25 | # and development dependencies are only needed for development (ie running rake tasks, tests, etc) 26 | # gem.add_runtime_dependency 'jabber4r', '> 0.1' 27 | # gem.add_development_dependency 'rspec', '> 1.2.3' 28 | end 29 | Jeweler::RubygemsDotOrgTasks.new 30 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | addressable (2.3.8) 5 | builder (3.2.2) 6 | descendants_tracker (0.0.4) 7 | thread_safe (~> 0.3, >= 0.3.1) 8 | eventmachine (1.0.7) 9 | faraday (0.9.1) 10 | multipart-post (>= 1.2, < 3) 11 | git (1.2.9.1) 12 | github_api (0.12.3) 13 | addressable (~> 2.3) 14 | descendants_tracker (~> 0.0.4) 15 | faraday (~> 0.8, < 0.10) 16 | hashie (>= 3.3) 17 | multi_json (>= 1.7.5, < 2.0) 18 | nokogiri (~> 1.6.3) 19 | oauth2 20 | hashie (3.4.2) 21 | highline (1.7.2) 22 | jeweler (2.0.1) 23 | builder 24 | bundler (>= 1.0) 25 | git (>= 1.2.5) 26 | github_api 27 | highline (>= 1.6.15) 28 | nokogiri (>= 1.5.10) 29 | rake 30 | rdoc 31 | jwt (1.5.1) 32 | mini_portile (0.6.2) 33 | multi_json (1.11.2) 34 | multi_xml (0.5.5) 35 | multipart-post (2.0.0) 36 | nokogiri (1.6.6.2) 37 | mini_portile (~> 0.6.0) 38 | oauth2 (1.0.0) 39 | faraday (>= 0.8, < 0.10) 40 | jwt (~> 1.0) 41 | multi_json (~> 1.3) 42 | multi_xml (~> 0.5) 43 | rack (~> 1.2) 44 | rack (1.6.4) 45 | rake (10.4.2) 46 | rdoc (4.2.0) 47 | riot (0.12.7) 48 | rr 49 | rr (1.1.2) 50 | thread_safe (0.3.5) 51 | 52 | PLATFORMS 53 | ruby 54 | 55 | DEPENDENCIES 56 | bundler (>= 1.0.0) 57 | eventmachine 58 | jeweler (>= 1.5.2) 59 | riot (>= 0.12.1) 60 | 61 | BUNDLED WITH 62 | 1.10.5 63 | -------------------------------------------------------------------------------- /em-files.gemspec: -------------------------------------------------------------------------------- 1 | # Generated by jeweler 2 | # DO NOT EDIT THIS FILE DIRECTLY 3 | # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' 4 | # -*- encoding: utf-8 -*- 5 | # stub: em-files 0.2.4 ruby lib 6 | 7 | Gem::Specification.new do |s| 8 | s.name = "em-files" 9 | s.version = "0.2.4" 10 | 11 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 12 | s.require_paths = ["lib"] 13 | s.authors = ["Martin Poljak"] 14 | s.date = "2015-07-18" 15 | s.email = "martin@poljak.cz" 16 | s.extra_rdoc_files = [ 17 | "LICENSE.txt", 18 | "README.md" 19 | ] 20 | s.files = [ 21 | ".document", 22 | "Gemfile", 23 | "Gemfile.lock", 24 | "LICENSE.txt", 25 | "README.md", 26 | "Rakefile", 27 | "VERSION", 28 | "em-files.gemspec", 29 | "lib/em-files.rb", 30 | "test.rb" 31 | ] 32 | s.homepage = "https://github.com/martinkozak/em-files" 33 | s.licenses = ["MIT"] 34 | s.rubygems_version = "2.4.5" 35 | s.summary = "Sequenced file reader and writer through EventMachine. Solves problem of blocking disk IO when operating with large files." 36 | 37 | if s.respond_to? :specification_version then 38 | s.specification_version = 4 39 | 40 | if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then 41 | s.add_runtime_dependency(%q, [">= 0"]) 42 | s.add_development_dependency(%q, [">= 1.0.0"]) 43 | s.add_development_dependency(%q, [">= 1.5.2"]) 44 | s.add_development_dependency(%q, [">= 0.12.1"]) 45 | else 46 | s.add_dependency(%q, [">= 0"]) 47 | s.add_dependency(%q, [">= 1.0.0"]) 48 | s.add_dependency(%q, [">= 1.5.2"]) 49 | s.add_dependency(%q, [">= 0.12.1"]) 50 | end 51 | else 52 | s.add_dependency(%q, [">= 0"]) 53 | s.add_dependency(%q, [">= 1.0.0"]) 54 | s.add_dependency(%q, [">= 1.5.2"]) 55 | s.add_dependency(%q, [">= 0.12.1"]) 56 | end 57 | end 58 | 59 | -------------------------------------------------------------------------------- /test.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | # encoding: utf-8 3 | # (c) 2011 Martin Kozák 4 | 5 | $:.push("./lib") 6 | require 'em-files' 7 | require "riot" 8 | 9 | test = Array::new(5) 10 | 11 | EM::run do 12 | 13 | # Test 1 14 | test[0] = EM::File::open("./~test1", "w") 15 | test[0].close() 16 | 17 | # Test 2 18 | EM::File::open("./~test1", "w") do |io| 19 | io.write("x" * 300000) do |len| 20 | test[1] = len 21 | io.close() 22 | end 23 | end 24 | 25 | EM::add_timer(1) do 26 | # Test 3 27 | EM::File::open("./~test1", "r") do |io| 28 | io.read do |data| 29 | test[2] = data 30 | io.close() 31 | end 32 | end 33 | end 34 | 35 | # Test 4 36 | EM::File::write("./~test2", "x" * 300000) do |len| 37 | test[3] = len 38 | end 39 | 40 | EM::add_timer(1) do 41 | # Test 5 42 | EM::File::read("./~test2") do |data| 43 | test[4] = data 44 | end 45 | end 46 | 47 | EM::add_timer(2) do 48 | EM::stop 49 | end 50 | 51 | end 52 | 53 | 54 | context "EM::Files (instance methods)" do 55 | setup { test } 56 | 57 | asserts("#open returns EM::File object") do 58 | topic[0].kind_of? EM::File 59 | end 60 | asserts("file size produced by #write is equivalent to reported written data length") do 61 | topic[1] == File.size?("./~test1") 62 | end 63 | asserts("file content produced by #write is correct and #read works well") do 64 | topic[2] == "x" * 300000 65 | end 66 | 67 | teardown do 68 | File.unlink("./~test1") 69 | end 70 | end 71 | 72 | context "EM::Files (class methods)" do 73 | setup { test } 74 | 75 | asserts("file size produced by #write is equivalent to reported written data length") do 76 | topic[1] == File.size?("./~test2") 77 | end 78 | asserts("file content produced by #write is correct and #read works well") do 79 | topic[2] == "x" * 300000 80 | end 81 | 82 | teardown do 83 | File.unlink("./~test2") 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | EventMachine Files 2 | ================== 3 | 4 | **em-files** solve problem of blocking disk IO when operating with 5 | large files. Use [EventMachine][4] for multiplexing reads and writes 6 | to small blocks performed in standalone EM ticks. They speed down the 7 | file IO operations of sure, but allow running other tasks with them 8 | simultaneously (from EM point of view). 9 | 10 | There is, of sure, question whether this all has sense as `EM::defer` 11 | is available for handling these blocking tasks. But sometimes are 12 | situations, in which it's undesirable to execute them in separate thread. 13 | 14 | API is similar to classic Ruby file IO represented by [File][1] class. 15 | See an example: 16 | ```ruby 17 | require "em-files" 18 | EM::run do 19 | EM::File::open("some_file.txt", "r") do |io| 20 | io.read(1024) do |data| # writing works by very similar 21 | # way, of sure 22 | puts data 23 | io.close() 24 | # it's necessary to do it in block too, because reading 25 | # is evented 26 | end 27 | end 28 | end 29 | ``` 30 | 31 | Support of Ruby API is limited to `#open`, `#close`, `#read` and `#write` 32 | methods only, so for special operations use simply: 33 | 34 | ```ruby 35 | EM::File::open("some_file.txt", "r") do |io| 36 | io.native # returns native Ruby File class object 37 | end 38 | ``` 39 | 40 | ### Special Uses 41 | 42 | It's possible to use also another IO objects than `File` object by 43 | giving appropriate IO instance instead of filename to methods: 44 | 45 | ```ruby 46 | require "em-files" 47 | require "stringio" 48 | 49 | io = StringIO::new 50 | 51 | EM::run do 52 | EM::File::open(io) do |io| 53 | # some multiplexed operations 54 | end 55 | end 56 | ``` 57 | 58 | By this way you can also perform for example more time consuming 59 | operations by simple way (if they can be processed in block manner) 60 | using filters: 61 | 62 | ```ruby 63 | require "em-files" 64 | require "zlib" 65 | 66 | zip = Zlib::Deflate::new 67 | filter = Proc::new { |chunk| zip.deflate(chunk, Zlib::SYNC_FLUSH) } 68 | data = "..." # some data bigger than big 69 | 70 | EM::run do 71 | EM::File::write(data, filter) # done in several ticks 72 | end 73 | ``` 74 | 75 | `#write` supports also copying data from another IO stream because it 76 | uses `StringIO` internally. Simply give it IO object instead of 77 | `String`. It will read it until EOF will occur. 78 | 79 | 80 | Copyright 81 | --------- 82 | 83 | Copyright © 2011 – 2015 [Martin Poljak][3]. See `LICENSE.txt` for further details. 84 | 85 | [1]: http://www.ruby-doc.org/core/classes/File.html 86 | [2]: http://github.com/martinkozak/em-files/issues 87 | [3]: http://www.martinpoljak.net/ 88 | [4]: http://rubyeventmachine.com/ 89 | -------------------------------------------------------------------------------- /lib/em-files.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # (c) 2011 Martin Kozák (martinkozak@martinkozak.net) 3 | 4 | require "eventmachine" 5 | require "stringio" 6 | 7 | ## 8 | # Main EventMachine module. 9 | # @see http://rubyeventmachine.com/ 10 | # 11 | 12 | module EM 13 | 14 | ## 15 | # Sequenced file reader and writer. 16 | # 17 | 18 | class File 19 | 20 | ## 21 | # Holds the default size of block operated during one tick. 22 | # 23 | 24 | RWSIZE = 65536 25 | 26 | ## 27 | # Opens the file. 28 | # 29 | # In opposite to appropriate Ruby method, "block syntax" is only 30 | # syntactic sugar, file isn't closed after return from block 31 | # because processing is asynchronous so it doesn't know when 32 | # is convenient to close the file. 33 | # 34 | # @param [String, IO, StringIO] filepath path to file or IO object 35 | # @param [String] mode file access mode (see equivalent Ruby method) 36 | # @param [Integer] rwsize size of block operated during one tick 37 | # @param [Proc] block syntactic sugar for wrapping File access object 38 | # @return [File] file access object 39 | # @yield [File] file access object 40 | # 41 | 42 | def self.open(filepath, mode = "r", rwsize = self::RWSIZE, &block) # 64 kilobytes 43 | rwsize = self::RWSIZE if rwsize.nil? 44 | 45 | file = self::new(filepath, mode, rwsize) 46 | if not block.nil? 47 | yield file 48 | end 49 | 50 | return file 51 | end 52 | 53 | ## 54 | # Reads whole content of the file. Be warn, it reads it in 55 | # binary mode. If IO object is given instead of filepath, uses 56 | # it as native one and +mode+ argument is ignored. 57 | # 58 | # @param [String, IO, StringIO] filepath path to file or IO object 59 | # @param [Integer] rwsize size of block operated during one tick 60 | # @param [Proc] filter filter which for postprocessing each 61 | # read chunk 62 | # @param [Proc] block block for giving back the result 63 | # @yield [String] read data 64 | # 65 | 66 | 67 | def self.read(filepath, rwsize = self::RWSIZE, filter = nil, &block) 68 | rwsize = self::RWSIZE if rwsize.nil? 69 | self::open(filepath, "rb", rwsize) do |io| 70 | io.read(nil, filter) do |out| 71 | io.close() 72 | yield out 73 | end 74 | end 75 | end 76 | 77 | ## 78 | # Writes data to file and closes it. Writes them in binary mode. 79 | # If IO object is given instead of filepath, uses it as native 80 | # one and +mode+ argument is ignored. 81 | # 82 | # @param [String, IO, StringIO] filepath path to file or IO object 83 | # @param [String] data data for write 84 | # @param [Integer] rwsize size of block operated during one tick 85 | # @param [Proc] filter filter which for preprocessing each 86 | # written chunk 87 | # @param [Proc] block block called when writing is finished with 88 | # written bytes size count as parameter 89 | # @yield [Integer] really written data length 90 | # 91 | 92 | def self.write(filepath, data = "", rwsize = self::RWSIZE, filter = nil, &block) 93 | rwsize = self::RWSIZE if rwsize.nil? 94 | self::open(filepath, "wb", rwsize) do |io| 95 | io.write(data, filter) do |length| 96 | io.close() 97 | if not block.nil? 98 | yield length 99 | end 100 | end 101 | end 102 | end 103 | 104 | ### 105 | 106 | ## 107 | # Holds file object. 108 | # @return [IO] 109 | # 110 | 111 | attr_accessor :native 112 | @native 113 | 114 | ## 115 | # Indicates block size for operate with in one tick. 116 | # @return [Integer] 117 | # 118 | 119 | attr_accessor :rw_len 120 | @rw_len 121 | 122 | ## 123 | # Holds mode of the object. 124 | # @return [String] 125 | # 126 | 127 | attr_reader :mode 128 | @mode 129 | 130 | ## 131 | # Constructor. If IO object is given instead of filepath, uses 132 | # it as native one and +mode+ argument is ignored. 133 | # 134 | # @param [String, IO, StringIO] filepath path to file or IO object 135 | # @param [String] mode file access mode (see equivalent Ruby method) 136 | # @param [Integer] rwsize size of block operated during one tick 137 | # 138 | 139 | def initialize(filepath, mode = "r", rwsize = self.class::RWSIZE) 140 | @mode = mode 141 | @rw_len = rwsize 142 | 143 | rwsize = self::RWSIZE if rwsize.nil? 144 | 145 | # If filepath is directly IO, uses it 146 | if filepath.kind_of? IO 147 | @native = filepath 148 | else 149 | @native = ::File::open(filepath, mode) 150 | end 151 | 152 | end 153 | 154 | ## 155 | # Reads data from file. 156 | # 157 | # It will reopen the file if +EBADF: Bad file descriptor+ of 158 | # +File+ class IO object will occur. 159 | # 160 | # @overload read(length, &block) 161 | # Reads specified amount of data from file. 162 | # @param [Integer] length length for read from file 163 | # @param [Proc] filter filter which for postprocessing each 164 | # read chunk 165 | # @param [Proc] block callback for returning the result 166 | # @yield [String] read data 167 | # @overload read(&block) 168 | # Reads whole content of file. 169 | # @param [Proc] filter filter which for processing each block 170 | # @param [Proc] block callback for returning the result 171 | # @yield [String] read data 172 | # 173 | 174 | def read(length = nil, filter = nil, &block) 175 | buffer = "" 176 | pos = 0 177 | 178 | # Arguments 179 | if length.kind_of? Proc 180 | filter = length 181 | end 182 | 183 | 184 | worker = Proc::new do 185 | 186 | # Sets length for read 187 | if not length.nil? 188 | rlen = length - buffer.length 189 | if rlen > @rw_len 190 | rlen = @rw_len 191 | end 192 | else 193 | rlen = @rw_len 194 | end 195 | 196 | # Reads 197 | begin 198 | chunk = @native.read(rlen) 199 | if not filter.nil? 200 | chunk = filter.call(chunk) 201 | end 202 | buffer << chunk 203 | rescue Errno::EBADF 204 | if @native.kind_of? ::File 205 | self.reopen! 206 | @native.seek(pos) 207 | redo 208 | else 209 | raise 210 | end 211 | end 212 | 213 | pos = @native.pos 214 | 215 | # Returns or continues work 216 | if @native.eof? or (buffer.length == length) 217 | if not block.nil? 218 | yield buffer # returns result 219 | end 220 | else 221 | EM::next_tick { worker.call() } # continues work 222 | end 223 | 224 | end 225 | 226 | worker.call() 227 | end 228 | 229 | ## 230 | # Reopens the file with the original mode. 231 | # 232 | 233 | def reopen! 234 | @native = ::File.open(@native.path, @mode) 235 | end 236 | 237 | ## 238 | # Writes data to file. Supports writing both strings or copying 239 | # from another IO object. Returns length of written data to 240 | # callback if filename given or current position of output 241 | # string if IO used. 242 | # 243 | # It will reopen the file if +EBADF: Bad file descriptor+ of 244 | # +File+ class IO object will occur on +File+ object. 245 | # 246 | # @param [String, IO, StringIO] data data for write or IO object 247 | # @param [Proc] filter filter which for preprocessing each 248 | # written chunk 249 | # @param [Proc] block callback called when finish and for giving 250 | # back the length of written data 251 | # @yield [Integer] length of really written data 252 | # 253 | 254 | def write(data, filter = nil, &block) 255 | pos = 0 256 | 257 | if data.kind_of? IO 258 | io = data 259 | else 260 | io = StringIO::new(data) 261 | end 262 | 263 | worker = Proc::new do 264 | 265 | # Writes 266 | begin 267 | chunk = io.read(@rw_len) 268 | if not filter.nil? 269 | chunk = filter.call(chunk) 270 | end 271 | @native.write(chunk) 272 | rescue Errno::EBADF 273 | if @native.kind_of? File 274 | self.reopen! 275 | @native.seek(pos) 276 | redo 277 | else 278 | raise 279 | end 280 | end 281 | 282 | pos = @native.pos 283 | 284 | # Returns or continues work 285 | if io.eof? 286 | if not block.nil? 287 | yield pos # returns result 288 | end 289 | else 290 | EM::next_tick { worker.call() } # continues work 291 | end 292 | 293 | end 294 | 295 | worker.call() 296 | end 297 | 298 | ## 299 | # Closes the file. 300 | # 301 | 302 | def close 303 | @native.close 304 | end 305 | end 306 | end 307 | --------------------------------------------------------------------------------