├── .gitignore ├── .rspec ├── .travis.yml ├── CHANGES.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── ext ├── .gitignore ├── extconf.rb └── iobuffer.c ├── iobuffer.gemspec ├── lib ├── .gitignore ├── iobuffer.rb └── iobuffer │ └── version.rb ├── spec └── buffer_spec.rb └── tasks ├── extension.rake └── rspec.rake /.gitignore: -------------------------------------------------------------------------------- 1 | pkg 2 | .bundle 3 | tmp 4 | Gemfile.lock 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | --backtrace 4 | --default_path spec 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | rvm: 2 | - 1.8.7 3 | - 1.9.2 4 | - 1.9.3 5 | - ree 6 | - ruby-head 7 | - rbx-18mode 8 | - rbx-19mode 9 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 1.1.2 2 | ----- 3 | * Bugfixes for native extension compilation 4 | * Bugfixes for internal corruption 5 | 6 | 1.1.1 7 | ----- 8 | * Fix problems with native extension not compiling 9 | 10 | 1.1.0 11 | ----- 12 | * IO::Buffer#read_frame for reading delimited data 13 | * Bugfixes (#6) 14 | * Some basic gem modernization 15 | 16 | 1.0.0 17 | ----- 18 | * Switch to rake-compiler for building the C extension 19 | * Define IO::Buffer::MAX_SIZE (1 GiB) as the maximum buffer size 20 | * Raise ArgumentError instead of RangeError if the buffer size is too big 21 | 22 | 0.1.3 23 | ----- 24 | * Fix botched release :( 25 | * Update gemspec so it only globs .c and .rb files, to prevent future 26 | botched releases containing .o or .so files. 27 | 28 | 0.1.2 29 | ----- 30 | * Tuneable node size 31 | 32 | 0.1.1 33 | ----- 34 | * Ruby 1.8.7 compatibility fix 35 | 36 | 0.1.0 37 | ----- 38 | * Initial release 39 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source :rubygems 2 | 3 | # Specify your gem's dependencies in iobuffer.gemspec 4 | gemspec -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C)2007-12 Tony Arcieri 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | IO::Buffer 2 | ========== 3 | [![Build Status](http://travis-ci.org/tarcieri/iobuffer.png)](http://travis-ci.org/tarcieri/iobuffer) 4 | 5 | IO::Buffer is a fast byte queue which is primarily intended for non-blocking 6 | I/O applications but is suitable wherever buffering is required. IO::Buffer 7 | is compatible with Ruby 1.8/1.9 and Rubinius. 8 | 9 | Usage 10 | ----- 11 | 12 | IO::Buffer provides a subset of the methods available in Strings: 13 | 14 | ```ruby 15 | >> buf = IO::Buffer.new 16 | => # 17 | >> buf << "foo" 18 | => "foo" 19 | >> buf << "bar" 20 | => "bar" 21 | >> buf.to_str 22 | => "foobar" 23 | >> buf.size 24 | => 6 25 | ``` 26 | 27 | The IO::Buffer#<< method is an alias for IO::Buffer#append. A prepend method 28 | is also available: 29 | 30 | ```ruby 31 | >> buf = IO::Buffer.new 32 | => # 33 | >> buf.append "bar" 34 | => "bar" 35 | >> buf.prepend "foo" 36 | => "foo" 37 | >> buf.append "baz" 38 | => "baz" 39 | >> buf.to_str 40 | => "foobarbaz" 41 | ``` 42 | 43 | IO::Buffer#read can be used to retrieve the contents of a buffer. You can mix 44 | reads alongside adding more data to the buffer: 45 | 46 | ```ruby 47 | >> buf = IO::Buffer.new 48 | => # 49 | >> buf << "foo" 50 | => "foo" 51 | >> buf.read 2 52 | => "fo" 53 | >> buf << "bar" 54 | => "bar" 55 | >> buf.read 2 56 | => "ob" 57 | >> buf << "baz" 58 | => "baz" 59 | >> buf.read 3 60 | => "arb" 61 | ``` 62 | 63 | Finally, IO::Buffer provides methods for performing non-blocking I/O with the 64 | contents of the buffer. The IO::Buffer#read_from(IO) method will read as much 65 | data as possible from the given IO object until the read would block. 66 | 67 | The IO::Buffer#write_to(IO) method writes the contents of the buffer to the 68 | given IO object until either the buffer is empty or the write would block: 69 | 70 | ```ruby 71 | >> buf = IO::Buffer.new 72 | => # 73 | >> file = File.open("README") 74 | => # 75 | >> buf.read_from(file) 76 | => 1713 77 | >> buf.to_str 78 | => "= IO::Buffer\n\nIO::Buffer is a fast byte queue... 79 | ``` 80 | 81 | If the file descriptor is not ready for I/O, the Errno::EAGAIN exception is 82 | raised indicating no I/O was performed. 83 | 84 | Contributing 85 | ------------ 86 | 87 | * Fork this repository on github 88 | * Make your changes and send me a pull request 89 | * If I like them I'll merge them 90 | * If I've accepted a patch, feel free to ask for commit access 91 | 92 | License 93 | ------- 94 | 95 | Copyright (c) 2007-12 Tony Arcieri. Distributed under the MIT License. See 96 | LICENSE.txt for further details. 97 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require "bundler/gem_tasks" 3 | require "rake/clean" 4 | 5 | Dir[File.expand_path("../tasks/**/*.rake", __FILE__)].each { |task| load task } 6 | 7 | task :default => %w(compile spec) 8 | 9 | CLEAN.include "**/*.o", "**/*.so", "**/*.bundle", "pkg", "tmp" 10 | -------------------------------------------------------------------------------- /ext/.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | iobuffer.* 3 | mkmf.log 4 | conftest.dSYM 5 | -------------------------------------------------------------------------------- /ext/extconf.rb: -------------------------------------------------------------------------------- 1 | require 'mkmf' 2 | 3 | dir_config("iobuffer") 4 | have_library("c", "main") 5 | if have_macro("HAVE_RB_IO_T", "rubyio.h") 6 | have_struct_member("rb_io_t", "fd", "rubyio.h") 7 | end 8 | 9 | create_makefile("iobuffer_ext") 10 | -------------------------------------------------------------------------------- /ext/iobuffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2007-12 Tony Arcieri 3 | * You may redistribute this under the terms of the MIT license. 4 | * See LICENSE for details 5 | */ 6 | 7 | #include "ruby.h" 8 | #include "rubyio.h" 9 | 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | /* 1 GiB maximum buffer size */ 18 | #define MAX_BUFFER_SIZE 0x40000000 19 | 20 | /* Macro for retrieving the file descriptor from an FPTR */ 21 | #if !HAVE_RB_IO_T_FD 22 | #define FPTR_TO_FD(fptr) fileno(fptr->f) 23 | #else 24 | #define FPTR_TO_FD(fptr) fptr->fd 25 | #endif 26 | 27 | /* Default number of bytes in each node's buffer. Should be >= MTU */ 28 | #define DEFAULT_NODE_SIZE 16384 29 | static unsigned default_node_size = DEFAULT_NODE_SIZE; 30 | 31 | struct buffer { 32 | unsigned size, node_size; 33 | struct buffer_node *head, *tail; 34 | struct buffer_node *pool_head, *pool_tail; 35 | 36 | }; 37 | 38 | struct buffer_node { 39 | unsigned start, end; 40 | struct buffer_node *next; 41 | unsigned char data[0]; 42 | }; 43 | 44 | static VALUE cIO_Buffer = Qnil; 45 | 46 | static VALUE IO_Buffer_allocate(VALUE klass); 47 | static void IO_Buffer_mark(struct buffer *); 48 | static void IO_Buffer_free(struct buffer *); 49 | 50 | static VALUE IO_Buffer_default_node_size(VALUE klass); 51 | static VALUE IO_Buffer_set_default_node_size(VALUE klass, VALUE size); 52 | static VALUE IO_Buffer_initialize(int argc, VALUE * argv, VALUE self); 53 | static VALUE IO_Buffer_clear(VALUE self); 54 | static VALUE IO_Buffer_size(VALUE self); 55 | static VALUE IO_Buffer_empty(VALUE self); 56 | static VALUE IO_Buffer_append(VALUE self, VALUE data); 57 | static VALUE IO_Buffer_prepend(VALUE self, VALUE data); 58 | static VALUE IO_Buffer_read(int argc, VALUE * argv, VALUE self); 59 | static VALUE IO_Buffer_read_frame(VALUE self, VALUE data, VALUE mark); 60 | static VALUE IO_Buffer_to_str(VALUE self); 61 | static VALUE IO_Buffer_read_from(VALUE self, VALUE io); 62 | static VALUE IO_Buffer_write_to(VALUE self, VALUE io); 63 | 64 | static struct buffer *buffer_new(void); 65 | static void buffer_clear(struct buffer * buf); 66 | static void buffer_free(struct buffer * buf); 67 | static void buffer_free_pool(struct buffer * buf); 68 | static void buffer_prepend(struct buffer * buf, char *str, unsigned len); 69 | static void buffer_append(struct buffer * buf, char *str, unsigned len); 70 | static void buffer_read(struct buffer * buf, char *str, unsigned len); 71 | static int buffer_read_frame(struct buffer * buf, VALUE str, char frame_mark); 72 | static void buffer_copy(struct buffer * buf, char *str, unsigned len); 73 | static int buffer_read_from(struct buffer * buf, int fd); 74 | static int buffer_write_to(struct buffer * buf, int fd); 75 | 76 | /* 77 | * High-performance I/O buffer intended for use in non-blocking programs 78 | * 79 | * Data is stored in as a memory-pooled linked list of equally sized chunks. 80 | * Routines are provided for high speed non-blocking reads and writes from 81 | * Ruby IO objects. 82 | */ 83 | void 84 | Init_iobuffer_ext() 85 | { 86 | cIO_Buffer = rb_define_class_under(rb_cIO, "Buffer", rb_cObject); 87 | rb_define_alloc_func(cIO_Buffer, IO_Buffer_allocate); 88 | 89 | rb_define_singleton_method(cIO_Buffer, "default_node_size", 90 | IO_Buffer_default_node_size, 0); 91 | rb_define_singleton_method(cIO_Buffer, "default_node_size=", 92 | IO_Buffer_set_default_node_size, 1); 93 | 94 | rb_define_method(cIO_Buffer, "initialize", IO_Buffer_initialize, -1); 95 | rb_define_method(cIO_Buffer, "clear", IO_Buffer_clear, 0); 96 | rb_define_method(cIO_Buffer, "size", IO_Buffer_size, 0); 97 | rb_define_method(cIO_Buffer, "empty?", IO_Buffer_empty, 0); 98 | rb_define_method(cIO_Buffer, "<<", IO_Buffer_append, 1); 99 | rb_define_method(cIO_Buffer, "append", IO_Buffer_append, 1); 100 | rb_define_method(cIO_Buffer, "write", IO_Buffer_append, 1); 101 | rb_define_method(cIO_Buffer, "prepend", IO_Buffer_prepend, 1); 102 | rb_define_method(cIO_Buffer, "read", IO_Buffer_read, -1); 103 | rb_define_method(cIO_Buffer, "read_frame", IO_Buffer_read_frame, 2); 104 | rb_define_method(cIO_Buffer, "to_str", IO_Buffer_to_str, 0); 105 | rb_define_method(cIO_Buffer, "read_from", IO_Buffer_read_from, 1); 106 | rb_define_method(cIO_Buffer, "write_to", IO_Buffer_write_to, 1); 107 | 108 | rb_define_const(cIO_Buffer, "MAX_SIZE", INT2NUM(MAX_BUFFER_SIZE)); 109 | } 110 | 111 | static VALUE 112 | IO_Buffer_allocate(VALUE klass) 113 | { 114 | return Data_Wrap_Struct(klass, IO_Buffer_mark, IO_Buffer_free, buffer_new()); 115 | } 116 | 117 | static void 118 | IO_Buffer_mark(struct buffer * buf) 119 | { 120 | /* Naively discard the memory pool whenever Ruby garbage collects */ 121 | buffer_free_pool(buf); 122 | } 123 | 124 | static void 125 | IO_Buffer_free(struct buffer * buf) 126 | { 127 | buffer_free(buf); 128 | } 129 | 130 | /** 131 | * call-seq: 132 | * IO_Buffer.default_node_size -> 4096 133 | * 134 | * Retrieves the current value of the default node size. 135 | */ 136 | static VALUE 137 | IO_Buffer_default_node_size(VALUE klass) 138 | { 139 | return UINT2NUM(default_node_size); 140 | } 141 | 142 | /* 143 | * safely converts node sizes from Ruby numerics to C and raising 144 | * ArgumentError or RangeError on invalid sizes 145 | */ 146 | static unsigned 147 | convert_node_size(VALUE size) 148 | { 149 | if ( 150 | rb_funcall(size, rb_intern("<"), 1, INT2NUM(1)) == Qtrue || 151 | rb_funcall(size, rb_intern(">"), 1, INT2NUM(MAX_BUFFER_SIZE)) == Qtrue 152 | ) 153 | rb_raise(rb_eArgError, "invalid buffer size"); 154 | 155 | return (unsigned) NUM2INT(size); 156 | } 157 | 158 | /** 159 | * call-seq: 160 | * IO_Buffer.default_node_size = 16384 161 | * 162 | * Sets the default node size for calling IO::Buffer.new with no arguments. 163 | */ 164 | static VALUE 165 | IO_Buffer_set_default_node_size(VALUE klass, VALUE size) 166 | { 167 | default_node_size = convert_node_size(size); 168 | 169 | return size; 170 | } 171 | 172 | /** 173 | * call-seq: 174 | * IO_Buffer.new(size = IO::Buffer.default_node_size) -> IO_Buffer 175 | * 176 | * Create a new IO_Buffer with linked segments of the given size 177 | */ 178 | static VALUE 179 | IO_Buffer_initialize(int argc, VALUE * argv, VALUE self) 180 | { 181 | VALUE node_size_obj; 182 | struct buffer *buf; 183 | 184 | if (rb_scan_args(argc, argv, "01", &node_size_obj) == 1) { 185 | Data_Get_Struct(self, struct buffer, buf); 186 | 187 | /* 188 | * Make sure we're not changing the buffer size after data 189 | * has been allocated 190 | */ 191 | assert(!buf->head); 192 | assert(!buf->pool_head); 193 | 194 | buf->node_size = convert_node_size(node_size_obj); 195 | } 196 | return Qnil; 197 | } 198 | 199 | /** 200 | * call-seq: 201 | * IO_Buffer#clear -> nil 202 | * 203 | * Clear all data from the IO_Buffer 204 | */ 205 | static VALUE 206 | IO_Buffer_clear(VALUE self) 207 | { 208 | struct buffer *buf; 209 | Data_Get_Struct(self, struct buffer, buf); 210 | 211 | buffer_clear(buf); 212 | 213 | return Qnil; 214 | } 215 | 216 | /** 217 | * call-seq: 218 | * IO_Buffer#size -> Integer 219 | * 220 | * Return the size of the buffer in bytes 221 | */ 222 | static VALUE 223 | IO_Buffer_size(VALUE self) 224 | { 225 | struct buffer *buf; 226 | Data_Get_Struct(self, struct buffer, buf); 227 | 228 | return INT2NUM(buf->size); 229 | } 230 | 231 | /** 232 | * call-seq: 233 | * IO_Buffer#empty? -> Boolean 234 | * 235 | * Is the buffer empty? 236 | */ 237 | static VALUE 238 | IO_Buffer_empty(VALUE self) 239 | { 240 | struct buffer *buf; 241 | Data_Get_Struct(self, struct buffer, buf); 242 | 243 | return buf->size > 0 ? Qfalse : Qtrue; 244 | } 245 | 246 | /** 247 | * call-seq: 248 | * IO_Buffer#append(data) -> String 249 | * 250 | * Append the given data to the end of the buffer 251 | */ 252 | static VALUE 253 | IO_Buffer_append(VALUE self, VALUE data) 254 | { 255 | struct buffer *buf; 256 | Data_Get_Struct(self, struct buffer, buf); 257 | 258 | /* Is this needed? Never seen anyone else do it... */ 259 | data = rb_convert_type(data, T_STRING, "String", "to_str"); 260 | buffer_append(buf, RSTRING_PTR(data), RSTRING_LEN(data)); 261 | 262 | return data; 263 | } 264 | 265 | /** 266 | * call-seq: 267 | * IO_Buffer#prepend(data) -> String 268 | * 269 | * Prepend the given data to the beginning of the buffer 270 | */ 271 | static VALUE 272 | IO_Buffer_prepend(VALUE self, VALUE data) 273 | { 274 | struct buffer *buf; 275 | Data_Get_Struct(self, struct buffer, buf); 276 | 277 | data = rb_convert_type(data, T_STRING, "String", "to_str"); 278 | buffer_prepend(buf, RSTRING_PTR(data), RSTRING_LEN(data)); 279 | 280 | return data; 281 | } 282 | 283 | /** 284 | * call-seq: 285 | * IO_Buffer#read(length = nil) -> String 286 | * 287 | * Read the specified abount of data from the buffer. If no value 288 | * is given the entire contents of the buffer are returned. Any data 289 | * read from the buffer is cleared. 290 | * The given length must be greater than 0 or an exception would raise. 291 | * If the buffer size is zero then an empty string is returned (regardless 292 | * the given length). 293 | */ 294 | static VALUE 295 | IO_Buffer_read(int argc, VALUE * argv, VALUE self) 296 | { 297 | VALUE length_obj, str; 298 | int length; 299 | struct buffer *buf; 300 | 301 | Data_Get_Struct(self, struct buffer, buf); 302 | 303 | if (rb_scan_args(argc, argv, "01", &length_obj) == 1) { 304 | length = NUM2INT(length_obj); 305 | if(length < 1) 306 | rb_raise(rb_eArgError, "length must be greater than zero"); 307 | if(length > buf->size) 308 | length = buf->size; 309 | } else 310 | length = buf->size; 311 | 312 | if(buf->size == 0) 313 | return rb_str_new2(""); 314 | 315 | str = rb_str_new(0, length); 316 | buffer_read(buf, RSTRING_PTR(str), length); 317 | 318 | return str; 319 | } 320 | 321 | /** 322 | * call-seq: 323 | * IO_Buffer#read_frame(str, mark) -> boolean 324 | * 325 | * Read up to and including the given frame marker (expressed a a 326 | * Fixnum 0-255) byte, copying into the supplied string object. If the mark is 327 | * not encountered before the end of the buffer, false is returned but data 328 | * is still copied into str. True is returned if the end of a frame is reached. 329 | * 330 | */ 331 | static VALUE 332 | IO_Buffer_read_frame(VALUE self, VALUE data, VALUE mark) 333 | { 334 | char mark_c = (char) NUM2INT(mark); 335 | struct buffer *buf; 336 | 337 | Data_Get_Struct(self, struct buffer, buf); 338 | 339 | if (buffer_read_frame(buf, data, mark_c)) { 340 | return Qtrue; 341 | } else { 342 | return Qfalse; 343 | } 344 | } 345 | 346 | /** 347 | * call-seq: 348 | * IO_Buffer#to_str -> String 349 | * 350 | * Convert the Buffer to a String. The original buffer is unmodified. 351 | */ 352 | static VALUE 353 | IO_Buffer_to_str(VALUE self) 354 | { 355 | VALUE str; 356 | struct buffer *buf; 357 | 358 | Data_Get_Struct(self, struct buffer, buf); 359 | 360 | str = rb_str_new(0, buf->size); 361 | buffer_copy(buf, RSTRING_PTR(str), buf->size); 362 | 363 | return str; 364 | } 365 | 366 | /** 367 | * call-seq: 368 | * IO_Buffer#read_from(io) -> Integer 369 | * 370 | * Perform a nonblocking read of the the given IO object and fill 371 | * the buffer with any data received. The call will read as much 372 | * data as it can until the read would block. 373 | */ 374 | static VALUE 375 | IO_Buffer_read_from(VALUE self, VALUE io) 376 | { 377 | struct buffer *buf; 378 | int ret; 379 | #if HAVE_RB_IO_T 380 | rb_io_t *fptr; 381 | #else 382 | OpenFile *fptr; 383 | #endif 384 | 385 | Data_Get_Struct(self, struct buffer, buf); 386 | GetOpenFile(rb_convert_type(io, T_FILE, "IO", "to_io"), fptr); 387 | rb_io_set_nonblock(fptr); 388 | 389 | ret = buffer_read_from(buf, FPTR_TO_FD(fptr)); 390 | return ret == -1 ? Qnil : INT2NUM(ret); 391 | } 392 | 393 | /** 394 | * call-seq: 395 | * IO_Buffer#write_to(io) -> Integer 396 | * 397 | * Perform a nonblocking write of the buffer to the given IO object. 398 | * As much data as possible is written until the call would block. 399 | * Any data which is written is removed from the buffer. 400 | */ 401 | static VALUE 402 | IO_Buffer_write_to(VALUE self, VALUE io) 403 | { 404 | struct buffer *buf; 405 | #if HAVE_RB_IO_T 406 | rb_io_t *fptr; 407 | #else 408 | OpenFile *fptr; 409 | #endif 410 | 411 | Data_Get_Struct(self, struct buffer, buf); 412 | GetOpenFile(rb_convert_type(io, T_FILE, "IO", "to_io"), fptr); 413 | rb_io_set_nonblock(fptr); 414 | 415 | return INT2NUM(buffer_write_to(buf, FPTR_TO_FD(fptr))); 416 | } 417 | 418 | /* 419 | * Ruby bindings end here. Below is the actual implementation of 420 | * the underlying byte queue ADT 421 | */ 422 | 423 | /* Create a new buffer */ 424 | static struct buffer * 425 | buffer_new(void) 426 | { 427 | struct buffer *buf; 428 | 429 | buf = (struct buffer *) xmalloc(sizeof(struct buffer)); 430 | buf->head = buf->tail = buf->pool_head = buf->pool_tail = 0; 431 | buf->size = 0; 432 | buf->node_size = default_node_size; 433 | 434 | return buf; 435 | } 436 | 437 | /* Clear all data from a buffer */ 438 | static void 439 | buffer_clear(struct buffer * buf) 440 | { 441 | /* Move everything into the buffer pool */ 442 | if (!buf->pool_tail) { 443 | buf->pool_head = buf->pool_tail = buf->head; 444 | } else { 445 | buf->pool_tail->next = buf->head; 446 | } 447 | 448 | buf->head = buf->tail = 0; 449 | buf->size = 0; 450 | } 451 | 452 | /* Free a buffer */ 453 | static void 454 | buffer_free(struct buffer * buf) 455 | { 456 | buffer_clear(buf); 457 | buffer_free_pool(buf); 458 | 459 | free(buf); 460 | } 461 | 462 | /* Free the memory pool */ 463 | static void 464 | buffer_free_pool(struct buffer * buf) 465 | { 466 | struct buffer_node *tmp; 467 | 468 | while (buf->pool_head) { 469 | tmp = buf->pool_head; 470 | buf->pool_head = tmp->next; 471 | free(tmp); 472 | } 473 | 474 | buf->pool_tail = 0; 475 | } 476 | 477 | /* Create a new buffer_node (or pull one from the memory pool) */ 478 | static struct buffer_node * 479 | buffer_node_new(struct buffer * buf) 480 | { 481 | struct buffer_node *node; 482 | 483 | /* Pull from the memory pool if available */ 484 | if (buf->pool_head) { 485 | node = buf->pool_head; 486 | buf->pool_head = node->next; 487 | 488 | if (node->next) 489 | node->next = 0; 490 | else 491 | buf->pool_tail = 0; 492 | } else { 493 | node = (struct buffer_node *) xmalloc(sizeof(struct buffer_node) + buf->node_size); 494 | node->next = 0; 495 | } 496 | 497 | node->start = node->end = 0; 498 | return node; 499 | } 500 | 501 | /* Free a buffer node (i.e. return it to the memory pool) */ 502 | static void 503 | buffer_node_free(struct buffer * buf, struct buffer_node * node) 504 | { 505 | node->next = buf->pool_head; 506 | buf->pool_head = node; 507 | 508 | if (!buf->pool_tail) { 509 | buf->pool_tail = node; 510 | } 511 | } 512 | 513 | /* Prepend data to the front of the buffer */ 514 | static void 515 | buffer_prepend(struct buffer * buf, char *str, unsigned len) 516 | { 517 | struct buffer_node *node, *tmp; 518 | buf->size += len; 519 | 520 | /* If it fits in the beginning of the head */ 521 | if (buf->head && buf->head->start >= len) { 522 | buf->head->start -= len; 523 | memcpy(buf->head->data + buf->head->start, str, len); 524 | } else { 525 | node = buffer_node_new(buf); 526 | node->next = buf->head; 527 | buf->head = node; 528 | if (!buf->tail) 529 | buf->tail = node; 530 | 531 | while (len > buf->node_size) { 532 | memcpy(node->data, str, buf->node_size); 533 | node->end = buf->node_size; 534 | 535 | tmp = buffer_node_new(buf); 536 | tmp->next = node->next; 537 | node->next = tmp; 538 | 539 | if (buf->tail == node) 540 | buf->tail = tmp; 541 | node = tmp; 542 | 543 | str += buf->node_size; 544 | len -= buf->node_size; 545 | } 546 | 547 | if (len > 0) { 548 | memcpy(node->data, str, len); 549 | node->end = len; 550 | } 551 | } 552 | } 553 | 554 | /* Append data to the front of the buffer */ 555 | static void 556 | buffer_append(struct buffer * buf, char *str, unsigned len) 557 | { 558 | unsigned nbytes; 559 | buf->size += len; 560 | 561 | /* If it fits in the remaining space in the tail */ 562 | if (buf->tail && len <= buf->node_size - buf->tail->end) { 563 | memcpy(buf->tail->data + buf->tail->end, str, len); 564 | buf->tail->end += len; 565 | return; 566 | } 567 | /* Empty list needs initialized */ 568 | if (!buf->head) { 569 | buf->head = buffer_node_new(buf); 570 | buf->tail = buf->head; 571 | } 572 | /* Build links out of the data */ 573 | while (len > 0) { 574 | nbytes = buf->node_size - buf->tail->end; 575 | if (len < nbytes) 576 | nbytes = len; 577 | 578 | memcpy(buf->tail->data + buf->tail->end, str, nbytes); 579 | str += nbytes; 580 | len -= nbytes; 581 | 582 | buf->tail->end += nbytes; 583 | 584 | if (len > 0) { 585 | buf->tail->next = buffer_node_new(buf); 586 | buf->tail = buf->tail->next; 587 | } 588 | } 589 | } 590 | 591 | /* Read data from the buffer (and clear what we've read) */ 592 | static void 593 | buffer_read(struct buffer * buf, char *str, unsigned len) 594 | { 595 | unsigned nbytes; 596 | struct buffer_node *tmp; 597 | 598 | while (buf->size > 0 && len > 0) { 599 | nbytes = buf->head->end - buf->head->start; 600 | if (len < nbytes) 601 | nbytes = len; 602 | 603 | memcpy(str, buf->head->data + buf->head->start, nbytes); 604 | str += nbytes; 605 | len -= nbytes; 606 | 607 | buf->head->start += nbytes; 608 | buf->size -= nbytes; 609 | 610 | if (buf->head->start == buf->head->end) { 611 | tmp = buf->head; 612 | buf->head = tmp->next; 613 | buffer_node_free(buf, tmp); 614 | 615 | if (!buf->head) 616 | buf->tail = 0; 617 | } 618 | } 619 | } 620 | 621 | /* 622 | * Read data from the buffer into str until byte frame_mark or empty. Bytes 623 | * are copied into str and removed if a complete frame is read, a true value 624 | * is returned 625 | */ 626 | static int 627 | buffer_read_frame(struct buffer * buf, VALUE str, char frame_mark) 628 | { 629 | unsigned nbytes = 0; 630 | struct buffer_node *tmp; 631 | 632 | while (buf->size > 0) { 633 | struct buffer_node *head = buf->head; 634 | char *loc, *s = head->data + head->start, *e = head->data + head->end; 635 | nbytes = e - s; 636 | 637 | loc = memchr(s, frame_mark, nbytes); 638 | 639 | if (loc) { 640 | nbytes = loc - s + 1; 641 | } 642 | 643 | /* Copy less than everything if we found a frame byte */ 644 | rb_str_cat(str, s, nbytes); 645 | 646 | /* Fixup the buffer pointers to indicate the bytes were consumed */ 647 | head->start += nbytes; 648 | buf->size -= nbytes; 649 | 650 | if (head->start == head->end) { 651 | buf->head = head->next; 652 | buffer_node_free(buf, head); 653 | 654 | if (!buf->head) 655 | buf->tail = 0; 656 | } 657 | 658 | if (loc) { 659 | return 1; 660 | } 661 | } 662 | 663 | return 0; 664 | } 665 | 666 | /* Copy data from the buffer without clearing it */ 667 | static void 668 | buffer_copy(struct buffer * buf, char *str, unsigned len) 669 | { 670 | unsigned nbytes; 671 | struct buffer_node *node; 672 | 673 | node = buf->head; 674 | while (node && len > 0) { 675 | nbytes = node->end - node->start; 676 | if (len < nbytes) 677 | nbytes = len; 678 | 679 | memcpy(str, node->data + node->start, nbytes); 680 | str += nbytes; 681 | len -= nbytes; 682 | 683 | if (node->start + nbytes == node->end) 684 | node = node->next; 685 | } 686 | } 687 | 688 | /* Write data from the buffer to a file descriptor */ 689 | static int 690 | buffer_write_to(struct buffer * buf, int fd) 691 | { 692 | int bytes_written, total_bytes_written = 0; 693 | struct buffer_node *tmp; 694 | 695 | while (buf->head) { 696 | bytes_written = write(fd, buf->head->data + buf->head->start, buf->head->end - buf->head->start); 697 | 698 | /* If the write failed... */ 699 | if (bytes_written < 0) { 700 | if (errno != EAGAIN) 701 | rb_sys_fail("write"); 702 | 703 | return total_bytes_written; 704 | } 705 | 706 | total_bytes_written += bytes_written; 707 | buf->size -= bytes_written; 708 | 709 | /* If the write blocked... */ 710 | if (bytes_written < buf->head->end - buf->head->start) { 711 | buf->head->start += bytes_written; 712 | return total_bytes_written; 713 | } 714 | /* Otherwise we wrote the whole buffer */ 715 | tmp = buf->head; 716 | buf->head = tmp->next; 717 | buffer_node_free(buf, tmp); 718 | 719 | if (!buf->head) 720 | buf->tail = 0; 721 | } 722 | 723 | return total_bytes_written; 724 | } 725 | 726 | /* Read data from a file descriptor to a buffer */ 727 | /* Append data to the front of the buffer */ 728 | static int 729 | buffer_read_from(struct buffer * buf, int fd) 730 | { 731 | int bytes_read, total_bytes_read = 0; 732 | unsigned nbytes; 733 | 734 | /* Empty list needs initialized */ 735 | if (!buf->head) { 736 | buf->head = buffer_node_new(buf); 737 | buf->tail = buf->head; 738 | } 739 | 740 | do { 741 | nbytes = buf->node_size - buf->tail->end; 742 | bytes_read = read(fd, buf->tail->data + buf->tail->end, nbytes); 743 | 744 | if (bytes_read == 0) { 745 | return -1; 746 | //When the file reaches EOF 747 | } else if (bytes_read < 0) { 748 | if (errno != EAGAIN) 749 | rb_sys_fail("read"); 750 | 751 | return total_bytes_read; 752 | } 753 | 754 | total_bytes_read += bytes_read; 755 | buf->tail->end += bytes_read; 756 | buf->size += bytes_read; 757 | 758 | if (buf->tail->end == buf->node_size) { 759 | buf->tail->next = buffer_node_new(buf); 760 | buf->tail = buf->tail->next; 761 | } 762 | } while (bytes_read == nbytes); 763 | 764 | return total_bytes_read; 765 | } 766 | -------------------------------------------------------------------------------- /iobuffer.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path('../lib', __FILE__) 3 | require "iobuffer/version" 4 | 5 | Gem::Specification.new do |gem| 6 | gem.name = "iobuffer" 7 | gem.version = IO::Buffer::VERSION 8 | gem.platform = Gem::Platform::RUBY 9 | gem.summary = "fast buffers for non-blocking IO" 10 | gem.description = gem.summary 11 | gem.licenses = ['MIT'] 12 | 13 | gem.authors = ['Tony Arcieri'] 14 | gem.email = ['tony.arcieri@gmail.com'] 15 | gem.homepage = 'https://github.com/tarcieri/iobuffer' 16 | 17 | gem.required_rubygems_version = '>= 1.3.6' 18 | 19 | gem.files = Dir['README.md', 'lib/iobuffer', 'lib/**/*', 'ext/**/*.{c,rb}'] 20 | gem.require_path = 'lib' 21 | 22 | gem.extensions = ["ext/extconf.rb"] 23 | 24 | gem.add_development_dependency 'rake-compiler' 25 | gem.add_development_dependency 'rake' 26 | gem.add_development_dependency 'rspec' 27 | end 28 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | iobuffer.* 2 | -------------------------------------------------------------------------------- /lib/iobuffer.rb: -------------------------------------------------------------------------------- 1 | require 'iobuffer/version' 2 | require 'iobuffer_ext' 3 | -------------------------------------------------------------------------------- /lib/iobuffer/version.rb: -------------------------------------------------------------------------------- 1 | class IO 2 | class Buffer 3 | VERSION = "1.1.2" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/buffer_spec.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../lib/iobuffer') 2 | 3 | describe IO::Buffer do 4 | before :each do 5 | @buffer = IO::Buffer.new 6 | @buffer.size.should == 0 7 | end 8 | 9 | it "appends data" do 10 | @buffer.append "foo" 11 | @buffer.size.should == 3 12 | 13 | @buffer << "bar" 14 | @buffer.size.should == 6 15 | 16 | @buffer.write "baz" 17 | @buffer.size.should == 9 18 | 19 | @buffer.read.should == "foobarbaz" 20 | @buffer.size.should == 0 21 | end 22 | 23 | it "prepends data" do 24 | @buffer.prepend "foo" 25 | @buffer.size.should == 3 26 | 27 | @buffer.prepend "bar" 28 | @buffer.size.should == 6 29 | 30 | @buffer.read.should == "barfoo" 31 | @buffer.size.should == 0 32 | end 33 | 34 | it "mixes prepending and appending properly" do 35 | source_data = %w{foo bar baz qux} 36 | actions = permutator([:append, :prepend] * 2) 37 | 38 | actions.each do |sequence| 39 | sequence.each_with_index do |entry, i| 40 | @buffer.send(entry, source_data[i]) 41 | end 42 | 43 | @buffer.size.should == sequence.size * 3 44 | 45 | i = 0 46 | expected = sequence.inject('') do |str, action| 47 | case action 48 | when :append 49 | str << source_data[i] 50 | when :prepend 51 | str = source_data[i] + str 52 | end 53 | 54 | i += 1 55 | str 56 | end 57 | 58 | @buffer.read.should == expected 59 | end 60 | end 61 | 62 | it "reads data in chunks properly" do 63 | @buffer.append "foobarbazqux" 64 | 65 | @buffer.read(1).should == 'f' 66 | @buffer.read(2).should == 'oo' 67 | @buffer.read(3).should == 'bar' 68 | @buffer.read(4).should == 'bazq' 69 | @buffer.read(1).should == 'u' 70 | @buffer.read(2).should == 'x' 71 | end 72 | 73 | it "converts to a string" do 74 | @buffer.append "foobar" 75 | @buffer.to_str == "foobar" 76 | end 77 | 78 | it "clears data" do 79 | @buffer.append "foo" 80 | @buffer.prepend "bar" 81 | 82 | @buffer.clear 83 | @buffer.size.should == 0 84 | @buffer.read.should == "" 85 | 86 | @buffer.prepend "foo" 87 | @buffer.prepend "bar" 88 | @buffer.append "baz" 89 | 90 | @buffer.clear 91 | @buffer.size.should == 0 92 | @buffer.read.should == "" 93 | end 94 | 95 | it "knows when it's empty" do 96 | @buffer.should be_empty 97 | @buffer.append "foo" 98 | @buffer.should_not be_empty 99 | end 100 | 101 | it "can set default node size" do 102 | IO::Buffer.default_node_size = 1 103 | IO::Buffer.default_node_size.should == 1 104 | (IO::Buffer.default_node_size = 4096).should == 4096 105 | (IO::Buffer.default_node_size = IO::Buffer::MAX_SIZE).should == IO::Buffer::MAX_SIZE 106 | end 107 | 108 | it "can be created with a different node size" do 109 | IO::Buffer.new(16384) 110 | end 111 | 112 | it "cannot set invalid node sizes" do 113 | proc { 114 | IO::Buffer.default_node_size = IO::Buffer::MAX_SIZE + 1 115 | }.should raise_error(ArgumentError) 116 | proc { 117 | IO::Buffer.default_node_size = 0 118 | }.should raise_error(ArgumentError) 119 | proc { 120 | IO::Buffer.new(IO::Buffer::MAX_SIZE + 1) 121 | }.should raise_error(ArgumentError) 122 | proc { 123 | IO::Buffer.new(0) 124 | }.should raise_error(ArgumentError) 125 | end 126 | 127 | it "Reads can read a single frame" do 128 | @buffer.append("foo\0bar") 129 | str = "" 130 | @buffer.read_frame(str, 0).should == true 131 | str.should == "foo\0" 132 | @buffer.size.should == 3 133 | end 134 | 135 | it "Reads a frame, then reads only some data" do 136 | @buffer.append("foo\0bar") 137 | str = "" 138 | @buffer.read_frame(str,0) 139 | str = "" 140 | #This will read only a partial frame 141 | @buffer.read_frame(str,0).should == false 142 | str.should == "bar" 143 | @buffer.size.should == 0 144 | end 145 | 146 | it "Returns nil when reading from a filehandle at EOF" do 147 | (rp, wp) = File.pipe 148 | 149 | wp.write("Foo") 150 | wp.flush 151 | @buffer.read_from(rp).should == 3 152 | @buffer.read_from(rp).should == 0 153 | wp.close 154 | @buffer.read_from(rp).should == nil 155 | end 156 | 157 | it "Maintains proper buffer size" do 158 | #TODO use more methods 159 | 160 | #Testing of normal append 161 | str = "clarp of the flarn" 162 | @buffer.append(str) 163 | s = @buffer.size 164 | s.should == str.length 165 | 166 | #Testing of read_from 167 | (rp, wp) = File.pipe 168 | wp.write(str) 169 | wp.close 170 | @buffer.read_from(rp) 171 | @buffer.size.should == 2*str.length 172 | end 173 | 174 | it "Can handle lots of data" do 175 | rp, wp = File.pipe 176 | srand 1 177 | 178 | total = 0 179 | 100.times do 180 | chunk_size = rand(2048) #We don't actually know the pipe buffer size! 181 | if rand > 0.5 182 | wp.write("x" * chunk_size) 183 | @buffer.read_from(rp) 184 | else 185 | @buffer.append("x" * chunk_size) 186 | end 187 | total += chunk_size 188 | end 189 | wp.close 190 | @buffer.read_from(rp) 191 | 192 | @buffer.size.should == total 193 | @buffer.read.should == "x" * total 194 | end 195 | 196 | ####### 197 | private 198 | ####### 199 | 200 | def permutator(input) 201 | output = [] 202 | return output if input.empty? 203 | 204 | (0..input.size - 1).inject([]) do |a, n| 205 | if a.empty? 206 | input.each { |x| output << [x] } 207 | else 208 | input.each { |x| output += a.map { |y| [x, *y] } } 209 | end 210 | 211 | output.dup 212 | end 213 | end 214 | end 215 | -------------------------------------------------------------------------------- /tasks/extension.rake: -------------------------------------------------------------------------------- 1 | require 'rake/extensiontask' 2 | Rake::ExtensionTask.new('iobuffer_ext') do |ext| 3 | ext.ext_dir = 'ext' 4 | end 5 | -------------------------------------------------------------------------------- /tasks/rspec.rake: -------------------------------------------------------------------------------- 1 | require 'rspec/core/rake_task' 2 | 3 | RSpec::Core::RakeTask.new 4 | 5 | RSpec::Core::RakeTask.new(:rcov) do |task| 6 | task.rcov = true 7 | end --------------------------------------------------------------------------------