├── sample ├── colors.ppm ├── gradation.pgm ├── langs.csv └── poi-xls2csv.sh ├── test-all.rb ├── test ├── util_tbtest.rb ├── test_revcmp.rb ├── test_customcmp.rb ├── test_json.rb ├── test_customeq.rb ├── test_catreader.rb ├── test_cmd_to_pnm.rb ├── test_zipper.rb ├── test_tsv.rb ├── test_cmd_nest.rb ├── test_numericcsv.rb ├── test_cmd_to_yaml.rb ├── test_headercsv.rb ├── test_ltsv.rb ├── test_cmd_to_tsv.rb ├── test_cmd_to_ltsv.rb ├── test_cmd_to_pp.rb ├── test_cmdutil.rb ├── test_cmd_newfield.rb ├── test_ndjson.rb ├── test_reader.rb ├── test_cmd_unnest.rb ├── test_cmd_grep.rb ├── test_cmd_rename.rb ├── test_cmd_mheader.rb ├── test_cmd_shape.rb ├── test_csv.rb ├── test_cmd_cut.rb ├── test_cmd_crop.rb ├── test_cmd_consecutive.rb ├── test_cmd_sort.rb ├── test_cmd_gsub.rb ├── test_cmd_to_json.rb ├── test_pager.rb ├── test_cmd_cross.rb ├── test_cmd_svn_log.rb ├── test_cmd_to_csv.rb ├── test_cmdtty.rb └── test_cmd_melt.rb ├── test-all-cov.rb ├── bin └── tb ├── lib ├── tb │ ├── revcmp.rb │ ├── customeq.rb │ ├── customcmp.rb │ ├── catreader.rb │ ├── hashwriter.rb │ ├── json.rb │ ├── cmd_to_pnm.rb │ ├── cmd_to_ltsv.rb │ ├── cmd_to_tsv.rb │ ├── cmdmain.rb │ ├── cmd_to_yaml.rb │ ├── numericwriter.rb │ ├── ndjson.rb │ ├── numericreader.rb │ ├── cmd_to_json.rb │ ├── zipper.rb │ ├── csv.rb │ ├── cmd_to_csv.rb │ ├── cmd_cat.rb │ ├── cmdtop.rb │ ├── cmd_to_pp.rb │ ├── pager.rb │ ├── tsv.rb │ ├── hashreader.rb │ ├── cmd_rename.rb │ ├── cmd_newfield.rb │ ├── cmd_cut.rb │ ├── cmd_sort.rb │ ├── cmd_consecutive.rb │ ├── enumerator.rb │ ├── cmd_mheader.rb │ ├── cmd_gsub.rb │ ├── headerwriter.rb │ ├── ropen.rb │ ├── headerreader.rb │ ├── cmd_nest.rb │ ├── cmd_search.rb │ ├── cmd_join.rb │ ├── cmd_group.rb │ ├── ex_enumerator.rb │ ├── cmd_shape.rb │ ├── cmd_help.rb │ └── ltsv.rb └── tb.rb └── tb.gemspec /sample/colors.ppm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akr/tb/HEAD/sample/colors.ppm -------------------------------------------------------------------------------- /sample/gradation.pgm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/akr/tb/HEAD/sample/gradation.pgm -------------------------------------------------------------------------------- /test-all.rb: -------------------------------------------------------------------------------- 1 | $VERBOSE = true 2 | 3 | $:.unshift "lib" 4 | 5 | r, w = IO.pipe 6 | w.close 7 | $stdin.reopen(r) 8 | r.close 9 | 10 | Dir.glob('test/test_*.rb') {|filename| 11 | load filename 12 | } 13 | -------------------------------------------------------------------------------- /test/util_tbtest.rb: -------------------------------------------------------------------------------- 1 | require 'stringio' 2 | 3 | def capture_stderr 4 | begin 5 | save_stderr = $stderr 6 | $stderr = StringIO.new(stderr='') 7 | yield 8 | ensure 9 | $stderr = save_stderr 10 | end 11 | stderr 12 | end 13 | -------------------------------------------------------------------------------- /test/test_revcmp.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestRevCmp < Test::Unit::TestCase 5 | def test_cmp 6 | assert_equal(2 <=> 1, Tb::RevCmp.new(1) <=> Tb::RevCmp.new(2)) 7 | assert_equal(1 <=> 2, Tb::RevCmp.new(2) <=> Tb::RevCmp.new(1)) 8 | assert_equal(3 <=> 3, Tb::RevCmp.new(3) <=> Tb::RevCmp.new(3)) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /test/test_customcmp.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestCustomCmp < Test::Unit::TestCase 5 | def test_revcmp 6 | cmp = lambda {|a, b| b <=> a } 7 | v1 = Tb::CustomCmp.new(1, &cmp) 8 | v2 = Tb::CustomCmp.new(2, &cmp) 9 | v3 = Tb::CustomCmp.new(3, &cmp) 10 | assert_equal(2 <=> 1, v1 <=> v2) 11 | assert_equal(1 <=> 2, v2 <=> v1) 12 | assert_equal(3 <=> 3, v3 <=> v3) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/test_json.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbJSON < Test::Unit::TestCase 5 | def test_parse 6 | r = Tb::JSONReader.new(StringIO.new('[{"a":1, "b":2}, {"a":3, "b":4}]')) 7 | result = [] 8 | r.with_header {|header| 9 | result << header 10 | }.each {|obj| 11 | result << obj 12 | } 13 | assert_equal([%w[a b], {"a"=>1, "b"=>2}, {"a"=>3, "b"=>4}], result) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/test_customeq.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestCustomEq < Test::Unit::TestCase 5 | def test_decreasing 6 | rel = lambda {|a, b| a > b } 7 | v1 = Tb::CustomEq.new(1, &rel) 8 | v2 = Tb::CustomEq.new(2, &rel) 9 | v3 = Tb::CustomEq.new(3, &rel) 10 | assert_equal(false, v1 == v1) 11 | assert_equal(false, v1 == v2) 12 | assert_equal(false, v1 == v3) 13 | assert_equal(true, v2 == v1) 14 | assert_equal(false, v2 == v2) 15 | assert_equal(false, v2 == v3) 16 | assert_equal(true, v3 == v1) 17 | assert_equal(true, v3 == v2) 18 | assert_equal(false, v3 == v3) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /sample/langs.csv: -------------------------------------------------------------------------------- 1 | language,year 2 | FORTRAN,1955 3 | LISP,1958 4 | COBOL,1959 5 | ALGOL 58,1958 6 | APL,1962 7 | Simula,1962 8 | SNOBOL,1962 9 | BASIC,1964 10 | PL/I,1964 11 | BCPL,1967 12 | Logo,1968 13 | B,1969 14 | Pascal,1970 15 | Forth,1970 16 | C,1972 17 | Smalltalk,1972 18 | Prolog,1972 19 | ML,1973 20 | Scheme,1975 21 | SQL,1978 22 | C++,1980 23 | Objective-C,1983 24 | Ada,1983 25 | Common Lisp,1984 26 | Eiffel,1985 27 | Erlang,1986 28 | Perl,1987 29 | Tcl,1988 30 | Haskell,1990 31 | Python,1991 32 | Visual Basic,1991 33 | Ruby,1993 34 | Lua,1993 35 | CLOS,1994 36 | Java,1995 37 | Delphi,1995 38 | JavaScript,1995 39 | PHP,1995 40 | D,1999 41 | C#,2001 42 | F#,2002 43 | Groovy,2003 44 | Scala,2003 45 | Clojure,2007 46 | Go,2009 47 | -------------------------------------------------------------------------------- /test/test_catreader.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | require 'tmpdir' 4 | 5 | class TestTbCatReader < Test::Unit::TestCase 6 | def test_open 7 | Dir.mktmpdir {|d| 8 | open(i1="#{d}/i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 9 | a,b 10 | 1,2 11 | End 12 | open(i2="#{d}/i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 13 | b,a 14 | 3,4 15 | End 16 | Tb::CatReader.open([i1, i2]) {|r| 17 | result = [] 18 | r.with_header {|header| 19 | result << header 20 | }.each {|pairs| 21 | result << pairs.to_a 22 | } 23 | assert_equal([%w[a b], [['a','1'], ['b','2']], [['b','3'],['a','4']]], result) 24 | } 25 | } 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /test/test_cmd_to_pnm.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdToPNM < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.ppm", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | P1 21 | 2 3 22 | 10 23 | 11 24 | 01 25 | End 26 | Tb::Cmd.main_to_pnm(['-o', o="o.ppm", i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | P1 29 | 2 3 30 | 10 31 | 11 32 | 01 33 | End 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/test_zipper.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestZipper < Test::Unit::TestCase 5 | def test_basic 6 | z = Tb::Zipper.new([Tb::Func::Sum, Tb::Func::Min]) 7 | assert_equal([5,2], z.aggregate(z.call(z.start([2,3]), z.start([3,2])))) 8 | end 9 | 10 | def test_argerr 11 | z = Tb::Zipper.new([Tb::Func::Sum, Tb::Func::Min]) 12 | assert_raise(ArgumentError) { z.start([]) } 13 | assert_raise(ArgumentError) { z.start([1]) } 14 | assert_raise(ArgumentError) { z.start([1,2,3]) } 15 | assert_raise(ArgumentError) { z.call([1], [3]) } 16 | assert_raise(ArgumentError) { z.call([1], [3,4]) } 17 | assert_raise(ArgumentError) { z.call([1,2], [3]) } 18 | assert_raise(ArgumentError) { z.aggregate([]) } 19 | assert_raise(ArgumentError) { z.aggregate([1]) } 20 | assert_raise(ArgumentError) { z.aggregate([1,2,3]) } 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/test_tsv.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbTSV < Test::Unit::TestCase 5 | def parse_tsv(tsv) 6 | Tb::HeaderTSVReader.new(StringIO.new(tsv)).to_a 7 | end 8 | 9 | def generate_tsv(ary) 10 | writer = Tb::HeaderTSVWriter.new(out = '') 11 | ary.each {|h| writer.put_hash h } 12 | writer.finish 13 | out 14 | end 15 | 16 | def test_parse 17 | tsv = "a\tb\n1\t2\n" 18 | ary = parse_tsv(tsv) 19 | assert_equal( 20 | [{"a"=>"1", "b"=>"2"}], 21 | ary) 22 | end 23 | 24 | def test_parse2 25 | tsv = "a\tb\n" + "1\t2\n" + "3\t4\n" 26 | ary = parse_tsv(tsv) 27 | assert_equal( 28 | [{"a"=>"1", "b"=>"2"}, {"a"=>"3", "b"=>"4"}], 29 | ary) 30 | end 31 | 32 | def test_generate_tsv 33 | t = [{'a' => 'foo', 'b' => 'bar'}] 34 | assert_equal("a\tb\nfoo\tbar\n", generate_tsv(t)) 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /test/test_cmd_nest.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdNest < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c 21 | 0,1,2 22 | 0,3,4 23 | 4,5,6 24 | End 25 | Tb::Cmd.main_nest(['-o', o="o.csv", 'z,b,c', i]) 26 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 27 | a,z 28 | 0,"b,c 29 | 1,2 30 | 3,4 31 | " 32 | 4,"b,c 33 | 5,6 34 | " 35 | End 36 | end 37 | 38 | def test_field_not_found 39 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 40 | a,b,c 41 | 0,1,2 42 | 0,3,4 43 | 4,5,6 44 | End 45 | exc = assert_raise(SystemExit) { Tb::Cmd.main_nest(['-o', "o.csv", 'z,b,d', i]) } 46 | assert(!exc.success?) 47 | end 48 | 49 | end 50 | -------------------------------------------------------------------------------- /test/test_numericcsv.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbNumericCSV < Test::Unit::TestCase 5 | def test_reader 6 | csv = <<-'End'.gsub(/^\s*/, '') 7 | A,B,C 8 | a,b,c 9 | d,e,f 10 | End 11 | reader = Tb::NumericCSVReader.new(StringIO.new(csv)) 12 | assert_equal({"1"=>"A", "2"=>"B", "3"=>"C"}, reader.get_hash) 13 | assert_equal({"1"=>"a", "2"=>"b", "3"=>"c"}, reader.get_hash) 14 | assert_equal({"1"=>"d", "2"=>"e", "3"=>"f"}, reader.get_hash) 15 | assert_equal(nil, reader.get_hash) 16 | assert_equal(nil, reader.get_hash) 17 | end 18 | 19 | def test_writer 20 | arys = [] 21 | writer = Tb::NumericCSVWriter.new(arys) 22 | writer.put_hash({"1"=>"A", "2"=>"B", "3"=>"C"}) 23 | assert_equal(["A,B,C\n"], arys) 24 | writer.put_hash({"1"=>"a", "2"=>"b", "3"=>"c"}) 25 | assert_equal(["A,B,C\n", "a,b,c\n"], arys) 26 | writer.put_hash({"1"=>"d", "2"=>"e", "3"=>"f"}) 27 | assert_equal(["A,B,C\n", "a,b,c\n", "d,e,f\n"], arys) 28 | end 29 | 30 | def test_writer_invalid_field 31 | arys = [] 32 | writer = Tb::NumericCSVWriter.new(arys) 33 | assert_raise(ArgumentError) { writer.put_hash({"A"=>"1"}) } 34 | end 35 | 36 | end 37 | -------------------------------------------------------------------------------- /test/test_cmd_to_yaml.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdToYAML < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c 21 | 0,1,2 22 | 4,5,6 23 | End 24 | Tb::Cmd.main_to_yaml(['-o', o="o.yaml", i]) 25 | assert_equal( 26 | [ 27 | {'a' => '0', 'b' => '1', 'c' => '2'}, 28 | {'a' => '4', 'b' => '5', 'c' => '6'}, 29 | ], 30 | YAML.load(File.read(o))) 31 | end 32 | 33 | def test_twofile 34 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 35 | a,b 36 | 1,2 37 | 3,4 38 | End 39 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 40 | b,a 41 | 5,6 42 | 7,8 43 | End 44 | Tb::Cmd.main_to_yaml(['-o', o="o.csv", i1, i2]) 45 | assert_equal( 46 | [ 47 | {'a' => '1', 'b' => '2'}, 48 | {'a' => '3', 'b' => '4'}, 49 | {'a' => '6', 'b' => '5'}, 50 | {'a' => '8', 'b' => '7'}, 51 | ], 52 | YAML.load(File.read(o))) 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /test/test_headercsv.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbHeaderCSV < Test::Unit::TestCase 5 | def test_reader 6 | csv = <<-'End'.gsub(/^\s*/, '') 7 | A,B,C 8 | a,b,c 9 | d,e,f 10 | End 11 | reader = Tb::HeaderCSVReader.new(StringIO.new(csv)) 12 | assert_equal({"A"=>"a", "B"=>"b", "C"=>"c"}, reader.get_hash) 13 | assert_equal({"A"=>"d", "B"=>"e", "C"=>"f"}, reader.get_hash) 14 | assert_equal(nil, reader.get_hash) 15 | assert_equal(nil, reader.get_hash) 16 | end 17 | 18 | def test_writer 19 | arys = [] 20 | writer = Tb::HeaderCSVWriter.new(arys) 21 | assert_equal([], arys) 22 | writer.put_hash({"A"=>"a", "B"=>"b", "C"=>"c"}) 23 | assert_equal([], arys) 24 | writer.put_hash({"A"=>"d", "B"=>"e", "C"=>"f"}) 25 | assert_equal([], arys) 26 | writer.finish 27 | assert_equal(["A,B,C\n", "a,b,c\n", "d,e,f\n"], arys) 28 | end 29 | 30 | def test_writer_known_header 31 | arys = [] 32 | writer = Tb::HeaderCSVWriter.new(arys) 33 | writer.header_generator = lambda { %w[A B C] } 34 | assert_equal([], arys) 35 | writer.put_hash({"A"=>"a", "B"=>"b", "C"=>"c"}) 36 | assert_equal(["A,B,C\n", "a,b,c\n"], arys) 37 | writer.put_hash({"A"=>"d", "B"=>"e", "C"=>"f"}) 38 | assert_equal(["A,B,C\n", "a,b,c\n", "d,e,f\n"], arys) 39 | writer.finish 40 | assert_equal(["A,B,C\n", "a,b,c\n", "d,e,f\n"], arys) 41 | end 42 | 43 | end 44 | -------------------------------------------------------------------------------- /test/test_ltsv.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbLTSV < Test::Unit::TestCase 5 | def parse_ltsv(ltsv) 6 | Tb::LTSVReader.new(StringIO.new(ltsv)).to_a 7 | end 8 | 9 | def generate_ltsv(ary) 10 | writer = Tb::LTSVWriter.new(out = '') 11 | ary.each {|h| writer.put_hash h } 12 | writer.finish 13 | out 14 | end 15 | 16 | def test_escape_and_unescape 17 | 0x00.upto(0x7f) {|c| 18 | s = [c].pack("C") 19 | assert_equal(s, Tb.ltsv_unescape_string(Tb.ltsv_escape_key(s))) 20 | assert_equal(s, Tb.ltsv_unescape_string(Tb.ltsv_escape_value(s))) 21 | } 22 | end 23 | 24 | def test_parse 25 | r = Tb::LTSVReader.new(StringIO.new("a:1\tb:2\na:3\tb:4\n")) 26 | result = [] 27 | r.with_header {|header| 28 | result << header 29 | }.each {|obj| 30 | result << obj 31 | } 32 | assert_equal([%w[a b], {"a"=>"1", "b"=>"2"}, {"a"=>"3", "b"=>"4"}], result) 33 | end 34 | 35 | def test_parse2 36 | t = parse_ltsv("a:1\tb:2\n") 37 | assert_equal( 38 | [{"a"=>"1", "b"=>"2"}], 39 | t) 40 | end 41 | 42 | def test_generate_ltsv 43 | t = [{'a' => 'foo', 'b' => 'bar'}] 44 | assert_equal("a:foo\tb:bar\n", generate_ltsv(t)) 45 | end 46 | 47 | def test_generate_ltsv2 48 | t = [{'a' => 'foo', 'b' => 'bar'}, 49 | {'a' => 'q', 'b' => 'w'}] 50 | assert_equal("a:foo\tb:bar\na:q\tb:w\n", generate_ltsv(t)) 51 | end 52 | 53 | end 54 | -------------------------------------------------------------------------------- /test-all-cov.rb: -------------------------------------------------------------------------------- 1 | require "coverage.so" 2 | 3 | def expand_tab(str, tabstop=8) 4 | col = 0 5 | str.gsub(/(\t+)|[^\t]+/) { 6 | if $1 7 | ' ' * (($1.length * tabstop) - (col + 1) % tabstop) 8 | else 9 | $& 10 | end 11 | } 12 | end 13 | 14 | at_exit { 15 | r = Coverage.result 16 | fs = r.keys.sort.reject {|f| 17 | %r{lib/tb[/.]} !~ f 18 | } 19 | if !fs.empty? 20 | if $stdout.tty? 21 | out = IO.popen(['less', '-S', '-j20', '+/ 0:'], 'w') 22 | else 23 | out = $stdout 24 | end 25 | pat = nil 26 | fs[0].chars.to_a.reverse_each {|ch| 27 | if !pat 28 | pat = "#{Regexp.escape(ch)}?" 29 | else 30 | pat = "(?:#{Regexp.escape(ch)}#{pat})?" 31 | end 32 | } 33 | pat = Regexp.compile(pat) 34 | prefix_len = fs[0].length 35 | fs.each {|f| 36 | l = pat.match(f).end(0) 37 | prefix_len = l if l < prefix_len 38 | } 39 | prefix = fs[0][0, prefix_len] 40 | prefix.sub!(%r{[^/]+\z}, '') 41 | fs.each {|f| 42 | next if %r{lib/tb[/.]} !~ f 43 | f0 = f[prefix.length..-1] 44 | ns = r[f] 45 | max = ns.compact.max 46 | w = max.to_s.length 47 | fmt1 = "%s %#{w}d:%s" 48 | fmt2 = "%s #{" " * w}:%s" 49 | File.foreach(f).with_index {|line, i| 50 | line = expand_tab(line) 51 | if ns[i] 52 | out.puts fmt1 % [f0, ns[i], line] 53 | else 54 | out.puts fmt2 % [f0, line] 55 | end 56 | } 57 | } 58 | if out != $stdout 59 | out.close 60 | end 61 | end 62 | } 63 | Coverage.start 64 | 65 | load 'test-all.rb' 66 | -------------------------------------------------------------------------------- /bin/tb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Copyright (C) 2011 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tb/cmdtop' 32 | 33 | Tb::Cmd.main(ARGV) 34 | -------------------------------------------------------------------------------- /test/test_cmd_to_tsv.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdToTSV < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c 21 | 0,1,2 22 | 4,5,6 23 | End 24 | Tb::Cmd.main_to_tsv(['-o', o="o.tsv", i]) 25 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 26 | a\tb\tc 27 | 0\t1\t2 28 | 4\t5\t6 29 | End 30 | end 31 | 32 | def test_numeric 33 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 34 | a,b,c 35 | 0,1,2 36 | 4,5,6 37 | End 38 | Tb::Cmd.main_to_tsv(['-o', o="o.tsv", '-N', i]) 39 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 40 | a\tb\tc 41 | 0\t1\t2 42 | 4\t5\t6 43 | End 44 | end 45 | 46 | def test_twofile 47 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 48 | a,b 49 | 1,2 50 | 3,4 51 | End 52 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 53 | b,a 54 | 5,6 55 | 7,8 56 | End 57 | Tb::Cmd.main_to_tsv(['-o', o="o.csv", i1, i2]) 58 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 59 | a\tb 60 | 1\t2 61 | 3\t4 62 | 6\t5 63 | 8\t7 64 | End 65 | end 66 | 67 | end 68 | -------------------------------------------------------------------------------- /test/test_cmd_to_ltsv.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdToLTSV < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c 21 | 0,1,2 22 | 4,5,6 23 | End 24 | Tb::Cmd.main_to_ltsv(['-o', o="o.ltsv", i]) 25 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 26 | a:0\tb:1\tc:2 27 | a:4\tb:5\tc:6 28 | End 29 | end 30 | 31 | def test_numeric 32 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 33 | a,b,c 34 | 0,1,2 35 | 4,5,6 36 | End 37 | Tb::Cmd.main_to_ltsv(['-o', o="o.ltsv", '-N', i]) 38 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 39 | 1:a\t2:b\t3:c 40 | 1:0\t2:1\t3:2 41 | 1:4\t2:5\t3:6 42 | End 43 | end 44 | 45 | def test_twofile 46 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 47 | a,b 48 | 1,2 49 | 3,4 50 | End 51 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 52 | b,a 53 | 5,6 54 | 7,8 55 | End 56 | Tb::Cmd.main_to_ltsv(['-o', o="o.ltsv", i1, i2]) 57 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 58 | a:1\tb:2 59 | a:3\tb:4 60 | b:5\ta:6 61 | b:7\ta:8 62 | End 63 | end 64 | 65 | end 66 | -------------------------------------------------------------------------------- /lib/tb/revcmp.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | class Tb::RevCmp 30 | def initialize(revcmp_object) 31 | @revcmp_object = revcmp_object 32 | end 33 | attr_reader :revcmp_object 34 | 35 | def <=> other 36 | other.revcmp_object <=> self.revcmp_object 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/test_cmd_to_pp.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | require_relative 'util_tbtest' 5 | 6 | class TestTbCmdToPP < Test::Unit::TestCase 7 | def setup 8 | Tb::Cmd.reset_option 9 | @curdir = Dir.pwd 10 | @tmpdir = Dir.mktmpdir 11 | Dir.chdir @tmpdir 12 | end 13 | def teardown 14 | Tb::Cmd.reset_option 15 | Dir.chdir @curdir 16 | FileUtils.rmtree @tmpdir 17 | end 18 | 19 | def test_basic 20 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 21 | a,b,c 22 | 0,1,2 23 | 4,5,6 24 | End 25 | Tb::Cmd.main_to_pp(['-o', o="o.pp", i]) 26 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 27 | { "a" => "0", "b" => "1", "c" => "2" } 28 | { "a" => "4", "b" => "5", "c" => "6" } 29 | End 30 | end 31 | 32 | def test_extend 33 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 34 | a,b 35 | 0,1,2,3 36 | End 37 | o = "o.pp" 38 | stderr = capture_stderr { 39 | Tb::Cmd.main_to_pp(['-o', o, i]) 40 | } 41 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 42 | { "a" => "0", "b" => "1" } 43 | End 44 | assert_match(/Header too short/, stderr) 45 | end 46 | 47 | def test_twofile 48 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 49 | a,b 50 | 1,2 51 | 3,4 52 | End 53 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 54 | b,a 55 | 5,6 56 | 7,8 57 | End 58 | Tb::Cmd.main_to_pp(['-o', o="o.csv", i1, i2]) 59 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 60 | {"a"=>"1", "b"=>"2"} 61 | {"a"=>"3", "b"=>"4"} 62 | {"b"=>"5", "a"=>"6"} 63 | {"b"=>"7", "a"=>"8"} 64 | End 65 | end 66 | 67 | end 68 | -------------------------------------------------------------------------------- /lib/tb/customeq.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | class Tb::CustomEq 30 | include Comparable 31 | 32 | def initialize(customeq_object, &eq) 33 | @customeq_object = customeq_object 34 | @eq = eq 35 | end 36 | attr_reader :customeq_object, :eq 37 | 38 | def ==(other) 39 | @eq.call(@customeq_object, other.customeq_object) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/tb/customcmp.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | class Tb::CustomCmp 30 | include Comparable 31 | 32 | def initialize(customcmp_object, &cmp) 33 | @customcmp_object = customcmp_object 34 | @cmp = cmp 35 | end 36 | attr_reader :customcmp_object, :cmp 37 | 38 | def <=> other 39 | @cmp.call(@customcmp_object, other.customcmp_object) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /sample/poi-xls2csv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # sample/poi-xls2csv.sh - script to run sample/poi-xls2csv.rb 4 | # 5 | # Copyright (C) 2011 Tanaka Akira 6 | # 7 | # Redistribution and use in source and binary forms, with or without 8 | # modification, are permitted provided that the following conditions 9 | # are met: 10 | # 11 | # 1. Redistributions of source code must retain the above copyright 12 | # notice, this list of conditions and the following disclaimer. 13 | # 2. Redistributions in binary form must reproduce the above 14 | # copyright notice, this list of conditions and the following 15 | # disclaimer in the documentation and/or other materials provided 16 | # with the distribution. 17 | # 3. The name of the author may not be used to endorse or promote 18 | # products derived from this software without specific prior 19 | # written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 22 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 25 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 27 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 30 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 31 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | 33 | # Ubuntu libjakarta-poi-java package installs /usr/share/java/jakarta-poi.jar. 34 | 35 | d="`dirname $0`" 36 | d="`cd $d; pwd`" 37 | d="`dirname $d`" 38 | 39 | exec jruby -Ku \ 40 | -I"$d/lib" \ 41 | -I/usr/share/java \ 42 | "$d/sample/poi-xls2csv.rb" \ 43 | "$@" 44 | -------------------------------------------------------------------------------- /test/test_cmdutil.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdUtil < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | #@curdir = Dir.pwd 9 | #@tmpdir = Dir.mktmpdir 10 | #Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | #Dir.chdir @curdir 15 | #FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_def_vhelp 19 | verbose_help = Tb::Cmd.instance_variable_get(:@verbose_help) 20 | verbose_help['foo'] = 'bar' 21 | begin 22 | assert_raise(ArgumentError) { Tb::Cmd.def_vhelp('foo', 'baz') } 23 | ensure 24 | verbose_help.delete 'foo' 25 | end 26 | end 27 | 28 | def test_smart_cmp_value 29 | assert_equal(0, Tb::Func.smart_cmp_value(0) <=> Tb::Func.smart_cmp_value(0)) 30 | assert_equal(1, Tb::Func.smart_cmp_value(10) <=> Tb::Func.smart_cmp_value(0)) 31 | assert_equal(-1, Tb::Func.smart_cmp_value(-10) <=> Tb::Func.smart_cmp_value(0)) 32 | assert_equal(0, Tb::Func.smart_cmp_value("a") <=> Tb::Func.smart_cmp_value("a")) 33 | assert_equal(1, Tb::Func.smart_cmp_value("z") <=> Tb::Func.smart_cmp_value("a")) 34 | assert_equal(-1, Tb::Func.smart_cmp_value("a") <=> Tb::Func.smart_cmp_value("b")) 35 | assert_equal(1, Tb::Func.smart_cmp_value("08") <=> Tb::Func.smart_cmp_value("7")) 36 | assert_raise(ArgumentError) { Tb::Func.smart_cmp_value(Object.new) } 37 | end 38 | 39 | def test_parse_aggregator_spec2 40 | assert_raise(ArgumentError) { parse_aggregator_spec2("foo") } 41 | end 42 | 43 | def test_with_output_preserve_mtime 44 | Dir.mktmpdir {|d| 45 | fn = "#{d}/a" 46 | File.open(fn, "w") {|f| f.print "foo" } 47 | t0 = Time.utc(2000) 48 | File.utime(t0, t0, fn) 49 | t1 = File.stat(fn).mtime 50 | with_output(fn) {|f| 51 | f.print "foo" 52 | } 53 | t2 = File.stat(fn).mtime 54 | assert_equal(t1, t2) 55 | } 56 | end 57 | 58 | end 59 | -------------------------------------------------------------------------------- /test/test_cmd_newfield.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdNewfield < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b 21 | 1,2 22 | 3,4 23 | End 24 | Tb::Cmd.main_newfield(['-o', o="o.csv", 'c', 'z', i]) 25 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 26 | c,a,b 27 | z,1,2 28 | z,3,4 29 | End 30 | end 31 | 32 | def test_rubyexp 33 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 34 | a,b 35 | 1,2 36 | 3,4 37 | End 38 | Tb::Cmd.main_newfield(['-o', o="o.csv", 'c', '--ruby', '_["a"].to_i + _["b"].to_i', i]) 39 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 40 | c,a,b 41 | 3,1,2 42 | 7,3,4 43 | End 44 | end 45 | 46 | def test_no_new_field_name 47 | exc = assert_raise(SystemExit) { Tb::Cmd.main_newfield([]) } 48 | assert(!exc.success?) 49 | end 50 | 51 | def test_no_newfield_value 52 | exc = assert_raise(SystemExit) { Tb::Cmd.main_newfield(['foo']) } 53 | assert(!exc.success?) 54 | end 55 | 56 | def test_twofile 57 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 58 | a,b 59 | 1,2 60 | 3,4 61 | End 62 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 63 | b,a 64 | 5,6 65 | 7,8 66 | End 67 | Tb::Cmd.main_newfield(['-o', o="o.csv", 'c', '--ruby', '_["a"].to_i - _["b"].to_i', i1, i2]) 68 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 69 | c,a,b 70 | -1,1,2 71 | -1,3,4 72 | 1,6,5 73 | 1,8,7 74 | End 75 | end 76 | 77 | end 78 | -------------------------------------------------------------------------------- /lib/tb/catreader.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/catreader.rb - Tb::CatReader class 2 | # 3 | # Copyright (C) 2011-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | module Tb::CatReader 32 | def self.open(filenames, numeric=false, with_filename=false) 33 | readers = [] 34 | filenames.each {|f| 35 | r = Tb.open_reader(f, numeric) 36 | if with_filename 37 | r = r.newfield("filename") { f } 38 | end 39 | readers << r 40 | } 41 | r = readers.first.cat(*readers[1..-1]) 42 | if block_given? 43 | yield r 44 | else 45 | r 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/tb/hashwriter.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/hashwriterm.rb - writer mixin for table containing hashes 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tempfile' 32 | 33 | class Tb::HashWriter 34 | def initialize(put_hash, put_finish=nil) 35 | @put_hash = put_hash 36 | @put_finish = put_finish 37 | end 38 | 39 | def header_required? 40 | false 41 | end 42 | 43 | def header_generator=(gen) 44 | end 45 | 46 | def put_hash(hash) 47 | @put_hash.call hash 48 | nil 49 | end 50 | 51 | def finish 52 | @put_finish.call if @put_finish 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/tb/json.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | require 'json' 30 | 31 | module Tb 32 | class JSONReader < Tb::HashReader 33 | def initialize(io) 34 | ary = JSON.parse(io.read) 35 | super lambda { ary.shift } 36 | end 37 | end 38 | 39 | class JSONWriter < Tb::HashWriter 40 | def initialize(io) 41 | io << "[\n" 42 | sep = "" 43 | super lambda {|hash| 44 | io << sep << JSON.pretty_generate(hash) 45 | sep = ",\n" 46 | }, 47 | lambda { 48 | io << "\n]\n" 49 | } 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_pnm.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-pnm' 30 | 31 | def (Tb::Cmd).op_to_pnm 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-pnm [OPTS] [TABLE]\n" + 34 | "Convert a table to PNM (Portable Anymap: PPM, PGM, PBM)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_pnm(argv) 40 | op_to_pnm.parse!(argv) 41 | exit_if_help('to-pnm') 42 | argv = ['-'] if argv.empty? 43 | reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 44 | with_output {|out| 45 | reader.write_to_pnm(out) 46 | } 47 | end 48 | 49 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_ltsv.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2013-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-ltsv' 30 | 31 | def (Tb::Cmd).op_to_ltsv 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-ltsv [OPTS] [TABLE]\n" + 34 | "Convert a table to LTSV (Labeled Tab Separated Values)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_ltsv(argv) 40 | op_to_ltsv.parse!(argv) 41 | exit_if_help('to-ltsv') 42 | argv = ['-'] if argv.empty? 43 | reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 44 | with_output {|out| 45 | reader.write_to_ltsv(out) 46 | } 47 | end 48 | 49 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_tsv.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-tsv' 30 | 31 | def (Tb::Cmd).op_to_tsv 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-tsv [OPTS] [TABLE]\n" + 34 | "Convert a table to TSV (Tab Separated Values)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_tsv(argv) 40 | op_to_tsv.parse!(argv) 41 | exit_if_help('to-tsv') 42 | argv = ['-'] if argv.empty? 43 | reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 44 | with_output {|out| 45 | reader.write_to_tsv(out, !Tb::Cmd.opt_N) 46 | } 47 | end 48 | 49 | -------------------------------------------------------------------------------- /test/test_ndjson.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | 4 | class TestTbNDJSON < Test::Unit::TestCase 5 | def parse_ndjson(ndjson) 6 | Tb::NDJSONReader.new(StringIO.new(ndjson)).to_a 7 | end 8 | 9 | def generate_ndjson(ary) 10 | writer = Tb::NDJSONWriter.new(out = '') 11 | ary.each {|h| writer.put_hash h } 12 | writer.finish 13 | out 14 | end 15 | 16 | def test_reader 17 | ndjson = <<-'End' 18 | {"A":"a","B":"b","C":"c"} 19 | {"A":"d","B":"e","C":"f"} 20 | End 21 | reader = Tb::NDJSONReader.new(StringIO.new(ndjson)) 22 | assert_equal({"A"=>"a", "B"=>"b", "C"=>"c"}, reader.get_hash) 23 | assert_equal({"A"=>"d", "B"=>"e", "C"=>"f"}, reader.get_hash) 24 | assert_equal(nil, reader.get_hash) 25 | assert_equal(nil, reader.get_hash) 26 | end 27 | 28 | def test_reader_header 29 | ndjson = <<-'End' 30 | {"A":"a"} 31 | {"A":"d","B":"e","C":"f"} 32 | End 33 | reader = Tb::NDJSONReader.new(StringIO.new(ndjson)) 34 | assert_equal(%w[A B C], reader.get_named_header) 35 | assert_equal({"A"=>"a"}, reader.get_hash) 36 | assert_equal({"A"=>"d", "B"=>"e", "C"=>"f"}, reader.get_hash) 37 | assert_equal(nil, reader.get_hash) 38 | assert_equal(nil, reader.get_hash) 39 | end 40 | 41 | def test_writer 42 | arys = [] 43 | writer = Tb::NDJSONWriter.new(arys) 44 | writer.header_generator = lambda { flunk } 45 | assert_equal([], arys) 46 | writer.put_hash({"A"=>"a", "B"=>"b", "C"=>"c"}) 47 | assert_equal(['{"A":"a","B":"b","C":"c"}'+"\n"], arys) 48 | writer.put_hash({"A"=>"d", "B"=>"e", "C"=>"f"}) 49 | assert_equal(['{"A":"a","B":"b","C":"c"}'+"\n", '{"A":"d","B":"e","C":"f"}'+"\n"], arys) 50 | writer.finish 51 | assert_equal(['{"A":"a","B":"b","C":"c"}'+"\n", '{"A":"d","B":"e","C":"f"}'+"\n"], arys) 52 | end 53 | 54 | def test_newline_in_string 55 | assert_equal('{"r":"\r","n":"\n"}'+"\n", generate_ndjson([{"r"=>"\r", "n"=>"\n"}])) 56 | end 57 | 58 | def test_empty_line 59 | assert_equal([{"a"=>"1"}, {"a"=>"2"}], parse_ndjson('{"a":"1"}'+"\n\n" + '{"a":"2"}'+"\n")) 60 | end 61 | 62 | end 63 | -------------------------------------------------------------------------------- /test/test_reader.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | require 'tmpdir' 4 | 5 | class TestTbReader < Test::Unit::TestCase 6 | def test_open_prefix_csv 7 | Dir.mktmpdir {|d| 8 | open(ic="#{d}/c", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 9 | a,b 10 | 1,3 11 | End 12 | Tb.open_reader("csv:#{ic}") {|r| 13 | header = nil 14 | all = [] 15 | r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } } 16 | assert_equal(%w[a b], header) 17 | assert_equal([%w[1 3]], all) 18 | } 19 | } 20 | end 21 | 22 | def test_open_no_prefix_suffix_csv 23 | Dir.mktmpdir {|d| 24 | open(ic="#{d}/c", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 25 | a,b 26 | 1,3 27 | End 28 | Tb.open_reader(ic) {|r| 29 | header = nil 30 | all = [] 31 | r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } } 32 | assert_equal(%w[a b], header) 33 | assert_equal([%w[1 3]], all) 34 | } 35 | } 36 | end 37 | 38 | def test_open_prefix_tsv 39 | Dir.mktmpdir {|d| 40 | open(it="#{d}/t", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 41 | a\tb 42 | 1\t3 43 | End 44 | Tb.open_reader("tsv:#{it}") {|r| 45 | header = nil 46 | all = [] 47 | r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } } 48 | assert_equal(%w[a b], header) 49 | assert_equal([%w[1 3]], all) 50 | } 51 | } 52 | end 53 | 54 | def test_open_prefix_json 55 | Dir.mktmpdir {|d| 56 | open(ij="#{d}/j", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 57 | [{"a":1,"b":3}] 58 | End 59 | Tb.open_reader("json:#{ij}") {|r| 60 | header = nil 61 | all = [] 62 | r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } } 63 | assert_equal(%w[a b], header) 64 | assert_equal([[1, 3]], all) 65 | } 66 | } 67 | end 68 | 69 | def test_open_nonstring 70 | assert_raise(ArgumentError) { Tb.open_reader(Object.new) } 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/tb/cmdmain.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | def (Tb::Cmd).main_body(argv) 30 | subcommand = argv.shift 31 | if subcommand == '-h' || subcommand == '--help' 32 | main_help(argv) 33 | elsif Tb::Cmd.subcommands.include?(subcommand) 34 | self.subcommand_send("main", subcommand, argv) 35 | elsif subcommand == nil 36 | usage_list_subcommands 37 | true 38 | else 39 | err "unexpected subcommand: #{subcommand.inspect}" 40 | end 41 | end 42 | 43 | def (Tb::Cmd).main(argv) 44 | main_body(argv) 45 | rescue SystemExit 46 | $stderr.puts $!.message if $!.message != 'exit' 47 | raise 48 | rescue Errno::EPIPE 49 | exit false 50 | end 51 | 52 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_yaml.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-yaml' 30 | 31 | def (Tb::Cmd).op_to_yaml 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-yaml [OPTS] [TABLE]\n" + 34 | "Convert a table to YAML (YAML Ain't a Markup Language)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_yaml(argv) 40 | require 'yaml' 41 | op_to_yaml.parse!(argv) 42 | exit_if_help('to-yaml') 43 | argv = ['-'] if argv.empty? 44 | reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 45 | ary = reader.to_a 46 | with_output {|out| 47 | YAML.dump(ary, out) 48 | out.puts 49 | } 50 | end 51 | 52 | 53 | -------------------------------------------------------------------------------- /test/test_cmd_unnest.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdUnnest < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c 21 | foo,"b1,b2 22 | 1,2 23 | 3,4 24 | ",baz 25 | qux,"b1,b2 26 | 5,6 27 | ",quuux 28 | End 29 | Tb::Cmd.main_unnest(['-o', o="o.csv", 'b', i]) 30 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 31 | a,b1,b2,c 32 | foo,1,2,baz 33 | foo,3,4,baz 34 | qux,5,6,quuux 35 | End 36 | end 37 | 38 | def test_opt_outer1 39 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 40 | a,b,c 41 | foo,"b1,b2 42 | 1,2 43 | ",baz 44 | f,"b1,b2 45 | ",g 46 | qux,,quuux 47 | End 48 | Tb::Cmd.main_unnest(['-o', o="o.csv", '--outer', 'b', i]) 49 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 50 | a,b1,b2,c 51 | foo,1,2,baz 52 | f,,,g 53 | qux,,,quuux 54 | End 55 | end 56 | 57 | def test_opt_no_outer 58 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 59 | a,b,c 60 | foo,"b1,b2 61 | 1,2 62 | ",baz 63 | f,"b1,b2 64 | ",g 65 | qux,,quuux 66 | End 67 | Tb::Cmd.main_unnest(['-o', o="o.csv", 'b', i]) 68 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 69 | a,b1,b2,c 70 | foo,1,2,baz 71 | End 72 | end 73 | 74 | def test_no_target_field 75 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 76 | a,b,c 77 | foo,"b1,b2 78 | 1,2 79 | 3,4 80 | ",baz 81 | qux,"b1,b2 82 | 5,6 83 | ",quuux 84 | End 85 | exc = assert_raise(SystemExit) { Tb::Cmd.main_unnest(['-o', "o.csv", 'bb', i]) } 86 | assert(!exc.success?) 87 | end 88 | 89 | end 90 | -------------------------------------------------------------------------------- /test/test_cmd_grep.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdSearch < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c,d 21 | 0,1,2,3 22 | 4,5,6,7 23 | 8,9,a,b 24 | c,d,e,f 25 | End 26 | Tb::Cmd.main_search(['-o', o="o.csv", '[6f]', i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | a,b,c,d 29 | 4,5,6,7 30 | c,d,e,f 31 | End 32 | end 33 | 34 | def test_no_regexp 35 | exc = assert_raise(SystemExit) { Tb::Cmd.main_search([]) } 36 | assert(!exc.success?) 37 | end 38 | 39 | def test_opt_e 40 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 41 | a,b,c,d 42 | 0,1,2,3 43 | 4,5,6,7 44 | 8,9,a,b 45 | c,d,e,f 46 | End 47 | Tb::Cmd.main_search(['-o', o="o.csv", '-e', '[6f]', i]) 48 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 49 | a,b,c,d 50 | 4,5,6,7 51 | c,d,e,f 52 | End 53 | end 54 | 55 | def test_ruby_pred 56 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 57 | a,b,c,d 58 | 0,1,2,3 59 | 4,5,6,7 60 | 8,9,a,b 61 | c,d,e,f 62 | End 63 | Tb::Cmd.main_search(['-o', o="o.csv", '--ruby', '_["b"] == "5"', i]) 64 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 65 | a,b,c,d 66 | 4,5,6,7 67 | End 68 | end 69 | 70 | def test_twofile 71 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 72 | a,b 73 | 1,2 74 | 3,4 75 | End 76 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 77 | b,a 78 | 5,6 79 | 7,8 80 | End 81 | Tb::Cmd.main_search(['-o', o="o.csv", '[46]', i1, i2]) 82 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 83 | a,b 84 | 3,4 85 | 6,5 86 | End 87 | end 88 | 89 | end 90 | -------------------------------------------------------------------------------- /lib/tb/numericwriter.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/numericwriterm.rb - writer mixin for table without header 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tempfile' 32 | 33 | class Tb::NumericWriter 34 | def initialize(put_array, put_finish=nil) 35 | @put_array = put_array 36 | @put_finish = put_finish 37 | end 38 | 39 | def header_required? 40 | false 41 | end 42 | 43 | def header_generator=(gen) 44 | end 45 | 46 | def put_hash(hash) 47 | ary = [] 48 | hash.each {|k, v| 49 | if /\A[1-9][0-9]*\z/ !~ k 50 | raise ArgumentError, "numeric field name expected: #{k.inspect}" 51 | end 52 | ary[k.to_i-1] = v 53 | } 54 | @put_array.call ary 55 | nil 56 | end 57 | 58 | def finish 59 | @put_finish.call if @put_finish 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /test/test_cmd_rename.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdRename < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c,d 21 | 0,1,2,3 22 | 4,5,6,7 23 | 8,9,a,b 24 | c,d,e,f 25 | End 26 | Tb::Cmd.main_rename(['-o', o="o.csv", 'b,x,c,b', i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | a,x,b,d 29 | 0,1,2,3 30 | 4,5,6,7 31 | 8,9,a,b 32 | c,d,e,f 33 | End 34 | end 35 | 36 | def test_no_rename_fields 37 | exc = assert_raise(SystemExit) { Tb::Cmd.main_rename([]) } 38 | assert(!exc.success?) 39 | end 40 | 41 | def test_empty 42 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 43 | a,b,c,d 44 | 0,1,2,3 45 | 4,5,6,7 46 | 8,9,a,b 47 | c,d,e,f 48 | End 49 | Tb::Cmd.main_rename(['-o', o="o.csv", '', i]) 50 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 51 | a,b,c,d 52 | 0,1,2,3 53 | 4,5,6,7 54 | 8,9,a,b 55 | c,d,e,f 56 | End 57 | end 58 | 59 | def test_twofile 60 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 61 | a,b 62 | 1,2 63 | 3,4 64 | End 65 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 66 | b,a 67 | 5,6 68 | 7,8 69 | End 70 | Tb::Cmd.main_rename(['-o', o="o.csv", 'a,c', i1, i2]) 71 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 72 | c,b 73 | 1,2 74 | 3,4 75 | 6,5 76 | 8,7 77 | End 78 | end 79 | 80 | def test_field_not_found 81 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 82 | a,b 83 | 1,2 84 | 3,4 85 | End 86 | exc = assert_raise(SystemExit) { Tb::Cmd.main_rename(['-o', "o.csv", 'z,c', i]) } 87 | assert(!exc.success?) 88 | assert_match(/field not found/, exc.message) 89 | end 90 | 91 | end 92 | -------------------------------------------------------------------------------- /lib/tb/ndjson.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | require 'json' 30 | 31 | # NDJSON is Newline Delimited JSON 32 | # http://ndjson.org/ 33 | # 34 | # Tb::NDJSONReader accepts empty lines. 35 | 36 | module Tb 37 | class NDJSONReader < Tb::HashReader 38 | # io.gets should returns a string. 39 | def initialize(io) 40 | super lambda { 41 | while true 42 | line = io.gets 43 | if line.nil? || /\S/ =~ line 44 | break 45 | end 46 | end 47 | if line 48 | JSON.parse(line) 49 | else 50 | nil 51 | end 52 | } 53 | end 54 | end 55 | 56 | class NDJSONWriter < Tb::HashWriter 57 | def initialize(io) 58 | super lambda {|hash| 59 | io << (JSON.generate(hash) + "\n") 60 | } 61 | end 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/tb/numericreader.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/numericreaderm.rb - reader mixin for table without header 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | class Tb::NumericReader 32 | include Tb::EnumerableWithEach 33 | 34 | def initialize(get_array) 35 | @get_array = get_array 36 | end 37 | 38 | def header_known? 39 | false 40 | end 41 | 42 | def get_named_header 43 | [] 44 | end 45 | 46 | def get_hash 47 | ary = @get_array.call 48 | if !ary 49 | return nil 50 | end 51 | hash = {} 52 | ary.each_with_index {|v, i| 53 | field = (i+1).to_s 54 | hash[field] = v 55 | } 56 | hash 57 | end 58 | 59 | def header_and_each(header_proc) 60 | header_proc.call(get_named_header) if header_proc 61 | while hash = get_hash 62 | yield hash 63 | end 64 | nil 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /test/test_cmd_mheader.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdMheader < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def with_stderr(io) 19 | save = $stderr 20 | $stderr = io 21 | begin 22 | yield 23 | ensure 24 | $stderr = save 25 | end 26 | end 27 | 28 | def test_basic 29 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 30 | ,2000,2000,2001,2001 31 | name,aaaa,bbbb,aaaa,bbbb 32 | x,1,2,3,4 33 | y,5,6,7,8 34 | End 35 | Tb::Cmd.main_mheader(['-o', o="o.csv", i]) 36 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 37 | name,2000 aaaa,2000 bbbb,2001 aaaa,2001 bbbb 38 | x,1,2,3,4 39 | y,5,6,7,8 40 | End 41 | end 42 | 43 | def test_opt_c 44 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 45 | a,b,c 46 | 1,2,3 47 | 4,5,6 48 | End 49 | Tb::Cmd.main_mheader(['-o', o="o.csv", '-c', '2', i]) 50 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 51 | a 1,b 2,c 3 52 | 4,5,6 53 | End 54 | end 55 | 56 | def test_no_unique_header 57 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 58 | a,a 59 | 1,1 60 | End 61 | o = nil 62 | File.open(log="log", "w") {|logf| 63 | with_stderr(logf) { 64 | Tb::Cmd.main_mheader(['-o', o="o.csv", i]) 65 | } 66 | } 67 | assert_equal('', File.read(o)) 68 | assert_match(/unique header fields not recognized/, File.read(log)) 69 | end 70 | 71 | def test_twofile 72 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 73 | a,b 74 | 1,2 75 | 3,4 76 | End 77 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 78 | b,a 79 | 5,6 80 | 7,8 81 | End 82 | Tb::Cmd.main_mheader(['-o', o="o.csv", '-c', '2', i1, i2]) 83 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 84 | a 1,b 2 85 | 3,4 86 | b,a 87 | 5,6 88 | 7,8 89 | End 90 | end 91 | 92 | end 93 | -------------------------------------------------------------------------------- /lib/tb.rb: -------------------------------------------------------------------------------- 1 | # lib/tb.rb - entry file for table library 2 | # 3 | # Copyright (C) 2010-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tempfile' 32 | 33 | module Tb 34 | end 35 | 36 | require 'pp' 37 | require 'tb/enumerable' 38 | require 'tb/enumerator' 39 | require 'tb/func' 40 | require 'tb/zipper' 41 | 42 | require 'tb/headerreader' 43 | require 'tb/headerwriter' 44 | 45 | require 'tb/numericreader' 46 | require 'tb/numericwriter' 47 | 48 | require 'tb/hashreader' 49 | require 'tb/hashwriter' 50 | 51 | require 'tb/csv' 52 | require 'tb/tsv' 53 | require 'tb/ltsv' 54 | require 'tb/pnm' 55 | require 'tb/json' 56 | require 'tb/ndjson' 57 | 58 | require 'tb/ropen' 59 | require 'tb/catreader' 60 | require 'tb/search' 61 | require 'tb/ex_enumerable' 62 | require 'tb/ex_enumerator' 63 | require 'tb/fileenumerator' 64 | require 'tb/revcmp' 65 | require 'tb/customcmp' 66 | require 'tb/customeq' 67 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_json.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-json' 30 | 31 | def (Tb::Cmd).op_to_json 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-json [OPTS] [TABLE]\n" + 34 | "Convert a table to JSON (JavaScript Object Notation)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_json(argv) 40 | require 'json' 41 | op_to_json.parse!(argv) 42 | exit_if_help('to-json') 43 | argv = ['-'] if argv.empty? 44 | with_output {|out| 45 | out.print "[" 46 | sep = nil 47 | argv.each {|filename| 48 | sep = ",\n\n" if sep 49 | tablereader_open(filename) {|tblreader| 50 | tblreader.each {|pairs| 51 | out.print sep if sep 52 | out.print JSON.pretty_generate(pairs) 53 | sep = ",\n" 54 | } 55 | } 56 | } 57 | out.puts "]" 58 | } 59 | end 60 | 61 | -------------------------------------------------------------------------------- /lib/tb/zipper.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | class Tb::Zipper 30 | def initialize(ops) 31 | @ops = ops 32 | end 33 | 34 | def start(ary) 35 | if ary.length != @ops.length 36 | raise ArgumentError, "expect an array which lengths are #{@ops.length}" 37 | end 38 | @ops.map.with_index {|op, i| 39 | op.start(ary[i]) 40 | } 41 | end 42 | 43 | def call(ary1, ary2) 44 | if ary1.length != @ops.length || ary2.length != @ops.length 45 | raise ArgumentError, "expect an array of arrays which lengths are #{@ops.length}" 46 | end 47 | @ops.zip(ary1, ary2).map {|op, v1, v2| 48 | op.call(v1, v2) 49 | } 50 | end 51 | 52 | def aggregate(ary) 53 | if ary.length != @ops.length 54 | raise ArgumentError, "expect an array which lengths are #{@ops.length}" 55 | end 56 | @ops.map.with_index {|op, i| 57 | op.aggregate(ary[i]) 58 | } 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/tb/csv.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/csv.rb - CSV related fetures for table library 2 | # 3 | # Copyright (C) 2010-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'csv' 32 | 33 | module Tb 34 | def Tb.csv_encode_row(ary) 35 | ary.to_csv 36 | end 37 | 38 | class HeaderCSVReader < HeaderReader 39 | def initialize(io) 40 | aryreader = CSV.new(io) 41 | super lambda { aryreader.shift } 42 | end 43 | end 44 | 45 | class HeaderCSVWriter < HeaderWriter 46 | # io is an object which has "<<" method. 47 | def initialize(io) 48 | super lambda {|ary| io << ary.to_csv} 49 | end 50 | end 51 | 52 | class NumericCSVReader < NumericReader 53 | def initialize(io) 54 | aryreader = CSV.new(io) 55 | super lambda { aryreader.shift } 56 | end 57 | end 58 | 59 | class NumericCSVWriter < NumericWriter 60 | # io is an object which has "<<" method. 61 | def initialize(io) 62 | super lambda {|ary| io << ary.to_csv } 63 | end 64 | end 65 | 66 | end 67 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_csv.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-csv' 30 | 31 | def (Tb::Cmd).op_to_csv 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-csv [OPTS] [TABLE ...]\n" + 34 | "Convert a table to CSV (Comma Separated Values)." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_csv(argv) 40 | op_to_csv.parse!(argv) 41 | exit_if_help('to-csv') 42 | argv = ['-'] if argv.empty? 43 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 44 | header = [] 45 | ter = Tb::Enumerator.new {|y| 46 | creader.with_cumulative_header.each {|pairs, header1| 47 | header = header1 48 | y.yield pairs 49 | } 50 | }.to_fileenumerator 51 | er = Tb::Enumerator.new {|y| 52 | y.set_header header 53 | ter.each {|pairs| 54 | y.yield Hash[header.map {|f| [f, pairs[f]] }] 55 | } 56 | } 57 | with_output {|out| 58 | er.write_to_csv(out, !Tb::Cmd.opt_N) 59 | } 60 | end 61 | -------------------------------------------------------------------------------- /lib/tb/cmd_cat.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'cat' 30 | 31 | Tb::Cmd.default_option[:opt_cat_with_filename] = nil 32 | 33 | def (Tb::Cmd).op_cat 34 | op = OptionParser.new 35 | op.banner = "Usage: tb cat [OPTS] [TABLE ...]\n" + 36 | "Concatenate tables vertically." 37 | define_common_option(op, "hNo", "--no-pager") 38 | op.def_option('-H', '--with-filename', 'add filename column') { Tb::Cmd.opt_cat_with_filename = true } 39 | op 40 | end 41 | 42 | Tb::Cmd.def_vhelp('cat', <<'End') 43 | Example: 44 | 45 | % cat tst1.csv 46 | a,b,c 47 | 0,1,2 48 | 4,5,6 49 | % cat tst2.csv 50 | a,b,d 51 | U,V,W 52 | X,Y,Z 53 | % tb cat tst1.csv tst2.csv 54 | a,b,c,d 55 | 0,1,2 56 | 4,5,6 57 | U,V,,W 58 | X,Y,,Z 59 | End 60 | 61 | def (Tb::Cmd).main_cat(argv) 62 | op_cat.parse!(argv) 63 | exit_if_help('cat') 64 | argv = ['-'] if argv.empty? 65 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N, Tb::Cmd.opt_cat_with_filename) 66 | output_tbenum(creader) 67 | end 68 | -------------------------------------------------------------------------------- /test/test_cmd_shape.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdShape < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_ndjson 19 | File.open(i="i.ndjson", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | {"a":0, "b":1} 21 | {"a":4, "b":5, "c":6} 22 | End 23 | Tb::Cmd.main_shape(['-o', o="o.csv", i]) 24 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 25 | filename,records,min_pairs,max_pairs 26 | i.ndjson,2,2,3 27 | End 28 | end 29 | 30 | def test_csv 31 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 32 | a,b,c 33 | 0,1 34 | 4,5,6 35 | End 36 | Tb::Cmd.main_shape(['-o', o="o.csv", i]) 37 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 38 | filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields 39 | i.csv,2,2,3,3,2,3 40 | End 41 | end 42 | 43 | def test_output_ndjson 44 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 45 | a,b,c 46 | 0,1 47 | 4,5,6 48 | End 49 | Tb::Cmd.main_shape(['-o', o="o.ndjson", i]) 50 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 51 | {"filename":"i.csv","records":2,"min_pairs":2,"max_pairs":3,"header_fields":3,"min_fields":2,"max_fields":3} 52 | End 53 | end 54 | 55 | def test_extra_field 56 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 57 | a,b,c 58 | 0,1 59 | 4,5,6,7 60 | End 61 | Tb::Cmd.main_shape(['-o', o="o.csv", i]) 62 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 63 | filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields 64 | i.csv,2,2,3,3,2,4 65 | End 66 | end 67 | 68 | def test_twofile 69 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 70 | a 71 | 1 72 | 3 73 | End 74 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 75 | b,a 76 | 5,6 77 | 7,8 78 | End 79 | Tb::Cmd.main_shape(['-o', o="o.csv", i1, i2]) 80 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 81 | filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields 82 | i1.csv,2,1,1,1,1,1 83 | i2.csv,2,2,2,2,2,2 84 | End 85 | end 86 | 87 | end 88 | -------------------------------------------------------------------------------- /lib/tb/cmdtop.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2013 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | require 'tb' 30 | require 'optparse' 31 | require 'pathname' 32 | require 'etc' 33 | require 'time' 34 | require 'enumerator' 35 | require 'digest' 36 | require 'tb/pager' 37 | require 'tb/cmdutil' 38 | require 'tb/cmd_help' 39 | require 'tb/cmd_to_csv' 40 | require 'tb/cmd_to_tsv' 41 | require 'tb/cmd_to_ltsv' 42 | require 'tb/cmd_to_pnm' 43 | require 'tb/cmd_to_json' 44 | require 'tb/cmd_to_yaml' 45 | require 'tb/cmd_to_pp' 46 | require 'tb/cmd_search' 47 | require 'tb/cmd_gsub' 48 | require 'tb/cmd_sort' 49 | require 'tb/cmd_cut' 50 | require 'tb/cmd_rename' 51 | require 'tb/cmd_newfield' 52 | require 'tb/cmd_cat' 53 | require 'tb/cmd_join' 54 | require 'tb/cmd_consecutive' 55 | require 'tb/cmd_group' 56 | require 'tb/cmd_cross' 57 | require 'tb/cmd_melt' 58 | require 'tb/cmd_unmelt' 59 | require 'tb/cmd_nest' 60 | require 'tb/cmd_unnest' 61 | require 'tb/cmd_shape' 62 | require 'tb/cmd_mheader' 63 | require 'tb/cmd_crop' 64 | require 'tb/cmd_ls' 65 | require 'tb/cmd_tar' 66 | require 'tb/cmd_svn' 67 | require 'tb/cmd_git' 68 | require 'tb/cmdmain' 69 | 70 | Tb::Cmd.init_option 71 | -------------------------------------------------------------------------------- /test/test_csv.rb: -------------------------------------------------------------------------------- 1 | require 'tb' 2 | require 'test/unit' 3 | require_relative 'util_tbtest' 4 | 5 | class TestTbCSV < Test::Unit::TestCase 6 | def parse_csv(csv) 7 | Tb::HeaderCSVReader.new(StringIO.new(csv)).to_a 8 | end 9 | 10 | def generate_csv(ary) 11 | writer = Tb::HeaderCSVWriter.new(out = '') 12 | ary.each {|h| writer.put_hash h } 13 | writer.finish 14 | out 15 | end 16 | 17 | def test_parse 18 | t = parse_csv(<<-'End'.gsub(/^\s+/, '')) 19 | a,b 20 | 1,2 21 | 3,4 22 | End 23 | assert_equal( 24 | [{"a"=>"1", "b"=>"2"}, 25 | {"a"=>"3", "b"=>"4"}], 26 | t) 27 | end 28 | 29 | def test_parse_empty_line_before_header 30 | empty_line = "\n" 31 | t = parse_csv(empty_line + <<-'End'.gsub(/^\s+/, '')) 32 | a,b 33 | 1,2 34 | 3,4 35 | End 36 | assert_equal( 37 | [{"a"=>"1", "b"=>"2"}, 38 | {"a"=>"3", "b"=>"4"}], 39 | t) 40 | end 41 | 42 | def test_parse_empty_value 43 | t = parse_csv(<<-'End'.gsub(/^\s+/, '')) 44 | a,b,c 45 | 1,,2 46 | 3,"",4 47 | End 48 | assert_equal( 49 | [{"a"=>"1", "b"=>nil, "c"=>"2"}, 50 | {"a"=>"3", "b"=>"", "c"=>"4"}], 51 | t) 52 | end 53 | 54 | def test_parse_newline 55 | t = parse_csv("\n") 56 | assert_equal([], t) 57 | end 58 | 59 | def test_generate 60 | t = [{'a' => 1, 'b' => 2}, 61 | {'a' => 3, 'b' => 4}] 62 | assert_equal(<<-'End'.gsub(/^\s+/, ''), generate_csv(t)) 63 | a,b 64 | 1,2 65 | 3,4 66 | End 67 | end 68 | 69 | def test_generate_empty 70 | t = [{'a' => 1, 'b' => nil, 'c' => 2}, 71 | {'a' => 3, 'b' => '', 'c' => 4}] 72 | assert_equal(<<-'End'.gsub(/^\s+/, ''), generate_csv(t)) 73 | a,b,c 74 | 1,,2 75 | 3,"",4 76 | End 77 | end 78 | 79 | def test_parse_ambiguous_header 80 | t = nil 81 | stderr = capture_stderr { 82 | t = parse_csv(<<-'End'.gsub(/^\s+/, '')) 83 | a,b,a,b,c 84 | 0,1,2,3,4 85 | 5,6,7,8,9 86 | End 87 | } 88 | assert_equal( 89 | [{"a"=>"0", "b"=>"1", "c"=>"4"}, 90 | {"a"=>"5", "b"=>"6", "c"=>"9"}], 91 | t) 92 | assert_match(/Ambiguous header field/, stderr) 93 | end 94 | 95 | def test_parse_empty_header_field 96 | t = nil 97 | stderr = capture_stderr { 98 | t = parse_csv(<<-'End'.gsub(/^\s+/, '')) 99 | a,,c 100 | 0,1,2 101 | 5,6,7 102 | End 103 | } 104 | assert_equal( 105 | [{"a"=>"0", "c"=>"2"}, 106 | {"a"=>"5", "c"=>"7"}], 107 | t) 108 | assert_match(/Empty header field/, stderr) 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /lib/tb/cmd_to_pp.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'to-pp' 30 | 31 | def (Tb::Cmd).op_to_pp 32 | op = OptionParser.new 33 | op.banner = "Usage: tb to-pp [OPTS] [TABLE]\n" + 34 | "Convert a table to pretty printed format." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | def (Tb::Cmd).main_to_pp(argv) 40 | op_to_pp.parse!(argv) 41 | exit_if_help('to-pp') 42 | argv.unshift '-' if argv.empty? 43 | with_output {|out| 44 | argv.each {|filename| 45 | tablereader_open(filename) {|tblreader| 46 | tblreader.each {|pairs| 47 | a = pairs.reject {|f, v| v.nil? } 48 | q = PP.new(out, 79) 49 | q.guard_inspect_key { 50 | q.group(1, '{', '}') { 51 | q.seplist(a, nil, :each) {|kv| 52 | k, v = kv 53 | q.group { 54 | q.pp k 55 | q.text '=>' 56 | q.group(1) { 57 | q.breakable '' 58 | q.pp v 59 | } 60 | } 61 | } 62 | } 63 | } 64 | q.flush 65 | out << "\n" 66 | } 67 | } 68 | } 69 | } 70 | end 71 | 72 | -------------------------------------------------------------------------------- /test/test_cmd_cut.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | require_relative 'util_tbtest' 5 | 6 | class TestTbCmdCut < Test::Unit::TestCase 7 | def setup 8 | Tb::Cmd.reset_option 9 | @curdir = Dir.pwd 10 | @tmpdir = Dir.mktmpdir 11 | Dir.chdir @tmpdir 12 | end 13 | def teardown 14 | Tb::Cmd.reset_option 15 | Dir.chdir @curdir 16 | FileUtils.rmtree @tmpdir 17 | end 18 | 19 | def test_basic 20 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 21 | a,b,c,d 22 | 0,1,2,3 23 | 4,5,6,7 24 | 8,9,a,b 25 | c,d,e,f 26 | End 27 | Tb::Cmd.main_cut(['-o', o="o.csv", 'b,d', i]) 28 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 29 | b,d 30 | 1,3 31 | 5,7 32 | 9,b 33 | d,f 34 | End 35 | end 36 | 37 | def test_no_cut_fields 38 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cut([]) } 39 | assert(!exc.success?) 40 | end 41 | 42 | def test_opt_v 43 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 44 | a,b,c,d 45 | 0,1,2,3 46 | 4,5,6,7 47 | 8,9,a,b 48 | c,d,e,f 49 | End 50 | Tb::Cmd.main_cut(['-o', o="o.csv", '-v', 'b,d', i]) 51 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 52 | a,c 53 | 0,2 54 | 4,6 55 | 8,a 56 | c,e 57 | End 58 | end 59 | 60 | def test_extend 61 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 62 | a,b 63 | 0,1,2,3 64 | End 65 | o = "o.csv" 66 | stderr = capture_stderr { 67 | Tb::Cmd.main_cut(['-o', o, 'a,2,1', i]) 68 | } 69 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 70 | a,2,1 71 | 0 72 | End 73 | assert_match(/Header too short/, stderr) 74 | end 75 | 76 | def test_zero 77 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 78 | a,b 79 | 0,1,2,3 80 | End 81 | o = "o.csv" 82 | stderr = capture_stderr { 83 | Tb::Cmd.main_cut(['-o', o, '0', i]) 84 | } 85 | assert_equal("0\n\n", File.read(o)) 86 | assert_match(/Header too short/, stderr) 87 | end 88 | 89 | def test_twofile 90 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 91 | a,b 92 | 1,2 93 | 3,4 94 | End 95 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 96 | b,a 97 | 5,6 98 | 7,8 99 | End 100 | Tb::Cmd.main_cut(['-o', o="o.csv", 'a', i1, i2]) 101 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 102 | a 103 | 1 104 | 3 105 | 6 106 | 8 107 | End 108 | end 109 | 110 | end 111 | -------------------------------------------------------------------------------- /lib/tb/pager.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require 'io/console' 3 | rescue LoadError 4 | end 5 | 6 | class Tb::Pager 7 | def self.open 8 | pager = self.new 9 | begin 10 | yield pager 11 | ensure 12 | pager.close 13 | end 14 | end 15 | 16 | def initialize 17 | if $stdout.tty? 18 | @io = nil 19 | @buf = '' 20 | else 21 | @io = $stdout 22 | @buf = nil 23 | end 24 | end 25 | 26 | def <<(obj) 27 | write obj.to_s 28 | self 29 | end 30 | 31 | def print(*args) 32 | s = '' 33 | args.map {|a| s << a.to_s } 34 | write s 35 | nil 36 | end 37 | 38 | def printf(format, *args) 39 | write sprintf(format, *args) 40 | nil 41 | end 42 | 43 | def putc(ch) 44 | if Integer === ch 45 | write [ch].pack("C") 46 | else 47 | write ch.to_s 48 | end 49 | ch 50 | end 51 | 52 | def puts(*objs) 53 | if objs.empty? 54 | write "\n" 55 | else 56 | objs.each {|o| 57 | o = o.to_s 58 | write o 59 | write "\n" if /\n\z/ !~ o 60 | } 61 | end 62 | nil 63 | end 64 | 65 | def write_nonblock(str) 66 | write str.to_s 67 | end 68 | 69 | def expand_tab(str, tabstop=8) 70 | col = 0 71 | str.gsub(/(\t+)|[^\t]+/) { 72 | if $1 73 | ' ' * (($1.length * tabstop) - (col + 1) % tabstop) 74 | else 75 | $& 76 | end 77 | } 78 | end 79 | 80 | DEFAULT_LINES = 24 81 | DEFAULT_COLUMNS = 80 82 | 83 | def winsize 84 | if $stdout.respond_to? :winsize 85 | lines, columns = $stdout.winsize 86 | return [lines, columns] if lines != 0 && columns != 0 87 | end 88 | [DEFAULT_LINES, DEFAULT_COLUMNS] 89 | end 90 | 91 | def single_screen?(str) 92 | lines, columns = winsize 93 | n = 0 94 | str.each_line {|line| 95 | line = expand_tab(line).chomp 96 | cols = line.length # xxx: 1 column/character assumed. 97 | cols = 1 if cols == 0 98 | m = (cols + columns - 1) / columns # termcap am capability is assumed. 99 | n += m 100 | } 101 | n <= lines-1 102 | end 103 | 104 | def write(str) 105 | str = str.to_s 106 | if !@io 107 | @buf << str 108 | if !single_screen?(@buf) 109 | @io = IO.popen(ENV['PAGER'] || 'more', 'w') 110 | @io << @buf 111 | @buf = nil 112 | end 113 | else 114 | @io << str 115 | end 116 | end 117 | 118 | def flush 119 | @io.flush if @io 120 | self 121 | end 122 | 123 | def close 124 | if !@io 125 | $stdout.print @buf 126 | else 127 | # don't need to ouput @buf because @buf is nil. 128 | @io.close if @io != $stdout 129 | end 130 | nil 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /test/test_cmd_crop.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdCrop < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | ,, 21 | ,a,b,, 22 | ,0,1,,2, 23 | ,,4,,, 24 | ,,, 25 | 26 | End 27 | Tb::Cmd.main_crop(['-o', o="o.csv", i]) 28 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 29 | a,b 30 | 0,1,,2 31 | ,4 32 | End 33 | end 34 | 35 | def test_a1_range 36 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 37 | a,b,c,d 38 | 0,1,2,3 39 | 4,5,6,7 40 | 8,9,a,b 41 | c,d,e,f 42 | End 43 | Tb::Cmd.main_crop(['-o', o="o.csv", '-r', 'B1:C2', i]) 44 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 45 | b,c 46 | 1,2 47 | End 48 | end 49 | 50 | def test_r1c1_range 51 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 52 | a,b,c,d 53 | 0,1,2,3 54 | 4,5,6,7 55 | 8,9,a,b 56 | c,d,e,f 57 | End 58 | Tb::Cmd.main_crop(['-o', o="o.csv", '-r', 'R2C1:R3C2', i]) 59 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 60 | 0,1 61 | 4,5 62 | End 63 | end 64 | 65 | def test_twofile 66 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 67 | a,b 68 | 1,2 69 | 3,4 70 | End 71 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 72 | b,a 73 | 5,6 74 | 7,8 75 | End 76 | Tb::Cmd.main_crop(['-o', o="o.csv", '-r', 'B2:B4', i1, i2]) 77 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 78 | 2 79 | 4 80 | a 81 | End 82 | end 83 | end 84 | 85 | class TestTbCmdCropNoTmpDir < Test::Unit::TestCase 86 | def test_invalid_range 87 | assert_raise(ArgumentError) { Tb::Cmd.main_crop(['-r', 'foo']) } 88 | end 89 | 90 | def test_decode_a1_addressing_col 91 | assert_equal(1, Tb::Cmd.decode_a1_addressing_col("A")) 92 | assert_equal(26, Tb::Cmd.decode_a1_addressing_col("Z")) 93 | ("A".."Z").each_with_index {|ch, i| 94 | assert_equal(i+1, Tb::Cmd.decode_a1_addressing_col(ch)) 95 | } 96 | assert_equal(27, Tb::Cmd.decode_a1_addressing_col("AA")) 97 | assert_equal(256, Tb::Cmd.decode_a1_addressing_col("IV")) 98 | assert_equal(16384, Tb::Cmd.decode_a1_addressing_col("XFD")) 99 | end 100 | 101 | end 102 | -------------------------------------------------------------------------------- /lib/tb/tsv.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/tsv.rb - TSV related fetures for table library 2 | # 3 | # Copyright (C) 2010-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'stringio' 32 | 33 | module Tb 34 | def Tb.tsv_fields_join(values) 35 | values.map {|v| v.to_s.gsub(/[\t\r\n]/, ' ') }.join("\t") + "\n" 36 | end 37 | 38 | def Tb.tsv_fields_split(line) 39 | line = line.chomp("\n") 40 | line = line.chomp("\r") 41 | line.split(/\t/, -1) 42 | end 43 | 44 | class HeaderTSVReader < HeaderReader 45 | def initialize(io) 46 | super lambda { 47 | line = io.gets 48 | if line 49 | Tb.tsv_fields_split(line) 50 | else 51 | nil 52 | end 53 | } 54 | end 55 | end 56 | 57 | class HeaderTSVWriter < HeaderWriter 58 | # io is an object which has "<<" method. 59 | def initialize(io) 60 | super lambda {|ary| 61 | io << Tb.tsv_fields_join(ary) 62 | } 63 | end 64 | end 65 | 66 | class NumericTSVReader < NumericReader 67 | def initialize(io) 68 | super lambda { 69 | line = io.gets 70 | if line 71 | Tb.tsv_fields_split(line) 72 | else 73 | nil 74 | end 75 | } 76 | end 77 | end 78 | 79 | class NumericTSVWriter < NumericWriter 80 | def initialize(io) 81 | super lambda {|ary| 82 | io << Tb.tsv_fields_join(ary) 83 | } 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /lib/tb/hashreader.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/hashreaderm.rb - reader mixin for table containing hashes 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tempfile' 32 | 33 | class Tb::HashReader 34 | include Tb::EnumerableWithEach 35 | 36 | def initialize(get_hash) 37 | @get_hash = get_hash 38 | end 39 | 40 | def header_known? 41 | false 42 | end 43 | 44 | def get_named_header 45 | if defined? @hashreader_header_complete 46 | return @hashreader_header_complete 47 | end 48 | @hashreader_buffer = [] 49 | while hash = @get_hash.call 50 | update_header hash 51 | @hashreader_buffer << hash 52 | end 53 | update_header nil 54 | end 55 | 56 | def get_hash 57 | if defined? @hashreader_buffer 58 | return @hashreader_buffer.shift 59 | end 60 | hash = @get_hash.call 61 | update_header hash 62 | hash 63 | end 64 | 65 | def update_header(hash) 66 | unless defined? @hashreader_header_partial 67 | @hashreader_header_partial = [] 68 | end 69 | if hash 70 | @hashreader_header_partial.concat(hash.keys - @hashreader_header_partial) 71 | else 72 | @hashreader_header_complete = @hashreader_header_partial 73 | end 74 | end 75 | 76 | def header_and_each(header_proc) 77 | header_proc.call(get_named_header) if header_proc 78 | while hash = get_hash 79 | yield hash 80 | end 81 | nil 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /test/test_cmd_consecutive.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | require_relative 'util_tbtest' 5 | 6 | class TestTbCmdConsecutive < Test::Unit::TestCase 7 | def setup 8 | Tb::Cmd.reset_option 9 | @curdir = Dir.pwd 10 | @tmpdir = Dir.mktmpdir 11 | Dir.chdir @tmpdir 12 | end 13 | def teardown 14 | Tb::Cmd.reset_option 15 | Dir.chdir @curdir 16 | FileUtils.rmtree @tmpdir 17 | end 18 | 19 | def test_basic 20 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 21 | a,b,c 22 | 1,2,3 23 | 4,5,6 24 | 7,8,9 25 | End 26 | Tb::Cmd.main_consecutive(['-o', o="o.csv", i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | a_1,a_2,b_1,b_2,c_1,c_2 29 | 1,4,2,5,3,6 30 | 4,7,5,8,6,9 31 | End 32 | end 33 | 34 | def test_numeric 35 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 36 | a,b,c 37 | 1,2,3 38 | 4,5,6 39 | 7,8,9 40 | End 41 | Tb::Cmd.main_consecutive(['-o', o="o.csv", '-N', i]) 42 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 43 | a,1,b,2,c,3 44 | 1,4,2,5,3,6 45 | 4,7,5,8,6,9 46 | End 47 | end 48 | 49 | def test_extend 50 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 51 | a,b 52 | 1,2,3 53 | 4,5,6 54 | 7,8,9 55 | End 56 | o = "o.csv" 57 | stderr = capture_stderr { 58 | Tb::Cmd.main_consecutive(['-o', o, i]) 59 | } 60 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 61 | a_1,a_2,b_1,b_2 62 | 1,4,2,5 63 | 4,7,5,8 64 | End 65 | assert_match(/Header too short/, stderr) 66 | end 67 | 68 | def test_n3 69 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 70 | a,b,c 71 | 1,2,3 72 | 4,5,6 73 | 7,8,9 74 | End 75 | Tb::Cmd.main_consecutive(['-o', o="o.csv", '-n3', i]) 76 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 77 | a_1,a_2,a_3,b_1,b_2,b_3,c_1,c_2,c_3 78 | 1,4,7,2,5,8,3,6,9 79 | End 80 | end 81 | 82 | def test_n4 83 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 84 | a,b,c 85 | 1,2,3 86 | 4,5,6 87 | 7,8,9 88 | End 89 | Tb::Cmd.main_consecutive(['-o', o="o.csv", '-n4', i]) 90 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 91 | a_1,a_2,a_3,a_4,b_1,b_2,b_3,b_4,c_1,c_2,c_3,c_4 92 | End 93 | end 94 | 95 | def test_header_only 96 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 97 | a,b,c 98 | End 99 | Tb::Cmd.main_consecutive(['-o', o="o.csv", i]) 100 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 101 | a_1,a_2,b_1,b_2,c_1,c_2 102 | End 103 | end 104 | 105 | end 106 | -------------------------------------------------------------------------------- /test/test_cmd_sort.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdSort < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b 21 | 1,4 22 | 0,3 23 | 3,2 24 | End 25 | Tb::Cmd.main_sort(['-o', o="o.csv", i]) 26 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 27 | a,b 28 | 0,3 29 | 1,4 30 | 3,2 31 | End 32 | end 33 | 34 | def test_numeric 35 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 36 | a,b 37 | 1,4 38 | 0,3 39 | 3,2 40 | End 41 | Tb::Cmd.main_sort(['-o', o="o.csv", '-N', i]) 42 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 43 | 0,3 44 | 1,4 45 | 3,2 46 | a,b 47 | End 48 | end 49 | 50 | def test_opt_f 51 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 52 | a,b 53 | 1,4 54 | 0,3 55 | 3,2 56 | End 57 | Tb::Cmd.main_sort(['-o', o="o.csv", '-f', 'b', i]) 58 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 59 | a,b 60 | 3,2 61 | 0,3 62 | 1,4 63 | End 64 | end 65 | 66 | def test_cmp 67 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 68 | a,b,c 69 | 10,a20b0,11 70 | 1,2e1,3 71 | 4,,6 72 | 7,8,9 73 | End 74 | Tb::Cmd.main_sort(['-o', o="o.csv", '-f', 'b', i]) 75 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 76 | a,b,c 77 | 4,,6 78 | 7,8,9 79 | 1,2e1,3 80 | 10,a20b0,11 81 | End 82 | end 83 | 84 | def test_twofile 85 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 86 | a,b 87 | 1,2 88 | 3,4 89 | End 90 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 91 | b,a 92 | 5,0 93 | 7,8 94 | End 95 | Tb::Cmd.main_sort(['-o', o="o.csv", i1, i2]) 96 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 97 | a,b 98 | 0,5 99 | 1,2 100 | 3,4 101 | 8,7 102 | End 103 | end 104 | 105 | def test_reverse 106 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 107 | a,b 108 | 1,4 109 | 0,3 110 | 3,2 111 | End 112 | Tb::Cmd.main_sort(['-o', o="o.csv", '-r', i]) 113 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 114 | a,b 115 | 3,2 116 | 1,4 117 | 0,3 118 | End 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /test/test_cmd_gsub.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | require_relative 'util_tbtest' 5 | 6 | class TestTbCmdGsub < Test::Unit::TestCase 7 | def setup 8 | Tb::Cmd.reset_option 9 | @curdir = Dir.pwd 10 | @tmpdir = Dir.mktmpdir 11 | Dir.chdir @tmpdir 12 | end 13 | def teardown 14 | Tb::Cmd.reset_option 15 | Dir.chdir @curdir 16 | FileUtils.rmtree @tmpdir 17 | end 18 | 19 | def test_basic 20 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 21 | a,b,c 22 | foo,bar,baz 23 | qux,quuux 24 | End 25 | Tb::Cmd.main_gsub(['-o', o="o.csv", '[au]', 'YY', i]) 26 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 27 | a,b,c 28 | foo,bYYr,bYYz 29 | qYYx,qYYYYYYx 30 | End 31 | end 32 | 33 | def test_no_regexp 34 | exc = assert_raise(SystemExit) { Tb::Cmd.main_gsub([]) } 35 | assert(!exc.success?) 36 | end 37 | 38 | def test_no_subst 39 | exc = assert_raise(SystemExit) { Tb::Cmd.main_gsub(['foo']) } 40 | assert(!exc.success?) 41 | end 42 | 43 | def test_opt_e 44 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 45 | a,b,c 46 | foo,bar,baz 47 | qux,quuux 48 | End 49 | Tb::Cmd.main_gsub(['-o', o="o.csv", '-e', '[au]', 'YY', i]) 50 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 51 | a,b,c 52 | foo,bYYr,bYYz 53 | qYYx,qYYYYYYx 54 | End 55 | end 56 | 57 | def test_opt_f 58 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 59 | a,b,c 60 | foo,bar,baz 61 | qux,quuux 62 | End 63 | Tb::Cmd.main_gsub(['-o', o="o.csv", '-f', 'b', '[au]', 'YY', i]) 64 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 65 | a,b,c 66 | foo,bYYr,baz 67 | qux,qYYYYYYx 68 | End 69 | end 70 | 71 | def test_opt_f_extend 72 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 73 | a,b 74 | foo,bar,baz 75 | End 76 | o = "o.csv" 77 | stderr = capture_stderr { 78 | Tb::Cmd.main_gsub(['-o', o, '-f', '1', 'baz', 'Y', i]) 79 | } 80 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 81 | a,b 82 | foo,bar 83 | End 84 | assert_match(/Header too short/, stderr) 85 | end 86 | 87 | def test_twofile 88 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 89 | a,b 90 | 1,2 91 | 3,4 92 | End 93 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 94 | b,a 95 | 5,6 96 | 7,8 97 | End 98 | Tb::Cmd.main_gsub(['-o', o="o.csv", '[46]', 'z', i1, i2]) 99 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 100 | a,b 101 | 1,2 102 | 3,z 103 | z,5 104 | 8,7 105 | End 106 | end 107 | 108 | end 109 | -------------------------------------------------------------------------------- /lib/tb/cmd_rename.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'rename' 30 | 31 | def (Tb::Cmd).op_rename 32 | op = OptionParser.new 33 | op.banner = "Usage: tb rename [OPTS] SRC,DST,... [TABLE]\n" + 34 | "Rename field names." 35 | define_common_option(op, "ho", "--no-pager") 36 | op 37 | end 38 | 39 | Tb::Cmd.def_vhelp('rename', <<'End') 40 | Example: 41 | 42 | % cat tst.csv 43 | foo,bar,baz 44 | 1,2,3 45 | 4,5,6 46 | 7,8,9 47 | % tb rename bar,qux tst.csv 48 | foo,qux,baz 49 | 1,2,3 50 | 4,5,6 51 | 7,8,9 52 | % tb rename bar,qux,baz,quux tst.csv 53 | foo,qux,quux 54 | 1,2,3 55 | 4,5,6 56 | 7,8,9 57 | End 58 | 59 | def (Tb::Cmd).main_rename(argv) 60 | op_rename.parse!(argv) 61 | exit_if_help('rename') 62 | err('rename fields not given.') if argv.empty? 63 | fs = split_field_list_argument(argv.shift) 64 | argv = ['-'] if argv.empty? 65 | h = {} 66 | fs.each_slice(2) {|sf, df| h[sf] = df } 67 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 68 | er = Tb::Enumerator.new {|y| 69 | header = nil 70 | creader.with_header {|header0| 71 | header = header0 72 | h.each {|sf, df| 73 | unless header.include? sf 74 | err "field not found: #{sf.inspect}" 75 | end 76 | } 77 | y.set_header header.map {|f| h.fetch(f, f) } 78 | }.each {|pairs| 79 | y.yield Hash[pairs.map {|f, v| [h.fetch(f, f), v] }] 80 | } 81 | } 82 | output_tbenum(er) 83 | end 84 | 85 | -------------------------------------------------------------------------------- /lib/tb/cmd_newfield.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'newfield' 30 | 31 | Tb::Cmd.default_option[:opt_newfield_ruby] = nil 32 | 33 | def (Tb::Cmd).op_newfield 34 | op = OptionParser.new 35 | op.banner = "Usage: tb newfield [OPTS] FIELD VALUE [TABLE]\n" + 36 | "Add a field." 37 | define_common_option(op, "ho", "--no-pager") 38 | op.def_option('--ruby RUBY-EXP', 'ruby expression to generate values. A hash is given as _. no VALUE argument.') {|ruby_exp| 39 | Tb::Cmd.opt_newfield_ruby = ruby_exp 40 | } 41 | op 42 | end 43 | 44 | Tb::Cmd.def_vhelp('newfield', <<'End') 45 | Example: 46 | 47 | % cat tst.csv 48 | a,b,c 49 | 0,1,2 50 | 4,5,6 51 | % tb newfield foo bar tst.csv 52 | foo,a,b,c 53 | bar,0,1,2 54 | bar,4,5,6 55 | % tb newfield foo --ruby '_["b"] + _["c"]' tst.csv 56 | foo,a,b,c 57 | 12,0,1,2 58 | 56,4,5,6 59 | End 60 | 61 | def (Tb::Cmd).main_newfield(argv) 62 | op_newfield.parse!(argv) 63 | exit_if_help('newfield') 64 | err('no new field name given.') if argv.empty? 65 | field = argv.shift 66 | if Tb::Cmd.opt_newfield_ruby 67 | rubyexp = Tb::Cmd.opt_newfield_ruby 68 | pr = eval("lambda {|_| #{rubyexp} }") 69 | else 70 | err('no value given.') if argv.empty? 71 | value = argv.shift 72 | pr = lambda {|_| value } 73 | end 74 | argv = ['-'] if argv.empty? 75 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 76 | er = creader.newfield(field) {|pairs| pr.call(pairs) } 77 | output_tbenum(er) 78 | end 79 | 80 | 81 | -------------------------------------------------------------------------------- /test/test_cmd_to_json.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | begin 5 | require 'json' 6 | rescue LoadError 7 | end 8 | 9 | class TestTbCmdToJSON < Test::Unit::TestCase 10 | def setup 11 | Tb::Cmd.reset_option 12 | @curdir = Dir.pwd 13 | @tmpdir = Dir.mktmpdir 14 | Dir.chdir @tmpdir 15 | end 16 | def teardown 17 | Tb::Cmd.reset_option 18 | Dir.chdir @curdir 19 | FileUtils.rmtree @tmpdir 20 | end 21 | 22 | def test_basic 23 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 24 | a,b,c 25 | 0,1,2 26 | 4,5,6 27 | End 28 | Tb::Cmd.main_to_json(['-o', o="o.json", i]) 29 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 30 | [{ 31 | "a": "0", 32 | "b": "1", 33 | "c": "2" 34 | }, 35 | { 36 | "a": "4", 37 | "b": "5", 38 | "c": "6" 39 | }] 40 | End 41 | end 42 | 43 | def test_json_to_json 44 | File.open(i="i.json", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 45 | [{ 46 | "a": "0", 47 | "b": "1", 48 | "c": "2" 49 | }, 50 | { 51 | "a": "4", 52 | "b": "5", 53 | "c": "6" 54 | }] 55 | End 56 | Tb::Cmd.main_to_json(['-o', o="o.json", i]) 57 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 58 | [{ 59 | "a": "0", 60 | "b": "1", 61 | "c": "2" 62 | }, 63 | { 64 | "a": "4", 65 | "b": "5", 66 | "c": "6" 67 | }] 68 | End 69 | end 70 | 71 | def test_ltsv_to_json1 72 | File.open(i="i.ltsv", "w") {|f| f << "foo:bar\n" } 73 | Tb::Cmd.main_to_json(['-o', o="o.json", i]) 74 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 75 | [{"foo":"bar"}] 76 | End 77 | end 78 | 79 | def test_ltsv_to_json2 80 | File.open(i="i.ltsv", "w") {|f| f << "foo:bar\nbaz:qux\n" } 81 | Tb::Cmd.main_to_json(['-o', o="o.json", i]) 82 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 83 | [{"foo":"bar"}, {"baz":"qux"}] 84 | End 85 | end 86 | 87 | def test_twofile 88 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 89 | a,b 90 | 1,2 91 | 3,4 92 | End 93 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 94 | b,a 95 | 5,6 96 | 7,8 97 | End 98 | Tb::Cmd.main_to_json(['-o', o="o.csv", i1, i2]) 99 | assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, '')) 100 | [{ 101 | "a": "1", 102 | "b": "2" 103 | }, 104 | { 105 | "a": "3", 106 | "b": "4" 107 | }, 108 | { 109 | "b": "5", 110 | "a": "6" 111 | }, 112 | { 113 | "b": "7", 114 | "a": "8" 115 | }] 116 | End 117 | end 118 | 119 | end if defined?(JSON) 120 | -------------------------------------------------------------------------------- /lib/tb/cmd_cut.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'cut' 30 | 31 | Tb::Cmd.default_option[:opt_cut_v] = nil 32 | 33 | def (Tb::Cmd).op_cut 34 | op = OptionParser.new 35 | op.banner = "Usage: tb cut [OPTS] FIELD,... [TABLE]\n" + 36 | "Select columns." 37 | define_common_option(op, "hNo", "--no-pager") 38 | op.def_option('-v', 'invert match') { Tb::Cmd.opt_cut_v = true } 39 | op 40 | end 41 | 42 | Tb::Cmd.def_vhelp('cut', <<'End') 43 | Example: 44 | 45 | % cat tst.csv 46 | a,b,c 47 | 0,1,2 48 | 4,5,6 49 | % tb cut a,c tst.csv 50 | a,c 51 | 0,2 52 | 4,6 53 | % tb cut -v a tst.csv 54 | b,c 55 | 1,2 56 | 5,6 57 | End 58 | 59 | def (Tb::Cmd).main_cut(argv) 60 | op_cut.parse!(argv) 61 | exit_if_help('cut') 62 | err('no fields given.') if argv.empty? 63 | fs = split_field_list_argument(argv.shift) 64 | argv = ['-'] if argv.empty? 65 | Tb::CatReader.open(argv, Tb::Cmd.opt_N) {|tblreader| 66 | if Tb::Cmd.opt_cut_v 67 | er = Tb::Enumerator.new {|y| 68 | tblreader.with_header {|header0| 69 | if header0 70 | y.set_header header0 - fs 71 | end 72 | }.each {|pairs| 73 | y.yield pairs.reject {|k, v| fs.include? k } 74 | } 75 | } 76 | output_tbenum(er) 77 | else 78 | er = Tb::Enumerator.new {|y| 79 | y.set_header fs 80 | tblreader.each {|pairs| 81 | y.yield pairs.reject {|k, v| !fs.include?(k) } 82 | } 83 | } 84 | output_tbenum(er) 85 | end 86 | } 87 | end 88 | 89 | -------------------------------------------------------------------------------- /test/test_pager.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbPager < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def with_env(k, v) 19 | save = ENV[k] 20 | begin 21 | ENV[k] = v 22 | yield 23 | ensure 24 | ENV[k] = save 25 | end 26 | end 27 | 28 | def with_stdout(io) 29 | save = $stdout 30 | $stdout = io 31 | begin 32 | yield 33 | ensure 34 | $stdout = save 35 | end 36 | end 37 | 38 | def reader_thread(io) 39 | Thread.new { 40 | r = '' 41 | loop { 42 | begin 43 | r << io.readpartial(4096) 44 | rescue EOFError, Errno::EIO 45 | break 46 | end 47 | } 48 | r 49 | } 50 | end 51 | 52 | def test_notty 53 | open("tst", 'w') {|f| 54 | with_stdout(f) { 55 | Tb::Pager.open {|pager| 56 | pager.print "a" 57 | } 58 | } 59 | } 60 | assert_equal("a", File.read("tst")) 61 | end 62 | 63 | def test_printf 64 | open("tst", 'w') {|f| 65 | with_stdout(f) { 66 | Tb::Pager.open {|pager| 67 | pager.printf "%x", 255 68 | } 69 | } 70 | } 71 | assert_equal("ff", File.read("tst")) 72 | end 73 | 74 | def test_putc_int 75 | open("tst", 'w') {|f| 76 | with_stdout(f) { 77 | Tb::Pager.open {|pager| 78 | pager.putc 33 79 | } 80 | } 81 | } 82 | assert_equal("!", File.read("tst")) 83 | end 84 | 85 | def test_putc_str 86 | open("tst", 'w') {|f| 87 | with_stdout(f) { 88 | Tb::Pager.open {|pager| 89 | pager.putc "a" 90 | } 91 | } 92 | } 93 | assert_equal("a", File.read("tst")) 94 | end 95 | 96 | def test_puts_noarg 97 | open("tst", 'w') {|f| 98 | with_stdout(f) { 99 | Tb::Pager.open {|pager| 100 | pager.puts 101 | } 102 | } 103 | } 104 | assert_equal("\n", File.read("tst")) 105 | end 106 | 107 | def test_puts 108 | open("tst", 'w') {|f| 109 | with_stdout(f) { 110 | Tb::Pager.open {|pager| 111 | pager.puts "foo" 112 | } 113 | } 114 | } 115 | assert_equal("foo\n", File.read("tst")) 116 | end 117 | 118 | def test_write_nonblock 119 | open("tst", 'w') {|f| 120 | with_stdout(f) { 121 | Tb::Pager.open {|pager| 122 | pager.write_nonblock "foo" 123 | } 124 | } 125 | } 126 | assert_equal("foo", File.read("tst")) 127 | end 128 | 129 | def test_flush 130 | open("tst", 'w') {|f| 131 | with_stdout(f) { 132 | Tb::Pager.open {|pager| 133 | pager.write "foo" 134 | pager.flush 135 | } 136 | } 137 | } 138 | assert_equal("foo", File.read("tst")) 139 | end 140 | 141 | end 142 | -------------------------------------------------------------------------------- /lib/tb/cmd_sort.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'sort' 30 | 31 | Tb::Cmd.default_option[:opt_sort_f] = nil 32 | Tb::Cmd.default_option[:opt_sort_r] = nil 33 | 34 | def (Tb::Cmd).op_sort 35 | op = OptionParser.new 36 | op.banner = "Usage: tb sort [OPTS] [TABLE]\n" + 37 | "Sort rows." 38 | define_common_option(op, "hNo", "--no-pager") 39 | op.def_option('-f FIELD,...', 'specify sort keys') {|fs| Tb::Cmd.opt_sort_f = fs } 40 | op.def_option('-r', '--reverse', 'reverse order') { Tb::Cmd.opt_sort_r = true } 41 | op 42 | end 43 | 44 | Tb::Cmd.def_vhelp('sort', <<'End') 45 | Example: 46 | 47 | % cat tst.csv 48 | name,value 49 | foo,1 50 | bar,8 51 | baz,2 52 | % tb sort tst.csv 53 | name,value 54 | bar,8 55 | baz,2 56 | foo,1 57 | % tb sort -f value tst.csv 58 | name,value 59 | foo,1 60 | baz,2 61 | bar,8 62 | End 63 | 64 | def (Tb::Cmd).main_sort(argv) 65 | op_sort.parse!(argv) 66 | exit_if_help('sort') 67 | argv = ['-'] if argv.empty? 68 | if Tb::Cmd.opt_sort_f 69 | fs = split_field_list_argument(Tb::Cmd.opt_sort_f) 70 | else 71 | fs = nil 72 | end 73 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 74 | header = [] 75 | if fs 76 | blk = lambda {|pairs| fs.map {|f| Tb::Func.smart_cmp_value(pairs[f]) } } 77 | else 78 | blk = lambda {|pairs| header.map {|f| Tb::Func.smart_cmp_value(pairs[f]) } } 79 | end 80 | if Tb::Cmd.opt_sort_r 81 | blk1 = blk 82 | blk = lambda {|pairs| Tb::RevCmp.new(blk1.call(pairs)) } 83 | end 84 | er = Tb::Enumerator.new {|y| 85 | creader.with_cumulative_header {|header0| 86 | if header0 87 | y.set_header(header0) 88 | end 89 | }.each {|pairs, header1| 90 | header = header1 91 | y.yield pairs 92 | } 93 | }.extsort_by(&blk) 94 | output_tbenum(er) 95 | end 96 | 97 | 98 | -------------------------------------------------------------------------------- /lib/tb/cmd_consecutive.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'consecutive' 30 | 31 | Tb::Cmd.default_option[:opt_consecutive_n] = 2 32 | 33 | def (Tb::Cmd).op_consecutive 34 | op = OptionParser.new 35 | op.banner = "Usage: tb consecutive [OPTS] [TABLE ...]\n" + 36 | "Concatenate consecutive rows." 37 | define_common_option(op, "hNo", "--no-pager") 38 | op.def_option('-n NUM', 'gather NUM records. (default: 2)') {|n| Tb::Cmd.opt_consecutive_n = n.to_i } 39 | op 40 | end 41 | 42 | Tb::Cmd.def_vhelp('consecutive', <<'End') 43 | Example: 44 | 45 | % cat tst.csv 46 | a,b,c 47 | 0,1,2 48 | 4,5,6 49 | 7,8,9 50 | % tb consecutive tstcsv 51 | a_1,a_2,b_1,b_2,c_1,c_2 52 | 0,4,1,5,2,6 53 | 4,7,5,8,6,9 54 | End 55 | 56 | def (Tb::Cmd).main_consecutive(argv) 57 | op_consecutive.parse!(argv) 58 | exit_if_help('consecutive') 59 | argv = ['-'] if argv.empty? 60 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 61 | er = Tb::Enumerator.new {|y| 62 | buf = [] 63 | empty = true 64 | creader.with_cumulative_header {|header0| 65 | if header0 66 | y.set_header header0.map {|f| (1..Tb::Cmd.opt_consecutive_n).map {|i| "#{f}_#{i}" } }.flatten(1) 67 | end 68 | }.each {|pairs, header| 69 | buf << pairs 70 | if buf.length == Tb::Cmd.opt_consecutive_n 71 | pairs2 = {} 72 | header.each {|f| 73 | Tb::Cmd.opt_consecutive_n.times {|i| 74 | ps = buf[i] 75 | next if !ps.has_key?(f) 76 | v = ps[f] 77 | if Tb::Cmd.opt_N 78 | pairs2[((f.to_i-1) * Tb::Cmd.opt_consecutive_n + i + 1).to_s] = v 79 | else 80 | pairs2["#{f}_#{i+1}"] = v 81 | end 82 | } 83 | } 84 | empty = false 85 | y.yield pairs2 86 | buf.shift 87 | end 88 | } 89 | } 90 | output_tbenum(er) 91 | end 92 | -------------------------------------------------------------------------------- /lib/tb/enumerator.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | class Tb::Yielder 30 | def initialize(header_proc, base_yielder) 31 | @header_proc_called = false 32 | @header_proc = header_proc 33 | @base_yielder = base_yielder 34 | end 35 | attr_reader :header_proc_called 36 | 37 | def set_header(header) 38 | raise ArgumentError, "set_header called twice" if @header_proc_called 39 | @header_proc_called = true 40 | @header_proc.call(header) if @header_proc 41 | end 42 | 43 | def yield(*args) 44 | if !@header_proc_called 45 | set_header(nil) 46 | end 47 | unless args.is_a?(Array) && args.length == 1 && args[0].is_a?(Hash) 48 | raise "unexpected args: #{args.inspect}" 49 | end 50 | @base_yielder.yield(*args) 51 | end 52 | alias << yield 53 | end 54 | 55 | class Tb::Enumerator < Enumerator 56 | include Tb::EnumerableWithEach 57 | 58 | def self.from_header_and_values(header, *values_list) 59 | Tb::Enumerator.new {|y| 60 | y.set_header header 61 | values_list.each {|values| 62 | y.yield Hash[header.zip(values)] 63 | } 64 | } 65 | end 66 | 67 | def self.new(&enumerator_proc) 68 | super() {|y| 69 | header_proc = Thread.current[:tb_enumerator_header_proc] 70 | Thread.current[:tb_enumerator_header_proc] = nil 71 | ty = Tb::Yielder.new(header_proc, y) 72 | enumerator_proc.call(ty) 73 | if header_proc && !ty.header_proc_called 74 | header_proc.call(nil) 75 | end 76 | } 77 | end 78 | 79 | def header_and_each(header_proc, &each_proc) 80 | old = Thread.current[:tb_enumerator_header_proc] 81 | begin 82 | Thread.current[:tb_enumerator_header_proc] = header_proc 83 | Enumerator.instance_method(:each).bind(self).call(&each_proc) 84 | ensure 85 | Thread.current[:tb_enumerator_header_proc] = old 86 | end 87 | nil 88 | end 89 | 90 | end 91 | -------------------------------------------------------------------------------- /lib/tb/cmd_mheader.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'mheader' 30 | 31 | Tb::Cmd.default_option[:opt_mheader_count] = nil 32 | 33 | def (Tb::Cmd).op_mheader 34 | op = OptionParser.new 35 | op.banner = "Usage: tb mheader [OPTS] [TABLE]\n" + 36 | "Collapse multi rows header." 37 | define_common_option(op, "ho", "--no-pager") 38 | op.def_option('-c N', 'number of header records') {|arg| Tb::Cmd.opt_mheader_count = arg.to_i } 39 | op 40 | end 41 | 42 | Tb::Cmd.def_vhelp('mheader', <<'End') 43 | Example: 44 | 45 | % cat tst.csv 46 | foo,foo,bar,bar 47 | baz,qux,baz,qux 48 | 1,2,3,4 49 | 5,6,7,8 50 | % tb mheader tst.csv 51 | foo baz,foo qux,bar baz,bar qux 52 | 1,2,3,4 53 | 5,6,7,8 54 | End 55 | 56 | 57 | def (Tb::Cmd).main_mheader(argv) 58 | op_mheader.parse!(argv) 59 | exit_if_help('mheader') 60 | argv = ['-'] if argv.empty? 61 | header = [] 62 | if Tb::Cmd.opt_mheader_count 63 | c = Tb::Cmd.opt_mheader_count 64 | header_end_p = lambda { 65 | c -= 1 66 | c == 0 ? header.map {|a| a.compact.join(' ').strip } : nil 67 | } 68 | else 69 | header_end_p = lambda { 70 | h2 = header.map {|a| a.compact.join(' ').strip }.uniq 71 | header.length == h2.length ? h2 : nil 72 | } 73 | end 74 | creader = Tb::CatReader.open(argv, true) 75 | er = Tb::Enumerator.new {|y| 76 | creader.each {|pairs| 77 | if header 78 | ary = [] 79 | pairs.each {|f, v| ary[f.to_i-1] = v } 80 | ary.each_with_index {|v,i| 81 | header[i] ||= [] 82 | header[i] << v if header[i].empty? || header[i].last != v 83 | } 84 | h2 = header_end_p.call 85 | if h2 86 | pairs2 = Hash[h2.map.with_index {|v, i| ["#{i+1}", v] }] 87 | y.yield pairs2 88 | header = nil 89 | end 90 | else 91 | y.yield pairs 92 | end 93 | } 94 | } 95 | Tb::Cmd.opt_N = true 96 | output_tbenum(er) 97 | if header 98 | warn "unique header fields not recognized." 99 | end 100 | end 101 | 102 | -------------------------------------------------------------------------------- /test/test_cmd_cross.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdCross < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | name,year,observ 21 | aaaa,2000,1 22 | bbbb,2001,3 23 | bbbb,2000,4 24 | cccc,2002,5 25 | End 26 | Tb::Cmd.main_cross(['-o', o="o.csv", 'year', 'name', i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | name,aaaa,bbbb,cccc 29 | year,count,count,count 30 | 2000,1,1, 31 | 2001,,1, 32 | 2002,,,1 33 | End 34 | end 35 | 36 | def test_no_hkey_fields 37 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cross([]) } 38 | assert(!exc.success?) 39 | end 40 | 41 | def test_no_vkey_fields 42 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cross(['hk']) } 43 | assert(!exc.success?) 44 | end 45 | 46 | def test_field_not_found 47 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 48 | name,year,observ 49 | aaaa,2000,1 50 | bbbb,2001,3 51 | End 52 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cross(['-o', "o.csv", 'foo', 'year', i]) } 53 | assert(!exc.success?) 54 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cross(['-o', "o.csv", 'name', 'bar', i]) } 55 | assert(!exc.success?) 56 | end 57 | 58 | def test_compact 59 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 60 | name,year,observ 61 | aaaa,2000,1 62 | bbbb,2001,3 63 | bbbb,2000,4 64 | cccc,2002,5 65 | End 66 | Tb::Cmd.main_cross(['-o', o="o.csv", 'year', 'name', '-c', i]) 67 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 68 | year,aaaa,bbbb,cccc 69 | 2000,1,1, 70 | 2001,,1, 71 | 2002,,,1 72 | End 73 | end 74 | 75 | def test_sum 76 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 77 | name,year,observ 78 | aaaa,2000,1 79 | bbbb,2001,3 80 | bbbb,2000,4 81 | cccc,2002,5 82 | aaaa,2000,2 83 | End 84 | Tb::Cmd.main_cross(['-o', o="o.csv", 'year', 'name', '-a', 'sum(observ)', i]) 85 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 86 | name,aaaa,bbbb,cccc 87 | year,sum(observ),sum(observ),sum(observ) 88 | 2000,3,4, 89 | 2001,,3, 90 | 2002,,,5 91 | End 92 | end 93 | 94 | def test_twofile 95 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 96 | a,b 97 | 1,2 98 | 3,4 99 | End 100 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 101 | b,a 102 | 5,6 103 | 7,8 104 | End 105 | Tb::Cmd.main_cross(['-o', o="o.csv", 'b', 'a', i1, i2]) 106 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 107 | a,1,3,6,8 108 | b,count,count,count,count 109 | 2,1,,, 110 | 4,,1,, 111 | 5,,,1, 112 | 7,,,,1 113 | End 114 | end 115 | 116 | def test_invalid_aggregator 117 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 118 | a,b 119 | 1,2 120 | 3,4 121 | End 122 | exc = assert_raise(SystemExit) { Tb::Cmd.main_cross(['-o', "o.csv", 'a', 'b', '-a', 'foo', i]) } 123 | assert(!exc.success?) 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/tb/cmd_gsub.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'gsub' 30 | 31 | Tb::Cmd.default_option[:opt_gsub_e] = nil 32 | Tb::Cmd.default_option[:opt_gsub_f] = nil 33 | 34 | def (Tb::Cmd).op_gsub 35 | op = OptionParser.new 36 | op.banner = "Usage: tb gsub [OPTS] REGEXP STRING [TABLE ...]\n" + 37 | "Substitute cells." 38 | define_common_option(op, "hNo", "--no-pager") 39 | op.def_option('-f FIELD', 'target field') {|field| Tb::Cmd.opt_gsub_f = field } 40 | op.def_option('-e REGEXP', 'specify regexp, possibly begins with a hyphen') {|pattern| Tb::Cmd.opt_gsub_e = pattern } 41 | op 42 | end 43 | 44 | Tb::Cmd.def_vhelp('gsub', <<'End') 45 | Example: 46 | 47 | % cat tst.csv 48 | foo,bar 49 | baz,qux 50 | hoge,moga 51 | % tb gsub o X tst.csv 52 | foo,bar 53 | baz,qux 54 | hXge,mXga 55 | % tb gsub -f foo o X tst.csv 56 | foo,bar 57 | baz,qux 58 | hXge,moga 59 | % tb gsub '[aeiou]' '{\&}' tst.csv 60 | foo,bar 61 | b{a}z,q{u}x 62 | h{o}g{e},m{o}g{a} 63 | End 64 | 65 | def (Tb::Cmd).main_gsub(argv) 66 | op_gsub.parse!(argv) 67 | exit_if_help('gsub') 68 | if Tb::Cmd.opt_gsub_e 69 | re = Regexp.new(Tb::Cmd.opt_gsub_e) 70 | else 71 | err('no regexp given.') if argv.empty? 72 | re = Regexp.new(argv.shift) 73 | end 74 | err('no substitution given.') if argv.empty? 75 | repl = argv.shift 76 | argv = ['-'] if argv.empty? 77 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 78 | er = Tb::Enumerator.new {|y| 79 | creader.with_cumulative_header {|header0| 80 | if header0 81 | y.set_header header0 82 | end 83 | }.each {|pairs, header| 84 | fs = header.dup 85 | fs.pop while !fs.empty? && !pairs.has_key?(fs.last) 86 | if Tb::Cmd.opt_gsub_f 87 | pairs2 = pairs.map {|f, v| 88 | if f == Tb::Cmd.opt_gsub_f 89 | v ||= '' 90 | [f, v.gsub(re, repl)] 91 | else 92 | [f, v] 93 | end 94 | } 95 | else 96 | pairs2 = pairs.map {|f, v| 97 | v ||= '' 98 | [f, v.gsub(re, repl)] 99 | } 100 | end 101 | y.yield Hash[pairs2] 102 | } 103 | } 104 | output_tbenum(er) 105 | end 106 | 107 | -------------------------------------------------------------------------------- /test/test_cmd_svn_log.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdSvnLog < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def with_stderr(io) 19 | save = $stderr 20 | $stderr = io 21 | begin 22 | yield 23 | ensure 24 | $stderr = save 25 | end 26 | end 27 | 28 | def test_basic 29 | system("svnadmin create repo") 30 | system("svn co -q file://#{@tmpdir}/repo .") 31 | File.open("foo", "w") {|f| f.puts "bar" } 32 | File.open("hoge", "w") {|f| f.puts "moge" } 33 | system("svn add -q foo hoge") 34 | system("svn commit -q -m baz foo hoge") 35 | system("svn update -q") # update the revision of the directory. 36 | Tb::Cmd.main_svn(['-o', o="o.csv"]) 37 | aa = CSV.read(o) 38 | assert_equal(2, aa.length) 39 | header, row = aa 40 | assert_match(/baz/, row[header.index "msg"]) 41 | end 42 | 43 | def test_verbose 44 | system("svnadmin create repo") 45 | system("svn co -q file://#{@tmpdir}/repo .") 46 | File.open("foo", "w") {|f| f.puts "bar" } 47 | File.open("hoge", "w") {|f| f.puts "moge" } 48 | system("svn add -q foo hoge") 49 | system("svn commit -q -m baz foo hoge") 50 | system("svn update -q") # update the revision of the directory. 51 | Tb::Cmd.main_svn(['-o', o="o.csv", '--', '-v']) 52 | aa = CSV.read(o) 53 | assert_equal(3, aa.length) 54 | header, *rows = aa 55 | rows = rows.sort_by {|rec| rows[header.index 'path'] } 56 | rows.each {|row| 57 | assert_match(/baz/, row[header.index "msg"]) 58 | } 59 | assert_match(/foo/, rows[0][header.index "path"]) 60 | assert_match(/hoge/, rows[1][header.index "path"]) 61 | end 62 | 63 | def test_log_xml 64 | system("svnadmin create repo") 65 | system("svn co -q file://#{@tmpdir}/repo .") 66 | File.open("foo", "w") {|f| f.puts "bar" } 67 | File.open("hoge", "w") {|f| f.puts "moge" } 68 | system("svn add -q foo hoge") 69 | system("svn commit -q -m baz foo hoge") 70 | system("svn update -q") # update the revision of the directory. 71 | ### 72 | Tb::Cmd.main_svn(['-o', o="o.csv"]) 73 | aa = CSV.read(o) 74 | assert_equal(2, aa.length) 75 | header, row = aa 76 | assert_match(/baz/, row[header.index "msg"]) 77 | ### 78 | system("svn log --xml > log.xml") 79 | FileUtils.rmtree('.svn') 80 | Tb::Cmd.main_svn(['-o', o="o.csv", '--svn-log-xml=log.xml']) 81 | aa = CSV.read(o) 82 | assert_equal(2, aa.length) 83 | header, row = aa 84 | assert_match(/baz/, row[header.index "msg"]) 85 | end 86 | 87 | def test_no_props 88 | system("svnadmin create repo") 89 | File.open("repo/hooks/pre-revprop-change", "w", 0755) {|f| f.print "#!/bin/sh\nexit 0\0" } 90 | system("svn co -q file://#{@tmpdir}/repo .") 91 | File.open("foo", "w") {|f| f.puts "bar" } 92 | system("svn add -q foo") 93 | system("svn commit -q -m baz foo") 94 | system("svn update -q") # update the revision of the directory. 95 | system("svn propdel -q svn:author --revprop -r 1 .") 96 | system("svn propdel -q svn:date --revprop -r 1 .") 97 | system("svn propdel -q svn:log --revprop -r 1 .") 98 | ### 99 | Tb::Cmd.main_svn(['-o', o="o.csv"]) 100 | aa = CSV.read(o) 101 | assert_equal(2, aa.length) 102 | header, row = aa 103 | assert_equal('(no author)', row[header.index "author"]) 104 | assert_equal('(no date)', row[header.index "date"]) 105 | assert_equal('', row[header.index "msg"]) 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /lib/tb/headerwriter.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/headerwriterm.rb - writer mixin for table with header 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'tempfile' 32 | 33 | class Tb::HeaderWriter 34 | def initialize(put_array) 35 | @put_array = put_array 36 | end 37 | 38 | def header_required? 39 | true 40 | end 41 | 42 | def header_generator=(gen) 43 | @header_generator = gen 44 | end 45 | 46 | def generate_header_if_possible 47 | return if defined? @header_use_buffer 48 | header = nil 49 | if defined? @header_generator 50 | header = @header_generator.call 51 | end 52 | if header 53 | @header_use_buffer = false 54 | @header = header 55 | @put_array.call @header 56 | else 57 | @header_use_buffer = true 58 | @header = [] 59 | @header_buffer = Tempfile.new('tb') 60 | end 61 | end 62 | 63 | def put_hash(hash) 64 | generate_header_if_possible 65 | if @header_use_buffer 66 | put_hash_buffer(hash) 67 | else 68 | put_hash_immediate(hash) 69 | end 70 | nil 71 | end 72 | 73 | def put_hash_buffer(hash) 74 | Marshal.dump(hash, @header_buffer) 75 | (hash.map {|k, v| k } - @header).each {|f| 76 | @header << f 77 | } 78 | end 79 | private :put_hash_buffer 80 | 81 | def finish 82 | generate_header_if_possible 83 | if @header_use_buffer == nil 84 | generate_header_if_possible 85 | end 86 | if @header_use_buffer 87 | @header_buffer.rewind 88 | @put_array.call @header 89 | begin 90 | while true 91 | hash = Marshal.load(@header_buffer) 92 | put_hash_immediate(hash) 93 | end 94 | rescue EOFError 95 | end 96 | @header_buffer.close! 97 | end 98 | end 99 | 100 | def put_hash_immediate(hash) 101 | ary = [] 102 | @header.each_with_index {|f, i| 103 | if pair = hash.find {|k, v| k == f } 104 | ary[i] = pair.last 105 | end 106 | } 107 | (hash.map {|k, v| k } - @header).each {|f| 108 | warn "unexpected field: #{f.inspect}" if /\A[1-9][0-9]*\z/ !~ f 109 | i = @header.length 110 | @header << f 111 | ary[i] = hash[f] 112 | } 113 | @put_array.call ary 114 | end 115 | private :put_hash_immediate 116 | end 117 | -------------------------------------------------------------------------------- /lib/tb/ropen.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/ropen.rb - Tb.open_reader 2 | # 3 | # Copyright (C) 2011-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | Tb::FormatHash = { 32 | 'csv' => { :reader => Tb::HeaderCSVReader, :writer => Tb::HeaderCSVWriter}, 33 | 'ncsv' => { :reader => Tb::NumericCSVReader, :writer => Tb::NumericCSVWriter}, 34 | 'tsv' => { :reader => Tb::HeaderTSVReader, :writer => Tb::HeaderTSVWriter}, 35 | 'ntsv' => { :reader => Tb::NumericTSVReader, :writer => Tb::NumericTSVWriter}, 36 | 'ltsv' => { :reader => Tb::LTSVReader, :writer => Tb::LTSVWriter}, 37 | 'pnm' => { :reader => Tb::PNMReader, :writer => Tb::PNMWriter}, 38 | 'ppm' => { :reader => Tb::PNMReader, :writer => Tb::PNMWriter}, 39 | 'pgm' => { :reader => Tb::PNMReader, :writer => Tb::PNMWriter}, 40 | 'pbm' => { :reader => Tb::PNMReader, :writer => Tb::PNMWriter}, 41 | 'json' => { :reader => Tb::JSONReader, :writer => Tb::JSONWriter}, 42 | 'ndjson' => { :reader => Tb::NDJSONReader, :writer => Tb::NDJSONWriter}, 43 | } 44 | 45 | def Tb.undecorate_filename(filename, numeric) 46 | if filename.respond_to?(:to_str) 47 | filename = filename.to_str 48 | elsif filename.respond_to?(:to_path) 49 | filename = filename.to_path 50 | else 51 | raise ArgumentError, "unexpected filename: #{filename.inspect}" 52 | end 53 | if /\A([a-z0-9]{2,}):/ =~ filename 54 | fmt = $1 55 | filename = $' 56 | err("unexpected format: #{fmt.inspect}") if !Tb::FormatHash.has_key?(fmt) 57 | elsif /\.([a-z0-9]+{2,})\z/ =~ filename 58 | fmt = $1 59 | fmt = 'csv' if !Tb::FormatHash.has_key?(fmt) 60 | else 61 | fmt = 'csv' 62 | end 63 | if numeric 64 | case fmt 65 | when 'csv' then fmt = 'ncsv' 66 | when 'tsv' then fmt = 'ntsv' 67 | end 68 | end 69 | return filename, fmt 70 | end 71 | 72 | def Tb.open_reader(filename, numeric=false) 73 | filename, fmt = Tb.undecorate_filename(filename, numeric) 74 | factory = Tb::FormatHash.fetch(fmt)[:reader] 75 | io_opened = nil 76 | if filename == '-' 77 | reader = factory.new($stdin) 78 | else 79 | io_opened = File.open(filename) 80 | reader = factory.new(io_opened) 81 | end 82 | if block_given? 83 | begin 84 | yield reader 85 | ensure 86 | if io_opened && !io_opened.closed? 87 | io_opened.close 88 | end 89 | end 90 | else 91 | reader 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/tb/headerreader.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/headerreaderm.rb - reader mixin for table with header 2 | # 3 | # Copyright (C) 2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | class Tb::HeaderReader 32 | include Tb::EnumerableWithEach 33 | 34 | def initialize(get_array) 35 | @get_array = get_array 36 | @header_array_hook = nil 37 | @row_array_hook = nil 38 | @enable_warning = true 39 | end 40 | attr_accessor :header_array_hook 41 | attr_accessor :row_array_hook 42 | attr_accessor :enable_warning 43 | 44 | def header_known? 45 | true 46 | end 47 | 48 | def read_header_once 49 | return if defined? @header 50 | begin 51 | @header = @get_array.call 52 | end while @header && @header.all? {|elt| elt.nil? || elt == '' } 53 | if !@header 54 | @header = [] 55 | end 56 | @header_array_hook.call(@header) if @header_array_hook 57 | h = Hash.new { [] } 58 | @header.each_with_index {|f, i| 59 | h[f] <<= i 60 | } 61 | if h.has_key? nil 62 | warn "Empty header field #{h[nil].map(&:succ).join(',')}" if @enable_warning 63 | end 64 | h.each {|f, is| 65 | if 1 < is.length 66 | warn "Ambiguous header field: field #{is.map(&:succ).join(',')} has same name #{f.inspect}" if @enable_warning 67 | is[1..-1].each {|i| 68 | @header[i] = nil 69 | } 70 | end 71 | } 72 | end 73 | private :read_header_once 74 | 75 | def get_named_header 76 | read_header_once 77 | @header.compact 78 | end 79 | 80 | def get_hash 81 | read_header_once 82 | ary = @get_array.call 83 | if !ary 84 | return nil 85 | end 86 | @row_array_hook.call(ary) if @row_array_hook 87 | hash = {} 88 | if @header.length < ary.length 89 | warn "Header too short: header has #{@header.length} fields but a record has #{ary.length} fields : #{ary[@header.length..-1].map(&:inspect).join(',')}" if @enable_warning 90 | ary[@header.length..-1] = [] 91 | end 92 | ary.each_with_index {|v, i| 93 | field = @header[i] 94 | if !field.nil? 95 | hash[field] = v 96 | end 97 | } 98 | hash 99 | end 100 | 101 | def header_and_each(header_proc) 102 | header_proc.call(get_named_header) if header_proc 103 | while hash = get_hash 104 | yield hash 105 | end 106 | nil 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /lib/tb/cmd_nest.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'nest' 30 | 31 | def (Tb::Cmd).op_nest 32 | op = OptionParser.new 33 | op.banner = "Usage: tb nest [OPTS] NEWFIELD,OLDFIELD1,OLDFIELD2,... [TABLE ...]\n" + 34 | "Nest fields." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | Tb::Cmd.def_vhelp('nest', <<'End') 40 | Example: 41 | 42 | % cat tst.csv 43 | name,author,length 44 | foo,A,3 45 | bar,A,5 46 | baz,B,2 47 | qux,B,8 48 | % tb nest item,name,length tst.csv 49 | author,item 50 | A,"name,length 51 | foo,3 52 | bar,5 53 | " 54 | B,"name,length 55 | baz,2 56 | qux,8 57 | " 58 | End 59 | 60 | 61 | def (Tb::Cmd).main_nest(argv) 62 | op_nest.parse!(argv) 63 | exit_if_help('nest') 64 | err('no fields given.') if argv.empty? 65 | fields = split_field_list_argument(argv.shift) 66 | newfield, *oldfields = fields 67 | oldfields_hash = {} 68 | oldfields.each {|f| oldfields_hash[f] = true } 69 | argv = ['-'] if argv.empty? 70 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 71 | er = Tb::Enumerator.new {|y| 72 | sorted = creader.with_header {|header0| 73 | oldfields.each {|f| 74 | if !header0.include?(f) 75 | err("field not found: #{f.inspect}") 76 | end 77 | } 78 | y.set_header(header0.reject {|f| oldfields_hash[f] } + [newfield]) 79 | }.map {|pairs| 80 | cv = pairs.reject {|f, v| 81 | oldfields_hash[f] 82 | }.map {|f, v| 83 | [Tb::Func.smart_cmp_value(f), Tb::Func.smart_cmp_value(v)] 84 | }.sort 85 | [cv, pairs] 86 | } 87 | 88 | nested = nil 89 | before_group = lambda {|(_, _)| 90 | nested = [] 91 | } 92 | body = lambda {|(_, pairs)| 93 | nested << pairs.reject {|f, v| !oldfields_hash[f] } 94 | } 95 | after_group = lambda {|(_, last_pairs)| 96 | nested_csv = "" 97 | nested_csv << oldfields.to_csv 98 | nested.each {|npairs| 99 | nested_csv << oldfields.map {|of| npairs[of] }.to_csv 100 | } 101 | assoc = last_pairs.reject {|f, v| oldfields_hash[f] }.to_a 102 | assoc << [newfield, nested_csv] 103 | pairs = Hash[assoc] 104 | y.yield pairs 105 | } 106 | sorted.detect_group_by(before_group, after_group) {|cv,| cv }.each(&body) 107 | } 108 | output_tbenum(er) 109 | end 110 | 111 | -------------------------------------------------------------------------------- /test/test_cmd_to_csv.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | require_relative 'util_tbtest' 5 | 6 | class TestTbCmdToCSV < Test::Unit::TestCase 7 | def setup 8 | Tb::Cmd.reset_option 9 | @curdir = Dir.pwd 10 | @tmpdir = Dir.mktmpdir 11 | Dir.chdir @tmpdir 12 | end 13 | def teardown 14 | Tb::Cmd.reset_option 15 | Dir.chdir @curdir 16 | FileUtils.rmtree @tmpdir 17 | end 18 | 19 | def with_stdin(io) 20 | save = $stdin 21 | $stdin = io 22 | begin 23 | yield 24 | ensure 25 | $stdin = save 26 | end 27 | end 28 | 29 | def with_stdout(io) 30 | save = $stdout 31 | $stdout = io 32 | begin 33 | yield 34 | ensure 35 | $stdout = save 36 | end 37 | end 38 | 39 | def test_basic 40 | File.open(i="i.tsv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 41 | a\tb\tc 42 | 0\t1\t2 43 | 4\t5\t6 44 | End 45 | Tb::Cmd.main_to_csv(['-o', o="o.csv", i]) 46 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 47 | a,b,c 48 | 0,1,2 49 | 4,5,6 50 | End 51 | end 52 | 53 | def test_short_header 54 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 55 | a,b 56 | 0,1,2 57 | 4,5,6 58 | End 59 | o = "o.csv" 60 | stderr = capture_stderr { 61 | Tb::Cmd.main_to_csv(['-o', o, i]) 62 | } 63 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 64 | a,b 65 | 0,1 66 | 4,5 67 | End 68 | assert_match(/Header too short/, stderr) 69 | end 70 | 71 | def test_numeric 72 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 73 | a 74 | 0,1,2 75 | 4,5,6 76 | End 77 | Tb::Cmd.main_to_csv(['-o', o="o.csv", '-N', i]) 78 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 79 | a,, 80 | 0,1,2 81 | 4,5,6 82 | End 83 | end 84 | 85 | def test_noarg 86 | File.open(i="i.tsv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 87 | a,b,c 88 | 0,1,2 89 | 4,5,6 90 | End 91 | o = nil 92 | File.open(i) {|input| 93 | with_stdin(input) { 94 | Tb::Cmd.main_to_csv(['-o', o="o.csv"]) 95 | } 96 | } 97 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 98 | a,b,c 99 | 0,1,2 100 | 4,5,6 101 | End 102 | end 103 | 104 | def test_pipeout 105 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 106 | a,b,c 107 | 0,1,2 108 | 4,5,6 109 | End 110 | r, w = IO.pipe 111 | th = Thread.new { r.read } 112 | with_stdout(w) { 113 | Tb::Cmd.main_to_csv([i]) 114 | w.close 115 | } 116 | result = th.value 117 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), result) 118 | a,b,c 119 | 0,1,2 120 | 4,5,6 121 | End 122 | ensure 123 | r.close if r && !r.closed? 124 | w.close if w && !w.closed? 125 | end 126 | 127 | def test_twofile 128 | File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 129 | a,b 130 | 1,2 131 | 3,4 132 | End 133 | File.open(i2="i2.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 134 | b,a 135 | 5,6 136 | 7,8 137 | End 138 | Tb::Cmd.main_to_csv(['-o', o="o.csv", i1, i2]) 139 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 140 | a,b 141 | 1,2 142 | 3,4 143 | 6,5 144 | 8,7 145 | End 146 | end 147 | 148 | def test_output_extension 149 | File.open(i="i.tsv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 150 | a\tb\tc 151 | 0\t1\t2 152 | 4\t5\t6 153 | End 154 | Tb::Cmd.main_to_csv(['-o', o="o.json", i]) 155 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 156 | a,b,c 157 | 0,1,2 158 | 4,5,6 159 | End 160 | end 161 | 162 | end 163 | -------------------------------------------------------------------------------- /lib/tb/cmd_search.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'search' 30 | 31 | Tb::Cmd.default_option[:opt_search_e] = nil 32 | Tb::Cmd.default_option[:opt_search_ruby] = nil 33 | Tb::Cmd.default_option[:opt_search_f] = nil 34 | Tb::Cmd.default_option[:opt_search_v] = nil 35 | 36 | def (Tb::Cmd).op_search 37 | op = OptionParser.new 38 | op.banner = "Usage: tb search [OPTS] REGEXP [TABLE ...]\n" + 39 | "Search rows using regexp or ruby expression." 40 | define_common_option(op, "hNo", "--no-pager") 41 | op.def_option('-f FIELD', 'search field') {|field| Tb::Cmd.opt_search_f = field } 42 | op.def_option('-e REGEXP', 'specify a regexp.') {|pattern| Tb::Cmd.opt_search_e = pattern } 43 | op.def_option('--ruby RUBY-EXP', 'predicate written in ruby. A hash is given as _. no usual regexp argument.') {|ruby_exp| Tb::Cmd.opt_search_ruby = ruby_exp } 44 | op.def_option('-v', 'ouput the records which doesn\'t match') { Tb::Cmd.opt_search_v = true } 45 | op 46 | end 47 | 48 | Tb::Cmd.def_vhelp('search', <<'End') 49 | Example: 50 | 51 | % cat tst.csv 52 | a,b,c 53 | 0,1,2 54 | 4,5,6 55 | % tb search 0 tst.csv 56 | a,b,c 57 | 0,1,2 58 | End 59 | 60 | def (Tb::Cmd).main_search(argv) 61 | op_search.parse!(argv) 62 | exit_if_help('search') 63 | if Tb::Cmd.opt_search_ruby 64 | pred = eval("lambda {|_| #{Tb::Cmd.opt_search_ruby} }") 65 | elsif Tb::Cmd.opt_search_e 66 | re = Regexp.new(Tb::Cmd.opt_search_e) 67 | pred = Tb::Cmd.opt_search_f ? 68 | lambda {|_| re =~ _[Tb::Cmd.opt_search_f] } : 69 | lambda {|_| _.any? {|k, v| re =~ v.to_s } } 70 | else 71 | err("no regexp given.") if argv.empty? 72 | re = Regexp.new(argv.shift) 73 | pred = Tb::Cmd.opt_search_f ? 74 | lambda {|_| re =~ _[Tb::Cmd.opt_search_f] } : 75 | lambda {|_| _.any? {|k, v| re =~ v.to_s } } 76 | end 77 | opt_v = Tb::Cmd.opt_search_v ? true : false 78 | argv = ['-'] if argv.empty? 79 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 80 | er = Tb::Enumerator.new {|y| 81 | header = nil 82 | creader.with_header {|header0| 83 | header = header0 84 | y.set_header header 85 | }.each {|pairs| 86 | h = {} 87 | pairs.each {|f, v| 88 | h[f] = v 89 | } 90 | found = pred.call(h) 91 | found = opt_v ^ !!(found) 92 | if found 93 | y.yield pairs 94 | end 95 | } 96 | } 97 | output_tbenum(er) 98 | end 99 | 100 | -------------------------------------------------------------------------------- /lib/tb/cmd_join.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'join' 30 | 31 | Tb::Cmd.default_option[:opt_join_outer_missing] = nil 32 | Tb::Cmd.default_option[:opt_join_retain_left] = nil 33 | Tb::Cmd.default_option[:opt_join_retain_right] = nil 34 | 35 | def (Tb::Cmd).op_join 36 | op = OptionParser.new 37 | op.banner = "Usage: tb join [OPTS] [TABLE1 TABLE2 ...]\n" + 38 | "Concatenate tables horizontally as left/right/full natural join." 39 | define_common_option(op, 'hNod', '--no-pager', '--debug') 40 | op.def_option('--outer', 'outer join') { 41 | Tb::Cmd.opt_join_retain_left = true 42 | Tb::Cmd.opt_join_retain_right = true 43 | } 44 | op.def_option('--left', 'left outer join') { 45 | Tb::Cmd.opt_join_retain_left = true 46 | Tb::Cmd.opt_join_retain_right = false 47 | } 48 | op.def_option('--right', 'right outer join') { 49 | Tb::Cmd.opt_join_retain_left = false 50 | Tb::Cmd.opt_join_retain_right = true 51 | } 52 | op.def_option('--outer-missing=DEFAULT', 'missing value for outer join') {|missing| 53 | if Tb::Cmd.opt_join_retain_left == nil 54 | Tb::Cmd.opt_join_retain_left = true 55 | Tb::Cmd.opt_join_retain_right = true 56 | end 57 | Tb::Cmd.opt_join_outer_missing = missing 58 | } 59 | op 60 | end 61 | 62 | Tb::Cmd.def_vhelp('join', <<'End') 63 | Example: 64 | 65 | % cat tst1.csv 66 | name,length 67 | A,20 68 | B,30 69 | C,25 70 | % cat tst2.csv 71 | name,weight 72 | A,5 73 | B,8 74 | % tb join tst1.csv tst2.csv 75 | name,length,weight 76 | A,20,5 77 | B,30,8 78 | % tb join --left tst1.csv tst2.csv 79 | name,length,weight 80 | A,20,5 81 | B,30,8 82 | C,25, 83 | % tb join --left --outer-missing=zzz tst1.csv tst2.csv 84 | name,length,weight 85 | A,20,5 86 | B,30,8 87 | C,25,zzz 88 | End 89 | 90 | def (Tb::Cmd).main_join(argv) 91 | op_join.parse!(argv) 92 | exit_if_help('join') 93 | retain_left = Tb::Cmd.opt_join_retain_left 94 | retain_right = Tb::Cmd.opt_join_retain_right 95 | err('two tables required at least.') if argv.length < 2 96 | result = tablereader_open(argv.shift) 97 | argv.each {|filename| 98 | tbl = tablereader_open(filename) 99 | $stderr.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= Tb::Cmd.opt_debug 100 | result = result.natjoin2_outer(tbl, Tb::Cmd.opt_join_outer_missing, retain_left, retain_right) 101 | } 102 | output_tbenum(result) 103 | end 104 | 105 | -------------------------------------------------------------------------------- /lib/tb/cmd_group.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'group' 30 | 31 | Tb::Cmd.default_option[:opt_group_fields] = [] 32 | 33 | def (Tb::Cmd).op_group 34 | op = OptionParser.new 35 | op.banner = "Usage: tb group [OPTS] KEY-FIELD1,... [TABLE ...]\n" + 36 | "Group and aggregate rows." 37 | define_common_option(op, "hNo", "--no-pager") 38 | op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]', 39 | '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| Tb::Cmd.opt_group_fields << arg } 40 | op.def_option('--no-pager', 'don\'t use pager') { Tb::Cmd.opt_no_pager = true } 41 | op 42 | end 43 | 44 | Tb::Cmd.def_vhelp('group', <<'End') 45 | Example: 46 | 47 | % cat tst.csv 48 | a,b,c 49 | A,X,2 50 | A,Y,3 51 | B,Y,4 52 | % tb group a tst.csv 53 | a 54 | A 55 | B 56 | % tb group a -a count tst.csv 57 | a,count 58 | A,2 59 | B,1 60 | % tb group a -a 'avg(c)' tst.csv 61 | a,avg(c) 62 | A,2.5 63 | B,4.0 64 | % tb group a,b tst.csv 65 | a,b 66 | A,X 67 | A,Y 68 | B,Y 69 | % tb group a,b -a count tst.csv 70 | a,b,count 71 | A,X,1 72 | A,Y,1 73 | B,Y,1 74 | End 75 | 76 | def (Tb::Cmd).main_group(argv) 77 | op_group.parse!(argv) 78 | exit_if_help('group') 79 | err("no key fields given.") if argv.empty? 80 | kfs = split_field_list_argument(argv.shift) 81 | opt_group_fields = kfs.map {|f| [f, Tb::Func::First, f] } + 82 | Tb::Cmd.opt_group_fields.map {|arg| 83 | aggregation_spec, new_field = split_field_list_argument(arg) 84 | new_field ||= aggregation_spec 85 | [new_field, 86 | *begin 87 | parse_aggregator_spec2(aggregation_spec) 88 | rescue ArgumentError 89 | err($!.message) 90 | end 91 | ] 92 | } 93 | argv = ['-'] if argv.empty? 94 | creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N) 95 | result = Tb::Enumerator.new {|y| 96 | op = Tb::Zipper.new(opt_group_fields.map {|dstf, func, srcf| func }) 97 | er = creader.extsort_reduce(op) {|pairs| 98 | [kfs.map {|f| Tb::Func.smart_cmp_value(pairs[f]) }, 99 | opt_group_fields.map {|dstf, func, srcf| func.start(srcf ? pairs[srcf] : true) } ] 100 | } 101 | fields = opt_group_fields.map {|dstf, func, srcf| dstf } 102 | y.set_header(fields) 103 | er.each {|_, vals| 104 | pairs = opt_group_fields.zip(vals).map {|(dstf, func, _), val| 105 | [dstf, func.aggregate(val)] 106 | } 107 | y.yield Hash[pairs] 108 | } 109 | } 110 | output_tbenum(result) 111 | end 112 | 113 | -------------------------------------------------------------------------------- /lib/tb/ex_enumerator.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | module Tb::ExEnumerator 30 | # :call-seq: 31 | # 32 | # Tb::ExEnumerator.merge_sorted(enumerator1, ...) {|key, enumerator1_or_nil, ...| ... } 33 | # 34 | # iterates over enumerators specified by arguments. 35 | # 36 | # The enumerators should yield an array which first element is a comparable key. 37 | # The enumerators should be sorted by the key in ascending order. 38 | # 39 | # Tb::Enumerator.merge_sorted iterates keys in all of the enumerators. 40 | # It yields an array which contains a key and enumerators which has the key. 41 | # The array contains nil if corresponding enumerator don't have the key. 42 | # 43 | # The block may or may not use +next+ method to advance enumerators. 44 | # Anyway Tb::Enumerator.merge_sorted advance enumerators until peeked key is 45 | # greater than the yielded key. 46 | # 47 | # If a enumerator has multiple elements with same key, 48 | # the block can read them using +next+ method. 49 | # +peek+ method should also be used to determine the key of the next element has 50 | # the current key or not. 51 | # Be careful to not consume extra elements with different key. 52 | # 53 | def (Tb::ExEnumerator).merge_sorted(*enumerators) # :yields: [key, enumerator1_or_nil, ...] 54 | while true 55 | has_min = false 56 | min = nil 57 | min_enumerators = [] 58 | enumerators.each {|kpe| 59 | begin 60 | key, = kpe.peek 61 | rescue StopIteration 62 | min_enumerators << nil 63 | next 64 | end 65 | if !has_min 66 | has_min = true 67 | min = key 68 | min_enumerators << kpe 69 | else 70 | cmp = key <=> min 71 | if cmp < 0 72 | min_enumerators.fill(nil) 73 | min_enumerators << kpe 74 | min = key 75 | elsif cmp == 0 76 | min_enumerators << kpe 77 | else 78 | min_enumerators << nil 79 | end 80 | end 81 | } 82 | if !has_min 83 | return 84 | end 85 | yield [min, *min_enumerators] 86 | min_enumerators.each {|kpe| 87 | next if !kpe 88 | while true 89 | begin 90 | key, = kpe.peek 91 | rescue StopIteration 92 | break 93 | end 94 | if (min <=> key) < 0 95 | break 96 | end 97 | kpe.next 98 | end 99 | } 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /test/test_cmdtty.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | begin 5 | require 'pty' 6 | require 'io/console' 7 | rescue LoadError 8 | end 9 | 10 | class TestTbCmdTTY < Test::Unit::TestCase 11 | def setup 12 | Tb::Cmd.reset_option 13 | @curdir = Dir.pwd 14 | @tmpdir = Dir.mktmpdir 15 | Dir.chdir @tmpdir 16 | end 17 | def teardown 18 | Tb::Cmd.reset_option 19 | Dir.chdir @curdir 20 | FileUtils.rmtree @tmpdir 21 | end 22 | 23 | def with_env(k, v) 24 | save = ENV[k] 25 | begin 26 | ENV[k] = v 27 | yield 28 | ensure 29 | ENV[k] = save 30 | end 31 | end 32 | 33 | def with_real_stdout(io) 34 | save = $stdout.dup 35 | $stdout.reopen(io) 36 | begin 37 | yield 38 | ensure 39 | $stdout.reopen(save) 40 | save.close 41 | end 42 | end 43 | 44 | def reader_thread(io) 45 | th = Thread.new { 46 | r = '' 47 | loop { 48 | begin 49 | r << io.readpartial(4096) 50 | rescue EOFError, Errno::EIO 51 | break 52 | end 53 | } 54 | r 55 | } 56 | sleep 0.1 if /freebsd/ =~ RUBY_PLATFORM # FreeBSD 8.2-RELEASE-p3 (amd64) hang ??? 57 | th 58 | end 59 | 60 | def test_ttyout_multiscreen 61 | return unless IO.instance_methods.include?(:raw!) 62 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 63 | a,b,c 64 | 0,1,2 65 | 4,5,6 66 | End 67 | with_env('PAGER', 'sed "s/^/foo:/"') { 68 | PTY.open {|m, s| 69 | s.raw! 70 | s.winsize = [2, 80] 71 | th = reader_thread(m) 72 | with_real_stdout(s) { 73 | Tb::Cmd.main_to_csv([i]) 74 | } 75 | s.close 76 | result = th.value 77 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), result) 78 | foo:a,b,c 79 | foo:0,1,2 80 | foo:4,5,6 81 | End 82 | } 83 | } 84 | end 85 | 86 | def test_ttyout_singlescreen 87 | return unless IO.instance_methods.include?(:raw!) 88 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 89 | a,b,c 90 | 0,1,2 91 | 4,5,6 92 | End 93 | with_env('PAGER', 'sed "s/^/foo:/"') { 94 | PTY.open {|m, s| 95 | s.raw! 96 | s.winsize = [24, 80] 97 | th = reader_thread(m) 98 | with_real_stdout(s) { 99 | Tb::Cmd.main_to_csv([i]) 100 | } 101 | s.close 102 | result = th.value 103 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), result) 104 | a,b,c 105 | 0,1,2 106 | 4,5,6 107 | End 108 | } 109 | } 110 | end 111 | 112 | def test_ttyout_tab 113 | return unless IO.instance_methods.include?(:raw!) 114 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 115 | a,b,c 116 | 0,\t,2 117 | End 118 | with_env('PAGER', 'sed "s/^/foo:/"') { 119 | PTY.open {|m, s| 120 | s.raw! 121 | s.winsize = [3, 10] 122 | th = reader_thread(m) 123 | with_real_stdout(s) { 124 | Tb::Cmd.main_to_csv([i]) 125 | } 126 | s.close 127 | result = th.value 128 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), result) 129 | foo:a,b,c 130 | foo:0,\t,2 131 | End 132 | } 133 | } 134 | end 135 | 136 | def test_ttyout_nottysize 137 | return unless IO.instance_methods.include?(:raw!) 138 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 139 | a,b,c 140 | 0,1,2 141 | End 142 | with_env('PAGER', 'sed "s/^/foo:/"') { 143 | PTY.open {|m, s| 144 | s.raw! 145 | s.winsize = [0, 0] 146 | th = reader_thread(m) 147 | with_real_stdout(s) { 148 | Tb::Cmd.main_to_csv([i]) 149 | } 150 | s.close 151 | result = th.value 152 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), result) 153 | a,b,c 154 | 0,1,2 155 | End 156 | } 157 | } 158 | end 159 | 160 | end if defined?(PTY) && defined?(PTY.open) 161 | -------------------------------------------------------------------------------- /lib/tb/cmd_shape.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2014 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'shape' 30 | 31 | def (Tb::Cmd).op_shape 32 | op = OptionParser.new 33 | op.banner = "Usage: tb shape [OPTS] [TABLE ...]\n" + 34 | "Show table size." 35 | define_common_option(op, "hNo", "--no-pager") 36 | op 37 | end 38 | 39 | Tb::Cmd.def_vhelp('shape', <<'End') 40 | Example: 41 | 42 | % cat tst.csv 43 | foo,bar,baz 44 | 1,2,3 45 | 4,5 46 | 6,7,8,9 47 | % tb shape tst.csv -o json:- 48 | [ 49 | { 50 | "filename": "tst.csv", 51 | "records": 3, 52 | "min_pairs": 2, 53 | "max_pairs": 3, 54 | "header_fields": 3, 55 | "min_fields": 2, 56 | "max_fields": 4 57 | } 58 | ] 59 | End 60 | 61 | def (Tb::Cmd).main_shape(argv) 62 | op_shape.parse!(argv) 63 | exit_if_help('shape') 64 | filenames = argv.empty? ? ['-'] : argv 65 | ter = Tb::Enumerator.new {|y| 66 | filenames.each {|filename| 67 | tablereader_open(filename) {|tblreader| 68 | tblreader.enable_warning = false if tblreader.respond_to? :enable_warning 69 | num_records = 0 70 | num_header_fields = nil 71 | min_num_fields = nil 72 | max_num_fields = nil 73 | if tblreader.respond_to? :header_array_hook 74 | tblreader.header_array_hook = lambda {|header| 75 | num_header_fields = header.length 76 | } 77 | end 78 | if tblreader.respond_to? :row_array_hook 79 | tblreader.row_array_hook = lambda {|ary| 80 | n = ary.length 81 | if min_num_fields.nil? 82 | min_num_fields = max_num_fields = n 83 | else 84 | min_num_fields = n if n < min_num_fields 85 | max_num_fields = n if max_num_fields < n 86 | end 87 | } 88 | end 89 | min_num_pairs = nil 90 | max_num_pairs = nil 91 | tblreader.each {|pairs| 92 | num_records += 1 93 | n = pairs.length 94 | if min_num_pairs.nil? 95 | min_num_pairs = max_num_pairs = n 96 | else 97 | min_num_pairs = n if n < min_num_pairs 98 | max_num_pairs = n if max_num_pairs < n 99 | end 100 | } 101 | h = { 102 | 'filename'=>filename, 103 | 'records'=>num_records, 104 | 'min_pairs'=>min_num_pairs, 105 | 'max_pairs'=>max_num_pairs, 106 | } 107 | h['header_fields'] = num_header_fields if num_header_fields 108 | h['min_fields'] = min_num_fields if min_num_fields 109 | h['max_fields'] = max_num_fields if max_num_fields 110 | y.yield(h) 111 | } 112 | } 113 | } 114 | Tb::Cmd.opt_N = false 115 | output_tbenum(ter) 116 | end 117 | 118 | -------------------------------------------------------------------------------- /test/test_cmd_melt.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'tb/cmdtop' 3 | require 'tmpdir' 4 | 5 | class TestTbCmdMelt < Test::Unit::TestCase 6 | def setup 7 | Tb::Cmd.reset_option 8 | @curdir = Dir.pwd 9 | @tmpdir = Dir.mktmpdir 10 | Dir.chdir @tmpdir 11 | end 12 | def teardown 13 | Tb::Cmd.reset_option 14 | Dir.chdir @curdir 15 | FileUtils.rmtree @tmpdir 16 | end 17 | 18 | def test_basic 19 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 20 | a,b,c,d 21 | 0,1,2,3 22 | 4,5,6,7 23 | 8,9,a,b 24 | c,d,e,f 25 | End 26 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', i]) 27 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 28 | a,b,variable,value 29 | 0,1,c,2 30 | 0,1,d,3 31 | 4,5,c,6 32 | 4,5,d,7 33 | 8,9,c,a 34 | 8,9,d,b 35 | c,d,c,e 36 | c,d,d,f 37 | End 38 | end 39 | 40 | def test_recnum 41 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 42 | a,b,c,d 43 | 0,1,2,3 44 | 4,5,6,7 45 | 8,9,a,b 46 | c,d,e,f 47 | End 48 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum', i]) 49 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 50 | recnum,a,b,variable,value 51 | 1,0,1,c,2 52 | 1,0,1,d,3 53 | 2,4,5,c,6 54 | 2,4,5,d,7 55 | 3,8,9,c,a 56 | 3,8,9,d,b 57 | 4,c,d,c,e 58 | 4,c,d,d,f 59 | End 60 | end 61 | 62 | def test_recnum_value 63 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 64 | a,b,c,d 65 | 0,1,2,3 66 | 4,5,6,7 67 | 8,9,a,b 68 | c,d,e,f 69 | End 70 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum=rec', i]) 71 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 72 | rec,a,b,variable,value 73 | 1,0,1,c,2 74 | 1,0,1,d,3 75 | 2,4,5,c,6 76 | 2,4,5,d,7 77 | 3,8,9,c,a 78 | 3,8,9,d,b 79 | 4,c,d,c,e 80 | 4,c,d,d,f 81 | End 82 | end 83 | 84 | def test_variable_field 85 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 86 | a,b,c,d 87 | 0,1,2,3 88 | 4,5,6,7 89 | 8,9,a,b 90 | c,d,e,f 91 | End 92 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--variable-field=foo', i]) 93 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 94 | a,b,foo,value 95 | 0,1,c,2 96 | 0,1,d,3 97 | 4,5,c,6 98 | 4,5,d,7 99 | 8,9,c,a 100 | 8,9,d,b 101 | c,d,c,e 102 | c,d,d,f 103 | End 104 | end 105 | 106 | def test_value_field 107 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 108 | a,b,c,d 109 | 0,1,2,3 110 | 4,5,6,7 111 | 8,9,a,b 112 | c,d,e,f 113 | End 114 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--value-field=bar', i]) 115 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 116 | a,b,variable,bar 117 | 0,1,c,2 118 | 0,1,d,3 119 | 4,5,c,6 120 | 4,5,d,7 121 | 8,9,c,a 122 | 8,9,d,b 123 | c,d,c,e 124 | c,d,d,f 125 | End 126 | end 127 | 128 | def test_melt_regexp 129 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 130 | a,b,c,d 131 | 0,1,2,3 132 | 4,5,6,7 133 | 8,9,a,b 134 | c,d,e,f 135 | End 136 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-regexp=[bd]', i]) 137 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 138 | a,variable,value 139 | 0,b,1 140 | 0,d,3 141 | 4,b,5 142 | 4,d,7 143 | 8,b,9 144 | 8,d,b 145 | c,b,d 146 | c,d,f 147 | End 148 | end 149 | 150 | def test_melt_list 151 | File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') } 152 | a,b,c,d 153 | 0,1,2,3 154 | 4,5,6,7 155 | 8,9,a,b 156 | c,d,e,f 157 | End 158 | Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-fields=b,d', i]) 159 | assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o)) 160 | a,variable,value 161 | 0,b,1 162 | 0,d,3 163 | 4,b,5 164 | 4,d,7 165 | 8,b,9 166 | 8,d,b 167 | c,b,d 168 | c,d,f 169 | End 170 | end 171 | 172 | end 173 | -------------------------------------------------------------------------------- /lib/tb/cmd_help.rb: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2011-2012 Tanaka Akira 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions 5 | # are met: 6 | # 7 | # 1. Redistributions of source code must retain the above copyright 8 | # notice, this list of conditions and the following disclaimer. 9 | # 2. Redistributions in binary form must reproduce the above 10 | # copyright notice, this list of conditions and the following 11 | # disclaimer in the documentation and/or other materials provided 12 | # with the distribution. 13 | # 3. The name of the author may not be used to endorse or promote 14 | # products derived from this software without specific prior 15 | # written permission. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 21 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 26 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | Tb::Cmd.subcommands << 'help' 30 | 31 | Tb::Cmd.default_option[:opt_help_s] = nil 32 | 33 | def (Tb::Cmd).usage_list_subcommands 34 | with_output {|f| 35 | f.print <<'End' 36 | Usage: 37 | End 38 | Tb::Cmd.subcommands.each {|subcommand| 39 | banner = self.subcommand_send("op", subcommand).banner 40 | usage = banner[/^Usage: (.*)/, 1] 41 | f.puts " " + usage 42 | } 43 | } 44 | end 45 | 46 | def (Tb::Cmd).list_summary_of_subcommands 47 | with_output {|f| 48 | f.print <<'End' 49 | End 50 | subcommand_maxlen = Tb::Cmd.subcommands.map {|subcommand| subcommand.length }.max 51 | fmt = "%-#{subcommand_maxlen}s : %s" 52 | Tb::Cmd.subcommands.each {|subcommand| 53 | banner = self.subcommand_send("op", subcommand).banner 54 | summary = banner[/^Usage: (.*)\n(.*)/, 2] 55 | f.puts(fmt % [subcommand, summary]) 56 | } 57 | } 58 | end 59 | 60 | def (Tb::Cmd).op_help 61 | op = OptionParser.new 62 | op.banner = "Usage: tb help [OPTS] [SUBCOMMAND]\n" + 63 | "Show help message of tb command." 64 | define_common_option(op, "hvo", "--no-pager") 65 | op.def_option('-s', 'show summary of subcommands') { Tb::Cmd.opt_help_s = true } 66 | op 67 | end 68 | 69 | Tb::Cmd.def_vhelp('help', <<'End') 70 | Example: 71 | 72 | tb -h : list subcommands 73 | tb help : list subcommands 74 | 75 | tb cat -h : succinct help of "cat" subcommand 76 | tb help cat : succinct help of "cat" subcommand 77 | tb cat -hh : verbose help of "cat" subcommand 78 | tb help -h cat : verbose help of "cat" subcommand 79 | 80 | tb help -h : succinct help of "help" subcommand 81 | tb help help : succinct help of "help" subcommand 82 | tb help -hh : verbose help of "help" subcommand 83 | tb help -h help : verbose help of "help" subcommand 84 | End 85 | 86 | def (Tb::Cmd).exit_if_help(subcommand) 87 | if 0 < Tb::Cmd.opt_help 88 | show_help(subcommand) 89 | exit 90 | end 91 | end 92 | 93 | def (Tb::Cmd).show_help(subcommand) 94 | if Tb::Cmd.subcommands.include?(subcommand) 95 | with_output {|f| 96 | f.puts self.subcommand_send("op", subcommand) 97 | if 2 <= Tb::Cmd.opt_help && Tb::Cmd.verbose_help[subcommand] 98 | f.puts 99 | f.puts Tb::Cmd.verbose_help[subcommand] 100 | end 101 | } 102 | true 103 | else 104 | err "unexpected subcommand: #{subcommand.inspect}" 105 | end 106 | end 107 | 108 | def (Tb::Cmd).main_help(argv) 109 | op_help.parse!(argv) 110 | if Tb::Cmd.opt_help_s 111 | list_summary_of_subcommands 112 | exit 113 | elsif argv.empty? 114 | if Tb::Cmd.opt_help == 0 115 | usage_list_subcommands 116 | return true 117 | else 118 | argv.unshift 'help' 119 | Tb::Cmd.opt_help -= 1 120 | end 121 | end 122 | Tb::Cmd.opt_help += 1 123 | subcommand = argv.shift 124 | exit_if_help(subcommand) 125 | end 126 | -------------------------------------------------------------------------------- /lib/tb/ltsv.rb: -------------------------------------------------------------------------------- 1 | # lib/tb/ltsv.rb - LTSV related fetures for table library 2 | # 3 | # Copyright (C) 2013-2014 Tanaka Akira 4 | # 5 | # Redistribution and use in source and binary forms, with or without 6 | # modification, are permitted provided that the following conditions 7 | # are met: 8 | # 9 | # 1. Redistributions of source code must retain the above copyright 10 | # notice, this list of conditions and the following disclaimer. 11 | # 2. Redistributions in binary form must reproduce the above 12 | # copyright notice, this list of conditions and the following 13 | # disclaimer in the documentation and/or other materials provided 14 | # with the distribution. 15 | # 3. The name of the author may not be used to endorse or promote 16 | # products derived from this software without specific prior 17 | # written permission. 18 | # 19 | # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 20 | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 23 | # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 25 | # GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 | # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 | # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | require 'stringio' 32 | 33 | module Tb 34 | def Tb.ltsv_escape_key(str) 35 | if /[\0-\x1f":\\\x7f]/ =~ str 36 | '"' + 37 | str.gsub(/[\0-\x1f":\\\x7f]/) { 38 | ch = $& 39 | case ch 40 | when "\0"; '\0' 41 | when "\a"; '\a' 42 | when "\b"; '\b' 43 | when "\f"; '\f' 44 | when "\n"; '\n' 45 | when "\r"; '\r' 46 | when "\t"; '\t' 47 | when "\v"; '\v' 48 | when "\e"; '\e' 49 | else 50 | "\\x%02X" % ch.ord 51 | end 52 | } + 53 | '"' 54 | else 55 | str 56 | end 57 | end 58 | 59 | def Tb.ltsv_escape_value(str) 60 | if /[\0-\x1f"\\\x7f]/ =~ str 61 | '"' + 62 | str.gsub(/[\0-\x1f"\\\x7f]/) { 63 | ch = $& 64 | case ch 65 | when "\0"; '\0' 66 | when "\a"; '\a' 67 | when "\b"; '\b' 68 | when "\f"; '\f' 69 | when "\n"; '\n' 70 | when "\r"; '\r' 71 | when "\t"; '\t' 72 | when "\v"; '\v' 73 | when "\e"; '\e' 74 | else 75 | "\\x%02X" % ch.ord 76 | end 77 | } + 78 | '"' 79 | else 80 | str 81 | end 82 | end 83 | 84 | def Tb.ltsv_unescape_string(str) 85 | if /\A\s*"(.*)"\s*\z/ =~ str 86 | $1.gsub(/\\([0abfnrtve]|x([0-9A-Fa-f][0-9A-Fa-f]))/) { 87 | if $2 88 | [$2].pack("H2") 89 | else 90 | case $1 91 | when "0"; "\0" 92 | when "a"; "\a" 93 | when "b"; "\b" 94 | when "f"; "\f" 95 | when "n"; "\n" 96 | when "r"; "\r" 97 | when "t"; "\t" 98 | when "v"; "\v" 99 | when "e"; "\e" 100 | else 101 | raise "[bug] unexpected escape" 102 | end 103 | end 104 | } 105 | else 106 | str 107 | end 108 | end 109 | 110 | def Tb.ltsv_split_line(line) 111 | line = line.chomp("\n") 112 | line = line.chomp("\r") 113 | ary = line.split(/\t/, -1) 114 | assoc = ary.map {|str| 115 | /:/ =~ str 116 | key = $` 117 | val = $' 118 | key = Tb.ltsv_unescape_string(key) 119 | val = Tb.ltsv_unescape_string(val) 120 | [key, val] 121 | } 122 | assoc 123 | end 124 | 125 | def Tb.ltsv_assoc_join(assoc) 126 | assoc.map {|key, val| 127 | Tb.ltsv_escape_key(key) + ':' + Tb.ltsv_escape_value(val) 128 | }.join("\t") 129 | end 130 | 131 | class LTSVReader < Tb::HashReader 132 | def initialize(io) 133 | super lambda { 134 | line = io.gets 135 | if line 136 | Hash[Tb.ltsv_split_line(line)] 137 | else 138 | nil 139 | end 140 | } 141 | end 142 | end 143 | 144 | class LTSVWriter < Tb::HashWriter 145 | def initialize(io) 146 | super lambda {|hash| 147 | io << (Tb.ltsv_assoc_join(hash) + "\n") 148 | } 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /tb.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'tb' 3 | s.version = '1.0' 4 | s.date = '2014-10-29' 5 | s.author = 'Tanaka Akira' 6 | s.email = 'akr@fsij.org' 7 | s.required_ruby_version = '>= 1.9.2' 8 | s.add_development_dependency 'test-unit', '~> 2.5.5' 9 | s.files = %w[ 10 | README 11 | bin/tb 12 | lib/tb.rb 13 | lib/tb/catreader.rb 14 | lib/tb/cmd_cat.rb 15 | lib/tb/cmd_consecutive.rb 16 | lib/tb/cmd_crop.rb 17 | lib/tb/cmd_cross.rb 18 | lib/tb/cmd_cut.rb 19 | lib/tb/cmd_git.rb 20 | lib/tb/cmd_group.rb 21 | lib/tb/cmd_gsub.rb 22 | lib/tb/cmd_help.rb 23 | lib/tb/cmd_join.rb 24 | lib/tb/cmd_ls.rb 25 | lib/tb/cmd_melt.rb 26 | lib/tb/cmd_mheader.rb 27 | lib/tb/cmd_nest.rb 28 | lib/tb/cmd_newfield.rb 29 | lib/tb/cmd_rename.rb 30 | lib/tb/cmd_search.rb 31 | lib/tb/cmd_shape.rb 32 | lib/tb/cmd_sort.rb 33 | lib/tb/cmd_svn.rb 34 | lib/tb/cmd_tar.rb 35 | lib/tb/cmd_to_csv.rb 36 | lib/tb/cmd_to_json.rb 37 | lib/tb/cmd_to_ltsv.rb 38 | lib/tb/cmd_to_pnm.rb 39 | lib/tb/cmd_to_pp.rb 40 | lib/tb/cmd_to_tsv.rb 41 | lib/tb/cmd_to_yaml.rb 42 | lib/tb/cmd_unmelt.rb 43 | lib/tb/cmd_unnest.rb 44 | lib/tb/cmdmain.rb 45 | lib/tb/cmdtop.rb 46 | lib/tb/cmdutil.rb 47 | lib/tb/csv.rb 48 | lib/tb/customcmp.rb 49 | lib/tb/customeq.rb 50 | lib/tb/enumerable.rb 51 | lib/tb/enumerator.rb 52 | lib/tb/ex_enumerable.rb 53 | lib/tb/ex_enumerator.rb 54 | lib/tb/fileenumerator.rb 55 | lib/tb/func.rb 56 | lib/tb/hashreader.rb 57 | lib/tb/hashwriter.rb 58 | lib/tb/headerreader.rb 59 | lib/tb/headerwriter.rb 60 | lib/tb/json.rb 61 | lib/tb/ltsv.rb 62 | lib/tb/ndjson.rb 63 | lib/tb/numericreader.rb 64 | lib/tb/numericwriter.rb 65 | lib/tb/pager.rb 66 | lib/tb/pnm.rb 67 | lib/tb/revcmp.rb 68 | lib/tb/ropen.rb 69 | lib/tb/search.rb 70 | lib/tb/tsv.rb 71 | lib/tb/zipper.rb 72 | sample/colors.ppm 73 | sample/excel2csv 74 | sample/gradation.pgm 75 | sample/langs.csv 76 | sample/poi-xls2csv.rb 77 | sample/poi-xls2csv.sh 78 | sample/tbplot 79 | tb.gemspec 80 | test-all-cov.rb 81 | test-all.rb 82 | ] 83 | s.test_files = %w[ 84 | test/test_catreader.rb 85 | test/test_cmd_cat.rb 86 | test/test_cmd_consecutive.rb 87 | test/test_cmd_crop.rb 88 | test/test_cmd_cross.rb 89 | test/test_cmd_cut.rb 90 | test/test_cmd_git_log.rb 91 | test/test_cmd_grep.rb 92 | test/test_cmd_group.rb 93 | test/test_cmd_gsub.rb 94 | test/test_cmd_help.rb 95 | test/test_cmd_join.rb 96 | test/test_cmd_ls.rb 97 | test/test_cmd_melt.rb 98 | test/test_cmd_mheader.rb 99 | test/test_cmd_nest.rb 100 | test/test_cmd_newfield.rb 101 | test/test_cmd_rename.rb 102 | test/test_cmd_shape.rb 103 | test/test_cmd_sort.rb 104 | test/test_cmd_svn_log.rb 105 | test/test_cmd_tar_tvf.rb 106 | test/test_cmd_to_csv.rb 107 | test/test_cmd_to_json.rb 108 | test/test_cmd_to_ltsv.rb 109 | test/test_cmd_to_pnm.rb 110 | test/test_cmd_to_pp.rb 111 | test/test_cmd_to_tsv.rb 112 | test/test_cmd_to_yaml.rb 113 | test/test_cmd_unmelt.rb 114 | test/test_cmd_unnest.rb 115 | test/test_cmdtty.rb 116 | test/test_cmdutil.rb 117 | test/test_csv.rb 118 | test/test_customcmp.rb 119 | test/test_customeq.rb 120 | test/test_ex_enumerable.rb 121 | test/test_fileenumerator.rb 122 | test/test_headercsv.rb 123 | test/test_json.rb 124 | test/test_ltsv.rb 125 | test/test_ndjson.rb 126 | test/test_numericcsv.rb 127 | test/test_pager.rb 128 | test/test_pnm.rb 129 | test/test_reader.rb 130 | test/test_revcmp.rb 131 | test/test_search.rb 132 | test/test_tbenum.rb 133 | test/test_tsv.rb 134 | test/test_zipper.rb 135 | test/util_tbtest.rb 136 | ] 137 | s.has_rdoc = true 138 | s.homepage = 'https://github.com/akr/tb' 139 | s.require_path = 'lib' 140 | s.executables << 'tb' 141 | s.license = 'BSD-3-Clause' 142 | s.summary = 'manipulation tool for tables: CSV, TSV, LTSV, JSON, NDJSON, etc.' 143 | s.description = <<'End' 144 | tb is a manipulation tool for tables. 145 | 146 | tb provides a command and a library for manipulating tables: 147 | Unix filter like operations (sort, cat, cut, ls, etc.), 148 | SQL like operations (join, group, etc.), 149 | other table operations (search, gsub, rename, cross, melt, unmelt, etc.), 150 | information extractions (git, svn, tar), 151 | and more. 152 | 153 | tb supports various table formats: CSV, TSV, JSON, NDJSON, LTSV, etc. 154 | End 155 | end 156 | --------------------------------------------------------------------------------