├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── eval.go ├── eval_test.go ├── go.mod ├── go.sum ├── lltsv.go ├── lltsv_test.go ├── main.go └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | pkg/* 2 | lltsv 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.7.0 (2020/11/12) 2 | 3 | Enhancements: 4 | 5 | * Use mattn-isatty instead of andrew-d/go-termutil #28 6 | * Introduce Go Modules #27 7 | * Use https instead of http in comment. #24 8 | * Add tests. #23 9 | 10 | ## 0.6.1 (2017/07/14) 11 | 12 | Fixes: 13 | 14 | * Show compiler version by -v. refs: #19 15 | * feature: introduced --ignore-key. refs #20 16 | 17 | ## 0.6.0 (2017/04/11) 18 | 19 | Fixes: 20 | 21 | * sort keys by default. refs: #16 22 | * build: fixed `go get` error for ghr. refs #15 23 | 24 | ## 0.5.6 (2017/03/21) 25 | 26 | Fixes: 27 | 28 | * feature: introduced not-equal operators for string. 29 | * expr: re-implemented by go package. 30 | 31 | ## 0.5.5 (2016/11/24) 32 | 33 | Fixes: 34 | 35 | * filter: exited when invalid filter expression was given (thanks to @cubicdaiya) 36 | 37 | ## 0.5.4 (2016/10/31) 38 | 39 | Fixes: 40 | 41 | * Typo fixed 42 | 43 | ## 0.5.3 (2016/10/31) 44 | 45 | Enhancements: 46 | 47 | * Introduce case-insentive comparing operators such as `==*`, `=~*`, `!~*` (thanks to @cubicdaiya) 48 | 49 | ## 0.5.2 (2016/10/28) 50 | 51 | Enhancements: 52 | 53 | * Enable perfect string match by '==' (thanks to @cubicdaiya) 54 | 55 | Changes: 56 | 57 | * Replace codegangsta/cli with urfave/cli (thanks to @b4b4r07) 58 | 59 | ## 0.5.1 (2016/08/25) 60 | 61 | Enhancements: 62 | 63 | * Fix typo 64 | 65 | ## 0.5.0 (2016/08/24) 66 | 67 | Enhancements: 68 | 69 | * Add new option -expr, -e (thanks to @cubicdaiya) 70 | 71 | ## 0.4.1 (2015/06/13) 72 | 73 | Enhancements: 74 | 75 | * filter: fixed panic error when the invalid filter expression is given. (thanks to @cubicdaiya) 76 | 77 | ## 0.4.0 (2015/10/20) 78 | 79 | Enhancements: 80 | 81 | * Add `filter` option (thanks to @hirose31) 82 | 83 | ## 0.3.1 (2014/11/21) 84 | 85 | Enhancements: 86 | 87 | * Add line feed to error messages 88 | 89 | ## 0.3.0 (2014/11/21) 90 | 91 | Enhancements: 92 | 93 | * Read from multiple files 94 | 95 | ## 0.2.0 (2014/08/13) 96 | 97 | Enhancements: 98 | 99 | * Read from a file (not only from STDIN) 100 | 101 | Changes: 102 | 103 | * Print all fields if -k option is not specified 104 | * Print fields in specified keys order 105 | 106 | ## 0.1.0 (2014/08/13) 107 | 108 | * Initial Release 109 | 110 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Naotoshi Seo 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DEBUG_FLAG = $(if $(DEBUG),-debug) 2 | 3 | build: 4 | GO111MODULE=on go build 5 | 6 | test: 7 | GO111MODULE=on go test -v ./... 8 | 9 | install: 10 | GO111MODULE=on go install 11 | 12 | fmt: 13 | GO111MODULE=on go fmt ./... 14 | 15 | lint: 16 | golint . 17 | 18 | pkg: 19 | go get github.com/mitchellh/gox/... 20 | go get github.com/tcnksm/ghr 21 | mkdir -p pkg && cd pkg && gox ../... 22 | 23 | clean: 24 | rm -f lltsv 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lltsv 2 | 3 | List specified keys of LTSV (Labeled Tab Separated Values) 4 | 5 | # Description 6 | 7 | `lltsv` is a command line tool written in golang to list specified keys of LTSV (Labeled Tab Separated Values) text. 8 | 9 | Example 1: 10 | 11 | ```bash 12 | $ echo "foo:aaa\tbar:bbb\tbaz:ccc" | lltsv -k foo,bar 13 | foo:aaa bar:bbb 14 | ``` 15 | 16 | The output is colorized as default when you outputs to a terminal. 17 | The coloring is disabled if you pipe or redirect outputs. 18 | 19 | Example 2: 20 | 21 | ```bash 22 | $ echo "foo:aaa\tbar:bbb\tbaz:ccc" | lltsv -k foo,bar -K 23 | aaa bbb 24 | ``` 25 | 26 | Eliminate labels with `-K` option. 27 | 28 | Example 3: 29 | 30 | ```bash 31 | $ lltsv -k foo,bar -K file*.log 32 | ``` 33 | 34 | Specify input files as arguments. 35 | 36 | Example4: 37 | 38 | ```bash 39 | $ lltsv -k resptime,status,uri -f 'resptime > 6' access_log 40 | $ lltsv -k resptime,status,uri -f 'resptime > 6' -f 'uri =~ ^/foo' access_log 41 | ``` 42 | 43 | Filter output with "-f" option. Available comparing operators are: 44 | 45 | ``` 46 | >= > == < <= (arithmetic (float64)) 47 | == ==* != !=* (string comparison (string)) 48 | =~ !~ =~* !~* (regular expression (string)) 49 | ``` 50 | 51 | The comparing operators terminated by __*__ behave in case-insensitive. 52 | 53 | You can specify multiple -f options (AND condition). 54 | 55 | Example5: 56 | 57 | ```bash 58 | $ lltsv -k resptime,upstream_resptime,diff -e 'diff = resptime - upstream_resptime' access_log 59 | $ lltsv -k resptime,upstream_resptime,diff_ms -e 'diff_ms = (resptime - upstream_resptime) * 1000' access_log 60 | ``` 61 | 62 | Evaluate value with "-e" option. Available operators are: 63 | 64 | ``` 65 | + - * / (arithmetic (float64)) 66 | ``` 67 | 68 | **How Useful?** 69 | 70 | LTSV format is not `awk` friendly (I think), but `lltsv` can help it: 71 | 72 | ```bash 73 | $ echo -e "time:2014-08-13T14:10:10Z\tstatus:200\ntime:2014-08-13T14:10:12Z\tstatus:500" \ 74 | | lltsv -k time,status -K | awk '$2 == 500' 75 | 2014-08-13T14:10:12Z 500 76 | ``` 77 | 78 | Useful! 79 | 80 | ## Installation 81 | 82 | Executable binaries are available at [releases](https://github.com/sonots/lltsv/releases). 83 | 84 | For example, for linux x86_64, 85 | 86 | ```bash 87 | $ wget https://github.com/sonots/lltsv/releases/download/v0.3.0/lltsv_linux_amd64 -O lltsv 88 | $ chmod a+x lltsv 89 | ``` 90 | 91 | If you have the go runtime installed, you may use go get. 92 | 93 | ```bash 94 | $ go get github.com/sonots/lltsv 95 | ``` 96 | 97 | ## Usage 98 | 99 | ``` 100 | $ lltsv -h 101 | NAME: 102 | lltsv - List specified keys of LTSV (Labeled Tab Separated Values) 103 | 104 | USAGE: 105 | lltsv [global options] command [command options] [arguments...] 106 | 107 | VERSION: 108 | 0.5.1 109 | 110 | AUTHOR(S): 111 | sonots 112 | 113 | COMMANDS: 114 | help, h Shows a list of commands or help for one command 115 | 116 | GLOBAL OPTIONS: 117 | --key, -k keys to output (multiple keys separated by ,) 118 | --no-key, -K output without keys (and without color) 119 | --ignore-key value, -i value ignored keys to output (multiple keys separated by ,) 120 | --filter, -f [--filter option --filter option] filter expression to output 121 | --expr, -e [--expr option --expr option] evaluate value by expression to output 122 | --help, -h show help 123 | --version, -v print the version 124 | ``` 125 | 126 | ## ToDo 127 | 128 | 1. write tests 129 | 130 | ## Build 131 | 132 | To build, use go get and make 133 | 134 | ``` 135 | $ go get -d github.com/sonots/lltsv 136 | $ cd $GOPATH/src/github.com/sonots/lltsv 137 | $ make 138 | ``` 139 | 140 | To release binaries, I use [gox](https://github.com/mitchellh/gox) and [ghr](https://github.com/tcnksm/ghr) 141 | 142 | ``` 143 | go get github.com/mitchellh/gox 144 | gox -build-toolchain # only first time 145 | go get github.com/tcnksm/ghr 146 | 147 | mkdir -p pkg && cd pkg && gox ../... 148 | ghr vX.X.X . 149 | ``` 150 | 151 | ## Contribution 152 | 153 | 1. Fork (https://github.com/sonots/lltsv/fork) 154 | 2. Create a feature branch 155 | 3. Commit your changes 156 | 4. Rebase your local changes against the master branch 157 | 5. Run test suite with the go test ./... command and confirm that it passes 158 | 6. Run gofmt -s 159 | 7. Create new Pull Request 160 | 161 | ## Copyright 162 | 163 | See [LICENSE](./LICENSE) 164 | 165 | ## Special Thanks 166 | 167 | This is a golang fork of perl version created by [id:key_amb](http://keyamb.hatenablog.com/). Thanks! 168 | 169 | MEMO: golang version was 5x faster than perl version 170 | -------------------------------------------------------------------------------- /eval.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "go/ast" 6 | "go/constant" 7 | "go/parser" 8 | "strconv" 9 | ) 10 | 11 | // Vars is a map for LTSV's label and value. 12 | type Vars map[string]string 13 | 14 | // ExprRunner is an expression runner for Go code. 15 | type ExprRunner struct { 16 | expr ast.Expr 17 | } 18 | 19 | // ExprContext is a context for ExprRunner. 20 | type ExprContext struct { 21 | vars Vars 22 | } 23 | 24 | func parseExpr(e string) (ast.Expr, error) { 25 | expr, err := parser.ParseExpr(e) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | return expr, nil 31 | } 32 | 33 | func evalExpr(expr ast.Expr, ctx *ExprContext) (constant.Value, error) { 34 | switch e := expr.(type) { 35 | case *ast.BasicLit: 36 | return evalBasicLit(e, ctx) 37 | case *ast.BinaryExpr: 38 | return evalBinaryExpr(e, ctx) 39 | case *ast.Ident: 40 | return evalIdent(e, ctx) 41 | case *ast.ParenExpr: 42 | return evalExpr(e.X, ctx) 43 | } 44 | 45 | return constant.MakeUnknown(), errors.New("unknown expr") 46 | } 47 | 48 | func evalBasicLit(expr *ast.BasicLit, ctx *ExprContext) (constant.Value, error) { 49 | return constant.MakeFromLiteral(expr.Value, expr.Kind, 0), nil 50 | } 51 | 52 | func evalBinaryExpr(expr *ast.BinaryExpr, ctx *ExprContext) (constant.Value, error) { 53 | x, err := evalExpr(expr.X, ctx) 54 | if err != nil { 55 | return constant.MakeUnknown(), err 56 | } 57 | 58 | y, err := evalExpr(expr.Y, ctx) 59 | if err != nil { 60 | return constant.MakeUnknown(), err 61 | } 62 | 63 | return constant.BinaryOp(x, expr.Op, y), nil 64 | } 65 | 66 | func evalIdent(expr *ast.Ident, ctx *ExprContext) (constant.Value, error) { 67 | name, ok := ctx.vars[expr.Name] 68 | if !ok { 69 | return constant.MakeUnknown(), errors.New("unknown variable name") 70 | } 71 | 72 | n, err := strconv.ParseFloat(name, 64) 73 | if err != nil { 74 | return constant.MakeUnknown(), errors.New("variable type must be numeric") 75 | } 76 | 77 | return constant.MakeFloat64(n), nil 78 | } 79 | -------------------------------------------------------------------------------- /eval_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEval(t *testing.T) { 10 | assert := assert.New(t) 11 | 12 | tests := []struct { 13 | expression string 14 | vars Vars 15 | result string 16 | }{ 17 | { // no space 18 | expression: "1+1", 19 | vars: nil, 20 | result: "2", 21 | }, 22 | { 23 | expression: "1 + 1", 24 | vars: nil, 25 | result: "2", 26 | }, 27 | { 28 | expression: "resptime - upstream_resptime", 29 | vars: Vars{"resptime": "5", "upstream_resptime": "3"}, 30 | result: "2", 31 | }, 32 | { 33 | expression: "(resptime - upstream_resptime) * 1000", 34 | vars: Vars{"resptime": "5", "upstream_resptime": "3"}, 35 | result: "2000", 36 | }, 37 | } 38 | 39 | for _, test := range tests { 40 | expr, err := parseExpr(test.expression) 41 | assert.Nil(err) 42 | 43 | ctx := &ExprContext{ 44 | vars: test.vars, 45 | } 46 | v, err := evalExpr(expr, ctx) 47 | assert.Nil(err) 48 | assert.Equal(test.result, v.String()) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sonots/lltsv 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/mattn/go-colorable v0.1.4 // indirect 7 | github.com/mattn/go-isatty v0.0.12 8 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b 9 | github.com/stretchr/testify v1.4.0 10 | github.com/urfave/cli v1.22.2 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= 7 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 8 | github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= 9 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 10 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 11 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 12 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= 13 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 17 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 18 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 19 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 22 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 23 | github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= 24 | github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 25 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= 26 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 27 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= 28 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 31 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 32 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 33 | -------------------------------------------------------------------------------- /lltsv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "os" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/mattn/go-isatty" 12 | "github.com/mgutz/ansi" 13 | ) 14 | 15 | type tFuncAppend func([]string, string, string) []string 16 | type tFuncFilter func(string) bool 17 | 18 | // Lltsv is a context for processing LTSV. 19 | type Lltsv struct { 20 | keys []string 21 | ignoreKeyMap map[string]struct{} 22 | noKey bool 23 | filters []string 24 | exprs []string 25 | funcAppend tFuncAppend 26 | funcFilters map[string]tFuncFilter 27 | exprRunners map[string]*ExprRunner 28 | } 29 | 30 | func newLltsv(keys []string, ignoreKeys []string, noKey bool, filters []string, exprs []string) *Lltsv { 31 | ignoreKeyMap := make(map[string]struct{}) 32 | for _, key := range ignoreKeys { 33 | ignoreKeyMap[key] = struct{}{} 34 | } 35 | return &Lltsv{ 36 | keys: keys, 37 | ignoreKeyMap: ignoreKeyMap, 38 | noKey: noKey, 39 | filters: filters, 40 | exprs: exprs, 41 | funcAppend: getFuncAppend(noKey), 42 | funcFilters: getFuncFilters(filters), 43 | exprRunners: getExprRunners(exprs), 44 | } 45 | } 46 | 47 | func (lltsv *Lltsv) scanAndWrite(file *os.File) error { 48 | scanner := bufio.NewScanner(file) 49 | for scanner.Scan() { 50 | line := scanner.Text() 51 | lvs, labels := lltsv.parseLtsv(line) 52 | lltsv.expr(lvs) 53 | 54 | if lltsv.filter(lvs) { 55 | ltsv := lltsv.restructLtsv(lvs, labels) 56 | os.Stdout.WriteString(ltsv + "\n") 57 | } 58 | 59 | } 60 | return scanner.Err() 61 | } 62 | 63 | func (lltsv *Lltsv) filter(lvs map[string]string) bool { 64 | shouldOutput := true 65 | 66 | for key, funcFilter := range lltsv.funcFilters { 67 | if !funcFilter(lvs[key]) { 68 | shouldOutput = false 69 | break 70 | } 71 | } 72 | 73 | return shouldOutput 74 | } 75 | 76 | func (lltsv *Lltsv) expr(lvs map[string]string) { 77 | ctx := &ExprContext{ 78 | vars: lvs, 79 | } 80 | for key, runner := range lltsv.exprRunners { 81 | v, err := evalExpr(runner.expr, ctx) 82 | if err == nil { 83 | lvs[key] = v.String() 84 | } 85 | } 86 | } 87 | 88 | // lvs: label and value pairs 89 | func (lltsv *Lltsv) restructLtsv(lvs map[string]string, labels []string) string { 90 | // Sort in the order of -k, or follow the order of the input file. 91 | orders := lltsv.keys 92 | if len(lltsv.keys) == 0 { 93 | orders = labels 94 | } 95 | // Make slice with enough capacity so that append does not newly create object 96 | // cf. https://golang.org/pkg/builtin/#append 97 | selected := make([]string, 0, len(orders)) 98 | for _, label := range orders { 99 | if _, ok := lltsv.ignoreKeyMap[label]; ok { 100 | continue 101 | } 102 | value := lvs[label] 103 | selected = lltsv.funcAppend(selected, label, value) 104 | } 105 | return strings.Join(selected, "\t") 106 | } 107 | 108 | func (lltsv *Lltsv) parseLtsv(line string) (map[string]string, []string) { 109 | columns := strings.Split(line, "\t") 110 | lvs := make(map[string]string) 111 | labels := make([]string, 0, len(columns)) 112 | for _, column := range columns { 113 | lv := strings.SplitN(column, ":", 2) 114 | if len(lv) < 2 { 115 | continue 116 | } 117 | label, value := lv[0], lv[1] 118 | lvs[label] = value 119 | labels = append(labels, label) 120 | } 121 | return lvs, labels 122 | } 123 | 124 | // Return function pointer to avoid `if` evaluation occurs in each iteration 125 | func getFuncAppend(noKey bool) tFuncAppend { 126 | if noKey { 127 | return func(selected []string, label string, value string) []string { 128 | return append(selected, value) 129 | } 130 | } 131 | 132 | if isatty.IsTerminal(os.Stdout.Fd()) { 133 | return func(selected []string, label string, value string) []string { 134 | return append(selected, ansi.Color(label, "green")+":"+ansi.Color(value, "magenta")) 135 | } 136 | } 137 | 138 | // if pipe or redirect 139 | return func(selected []string, label string, value string) []string { 140 | return append(selected, label+":"+value) 141 | } 142 | } 143 | 144 | func getFuncFilters(filters []string) map[string]tFuncFilter { 145 | funcFilters := map[string]tFuncFilter{} 146 | for _, f := range filters { 147 | token := strings.SplitN(f, " ", 3) 148 | if len(token) < 3 { 149 | log.Fatalf("filter expression is invalid: %s\n", f) 150 | } 151 | key := token[0] 152 | switch token[1] { 153 | case ">", ">=", "<=", "<": 154 | r, err := strconv.ParseFloat(token[2], 64) 155 | if err != nil { 156 | log.Fatal(err) 157 | } 158 | 159 | funcFilters[key] = func(val string) bool { 160 | num, err := strconv.ParseFloat(val, 64) 161 | if err != nil { 162 | log.Println(err) 163 | return false 164 | } 165 | switch token[1] { 166 | case ">": 167 | return num > r 168 | case ">=": 169 | return num >= r 170 | case "<=": 171 | return num <= r 172 | case "<": 173 | return num < r 174 | default: 175 | return false 176 | } 177 | } 178 | case "==": 179 | funcFilters[key] = func(val string) bool { 180 | return val == token[2] 181 | } 182 | case "==*": 183 | funcFilters[key] = func(val string) bool { 184 | return strings.ToLower(val) == strings.ToLower(token[2]) 185 | } 186 | case "!=": 187 | funcFilters[key] = func(val string) bool { 188 | return val != token[2] 189 | } 190 | case "!=*": 191 | funcFilters[key] = func(val string) bool { 192 | return strings.ToLower(val) != strings.ToLower(token[2]) 193 | } 194 | case "=~", "!~", "=~*", "!~*": 195 | if token[1] == "=~*" || token[1] == "!~*" { 196 | token[2] = strings.ToLower(token[2]) 197 | } 198 | re := regexp.MustCompile(token[2]) 199 | funcFilters[key] = func(val string) bool { 200 | switch token[1] { 201 | case "=~": 202 | return re.MatchString(val) 203 | case "!~": 204 | return !re.MatchString(val) 205 | case "=~*": 206 | return re.MatchString(strings.ToLower(val)) 207 | case "!~*": 208 | return !re.MatchString(strings.ToLower(val)) 209 | default: 210 | return false 211 | } 212 | } 213 | } 214 | } 215 | return funcFilters 216 | } 217 | 218 | func getExprRunners(exprs []string) map[string]*ExprRunner { 219 | funcExprs := make(map[string]*ExprRunner, len(exprs)) 220 | for _, f := range exprs { 221 | token := strings.SplitN(f, "=", 2) 222 | if len(token) != 2 { 223 | log.Printf("expression is invalid: %s\n", f) 224 | continue 225 | } 226 | 227 | expr, err := parseExpr(token[1]) 228 | if err != nil { 229 | log.Printf("expression is invalid: %s\n", f) 230 | continue 231 | } 232 | 233 | key := strings.Trim(token[0], " ") 234 | 235 | funcExprs[key] = &ExprRunner{ 236 | expr: expr, 237 | } 238 | } 239 | return funcExprs 240 | } 241 | -------------------------------------------------------------------------------- /lltsv_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestFilter(t *testing.T) { 10 | assert := assert.New(t) 11 | 12 | tests := []struct { 13 | filter string 14 | lvs map[string]string 15 | result bool 16 | }{ 17 | // arithmetic comparison 18 | { 19 | filter: "resptime > 6", 20 | lvs: map[string]string{"resptime": "10"}, 21 | result: true, 22 | }, 23 | { 24 | filter: "resptime >= 6", 25 | lvs: map[string]string{"resptime": "6"}, 26 | result: true, 27 | }, 28 | { 29 | filter: "resptime == 60", 30 | lvs: map[string]string{"resptime": "60"}, 31 | result: true, 32 | }, 33 | { 34 | filter: "resptime < 6", 35 | lvs: map[string]string{"resptime": "7"}, 36 | result: false, 37 | }, 38 | { 39 | filter: "resptime <= 6", 40 | lvs: map[string]string{"resptime": "7"}, 41 | result: false, 42 | }, 43 | // string comparison 44 | { 45 | filter: "uri == /top", 46 | lvs: map[string]string{"uri": "/top"}, 47 | result: true, 48 | }, 49 | { 50 | filter: "uri ==* /TOP", 51 | lvs: map[string]string{"uri": "/top"}, 52 | result: true, 53 | }, 54 | { 55 | filter: "uri != /top", 56 | lvs: map[string]string{"uri": "/bottom"}, 57 | result: true, 58 | }, 59 | { 60 | filter: "uri !=* /top", 61 | lvs: map[string]string{"uri": "/TOP"}, 62 | result: false, 63 | }, 64 | // regular expression 65 | { 66 | filter: "uri =~ ^/", 67 | lvs: map[string]string{"uri": "/top"}, 68 | result: true, 69 | }, 70 | { 71 | filter: "uri !~ ^/", 72 | lvs: map[string]string{"uri": "/top"}, 73 | result: false, 74 | }, 75 | { 76 | filter: "uri =~* ^/", 77 | lvs: map[string]string{"uri": "/TOP"}, 78 | result: true, 79 | }, 80 | { 81 | filter: "uri !~* /top", 82 | lvs: map[string]string{"uri": "/TOP"}, 83 | result: false, 84 | }, 85 | } 86 | 87 | for _, test := range tests { 88 | filters := getFuncFilters([]string{test.filter}) 89 | for k, filter := range filters { 90 | assert.Equal(test.result, filter(test.lvs[k])) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "strings" 8 | 9 | "github.com/urfave/cli" 10 | ) 11 | 12 | // os.Exit forcely kills process, so let me share this global variable to terminate at the last 13 | var exitCode = 0 14 | 15 | func main() { 16 | app := cli.NewApp() 17 | app.Name = "lltsv" 18 | app.Version = Version 19 | app.Usage = `List specified keys of LTSV (Labeled Tab Separated Values) 20 | 21 | Example1 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar 22 | foo:aaa bar:bbb 23 | 24 | The output is colorized as default when you outputs to a terminal. 25 | The coloring is disabled if you pipe or redirect outputs. 26 | 27 | Example2 $ echo "foo:aaa\tbar:bbb" | lltsv -k foo,bar -K 28 | aaa bbb 29 | 30 | Eliminate labels with "-K" option. 31 | 32 | Example3 $ lltsv -k foo,bar -K file*.log 33 | 34 | Specify input files as arguments. 35 | 36 | Example4 $ lltsv -k resptime,status,uri -f 'resptime > 6' access_log 37 | $ lltsv -k resptime,status,uri -f 'resptime > 6' -f 'uri =~ ^/foo' access_log 38 | 39 | Filter output with "-f" option. Available comparing operators are: 40 | 41 | >= > == < <= (arithmetic (float64)) 42 | == ==* != !=* (string comparison (string)) 43 | =~ !~ =~* !~* (regular expression (string)) 44 | 45 | The comparing operators terminated by * behave in case-insensitive. 46 | 47 | You can specify multiple -f options (AND condition). 48 | 49 | Example5 $ lltsv -k resptime,upstream_resptime,diff -f 'diff = resptime - upstream_resptime' access_log 50 | $ lltsv -k resptime,upstream_resptime,diff_ms -e 'diff_ms = (resptime - upstream_resptime) * 1000' access_log 51 | 52 | Evaluate value with "-e" option. Available operators are: 53 | 54 | + - * / (arithmetic (float64)) 55 | 56 | Homepage: https://github.com/sonots/lltsv` 57 | app.Author = "sonots" 58 | app.Email = "sonots@gmail.com" 59 | app.Flags = []cli.Flag{ 60 | cli.StringFlag{ 61 | Name: "key, k", 62 | Usage: "keys to output (multiple keys separated by ,)", 63 | }, 64 | cli.BoolFlag{ 65 | Name: "no-key, K", 66 | Usage: "output without keys (and without color)", 67 | }, 68 | cli.StringFlag{ 69 | Name: "ignore-key, i", 70 | Usage: "ignored keys to output (multiple keys separated by ,)", 71 | }, 72 | cli.StringSliceFlag{ 73 | Name: "filter, f", 74 | Usage: "filter expression to output", 75 | }, 76 | cli.StringSliceFlag{ 77 | Name: "expr, e", 78 | Usage: "evaluate value by expression to output", 79 | }, 80 | } 81 | app.Action = doMain 82 | 83 | cli.VersionPrinter = func(c *cli.Context) { 84 | fmt.Fprintf(app.Writer, `%v version %v 85 | Compiler: %s %s 86 | `, 87 | app.Name, 88 | app.Version, 89 | runtime.Compiler, 90 | runtime.Version()) 91 | } 92 | 93 | app.Run(os.Args) 94 | os.Exit(exitCode) 95 | } 96 | 97 | func doMain(c *cli.Context) error { 98 | keys := make([]string, 0, 0) // slice with length 0 99 | if c.String("key") != "" { 100 | keys = strings.Split(c.String("key"), ",") 101 | } 102 | noKey := c.Bool("no-key") 103 | filters := c.StringSlice("filter") 104 | exprs := c.StringSlice("expr") 105 | 106 | ignoreKeys := make([]string, 0, 0) 107 | // If -k,--key is specified, -i,--ignore-key is ignored. 108 | if len(keys) == 0 && c.String("ignore-key") != "" { 109 | ignoreKeys = strings.Split(c.String("ignore-key"), ",") 110 | } 111 | 112 | lltsv := newLltsv(keys, ignoreKeys, noKey, filters, exprs) 113 | 114 | if len(c.Args()) > 0 { 115 | for _, filename := range c.Args() { 116 | file, err := os.Open(filename) 117 | if err != nil { 118 | os.Stderr.WriteString("failed to open and read `" + filename + "`.\n") 119 | exitCode = 1 120 | return err 121 | } 122 | err = lltsv.scanAndWrite(file) 123 | file.Close() 124 | if err != nil { 125 | os.Stderr.WriteString("failed to process `" + filename + "`.\n") 126 | exitCode = 1 127 | return err 128 | } 129 | } 130 | } else { 131 | file := os.Stdin 132 | err := lltsv.scanAndWrite(file) 133 | file.Close() 134 | if err != nil { 135 | os.Stderr.WriteString("failed to process stdin.\n") 136 | exitCode = 1 137 | return err 138 | } 139 | } 140 | 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // Version is lltsv version string 4 | const Version string = "0.7.0" 5 | --------------------------------------------------------------------------------