├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── compiledgrok.go ├── example └── main.go ├── glide.lock ├── glide.yaml ├── grok.go ├── grok_test.go ├── grokpattern.go ├── patternmap.go ├── patterns.go ├── patterns ├── aws.go ├── bacula.go ├── bro.go ├── exim.go ├── firewalls.go ├── grok.go ├── haproxy.go ├── java.go ├── junos.go ├── linux-syslog.go ├── mcollective.go ├── mongodb.go ├── nagios.go ├── postgresql.go ├── rails.go ├── redis.go └── ruby.go └── vendor └── github.com └── trivago └── tgo ├── LICENSE ├── errorstack.go ├── metric.go ├── metricserver.go ├── runtime.go ├── tcontainer ├── arrays.go ├── bytepool.go ├── marshalmap.go └── trie.go ├── terrors └── simpleerror.go ├── tflag └── flag.go ├── tfmt ├── bgcolor.go ├── color.go └── cursor.go ├── thealthcheck └── healthcheckserver.go ├── tio ├── bufferedreader.go ├── bytestream.go ├── bytewriter.go ├── files.go ├── syscall.go └── syscall_windows.go ├── tlog ├── log.go ├── logcache.go ├── loglogger.go ├── lognull.go └── logreferrer.go ├── tmath ├── bits.go └── math.go ├── tnet ├── network.go ├── stoplistener.go └── thttp │ └── sockettransport.go ├── tos ├── exit.go ├── file.go ├── file_unix.go ├── file_unsupported.go ├── ids_darwin.go ├── ids_freebsd.go ├── ids_linux.go ├── ids_windows.go ├── pidfile.go └── user.go ├── treflect ├── reflection.go └── typeregistry.go ├── tstrings ├── parser.go └── strings.go ├── tsync ├── errors.go ├── fuse.go ├── mutex.go ├── queue.go ├── run.go ├── spinner.go ├── stack.go └── waitgroup.go └── ttesting └── expect.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | *.cov 26 | .coveralls.yml 27 | # added by GitSavvy 28 | .DS_Store 29 | 30 | # vendor 31 | vendor/**/* 32 | !vendor/**/ 33 | !vendor/**/*.go 34 | !vendor/**/*.s 35 | !vendor/**/LICENSE 36 | vendor/**/*_test.go 37 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | install: 3 | - go get -v golang.org/x/tools/cmd/cover 4 | - go get github.com/mattn/goveralls 5 | - go get github.com/golang/lint/golint 6 | script: 7 | - go test -v -tags=testcgo -covermode=count -coverprofile=profile.cov 8 | - golint -set_exit_status `go list ./... | grep -v vendor` 9 | after_success: 10 | - export PATH=$PATH:$HOME/gopath/bin 11 | - goveralls -coverprofile=profile.cov -service=travis-ci -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grok 2 | 3 | [![GoDoc](https://godoc.org/github.com/trivago/grok?status.svg)](https://godoc.org/github.com/trivago/grok) 4 | [![Build Status](https://travis-ci.org/trivago/grok.svg)](https://travis-ci.org/trivago/grok) 5 | [![Coverage Status](https://coveralls.io/repos/github/trivago/grok/badge.svg?branch=master)](https://coveralls.io/github/trivago/grok?branch=master) 6 | [![Go Report Card](http://goreportcard.com/badge/trivago/grok)](http:/goreportcard.com/report/trivago/grok) 7 | 8 | This is a fork of [github.com/vjeantet/grok](https://github.com/vjeantet/grok) with improved concurrency. 9 | This fork is not 100% API compatible but the underlying implementation is (mostly) the same. 10 | 11 | The main intention of this fork is to get rid of all the mutexes in this library to make it scale properly when using multiple go routines. Also as grok is an extension of the regexp package the function scheme of this library should be closer to golang's regexp package. 12 | 13 | ## Changes 14 | 15 | - All patterns have to be known at creation time 16 | - No storage of known grok expressions (has to be done be the user, similar to the go regexp package) 17 | - No Mutexes used anymore (this library now scales as it should) 18 | - No Graphsort required anymore to resolve dependencies 19 | - All known patterns text files have been converted to go maps 20 | - Structured code to make it easier to maintain 21 | - Added tgo.ttesting dependencies for easier to write unittests 22 | - Fixed type hint case sensitivity and added string type 23 | - Added []byte based functions 24 | 25 | ## Benchmarks 26 | 27 | Original version 28 | 29 | ```text 30 | BenchmarkNew-8 2000 899731 ns/op 720324 B/op 3438 allocs/op 31 | BenchmarkCaptures-8 10000 200695 ns/op 4570 B/op 5 allocs/op 32 | BenchmarkCapturesTypedFake-8 10000 197983 ns/op 4571 B/op 5 allocs/op 33 | BenchmarkCapturesTypedReal-8 10000 206392 ns/op 4754 B/op 16 allocs/op 34 | BenchmarkParallelCaptures-8 10000 208389 ns/op 4570 B/op 5 allocs/op (added locally) 35 | ``` 36 | 37 | This version 38 | 39 | ```text 40 | BenchmarkNew-8 5000 357586 ns/op 285374 B/op 1611 allocs/op 41 | BenchmarkCaptures-8 10000 200825 ns/op 4570 B/op 5 allocs/op 42 | BenchmarkCapturesTypedFake-8 10000 197306 ns/op 4570 B/op 5 allocs/op 43 | BenchmarkCapturesTypedReal-8 10000 194882 ns/op 4140 B/op 12 allocs/op 44 | BenchmarkParallelCaptures-8 30000 55583 ns/op 4576 B/op 5 allocs/op 45 | ``` 46 | 47 | Improvements 48 | 49 | ```text 50 | BenchmarkNew-8 +150% 51 | BenchmarkParallelCaptures-8 +274% 52 | ``` 53 | -------------------------------------------------------------------------------- /compiledgrok.go: -------------------------------------------------------------------------------- 1 | package grok 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | ) 8 | 9 | // CompiledGrok represents a compiled Grok expression. 10 | // Use Grok.Compile to generate a CompiledGrok object. 11 | type CompiledGrok struct { 12 | regexp *regexp.Regexp 13 | typeHints typeHintByKey 14 | removeEmpty bool 15 | } 16 | 17 | type typeHintByKey map[string]string 18 | 19 | // GetFields returns a list of all named fields in this grok expression 20 | func (compiled CompiledGrok) GetFields() []string { 21 | return compiled.regexp.SubexpNames() 22 | } 23 | 24 | // Match returns true if the given data matches the pattern. 25 | func (compiled CompiledGrok) Match(data []byte) bool { 26 | return compiled.regexp.Match(data) 27 | } 28 | 29 | // MatchString returns true if the given text matches the pattern. 30 | func (compiled CompiledGrok) MatchString(text string) bool { 31 | return compiled.regexp.MatchString(text) 32 | } 33 | 34 | // Parse processes the given data and returns a map containing the values of 35 | // all named fields as byte arrays. If a field is parsed more than once, the 36 | // last match is return. 37 | func (compiled CompiledGrok) Parse(data []byte) map[string][]byte { 38 | captures := make(map[string][]byte) 39 | 40 | if matches := compiled.regexp.FindSubmatch(data); len(matches) > 0 { 41 | for idx, key := range compiled.GetFields() { 42 | match := matches[idx] 43 | if compiled.omitField(key, match) { 44 | continue 45 | } 46 | captures[key] = match 47 | } 48 | } 49 | 50 | return captures 51 | } 52 | 53 | // ParseString processes the given text and returns a map containing the values of 54 | // all named fields as strings. If a field is parsed more than once, the 55 | // last match is return. 56 | func (compiled CompiledGrok) ParseString(text string) map[string]string { 57 | captures := make(map[string]string) 58 | 59 | if matches := compiled.regexp.FindStringSubmatch(text); len(matches) > 0 { 60 | for idx, key := range compiled.GetFields() { 61 | match := matches[idx] 62 | if compiled.omitStringField(key, match) { 63 | continue 64 | } 65 | captures[key] = match 66 | } 67 | } 68 | 69 | return captures 70 | } 71 | 72 | // ParseTyped processes the given data and returns a map containing the values 73 | // of all named fields converted to their corresponding types. If no typehint is 74 | // given, the value will be converted to string. 75 | func (compiled CompiledGrok) ParseTyped(data []byte) (map[string]interface{}, error) { 76 | captures := make(map[string]interface{}) 77 | 78 | if matches := compiled.regexp.FindSubmatch(data); len(matches) > 0 { 79 | for idx, key := range compiled.GetFields() { 80 | match := matches[idx] 81 | if compiled.omitField(key, match) { 82 | continue 83 | } 84 | 85 | if val, err := compiled.typeCast(string(match), key); err == nil { 86 | captures[key] = val 87 | } else { 88 | return nil, err 89 | } 90 | } 91 | } 92 | 93 | return captures, nil 94 | } 95 | 96 | // ParseStringTyped processes the given data and returns a map containing the 97 | // values of all named fields converted to their corresponding types. If no 98 | // typehint is given, the value will be converted to string. 99 | func (compiled CompiledGrok) ParseStringTyped(text string) (map[string]interface{}, error) { 100 | captures := make(map[string]interface{}) 101 | 102 | if matches := compiled.regexp.FindStringSubmatch(text); len(matches) > 0 { 103 | for idx, key := range compiled.GetFields() { 104 | match := matches[idx] 105 | if compiled.omitStringField(key, match) { 106 | continue 107 | } 108 | 109 | if val, err := compiled.typeCast(match, key); err == nil { 110 | captures[key] = val 111 | } else { 112 | return nil, err 113 | } 114 | } 115 | } 116 | 117 | return captures, nil 118 | } 119 | 120 | // ParseToMultiMap acts like Parse but allows multiple matches per field. 121 | func (compiled CompiledGrok) ParseToMultiMap(data []byte) map[string][][]byte { 122 | captures := make(map[string][][]byte) 123 | 124 | if matches := compiled.regexp.FindSubmatch(data); len(matches) > 0 { 125 | for idx, key := range compiled.GetFields() { 126 | match := matches[idx] 127 | if compiled.omitField(key, match) { 128 | continue 129 | } 130 | 131 | if values, exists := captures[key]; exists { 132 | captures[key] = append(values, match) 133 | } else { 134 | captures[key] = [][]byte{match} 135 | } 136 | } 137 | } 138 | 139 | return captures 140 | } 141 | 142 | // ParseStringToMultiMap acts like ParseString but allows multiple matches per 143 | // field. 144 | func (compiled CompiledGrok) ParseStringToMultiMap(text string) map[string][]string { 145 | captures := make(map[string][]string) 146 | 147 | if matches := compiled.regexp.FindStringSubmatch(text); len(matches) > 0 { 148 | for idx, key := range compiled.GetFields() { 149 | match := matches[idx] 150 | if compiled.omitStringField(key, match) { 151 | continue 152 | } 153 | 154 | if values, exists := captures[key]; exists { 155 | captures[key] = append(values, match) 156 | } else { 157 | captures[key] = []string{match} 158 | } 159 | } 160 | } 161 | 162 | return captures 163 | } 164 | 165 | // omitField return true if the field is to be omitted 166 | func (compiled CompiledGrok) omitField(key string, match []byte) bool { 167 | return len(key) == 0 || compiled.removeEmpty && len(match) == 0 168 | } 169 | 170 | // omitStringField return true if the field is to be omitted 171 | func (compiled CompiledGrok) omitStringField(key, match string) bool { 172 | return len(key) == 0 || compiled.removeEmpty && len(match) == 0 173 | } 174 | 175 | // typeCast casts a field based on a typehint 176 | func (compiled CompiledGrok) typeCast(match, key string) (interface{}, error) { 177 | typeName, hasTypeHint := compiled.typeHints[key] 178 | if !hasTypeHint { 179 | return match, nil 180 | } 181 | 182 | switch typeName { 183 | case "int": 184 | return strconv.Atoi(match) 185 | 186 | case "float": 187 | return strconv.ParseFloat(match, 64) 188 | 189 | case "string": 190 | return match, nil 191 | 192 | default: 193 | return nil, fmt.Errorf("ERROR the value %s cannot be converted to %s. Must be int, float, string or empty", match, typeName) 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/trivago/grok" 7 | ) 8 | 9 | func main() { 10 | fmt.Println("# Default Capture :") 11 | g, _ := grok.New(grok.Config{}) 12 | values, _ := g.ParseString("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) 13 | for k, v := range values { 14 | fmt.Printf("%+15s: %s\n", k, v) 15 | } 16 | 17 | fmt.Println("\n# Named Capture :") 18 | g, _ = grok.New(grok.Config{NamedCapturesOnly: true}) 19 | values, _ = g.ParseString("%{COMMONAPACHELOG}", `127.0.0.1 - - [23/Apr/2014:22:58:32 +0200] "GET /index.php HTTP/1.1" 404 207`) 20 | for k, v := range values { 21 | fmt.Printf("%+15s: %s\n", k, v) 22 | } 23 | 24 | fmt.Println("\n# Add custom patterns :") 25 | // We add 3 patterns to our Grok instance, to structure an IRC message 26 | g, _ = grok.New(grok.Config{ 27 | NamedCapturesOnly: true, 28 | Patterns: map[string]string{ 29 | "IRCUSER": `\A@(\w+)`, 30 | "IRCBODY": `.*`, 31 | "IRCMSG": `%{IRCUSER:user} .* : %{IRCBODY:message}`, 32 | }, 33 | }) 34 | values, _ = g.ParseString("%{IRCMSG}", `@vjeantet said : Hello !`) 35 | for k, v := range values { 36 | fmt.Printf("%+15s: %s\n", k, v) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: f47b9345c96159fb80780b5af5afb3a21deba466456810eede26855f33cbf4c1 2 | updated: 2017-07-10T22:30:23.041269589+02:00 3 | imports: [] 4 | testImports: 5 | - name: github.com/trivago/tgo 6 | version: bdfa63d0265af496c5bc30356dc31b19aa71447e 7 | subpackages: 8 | - ttesting 9 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/trivago/grok 2 | import: [] 3 | testImport: 4 | - package: github.com/trivago/tgo 5 | subpackages: 6 | - ttesting 7 | -------------------------------------------------------------------------------- /grok.go: -------------------------------------------------------------------------------- 1 | package grok 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | // Config is used to pass a set of configuration values to the grok.New function. 8 | type Config struct { 9 | NamedCapturesOnly bool 10 | SkipDefaultPatterns bool 11 | RemoveEmptyValues bool 12 | Patterns map[string]string 13 | } 14 | 15 | // Grok holds a cache of known pattern substitions and acts as a builder for 16 | // compiled grok patterns. All pattern substitutions must be passed at creation 17 | // time and cannot be changed during runtime. 18 | type Grok struct { 19 | patterns patternMap 20 | removeEmpty bool 21 | namedOnly bool 22 | } 23 | 24 | // New returns a Grok object that caches a given set of patterns and creates 25 | // compiled grok patterns based on the passed configuration settings. 26 | // You can use multiple grok objects that act independently. 27 | func New(config Config) (*Grok, error) { 28 | patterns := patternMap{} 29 | 30 | if !config.SkipDefaultPatterns { 31 | // Add default patterns first so they can be referenced later 32 | if err := patterns.addList(DefaultPatterns, config.NamedCapturesOnly); err != nil { 33 | return nil, err 34 | } 35 | } 36 | 37 | // Add passed patterns 38 | if err := patterns.addList(config.Patterns, config.NamedCapturesOnly); err != nil { 39 | return nil, err 40 | } 41 | 42 | return &Grok{ 43 | patterns: patterns, 44 | namedOnly: config.NamedCapturesOnly, 45 | removeEmpty: config.RemoveEmptyValues, 46 | }, nil 47 | } 48 | 49 | // Compile precompiles a given grok expression. This function should be used 50 | // when a grok expression is used more than once. 51 | func (grok Grok) Compile(pattern string) (*CompiledGrok, error) { 52 | grokPattern, err := newPattern(pattern, grok.patterns, grok.namedOnly) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | compiled, err := regexp.Compile(grokPattern.expression) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return &CompiledGrok{ 63 | regexp: compiled, 64 | typeHints: grokPattern.typeHints, 65 | removeEmpty: grok.removeEmpty, 66 | }, nil 67 | } 68 | 69 | // Match returns true if the given data matches the pattern. 70 | // The given pattern is compiled on every call to this function. 71 | // If you want to call this function more than once consider using Compile. 72 | func (grok Grok) Match(pattern string, data []byte) (bool, error) { 73 | complied, err := grok.Compile(pattern) 74 | if err != nil { 75 | return false, err 76 | } 77 | 78 | return complied.Match(data), nil 79 | } 80 | 81 | // MatchString returns true if the given text matches the pattern. 82 | // The given pattern is compiled on every call to this function. 83 | // If you want to call this function more than once consider using Compile. 84 | func (grok Grok) MatchString(pattern, text string) (bool, error) { 85 | complied, err := grok.Compile(pattern) 86 | if err != nil { 87 | return false, err 88 | } 89 | 90 | return complied.MatchString(text), nil 91 | } 92 | 93 | // Parse processes the given data and returns a map containing the values of 94 | // all named fields as byte arrays. If a field is parsed more than once, the 95 | // last match is return. 96 | // The given pattern is compiled on every call to this function. 97 | // If you want to call this function more than once consider using Compile. 98 | func (grok Grok) Parse(pattern string, data []byte) (map[string][]byte, error) { 99 | complied, err := grok.Compile(pattern) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | return complied.Parse(data), nil 105 | } 106 | 107 | // ParseString processes the given text and returns a map containing the values of 108 | // all named fields as strings. If a field is parsed more than once, the 109 | // last match is return. 110 | // The given pattern is compiled on every call to this function. 111 | // If you want to call this function more than once consider using Compile. 112 | func (grok Grok) ParseString(pattern, text string) (map[string]string, error) { 113 | complied, err := grok.Compile(pattern) 114 | if err != nil { 115 | return nil, err 116 | } 117 | 118 | return complied.ParseString(text), nil 119 | } 120 | 121 | // ParseTyped processes the given data and returns a map containing the values 122 | // of all named fields converted to their corresponding types. If no typehint is 123 | // given, the value will be converted to string. 124 | // The given pattern is compiled on every call to this function. 125 | // If you want to call this function more than once consider using Compile. 126 | func (grok Grok) ParseTyped(pattern string, data []byte) (map[string]interface{}, error) { 127 | complied, err := grok.Compile(pattern) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | return complied.ParseTyped(data) 133 | } 134 | 135 | // ParseStringTyped processes the given data and returns a map containing the 136 | // values of all named fields converted to their corresponding types. If no 137 | // typehint is given, the value will be converted to string. 138 | // The given pattern is compiled on every call to this function. 139 | // If you want to call this function more than once consider using Compile. 140 | func (grok Grok) ParseStringTyped(pattern, text string) (map[string]interface{}, error) { 141 | complied, err := grok.Compile(pattern) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | return complied.ParseStringTyped(text) 147 | } 148 | 149 | // ParseToMultiMap acts like Parse but allows multiple matches per field. 150 | // The given pattern is compiled on every call to this function. 151 | // If you want to call this function more than once consider using Compile. 152 | func (grok Grok) ParseToMultiMap(pattern string, data []byte) (map[string][][]byte, error) { 153 | complied, err := grok.Compile(pattern) 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | return complied.ParseToMultiMap(data), nil 159 | } 160 | 161 | // ParseStringToMultiMap acts like ParseString but allows multiple matches per 162 | // field. 163 | // The given pattern is compiled on every call to this function. 164 | // If you want to call this function more than once consider using Compile. 165 | func (grok Grok) ParseStringToMultiMap(pattern, text string) (map[string][]string, error) { 166 | complied, err := grok.Compile(pattern) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | return complied.ParseStringToMultiMap(text), nil 172 | } 173 | -------------------------------------------------------------------------------- /grokpattern.go: -------------------------------------------------------------------------------- 1 | package grok 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | type grokPattern struct { 10 | expression string 11 | typeHints typeHintByKey 12 | } 13 | 14 | var ( 15 | namedReference = regexp.MustCompile(`%{(\w+(?::\w+(?::\w+)?)?)}`) 16 | ) 17 | 18 | func newPattern(pattern string, knownPatterns patternMap, namedOnly bool) (*grokPattern, error) { 19 | typeHints := typeHintByKey{} 20 | 21 | for _, keys := range namedReference.FindAllStringSubmatch(pattern, -1) { 22 | 23 | names := strings.Split(keys[1], ":") 24 | refKey, refAlias := names[0], names[0] 25 | if len(names) > 1 { 26 | refAlias = names[1] 27 | } 28 | 29 | // Add type cast information only if type set, and not string 30 | if len(names) == 3 { 31 | if names[2] != "string" { 32 | typeHints[refAlias] = names[2] 33 | } 34 | } 35 | 36 | refPattern, patternExists := knownPatterns[refKey] 37 | if !patternExists { 38 | return nil, fmt.Errorf("no pattern found for %%{%s}", refKey) 39 | } 40 | 41 | var refExpression string 42 | if !namedOnly || (namedOnly && len(names) > 1) { 43 | refExpression = fmt.Sprintf("(?P<%s>%s)", refAlias, refPattern.expression) 44 | } else { 45 | refExpression = fmt.Sprintf("(%s)", refPattern.expression) 46 | } 47 | 48 | // Add new type Informations 49 | for key, typeName := range refPattern.typeHints { 50 | if _, hasTypeHint := typeHints[key]; !hasTypeHint { 51 | typeHints[key] = strings.ToLower(typeName) 52 | } 53 | } 54 | 55 | pattern = strings.Replace(pattern, keys[0], refExpression, -1) 56 | } 57 | 58 | return &grokPattern{ 59 | expression: pattern, 60 | typeHints: typeHints, 61 | }, nil 62 | } 63 | -------------------------------------------------------------------------------- /patternmap.go: -------------------------------------------------------------------------------- 1 | package grok 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type patternMap map[string]*grokPattern 9 | 10 | // resolve references inside a pattern so that all substitutions are added 11 | // in the correct order. 12 | func (knownPatterns *patternMap) resolve(key, pattern string, newPatterns map[string]string, namedOnly bool) error { 13 | for _, keys := range namedReference.FindAllStringSubmatch(pattern, -1) { 14 | names := strings.Split(keys[1], ":") 15 | refKey := names[0] 16 | 17 | if _, refKeyCompiled := (*knownPatterns)[refKey]; !refKeyCompiled { 18 | refPattern, refKeyFound := newPatterns[refKey] 19 | if !refKeyFound { 20 | return fmt.Errorf("no pattern found for %%{%s}", refKey) 21 | } 22 | knownPatterns.resolve(refKey, refPattern, newPatterns, namedOnly) 23 | } 24 | } 25 | return knownPatterns.add(key, pattern, namedOnly) 26 | } 27 | 28 | // add a list of patterns to the map 29 | func (knownPatterns *patternMap) addList(newPatterns map[string]string, namedOnly bool) error { 30 | for key, pattern := range newPatterns { 31 | if _, alreadyCompiled := (*knownPatterns)[key]; alreadyCompiled { 32 | continue 33 | } 34 | if err := knownPatterns.resolve(key, pattern, newPatterns, namedOnly); err != nil { 35 | return err 36 | } 37 | } 38 | 39 | return nil 40 | } 41 | 42 | // add a single pattern to the map 43 | func (knownPatterns *patternMap) add(name, pattern string, namedOnly bool) error { 44 | p, err := newPattern(pattern, *knownPatterns, namedOnly) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | (*knownPatterns)[name] = p 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /patterns.go: -------------------------------------------------------------------------------- 1 | package grok 2 | 3 | // DefaultPatterns is a collection of patterns that are added to each Grok 4 | // instance if not explicitly disabled. 5 | var DefaultPatterns = map[string]string{ 6 | "USERNAME": `[a-zA-Z0-9._-]+`, 7 | "USER": `%{USERNAME}`, 8 | "EMAILLOCALPART": `[a-zA-Z][a-zA-Z0-9_.+-=:]+`, 9 | "EMAILADDRESS": `%{EMAILLOCALPART}@%{HOSTNAME}`, 10 | "HTTPDUSER": `%{EMAILADDRESS}|%{USER}`, 11 | "INT": `(?:[+-]?(?:[0-9]+))`, 12 | "BASE10NUM": `([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)`, 13 | "NUMBER": `(?:%{BASE10NUM})`, 14 | "BASE16NUM": `(0[xX]?[0-9a-fA-F]+)`, 15 | "POSINT": `\b(?:[1-9][0-9]*)\b`, 16 | "NONNEGINT": `\b(?:[0-9]+)\b`, 17 | "WORD": `\b\w+\b`, 18 | "NOTSPACE": `\S+`, 19 | "SPACE": `\s*`, 20 | "DATA": `.*?`, 21 | "GREEDYDATA": `.*`, 22 | "QUOTEDSTRING": `"([^"\\]*(\\.[^"\\]*)*)"|\'([^\'\\]*(\\.[^\'\\]*)*)\'`, 23 | "UUID": `[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}`, 24 | "MAC": `(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})`, 25 | "CISCOMAC": `(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})`, 26 | "WINDOWSMAC": `(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})`, 27 | "COMMONMAC": `(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})`, 28 | "IPV6": `((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?`, 29 | "IPV4": `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`, 30 | "IP": `(?:%{IPV6}|%{IPV4})`, 31 | "HOSTNAME": `\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)`, 32 | "HOST": `%{HOSTNAME}`, 33 | "IPORHOST": `(?:%{IP}|%{HOSTNAME})`, 34 | "HOSTPORT": `%{IPORHOST}:%{POSINT}`, 35 | "PATH": `(?:%{UNIXPATH}|%{WINPATH})`, 36 | "UNIXPATH": `(/[\w_%!$@:.,-]?/?)(\S+)?`, 37 | "TTY": `(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))`, 38 | "WINPATH": `([A-Za-z]:|\\)(?:\\[^\\?*]*)+`, 39 | "URIPROTO": `[A-Za-z]+(\+[A-Za-z+]+)?`, 40 | "URIHOST": `%{IPORHOST}(?::%{POSINT:port})?`, 41 | "URIPATH": `(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+`, 42 | "URIPARAM": `\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*`, 43 | "URIPATHPARAM": `%{URIPATH}(?:%{URIPARAM})?`, 44 | "URI": `%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?`, 45 | "MONTH": `\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b`, 46 | "MONTHNUM": `(?:0?[1-9]|1[0-2])`, 47 | "MONTHNUM2": `(?:0[1-9]|1[0-2])`, 48 | "MONTHDAY": `(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])`, 49 | "DAY": `(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)`, 50 | "YEAR": `(\d\d){1,2}`, 51 | "HOUR": `(?:2[0123]|[01]?[0-9])`, 52 | "MINUTE": `(?:[0-5][0-9])`, 53 | "SECOND": `(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)`, 54 | "TIME": `([^0-9]?)%{HOUR}:%{MINUTE}(?::%{SECOND})([^0-9]?)`, 55 | "DATE_US": `%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}`, 56 | "DATE_EU": `%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}`, 57 | "ISO8601_TIMEZONE": `(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))`, 58 | "ISO8601_SECOND": `(?:%{SECOND}|60)`, 59 | "TIMESTAMP_ISO8601": `%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?`, 60 | "DATE": `%{DATE_US}|%{DATE_EU}`, 61 | "DATESTAMP": `%{DATE}[- ]%{TIME}`, 62 | "TZ": `(?:[PMCE][SD]T|UTC)`, 63 | "DATESTAMP_RFC822": `%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}`, 64 | "DATESTAMP_RFC2822": `%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}`, 65 | "DATESTAMP_OTHER": `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}`, 66 | "DATESTAMP_EVENTLOG": `%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}`, 67 | "HTTPDERROR_DATE": `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}`, 68 | "SYSLOGTIMESTAMP": `%{MONTH} +%{MONTHDAY} %{TIME}`, 69 | "PROG": `[\x21-\x5a\x5c\x5e-\x7e]+`, 70 | "SYSLOGPROG": `%{PROG:program}(?:\[%{POSINT:pid}\])?`, 71 | "SYSLOGHOST": `%{IPORHOST}`, 72 | "SYSLOGFACILITY": `<%{NONNEGINT:facility}.%{NONNEGINT:priority}>`, 73 | "HTTPDATE": `%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}`, 74 | "QS": `%{QUOTEDSTRING}`, 75 | "SYSLOGBASE": `%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:`, 76 | "COMMONAPACHELOG": `%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, 77 | "COMBINEDAPACHELOG": `%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}`, 78 | "HTTPD20_ERRORLOG": `\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}`, 79 | "HTTPD24_ERRORLOG": `\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}`, 80 | "HTTPD_ERRORLOG": `%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}`, 81 | "LOGLEVEL": `([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)`, 82 | } 83 | -------------------------------------------------------------------------------- /patterns/aws.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // AWS is a collection of common AWS patterns 4 | var AWS = map[string]string{ 5 | "S3_REQUEST_LINE": `(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})`, 6 | "S3_ACCESS_LOG": `%{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})`, 7 | "ELB_URIPATHPARAM": `%{URIPATH:path}(?:%{URIPARAM:params})?`, 8 | "ELB_URI": `%{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{ELB_URIPATHPARAM})?`, 9 | "ELB_REQUEST_LINE": `(?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})`, 10 | "ELB_ACCESS_LOG": `%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"`, 11 | } 12 | -------------------------------------------------------------------------------- /patterns/bacula.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Bacula is a collection of common Bacula patterns. See http://blog.bacula.org 4 | var Bacula = map[string]string{ 5 | "BACULA_TIMESTAMP": `%{MONTHDAY}-%{MONTH} %{HOUR}:%{MINUTE}`, 6 | "BACULA_HOST": `[a-zA-Z0-9-]+`, 7 | "BACULA_VOLUME": `%{USER}`, 8 | "BACULA_DEVICE": `%{USER}`, 9 | "BACULA_DEVICEPATH": `%{UNIXPATH}`, 10 | "BACULA_CAPACITY": `%{INT}{1,3}(,%{INT}{3})*`, 11 | "BACULA_VERSION": `%{USER}`, 12 | "BACULA_JOB": `%{USER}`, 13 | "BACULA_LOG_MAX_CAPACITY": `User defined maximum volume capacity %{BACULA_CAPACITY} exceeded on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\)`, 14 | "BACULA_LOG_END_VOLUME": `End of medium on Volume \"%{BACULA_VOLUME:volume}\" Bytes=%{BACULA_CAPACITY} Blocks=%{BACULA_CAPACITY} at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.`, 15 | "BACULA_LOG_NEW_VOLUME": `Created new Volume \"%{BACULA_VOLUME:volume}\" in catalog.`, 16 | "BACULA_LOG_NEW_LABEL": `Labeled new Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\).`, 17 | "BACULA_LOG_WROTE_LABEL": `Wrote label to prelabeled Volume \"%{BACULA_VOLUME:volume}\" on device \"%{BACULA_DEVICE}\" \(%{BACULA_DEVICEPATH}\)`, 18 | "BACULA_LOG_NEW_MOUNT": `New volume \"%{BACULA_VOLUME:volume}\" mounted on device \"%{BACULA_DEVICE:device}\" \(%{BACULA_DEVICEPATH}\) at %{MONTHDAY}-%{MONTH}-%{YEAR} %{HOUR}:%{MINUTE}.`, 19 | "BACULA_LOG_NOOPEN": `\s+Cannot open %{DATA}: ERR=%{GREEDYDATA:berror}`, 20 | "BACULA_LOG_NOOPENDIR": `\s+Could not open directory %{DATA}: ERR=%{GREEDYDATA:berror}`, 21 | "BACULA_LOG_NOSTAT": `\s+Could not stat %{DATA}: ERR=%{GREEDYDATA:berror}`, 22 | "BACULA_LOG_NOJOBS": `There are no more Jobs associated with Volume \"%{BACULA_VOLUME:volume}\". Marking it purged.`, 23 | "BACULA_LOG_ALL_RECORDS_PRUNED": `All records pruned from Volume \"%{BACULA_VOLUME:volume}\"; marking it \"Purged\"`, 24 | "BACULA_LOG_BEGIN_PRUNE_JOBS": `Begin pruning Jobs older than %{INT} month %{INT} days .`, 25 | "BACULA_LOG_BEGIN_PRUNE_FILES": `Begin pruning Files.`, 26 | "BACULA_LOG_PRUNED_JOBS": `Pruned %{INT} Jobs* for client %{BACULA_HOST:client} from catalog.`, 27 | "BACULA_LOG_PRUNED_FILES": `Pruned Files from %{INT} Jobs* for client %{BACULA_HOST:client} from catalog.`, 28 | "BACULA_LOG_ENDPRUNE": `End auto prune.`, 29 | "BACULA_LOG_STARTJOB": `Start Backup JobId %{INT}, Job=%{BACULA_JOB:job}`, 30 | "BACULA_LOG_STARTRESTORE": `Start Restore Job %{BACULA_JOB:job}`, 31 | "BACULA_LOG_USEDEVICE": `Using Device \"%{BACULA_DEVICE:device}\"`, 32 | "BACULA_LOG_DIFF_FS": `\s+%{UNIXPATH} is a different filesystem. Will not descend from %{UNIXPATH} into it.`, 33 | "BACULA_LOG_JOBEND": `Job write elapsed time = %{DATA:elapsed}, Transfer rate = %{NUMBER} (K|M|G)? Bytes/second`, 34 | "BACULA_LOG_NOPRUNE_JOBS": `No Jobs found to prune.`, 35 | "BACULA_LOG_NOPRUNE_FILES": `No Files found to prune.`, 36 | "BACULA_LOG_VOLUME_PREVWRITTEN": `Volume \"%{BACULA_VOLUME:volume}\" previously written, moving to end of data.`, 37 | "BACULA_LOG_READYAPPEND": `Ready to append to end of Volume \"%{BACULA_VOLUME:volume}\" size=%{INT}`, 38 | "BACULA_LOG_CANCELLING": `Cancelling duplicate JobId=%{INT}.`, 39 | "BACULA_LOG_MARKCANCEL": `JobId %{INT}, Job %{BACULA_JOB:job} marked to be canceled.`, 40 | "BACULA_LOG_CLIENT_RBJ": `shell command: run ClientRunBeforeJob \"%{GREEDYDATA:runjob}\"`, 41 | "BACULA_LOG_VSS": `(Generate )?VSS (Writer)?`, 42 | "BACULA_LOG_MAXSTART": `Fatal error: Job canceled because max start delay time exceeded.`, 43 | "BACULA_LOG_DUPLICATE": `Fatal error: JobId %{INT:duplicate} already running. Duplicate job not allowed.`, 44 | "BACULA_LOG_NOJOBSTAT": `Fatal error: No Job status returned from FD.`, 45 | "BACULA_LOG_FATAL_CONN": `Fatal error: bsock.c:133 Unable to connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?%{GREEDYDATA})`, 46 | "BACULA_LOG_NO_CONNECT": `Warning: bsock.c:127 Could not connect to (Client: %{BACULA_HOST:client}|Storage daemon) on %{HOSTNAME}:%{POSINT}. ERR=(?%{GREEDYDATA})`, 47 | "BACULA_LOG_NO_AUTH": `Fatal error: Unable to authenticate with File daemon at %{HOSTNAME}. Possible causes:`, 48 | "BACULA_LOG_NOSUIT": `No prior or suitable Full backup found in catalog. Doing FULL backup.`, 49 | "BACULA_LOG_NOPRIOR": `No prior Full backup Job record found.`, 50 | "BACULA_LOG_JOB": `(Error: )?Bacula %{BACULA_HOST} %{BACULA_VERSION} \(%{BACULA_VERSION}\):`, 51 | "BACULA_LOGLINE": `%{BACULA_TIMESTAMP:bts} %{BACULA_HOST:hostname} JobId %{INT:jobid}: (%{BACULA_LOG_MAX_CAPACITY}|%{BACULA_LOG_END_VOLUME}|%{BACULA_LOG_NEW_VOLUME}|%{BACULA_LOG_NEW_LABEL}|%{BACULA_LOG_WROTE_LABEL}|%{BACULA_LOG_NEW_MOUNT}|%{BACULA_LOG_NOOPEN}|%{BACULA_LOG_NOOPENDIR}|%{BACULA_LOG_NOSTAT}|%{BACULA_LOG_NOJOBS}|%{BACULA_LOG_ALL_RECORDS_PRUNED}|%{BACULA_LOG_BEGIN_PRUNE_JOBS}|%{BACULA_LOG_BEGIN_PRUNE_FILES}|%{BACULA_LOG_PRUNED_JOBS}|%{BACULA_LOG_PRUNED_FILES}|%{BACULA_LOG_ENDPRUNE}|%{BACULA_LOG_STARTJOB}|%{BACULA_LOG_STARTRESTORE}|%{BACULA_LOG_USEDEVICE}|%{BACULA_LOG_DIFF_FS}|%{BACULA_LOG_JOBEND}|%{BACULA_LOG_NOPRUNE_JOBS}|%{BACULA_LOG_NOPRUNE_FILES}|%{BACULA_LOG_VOLUME_PREVWRITTEN}|%{BACULA_LOG_READYAPPEND}|%{BACULA_LOG_CANCELLING}|%{BACULA_LOG_MARKCANCEL}|%{BACULA_LOG_CLIENT_RBJ}|%{BACULA_LOG_VSS}|%{BACULA_LOG_MAXSTART}|%{BACULA_LOG_DUPLICATE}|%{BACULA_LOG_NOJOBSTAT}|%{BACULA_LOG_FATAL_CONN}|%{BACULA_LOG_NO_CONNECT}|%{BACULA_LOG_NO_AUTH}|%{BACULA_LOG_NOSUIT}|%{BACULA_LOG_JOB}|%{BACULA_LOG_NOPRIOR})`, 52 | } 53 | -------------------------------------------------------------------------------- /patterns/bro.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Bro is a collection of common bro patterns. See https://www.bro.org/sphinx/script-reference/log-files.html 4 | var Bro = map[string]string{ 5 | "BRO_HTTP": `%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{INT:trans_depth}\t%{GREEDYDATA:method}\t%{GREEDYDATA:domain}\t%{GREEDYDATA:uri}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:user_agent}\t%{NUMBER:request_body_len}\t%{NUMBER:response_body_len}\t%{GREEDYDATA:status_code}\t%{GREEDYDATA:status_msg}\t%{GREEDYDATA:info_code}\t%{GREEDYDATA:info_msg}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:bro_tags}\t%{GREEDYDATA:username}\t%{GREEDYDATA:password}\t%{GREEDYDATA:proxied}\t%{GREEDYDATA:orig_fuids}\t%{GREEDYDATA:orig_mime_types}\t%{GREEDYDATA:resp_fuids}\t%{GREEDYDATA:resp_mime_types}`, 6 | "BRO_DNS": `%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{INT:trans_id}\t%{GREEDYDATA:query}\t%{GREEDYDATA:qclass}\t%{GREEDYDATA:qclass_name}\t%{GREEDYDATA:qtype}\t%{GREEDYDATA:qtype_name}\t%{GREEDYDATA:rcode}\t%{GREEDYDATA:rcode_name}\t%{GREEDYDATA:AA}\t%{GREEDYDATA:TC}\t%{GREEDYDATA:RD}\t%{GREEDYDATA:RA}\t%{GREEDYDATA:Z}\t%{GREEDYDATA:answers}\t%{GREEDYDATA:TTLs}\t%{GREEDYDATA:rejected}`, 7 | "BRO_CONN": `%{NUMBER:ts}\t%{NOTSPACE:uid}\t%{IP:orig_h}\t%{INT:orig_p}\t%{IP:resp_h}\t%{INT:resp_p}\t%{WORD:proto}\t%{GREEDYDATA:service}\t%{NUMBER:duration}\t%{NUMBER:orig_bytes}\t%{NUMBER:resp_bytes}\t%{GREEDYDATA:conn_state}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:missed_bytes}\t%{GREEDYDATA:history}\t%{GREEDYDATA:orig_pkts}\t%{GREEDYDATA:orig_ip_bytes}\t%{GREEDYDATA:resp_pkts}\t%{GREEDYDATA:resp_ip_bytes}\t%{GREEDYDATA:tunnel_parents}`, 8 | "BRO_FILES": `%{NUMBER:ts}\t%{NOTSPACE:fuid}\t%{IP:tx_hosts}\t%{IP:rx_hosts}\t%{NOTSPACE:conn_uids}\t%{GREEDYDATA:source}\t%{GREEDYDATA:depth}\t%{GREEDYDATA:analyzers}\t%{GREEDYDATA:mime_type}\t%{GREEDYDATA:filename}\t%{GREEDYDATA:duration}\t%{GREEDYDATA:local_orig}\t%{GREEDYDATA:is_orig}\t%{GREEDYDATA:seen_bytes}\t%{GREEDYDATA:total_bytes}\t%{GREEDYDATA:missing_bytes}\t%{GREEDYDATA:overflow_bytes}\t%{GREEDYDATA:timedout}\t%{GREEDYDATA:parent_fuid}\t%{GREEDYDATA:md5}\t%{GREEDYDATA:sha1}\t%{GREEDYDATA:sha256}\t%{GREEDYDATA:extracted}`, 9 | } 10 | -------------------------------------------------------------------------------- /patterns/exim.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Exim is a collection of common Exim patterns. See http://www.exim.org 4 | var Exim = map[string]string{ 5 | "EXIM_MSGID": `[0-9A-Za-z]{6}-[0-9A-Za-z]{6}-[0-9A-Za-z]{2}`, 6 | "EXIM_FLAGS": `(<=|[-=>*]>|[*]{2}|==)`, 7 | "EXIM_DATE": `%{YEAR:exim_year}-%{MONTHNUM:exim_month}-%{MONTHDAY:exim_day} %{TIME:exim_time}`, 8 | "EXIM_PID": `\[%{POSINT}\]`, 9 | "EXIM_QT": `((\d+y)?(\d+w)?(\d+d)?(\d+h)?(\d+m)?(\d+s)?)`, 10 | "EXIM_EXCLUDE_TERMS": `(Message is frozen|(Start|End) queue run| Warning: | retry time not reached | no (IP address|host name) found for (IP address|host) | unexpected disconnection while reading SMTP command | no immediate delivery: |another process is handling this message)`, 11 | "EXIM_REMOTE_HOST": `(H=(%{NOTSPACE:remote_hostname} )?(\(%{NOTSPACE:remote_heloname}\) )?\[%{IP:remote_host}\])`, 12 | "EXIM_INTERFACE": `(I=\[%{IP:exim_interface}\](:%{NUMBER:exim_interface_port}))`, 13 | "EXIM_PROTOCOL": `(P=%{NOTSPACE:protocol})`, 14 | "EXIM_MSG_SIZE": `(S=%{NUMBER:exim_msg_size})`, 15 | "EXIM_HEADER_ID": `(id=%{NOTSPACE:exim_header_id})`, 16 | "EXIM_SUBJECT": `(T=%{QS:exim_subject})`, 17 | } 18 | -------------------------------------------------------------------------------- /patterns/firewalls.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Firewalls is a collection of patterns for logs from different firwall vendors. 4 | var Firewalls = map[string]string{ 5 | // NetScreen firewall logs 6 | "NETSCREENSESSIONLOG": `%{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason}`, 7 | //== Cisco ASA == 8 | "CISCO_TAGGED_SYSLOG": `^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})? ?: %%{CISCOTAG:ciscotag}:`, 9 | "CISCOTIMESTAMP": `%{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME}`, 10 | "CISCOTAG": `[A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+)`, 11 | // Common Particles 12 | "CISCO_ACTION": `Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted`, 13 | "CISCO_REASON": `Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)*`, 14 | "CISCO_DIRECTION": `Inbound|inbound|Outbound|outbound`, 15 | "CISCO_INTERVAL": `first hit|%{INT}-second interval`, 16 | "CISCO_XLATE_TYPE": `static|dynamic`, 17 | // ASA-1-104001 18 | "CISCOFW104001": `\((?:Primary|Secondary)\) Switching to ACTIVE - %{GREEDYDATA:switch_reason}`, 19 | // ASA-1-104002 20 | "CISCOFW104002": `\((?:Primary|Secondary)\) Switching to STANDBY - %{GREEDYDATA:switch_reason}`, 21 | // ASA-1-104003 22 | "CISCOFW104003": `\((?:Primary|Secondary)\) Switching to FAILED\.`, 23 | // ASA-1-104004 24 | "CISCOFW104004": `\((?:Primary|Secondary)\) Switching to OK\.`, 25 | // ASA-1-105003 26 | "CISCOFW105003": `\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} waiting`, 27 | // ASA-1-105004 28 | "CISCOFW105004": `\((?:Primary|Secondary)\) Monitoring on [Ii]nterface %{GREEDYDATA:interface_name} normal`, 29 | // ASA-1-105005 30 | "CISCOFW105005": `\((?:Primary|Secondary)\) Lost Failover communications with mate on [Ii]nterface %{GREEDYDATA:interface_name}`, 31 | // ASA-1-105008 32 | "CISCOFW105008": `\((?:Primary|Secondary)\) Testing [Ii]nterface %{GREEDYDATA:interface_name}`, 33 | // ASA-1-105009 34 | "CISCOFW105009": `\((?:Primary|Secondary)\) Testing on [Ii]nterface %{GREEDYDATA:interface_name} (?:Passed|Failed)`, 35 | // ASA-2-106001 36 | "CISCOFW106001": `%{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface}`, 37 | // ASA-2-106006, ASA-2-106007, ASA-2-106010 38 | "CISCOFW106006_106007_106010": `%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}(\(%{DATA:src_fwuser}\))? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}(\(%{DATA:dst_fwuser}\))? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason})`, 39 | // ASA-3-106014 40 | "CISCOFW106014": `%{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\)`, 41 | // ASA-6-106015 42 | "CISCOFW106015": `%{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface}`, 43 | // ASA-1-106021 44 | "CISCOFW106021": `%{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}`, 45 | // ASA-4-106023 46 | "CISCOFW106023": `%{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src_interface}:%{DATA:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{DATA:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group "?%{DATA:policy_id}"? \[%{DATA:hashcode1}, %{DATA:hashcode2}\]`, 47 | // ASA-4-106100, ASA-4-106102, ASA-4-106103 48 | "CISCOFW106100_2_3": `access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]`, 49 | // ASA-5-106100 50 | "CISCOFW106100": `access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]`, 51 | // ASA-6-110002 52 | "CISCOFW110002": `%{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}`, 53 | // ASA-6-302010 54 | "CISCOFW302010": `%{INT:connection_count} in use, %{INT:connection_count_max} most used`, 55 | // ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 56 | "CISCOFW302013_302014_302015_302016": `%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( \(%{IP:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( \(%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}\))?(\(%{DATA:dst_fwuser}\))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( \(%{DATA:user}\))?`, 57 | // ASA-6-302020, ASA-6-302021 58 | "CISCOFW302020_302021": `%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:\(%{DATA:fwuser}\))? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( \(%{DATA:user}\))?`, 59 | // ASA-6-305011 60 | "CISCOFW305011": `%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port}`, 61 | // ASA-3-313001, ASA-3-313004, ASA-3-313008 62 | "CISCOFW313001_313004_313008": `%{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})?`, 63 | // ASA-4-313005 64 | "CISCOFW313005": `%{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}(\(%{DATA:err_src_fwuser}\))? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}(\(%{DATA:err_dst_fwuser}\))? \(type %{INT:err_icmp_type}, code %{INT:err_icmp_code}\) on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}(\(%{DATA:orig_src_fwuser}\))? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}(\(%{DATA:orig_dst_fwuser}\))?`, 65 | // ASA-5-321001 66 | "CISCOFW321001": `Resource '%{WORD:resource_name}' limit of %{POSINT:resource_limit} reached for system`, 67 | // ASA-4-402117 68 | "CISCOFW402117": `%{WORD:protocol}: Received a non-IPSec packet \(protocol= %{WORD:orig_protocol}\) from %{IP:src_ip} to %{IP:dst_ip}`, 69 | // ASA-4-402119 70 | "CISCOFW402119": `%{WORD:protocol}: Received an %{WORD:orig_protocol} packet \(SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}\) from %{IP:src_ip} \(user= %{DATA:user}\) to %{IP:dst_ip} that failed anti-replay checking`, 71 | // ASA-4-419001 72 | "CISCOFW419001": `%{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason}`, 73 | // ASA-4-419002 74 | "CISCOFW419002": `%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number`, 75 | // ASA-4-500004 76 | "CISCOFW500004": `%{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}`, 77 | // ASA-6-602303, ASA-6-602304 78 | "CISCOFW602303_602304": `%{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA \(SPI= %{DATA:spi}\) between %{IP:src_ip} and %{IP:dst_ip} \(user= %{DATA:user}\) has been %{CISCO_ACTION:action}`, 79 | // ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 80 | "CISCOFW710001_710002_710003_710005_710006": `%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}`, 81 | // ASA-6-713172 82 | "CISCOFW713172": `Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device`, 83 | // ASA-4-733100 84 | "CISCOFW733100": `\[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count}`, 85 | // Shorewall firewall logs 86 | "SHOREWALL": `(%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)`, 87 | //== End Shorewall 88 | } 89 | -------------------------------------------------------------------------------- /patterns/grok.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Grok is a collection of common grok patterns that extend the common defaults. 4 | var Grok = map[string]string{ 5 | "USERNAME": `[a-zA-Z0-9._-]+`, 6 | "USER": `%{USERNAME}`, 7 | "INT": `(?:[+-]?(?:[0-9]+))`, 8 | "BASE10NUM": `([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)`, 9 | "NUMBER": `(?:%{BASE10NUM})`, 10 | "BASE16NUM": `(0[xX]?[0-9a-fA-F]+)`, 11 | //BASE16FLOAT \b(?/(?>[\w_%!$@:.,-]+|\\.)*)+ 39 | "UNIXPATH": `(/[\w_%!$@:.,-]?/?)(\S+)?`, 40 | "TTY": `(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))`, 41 | //WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ 42 | "WINPATH": `([A-Za-z]:|\\)(?:\\[^\\?*]*)+`, 43 | 44 | "URIPROTO": `[A-Za-z]+(\+[A-Za-z+]+)?`, 45 | "URIHOST": `%{IPORHOST}(?::%{POSINT:port})?`, 46 | // uripath comes loosely from RFC1738, but mostly from what Firefox 47 | // doesn't turn into %XX 48 | "URIPATH": `(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+`, 49 | //URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? 50 | "URIPARAM": `\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*`, 51 | "URIPATHPARAM": `%{URIPATH}(?:%{URIPARAM})?`, 52 | "URI": `%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?`, 53 | 54 | // Months: January, Feb, 3, 03, 12, December 55 | "MONTH": `\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b`, 56 | "MONTHNUM": `(?:0?[1-9]|1[0-2])`, 57 | "MONTHDAY": `(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])`, 58 | 59 | // Days: Monday, Tue, Thu, etc... 60 | "DAY": `(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)`, 61 | 62 | // Years? 63 | //YEAR (?>\d\d){1,2} 64 | //c 65 | "YEAR": `(\d\d){1,2}`, 66 | 67 | "HOUR": `(?:2[0123]|[01]?[0-9])`, 68 | "MINUTE": `(?:[0-5][0-9])`, 69 | // '60' is a leap second in most time standards and thus is valid. 70 | "SECOND": `(?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)`, 71 | //TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) 72 | //c 73 | "TIME": `([^0-9]?)%{HOUR}:%{MINUTE}(?::%{SECOND})([^0-9]?)`, 74 | // datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) 75 | "DATE_US": `%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}`, 76 | "DATE_EU": `%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}`, 77 | "ISO8601_TIMEZONE": `(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))`, 78 | "ISO8601_SECOND": `(?:%{SECOND}|60)`, 79 | "TIMESTAMP_ISO8601": `%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?`, 80 | "DATE": `%{DATE_US}|%{DATE_EU}`, 81 | "DATESTAMP": `%{DATE}[- ]%{TIME}`, 82 | "TZ": `(?:[PMCE][SD]T|UTC|GMT)`, 83 | "DATESTAMP_RFC822": `%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}`, 84 | "DATESTAMP_OTHER": `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}`, 85 | 86 | // Syslog Dates: Month Day HH:MM:SS 87 | "SYSLOGTIMESTAMP": `%{MONTH} +%{MONTHDAY} %{TIME}`, 88 | "PROG": `(?:[\w._/%-]+)`, 89 | "SYSLOGPROG": `%{PROG:program}(?:\[%{POSINT:pid}\])?`, 90 | "SYSLOGHOST": `%{IPORHOST}`, 91 | "SYSLOGFACILITY": `<%{NONNEGINT:facility}.%{NONNEGINT:priority}>`, 92 | "HTTPDATE": `%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}`, 93 | 94 | // Shortcuts 95 | "QS": `%{QUOTEDSTRING}`, 96 | 97 | // Log formats 98 | "SYSLOGBASE": `%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:`, 99 | "COMMONAPACHELOG": `%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`, 100 | "COMBINEDAPACHELOG": `%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}`, 101 | 102 | // Log Levels 103 | "LOGLEVEL": `([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)`, 104 | } 105 | -------------------------------------------------------------------------------- /patterns/haproxy.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Haproxy is a collection of common HaProxy patterns. 4 | // These patterns were tested w/ haproxy-1.4.15 5 | // Documentation of the haproxy log formats can be found at the following links: 6 | // http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat 7 | // http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat 8 | var Haproxy = map[string]string{ 9 | "HAPROXYTIME": `(?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9])`, 10 | "HAPROXYDATE": `%{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds}`, 11 | 12 | // Override these default patterns to parse out what is captured in your haproxy.cfg 13 | "HAPROXYCAPTUREDREQUESTHEADERS": `%{DATA:captured_request_headers}`, 14 | "HAPROXYCAPTUREDRESPONSEHEADERS": `%{DATA:captured_response_headers}`, 15 | 16 | // Example: 17 | // These haproxy config lines will add data to the logs that are captured 18 | // by the patterns below. Place them in your custom patterns directory to 19 | // override the defaults. 20 | // 21 | // capture request header Host len 40 22 | // capture request header X-Forwarded-For len 50 23 | // capture request header Accept-Language len 50 24 | // capture request header Referer len 200 25 | // capture request header User-Agent len 200 26 | // 27 | // capture response header Content-Type len 30 28 | // capture response header Content-Encoding len 10 29 | // capture response header Cache-Control len 200 30 | // capture response header Last-Modified len 200 31 | // 32 | // HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent} 33 | // HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified} 34 | 35 | // parse a haproxy 'httplog' line 36 | "HAPROXYHTTPBASE": `%{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"`, 37 | 38 | "HAPROXYHTTP": `(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}`, 39 | 40 | // parse a haproxy 'tcplog' line 41 | "HAPROXYTCP": `(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue}`, 42 | } 43 | -------------------------------------------------------------------------------- /patterns/java.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Java is a collection of common java related patterns. 4 | var Java = map[string]string{ 5 | "JAVACLASS": `(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*`, 6 | //Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source' 7 | "JAVAFILE": `(?:[A-Za-z0-9_. -]+)`, 8 | //Allow special method 9 | "JAVAMETHOD": `(?:()|[a-zA-Z$_][a-zA-Z$_0-9]*)`, 10 | //Line number is optional in special cases 'Native method' or 'Unknown source' 11 | "JAVASTACKTRACEPART": `%{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)`, 12 | // Java Logs 13 | "JAVATHREAD": `(?:[A-Z]{2}-Processor[\d]+)`, 14 | "JAVACLASS_LOG": `(?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+`, 15 | "JAVAFILE_LOG": `(?:[A-Za-z0-9_.-]+)`, 16 | "JAVASTACKTRACEPART_LOG": `at %{JAVACLASS_LOG:class}\.%{WORD:method}\(%{JAVAFILE_LOG:file}:%{NUMBER:line}\)`, 17 | "JAVALOGMESSAGE": `(.*)`, 18 | // MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM 19 | "CATALINA_DATESTAMP": `%{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)`, 20 | // yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 21 | "TOMCAT_DATESTAMP": `20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE}`, 22 | "CATALINALOG": `%{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage}`, 23 | // 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened... 24 | "TOMCATLOG": `%{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage}`, 25 | } 26 | -------------------------------------------------------------------------------- /patterns/junos.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Junos is a collection of common Junos 11.4 RT_FLOW patterns. 4 | var Junos = map[string]string{ 5 | "RT_FLOW_EVENT": `(RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY)`, 6 | "RT_FLOW1": `%{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .*`, 7 | "RT_FLOW2": `%{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .*`, 8 | "RT_FLOW3": `%{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .*`, 9 | } 10 | -------------------------------------------------------------------------------- /patterns/linux-syslog.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // LinuxSyslog is a collection of syslog patterns used by the linux kernel. 4 | var LinuxSyslog = map[string]string{ 5 | "SYSLOG5424PRINTASCII": `[!-~]+`, 6 | "SYSLOGBASE2": `(?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource}+(?: %{SYSLOGPROG}:|)`, 7 | "SYSLOGPAMSESSION": `%{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})?`, 8 | "CRON_ACTION": `[A-Z ]+`, 9 | "CRONLOG": `%{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\)`, 10 | "SYSLOGLINE": `%{SYSLOGBASE2} %{GREEDYDATA:message}`, 11 | // IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424) 12 | "SYSLOG5424PRI": `<%{NONNEGINT:syslog5424_pri}>`, 13 | "SYSLOG5424SD": `\[%{DATA}\]+`, 14 | "SYSLOG5424BASE": `%{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)`, 15 | "SYSLOG5424LINE": `%{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}`, 16 | } 17 | -------------------------------------------------------------------------------- /patterns/mcollective.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // MCollective is a collection of common Marionette Collective patterns. 4 | // See https://github.com/puppetlabs/marionette-collective 5 | var MCollective = map[string]string{ 6 | "MCOLLECTIVEAUDIT": `%{TIMESTAMP_ISO8601:timestamp}:`, 7 | "MCOLLECTIVE": `., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level}`, 8 | } 9 | -------------------------------------------------------------------------------- /patterns/mongodb.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // MongoDB is a collection of common patterns for MongoDB related logs. 4 | var MongoDB = map[string]string{ 5 | "MONGO_LOG": `%{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message}`, 6 | "MONGO_QUERY": `\{ (?<={ ).*(?= } ntoreturn:) \}`, 7 | "MONGO_SLOWQUERY": `%{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?[0-9]+)ms`, 8 | "MONGO_WORDDASH": `\b[\w-]+\b`, 9 | "MONGO3_SEVERITY": `\w`, 10 | "MONGO3_COMPONENT": `%{WORD}|-`, 11 | "MONGO3_LOG": `%{TIMESTAMP_ISO8601:timestamp} %{MONGO3_SEVERITY:severity} %{MONGO3_COMPONENT:component}%{SPACE}(?:\[%{DATA:context}\])? %{GREEDYDATA:message}`, 12 | } 13 | -------------------------------------------------------------------------------- /patterns/nagios.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Nagios is a collection of patterns to process logfiles generated by Nagios. 4 | // While it does not, this set intends to cover all possible Nagios logs. 5 | // 6 | // Some more work needs to be done to cover all External Commands: 7 | // http://old.nagios.org/developerinfo/externalcommands/commandlist.php 8 | // 9 | // If you need some support on these rules please contact: 10 | // Jelle Smet http://smetj.net 11 | var Nagios = map[string]string{ 12 | "NAGIOSTIME": `\[%{NUMBER:nagios_epoch}\]`, 13 | 14 | // 15 | // Begin nagios log types 16 | // 17 | "NAGIOS_TYPE_CURRENT_SERVICE_STATE": `CURRENT SERVICE STATE`, 18 | "NAGIOS_TYPE_CURRENT_HOST_STATE": `CURRENT HOST STATE`, 19 | 20 | "NAGIOS_TYPE_SERVICE_NOTIFICATION": `SERVICE NOTIFICATION`, 21 | "NAGIOS_TYPE_HOST_NOTIFICATION": `HOST NOTIFICATION`, 22 | 23 | "NAGIOS_TYPE_SERVICE_ALERT": `SERVICE ALERT`, 24 | "NAGIOS_TYPE_HOST_ALERT": `HOST ALERT`, 25 | 26 | "NAGIOS_TYPE_SERVICE_FLAPPING_ALERT": `SERVICE FLAPPING ALERT`, 27 | "NAGIOS_TYPE_HOST_FLAPPING_ALERT": `HOST FLAPPING ALERT`, 28 | 29 | "NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT": `SERVICE DOWNTIME ALERT`, 30 | "NAGIOS_TYPE_HOST_DOWNTIME_ALERT": `HOST DOWNTIME ALERT`, 31 | 32 | "NAGIOS_TYPE_PASSIVE_SERVICE_CHECK": `PASSIVE SERVICE CHECK`, 33 | "NAGIOS_TYPE_PASSIVE_HOST_CHECK": `PASSIVE HOST CHECK`, 34 | 35 | "NAGIOS_TYPE_SERVICE_EVENT_HANDLER": `SERVICE EVENT HANDLER`, 36 | "NAGIOS_TYPE_HOST_EVENT_HANDLER": `HOST EVENT HANDLER`, 37 | 38 | "NAGIOS_TYPE_EXTERNAL_COMMAND": `EXTERNAL COMMAND`, 39 | "NAGIOS_TYPE_TIMEPERIOD_TRANSITION": `TIMEPERIOD TRANSITION`, 40 | // 41 | // End nagios log types 42 | // 43 | 44 | // 45 | // Begin external check types 46 | // 47 | "NAGIOS_EC_DISABLE_SVC_CHECK": `DISABLE_SVC_CHECK`, 48 | "NAGIOS_EC_ENABLE_SVC_CHECK": `ENABLE_SVC_CHECK`, 49 | "NAGIOS_EC_DISABLE_HOST_CHECK": `DISABLE_HOST_CHECK`, 50 | "NAGIOS_EC_ENABLE_HOST_CHECK": `ENABLE_HOST_CHECK`, 51 | "NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT": `PROCESS_SERVICE_CHECK_RESULT`, 52 | "NAGIOS_EC_PROCESS_HOST_CHECK_RESULT": `PROCESS_HOST_CHECK_RESULT`, 53 | "NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME": `SCHEDULE_SERVICE_DOWNTIME`, 54 | "NAGIOS_EC_SCHEDULE_HOST_DOWNTIME": `SCHEDULE_HOST_DOWNTIME`, 55 | "NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS": `DISABLE_HOST_SVC_NOTIFICATIONS`, 56 | "NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS": `ENABLE_HOST_SVC_NOTIFICATIONS`, 57 | "NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS": `DISABLE_HOST_NOTIFICATIONS`, 58 | "NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS": `ENABLE_HOST_NOTIFICATIONS`, 59 | "NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS": `DISABLE_SVC_NOTIFICATIONS`, 60 | "NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS": `ENABLE_SVC_NOTIFICATIONS`, 61 | // 62 | // End external check types 63 | // 64 | "NAGIOS_WARNING": `Warning:%{SPACE}%{GREEDYDATA:nagios_message}`, 65 | 66 | "NAGIOS_CURRENT_SERVICE_STATE": `%{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}`, 67 | "NAGIOS_CURRENT_HOST_STATE": `%{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message}`, 68 | 69 | "NAGIOS_SERVICE_NOTIFICATION": `%{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}`, 70 | "NAGIOS_HOST_NOTIFICATION": `%{NAGIOS_TYPE_HOST_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}`, 71 | 72 | "NAGIOS_SERVICE_ALERT": `%{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}`, 73 | "NAGIOS_HOST_ALERT": `%{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message}`, 74 | 75 | "NAGIOS_SERVICE_FLAPPING_ALERT": `%{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}`, 76 | "NAGIOS_HOST_FLAPPING_ALERT": `%{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message}`, 77 | 78 | "NAGIOS_SERVICE_DOWNTIME_ALERT": `%{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}`, 79 | "NAGIOS_HOST_DOWNTIME_ALERT": `%{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}`, 80 | 81 | "NAGIOS_PASSIVE_SERVICE_CHECK": `%{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}`, 82 | "NAGIOS_PASSIVE_HOST_CHECK": `%{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment}`, 83 | 84 | "NAGIOS_SERVICE_EVENT_HANDLER": `%{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}`, 85 | "NAGIOS_HOST_EVENT_HANDLER": `%{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}`, 86 | 87 | "NAGIOS_TIMEPERIOD_TRANSITION": `%{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}`, 88 | 89 | // 90 | // External checks 91 | // 92 | 93 | //Disable host & service check 94 | "NAGIOS_EC_LINE_DISABLE_SVC_CHECK": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}`, 95 | "NAGIOS_EC_LINE_DISABLE_HOST_CHECK": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}`, 96 | 97 | //Enable host & service check 98 | "NAGIOS_EC_LINE_ENABLE_SVC_CHECK": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service}`, 99 | "NAGIOS_EC_LINE_ENABLE_HOST_CHECK": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname}`, 100 | 101 | //Process host & service check 102 | "NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}`, 103 | "NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result}`, 104 | 105 | //Disable host & service notifications 106 | "NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}`, 107 | "NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}`, 108 | "NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service}`, 109 | 110 | //Enable host & service notifications 111 | "NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_SVC_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}`, 112 | "NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_NOTIFICATIONS:nagios_command};%{GREEDYDATA:nagios_hostname}`, 113 | "NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_NOTIFICATIONS:nagios_command};%{DATA:nagios_hostname};%{GREEDYDATA:nagios_service}`, 114 | 115 | //Schedule host & service downtime 116 | "NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME": `%{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment}`, 117 | 118 | //End matching line 119 | "NAGIOSLOGLINE": `%{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}|%{NAGIOS_EC_LINE_DISABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_HOST_NOTIFICATIONS}|%{NAGIOS_EC_LINE_DISABLE_SVC_NOTIFICATIONS}|%{NAGIOS_EC_LINE_ENABLE_SVC_NOTIFICATIONS})`, 120 | } 121 | -------------------------------------------------------------------------------- /patterns/postgresql.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // PostgreSQL is a collection of patterns to process the pg_log format. 4 | var PostgreSQL = map[string]string{ 5 | "POSTGRESQL": `%{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid}`, 6 | } 7 | -------------------------------------------------------------------------------- /patterns/rails.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Rails is a collection of patterns used for Ruby on Rails. 4 | var Rails = map[string]string{ 5 | "RUUID": `\h{32}`, 6 | // rails controller with action 7 | "RCONTROLLER": `(?[^#]+)#(?\w+)`, 8 | 9 | // this will often be the only line: 10 | "RAILS3HEAD": `(?m)Started %{WORD:verb} "%{URIPATHPARAM:request}" for %{IPORHOST:clientip} at (?%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE})`, 11 | // for some a strange reason, params are stripped of {} - not sure that's a good idea. 12 | "RPROCESSING": `\W*Processing by %{RCONTROLLER} as (?\S+)(?:\W*Parameters: {%{DATA:params}}\W*)?`, 13 | "RAILS3FOOT": `Completed %{NUMBER:response}%{DATA} in %{NUMBER:totalms}ms %{RAILS3PROFILE}%{GREEDYDATA}`, 14 | "RAILS3PROFILE": `(?:\(Views: %{NUMBER:viewms}ms \| ActiveRecord: %{NUMBER:activerecordms}ms|\(ActiveRecord: %{NUMBER:activerecordms}ms)?`, 15 | 16 | // putting it all together 17 | "RAILS3": `%{RAILS3HEAD}(?:%{RPROCESSING})?(?(?:%{DATA}\n)*)(?:%{RAILS3FOOT})?`, 18 | } 19 | -------------------------------------------------------------------------------- /patterns/redis.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Redis is a collection of patterns used for Redis logs. 4 | // See https://redis.io 5 | var Redis = map[string]string{ 6 | "REDISTIMESTAMP": `%{MONTHDAY} %{MONTH} %{TIME}`, 7 | "REDISLOG": `\[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* `, 8 | } 9 | -------------------------------------------------------------------------------- /patterns/ruby.go: -------------------------------------------------------------------------------- 1 | package patterns 2 | 3 | // Ruby is a collection of ruby log related patterns. 4 | var Ruby = map[string]string{ 5 | "RUBY_LOGLEVEL": `(?:DEBUG|FATAL|ERROR|WARN|INFO)`, 6 | "RUBY_LOGGER": `[DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message}`, 7 | } 8 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/errorstack.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tgo 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | // ErrorStack is a helper to store errors from multiple statements for 22 | // batch handling. Convenience functions to wrap function calls of the 23 | // form func() (, error) do exist for all golang base types. 24 | type ErrorStack struct { 25 | errors []error 26 | formatter ErrorStackFormatter 27 | } 28 | 29 | // ErrorStackFormatter is used by ErroStack to generate a single error string 30 | // from an array of errors. 31 | type ErrorStackFormatter func([]error) string 32 | 33 | // NewErrorStack creates a new error stack 34 | func NewErrorStack() ErrorStack { 35 | return ErrorStack{ 36 | errors: []error{}, 37 | formatter: ErrorStackFormatNumbered, 38 | } 39 | } 40 | 41 | // SetFormat set the format used when Error() is called. 42 | func (stack *ErrorStack) SetFormat(formatter ErrorStackFormatter) { 43 | stack.formatter = formatter 44 | } 45 | 46 | // Push adds a new error to the top of the error stack. 47 | // Returns if err != nil. 48 | func (stack *ErrorStack) Push(err error) bool { 49 | if err != nil { 50 | stack.errors = append(stack.errors, err) 51 | return true 52 | } 53 | return false 54 | } 55 | 56 | // Pushf adds a new error message to the top of the error stack 57 | func (stack *ErrorStack) Pushf(message string, args ...interface{}) { 58 | stack.errors = append(stack.errors, fmt.Errorf(message, args...)) 59 | } 60 | 61 | // PushAndDescribe behaves like Push but allows to prepend a text before 62 | // the error messages returned by err. The type of err will be lost. 63 | func (stack *ErrorStack) PushAndDescribe(message string, err error) bool { 64 | if err != nil { 65 | stack.errors = append(stack.errors, fmt.Errorf(message+" %s", err.Error())) 66 | return true 67 | } 68 | return false 69 | } 70 | 71 | // Pop removes an error from the top of the stack and returns it 72 | func (stack *ErrorStack) Pop() error { 73 | if len(stack.errors) == 0 { 74 | return nil 75 | } 76 | err := stack.errors[len(stack.errors)-1] 77 | stack.errors = stack.errors[:len(stack.errors)-1] 78 | return err 79 | } 80 | 81 | // Top returns the error on top of the stack (last error pushed) 82 | func (stack ErrorStack) Top() error { 83 | if len(stack.errors) == 0 { 84 | return nil 85 | } 86 | return stack.errors[len(stack.errors)-1] 87 | } 88 | 89 | // Error implements the Error interface 90 | func (stack ErrorStack) Error() string { 91 | if len(stack.errors) == 0 { 92 | return "" 93 | } 94 | 95 | return stack.formatter(stack.errors) 96 | } 97 | 98 | // Errors returns all gathered errors as an array 99 | func (stack ErrorStack) Errors() []error { 100 | return stack.errors 101 | } 102 | 103 | // Len returns the number of error on the stack 104 | func (stack ErrorStack) Len() int { 105 | return len(stack.errors) 106 | } 107 | 108 | // OrNil returns this object or nil of no errors are stored 109 | func (stack *ErrorStack) OrNil() error { 110 | if len(stack.errors) == 0 { 111 | return nil 112 | } 113 | return stack 114 | } 115 | 116 | // Clear removes all errors from the stack 117 | func (stack *ErrorStack) Clear() { 118 | stack.errors = []error{} 119 | } 120 | 121 | // ErrorStackFormatNumbered returns errors with a number prefix, separated by 122 | // newline. 123 | func ErrorStackFormatNumbered(errors []error) string { 124 | errString := "" 125 | lastIdx := len(errors) - 1 126 | for i := 0; i < lastIdx; i++ { 127 | errString = fmt.Sprintf("%s%d: %s\n", errString, i+1, errors[i].Error()) 128 | } 129 | errString = fmt.Sprintf("%s%d: %s", errString, lastIdx+1, errors[lastIdx].Error()) 130 | return errString 131 | } 132 | 133 | // ErrorStackFormatNumbered returns errors separated by newline 134 | func ErrorStackFormatNewline(errors []error) string { 135 | errString := "" 136 | lastIdx := len(errors) - 1 137 | for i := 0; i < lastIdx; i++ { 138 | errString = fmt.Sprintf("%s%s\n", errString, errors[i].Error()) 139 | } 140 | errString = fmt.Sprintf("%s%s", errString, errors[lastIdx].Error()) 141 | return errString 142 | } 143 | 144 | // ErrorStackFormatNumbered returns errors separated by comma 145 | func ErrorStackFormatCSV(errors []error) string { 146 | errString := "" 147 | lastIdx := len(errors) - 1 148 | for i := 0; i < lastIdx; i++ { 149 | errString = fmt.Sprintf("%s%s, ", errString, errors[i].Error()) 150 | } 151 | errString = fmt.Sprintf("%s%s", errString, errors[lastIdx].Error()) 152 | return errString 153 | } 154 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/metricserver.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tgo 16 | 17 | import ( 18 | "log" 19 | "net" 20 | "time" 21 | ) 22 | 23 | // MetricServer contains state information about the metric server process 24 | type MetricServer struct { 25 | metrics *Metrics 26 | running bool 27 | listen net.Listener 28 | updates *time.Ticker 29 | } 30 | 31 | // NewMetricServer creates a new server state for a metric server based on 32 | // the global Metric variable. 33 | func NewMetricServer() *MetricServer { 34 | return &MetricServer{ 35 | metrics: Metric, 36 | running: false, 37 | updates: time.NewTicker(time.Second), 38 | } 39 | } 40 | 41 | // NewMetricServerFor creates a new server state for a metric server based 42 | // on a custom Metrics variable 43 | func NewMetricServerFor(m *Metrics) *MetricServer { 44 | return &MetricServer{ 45 | metrics: m, 46 | running: false, 47 | updates: time.NewTicker(time.Second), 48 | } 49 | } 50 | 51 | func (server *MetricServer) handleMetricRequest(conn net.Conn) { 52 | defer conn.Close() 53 | 54 | data, err := server.metrics.Dump() 55 | if err != nil { 56 | conn.Write([]byte(err.Error())) 57 | } else { 58 | conn.Write(data) 59 | } 60 | conn.Write([]byte{'\n'}) 61 | conn.Close() 62 | } 63 | 64 | func (server *MetricServer) sysUpdate() { 65 | for server.running { 66 | _, running := <-server.updates.C 67 | if !running { 68 | return // ### return, timer has been stopped ### 69 | } 70 | 71 | server.metrics.UpdateSystemMetrics() 72 | } 73 | } 74 | 75 | // Start causes a metric server to listen for a specific address and port. 76 | // If this address/port is accessed a JSON containing all metrics will be 77 | // returned and the connection is closed. 78 | // You can use the standard go notation for addresses like ":80". 79 | func (server *MetricServer) Start(address string) { 80 | if server.running { 81 | return 82 | } 83 | 84 | var err error 85 | server.listen, err = net.Listen("tcp", address) 86 | if err != nil { 87 | log.Print("Metrics: ", err) 88 | time.AfterFunc(time.Second*5, func() { server.Start(address) }) 89 | return 90 | } 91 | 92 | server.running = true 93 | go server.sysUpdate() 94 | 95 | for server.running { 96 | client, err := server.listen.Accept() 97 | if err != nil { 98 | if server.running { 99 | log.Print("Metrics: ", err) 100 | } 101 | return // ### break ### 102 | } 103 | 104 | go server.handleMetricRequest(client) 105 | } 106 | } 107 | 108 | // Stop notifies the metric server to halt. 109 | func (server *MetricServer) Stop() { 110 | server.running = false 111 | server.updates.Stop() 112 | if server.listen != nil { 113 | if err := server.listen.Close(); err != nil { 114 | log.Print("Metrics: ", err) 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/runtime.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tgo 16 | 17 | import ( 18 | "log" 19 | "os" 20 | "runtime" 21 | "time" 22 | ) 23 | 24 | // ShutdownCallback holds the function that is called when RecoverShutdown detects 25 | // a panic and could recover. By default this functions sends an os.Interrupt 26 | // signal to the process. This function can be overwritten for customization. 27 | var ShutdownCallback = func() { 28 | proc, _ := os.FindProcess(os.Getpid()) 29 | proc.Signal(os.Interrupt) 30 | } 31 | 32 | // ReturnAfter calls a function. If that function does not return after the 33 | // given limit, the function returns regardless of the callback being done or 34 | // not. This guarantees the call to finish before or at the given limit. 35 | func ReturnAfter(runtimeLimit time.Duration, callback func()) bool { 36 | timeout := time.NewTimer(runtimeLimit) 37 | callOk := make(chan bool) 38 | 39 | go func() { 40 | callback() 41 | timeout.Stop() 42 | callOk <- true 43 | }() 44 | 45 | select { 46 | case <-timeout.C: 47 | return false 48 | 49 | case <-callOk: 50 | return true 51 | } 52 | } 53 | 54 | // RecoverShutdown will trigger a shutdown via os.Interrupt if a panic was issued. 55 | // A callstack will be printed like with RecoverTrace(). 56 | // Typically used as "defer RecoverShutdown()". 57 | func RecoverShutdown() { 58 | if r := recover(); r != nil { 59 | log.Printf("Panic shutdown: %s", r) 60 | for i := 0; i < 10; i++ { 61 | _, file, line, ok := runtime.Caller(i) 62 | if !ok { 63 | break // ### break, could not retrieve ### 64 | } 65 | log.Printf("\t%s:%d", file, line) 66 | } 67 | ShutdownCallback() 68 | } 69 | } 70 | 71 | // RecoverTrace will trigger a stack trace when a panic was recovered by this 72 | // function. Typically used as "defer RecoverTrace()". 73 | func RecoverTrace() { 74 | if r := recover(); r != nil { 75 | log.Printf("Panic ignored: %s", r) 76 | for i := 0; i < 10; i++ { 77 | _, file, line, ok := runtime.Caller(i) 78 | if !ok { 79 | break // ### break, could not retrieve ### 80 | } 81 | log.Printf("\t%s:%d", file, line) 82 | } 83 | } 84 | } 85 | 86 | // WithRecover can be used instead of RecoverTrace when using a function 87 | // without any parameters. E.g. "go WithRecover(myFunction)" 88 | func WithRecover(callback func()) { 89 | defer RecoverTrace() 90 | callback() 91 | } 92 | 93 | // WithRecoverShutdown can be used instead of RecoverShutdown when using a function 94 | // without any parameters. E.g. "go WithRecoverShutdown(myFunction)" 95 | func WithRecoverShutdown(callback func()) { 96 | defer RecoverShutdown() 97 | callback() 98 | } 99 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tcontainer/arrays.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcontainer 16 | 17 | import "sort" 18 | 19 | // Int64Slice is a typedef to allow sortable int64 slices 20 | type Int64Slice []int64 21 | 22 | func (s Int64Slice) Len() int { 23 | return len(s) 24 | } 25 | 26 | func (s Int64Slice) Less(i, j int) bool { 27 | return s[i] < s[j] 28 | } 29 | 30 | func (s Int64Slice) Swap(i, j int) { 31 | s[i], s[j] = s[j], s[i] 32 | } 33 | 34 | // Sort is a shortcut for sort.Sort(s) 35 | func (s Int64Slice) Sort() { 36 | sort.Sort(s) 37 | } 38 | 39 | // IsSorted is a shortcut for sort.IsSorted(s) 40 | func (s Int64Slice) IsSorted() bool { 41 | return sort.IsSorted(s) 42 | } 43 | 44 | // Set sets all values in this slice to the given value 45 | func (s Int64Slice) Set(v int64) { 46 | for i := range s { 47 | s[i] = v 48 | } 49 | } 50 | 51 | // Uint64Slice is a typedef to allow sortable uint64 slices 52 | type Uint64Slice []uint64 53 | 54 | func (s Uint64Slice) Len() int { 55 | return len(s) 56 | } 57 | 58 | func (s Uint64Slice) Less(i, j int) bool { 59 | return s[i] < s[j] 60 | } 61 | 62 | func (s Uint64Slice) Swap(i, j int) { 63 | s[i], s[j] = s[j], s[i] 64 | } 65 | 66 | // Sort is a shortcut for sort.Sort(s) 67 | func (s Uint64Slice) Sort() { 68 | sort.Sort(s) 69 | } 70 | 71 | // IsSorted is a shortcut for sort.IsSorted(s) 72 | func (s Uint64Slice) IsSorted() bool { 73 | return sort.IsSorted(s) 74 | } 75 | 76 | // Set sets all values in this slice to the given value 77 | func (s Uint64Slice) Set(v uint64) { 78 | for i := range s { 79 | s[i] = v 80 | } 81 | } 82 | 83 | // Float32Slice is a typedef to allow sortable float32 slices 84 | type Float32Slice []float32 85 | 86 | func (s Float32Slice) Len() int { 87 | return len(s) 88 | } 89 | 90 | func (s Float32Slice) Less(i, j int) bool { 91 | return s[i] < s[j] 92 | } 93 | 94 | func (s Float32Slice) Swap(i, j int) { 95 | s[i], s[j] = s[j], s[i] 96 | } 97 | 98 | // Sort is a shortcut for sort.Sort(s) 99 | func (s Float32Slice) Sort() { 100 | sort.Sort(s) 101 | } 102 | 103 | // IsSorted is a shortcut for sort.IsSorted(s) 104 | func (s Float32Slice) IsSorted() bool { 105 | return sort.IsSorted(s) 106 | } 107 | 108 | // Set sets all values in this slice to the given value 109 | func (s Float32Slice) Set(v float32) { 110 | for i := range s { 111 | s[i] = v 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tcontainer/bytepool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcontainer 16 | 17 | import ( 18 | "reflect" 19 | "runtime" 20 | "sync/atomic" 21 | "unsafe" 22 | ) 23 | 24 | const ( 25 | tiny = 64 26 | small = 512 27 | medium = 1024 28 | large = 1024 * 10 29 | huge = 1024 * 100 30 | 31 | tinyCount = 16384 // 1 MB 32 | smallCount = 2048 // 1 MB 33 | mediumCount = 1024 // 1 MB 34 | largeCount = 102 // ~1 MB 35 | hugeCount = 10 // ~1 MB 36 | ) 37 | 38 | type byteSlab struct { 39 | buffer []byte 40 | bufferSize uintptr 41 | stride uintptr 42 | basePtr *uintptr 43 | nextPtr *uintptr 44 | } 45 | 46 | // BytePool is a fragmentation friendly way to allocated byte slices. 47 | type BytePool struct { 48 | tinySlab byteSlab 49 | smallSlab byteSlab 50 | mediumSlab byteSlab 51 | largeSlab byteSlab 52 | hugeSlab byteSlab 53 | } 54 | 55 | func newByteSlab(size, count int) byteSlab { 56 | bufferSize := count * size 57 | buffer := make([]byte, bufferSize) 58 | basePtr := (*reflect.SliceHeader)(unsafe.Pointer(&buffer)).Data 59 | nextPtr := basePtr + uintptr(bufferSize) 60 | 61 | return byteSlab{ 62 | buffer: buffer, 63 | bufferSize: uintptr(bufferSize), 64 | stride: uintptr(size), 65 | basePtr: &basePtr, 66 | nextPtr: &nextPtr, 67 | } 68 | } 69 | 70 | func (slab *byteSlab) getSlice(size int) (chunk []byte) { 71 | chunkHeader := (*reflect.SliceHeader)(unsafe.Pointer(&chunk)) 72 | chunkHeader.Len = size 73 | chunkHeader.Cap = int(slab.stride) 74 | 75 | for { 76 | // WARNING: The following two lines are order sensitive 77 | basePtr := atomic.LoadUintptr(slab.basePtr) 78 | nextPtr := atomic.AddUintptr(slab.nextPtr, -slab.stride) 79 | lastPtr := basePtr + slab.bufferSize 80 | 81 | switch { 82 | case nextPtr < basePtr || nextPtr >= lastPtr: 83 | // out of range either means alloc while realloc or race between 84 | // base and next during realloc. In the latter case we lose a chunk. 85 | runtime.Gosched() 86 | 87 | case nextPtr == basePtr: 88 | // Last item: realloc 89 | buffer := make([]byte, slab.bufferSize) 90 | dataPtr := (*reflect.SliceHeader)(unsafe.Pointer(&buffer)).Data 91 | 92 | // WARNING: The following two lines are order sensitive 93 | atomic.StoreUintptr(slab.nextPtr, dataPtr+slab.bufferSize) 94 | atomic.StoreUintptr(slab.basePtr, dataPtr) 95 | 96 | slab.buffer = buffer // hold the buffer so that GC does not collect 97 | fallthrough 98 | 99 | default: 100 | chunkHeader.Data = nextPtr 101 | return 102 | } 103 | } 104 | } 105 | 106 | // NewBytePool creates a new BytePool with each slab using 1 MB of storage. 107 | // The pool contains 5 slabs of different sizes: 64B, 512B, 1KB, 10KB and 100KB. 108 | // Allocations above 100KB will be allocated directly. 109 | func NewBytePool() BytePool { 110 | return BytePool{ 111 | tinySlab: newByteSlab(tiny, tinyCount), 112 | smallSlab: newByteSlab(small, smallCount), 113 | mediumSlab: newByteSlab(medium, mediumCount), 114 | largeSlab: newByteSlab(large, largeCount), 115 | hugeSlab: newByteSlab(huge, hugeCount), 116 | } 117 | } 118 | 119 | // NewBytePoolWithSize creates a new BytePool with each slab size using n MB of 120 | // storage. See NewBytePool() for slab size details. 121 | func NewBytePoolWithSize(n int) BytePool { 122 | if n <= 0 { 123 | n = 1 124 | } 125 | return BytePool{ 126 | tinySlab: newByteSlab(tiny, tinyCount*n), 127 | smallSlab: newByteSlab(small, smallCount*n), 128 | mediumSlab: newByteSlab(medium, mediumCount*n), 129 | largeSlab: newByteSlab(large, largeCount*n), 130 | hugeSlab: newByteSlab(huge, hugeCount*n), 131 | } 132 | } 133 | 134 | // Get returns a slice allocated to a normalized size. 135 | // Sizes are organized in evenly sized buckets so that fragmentation is kept low. 136 | func (b *BytePool) Get(size int) []byte { 137 | switch { 138 | case size == 0: 139 | return []byte{} 140 | 141 | case size <= tiny: 142 | return b.tinySlab.getSlice(size) 143 | 144 | case size <= small: 145 | return b.smallSlab.getSlice(size) 146 | 147 | case size <= medium: 148 | return b.mediumSlab.getSlice(size) 149 | 150 | case size <= large: 151 | return b.largeSlab.getSlice(size) 152 | 153 | case size <= huge: 154 | return b.hugeSlab.getSlice(size) 155 | 156 | default: 157 | return make([]byte, size) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tcontainer/trie.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tcontainer 16 | 17 | // TrieNode represents a single node inside a trie. 18 | // Each node can contain a payload which can be retrieved after a successfull 19 | // match. In addition to that PathLen will contain the length of the match. 20 | type TrieNode struct { 21 | suffix []byte 22 | children []*TrieNode 23 | longestPath int 24 | PathLen int 25 | Payload interface{} 26 | } 27 | 28 | // NewTrie creates a new root TrieNode 29 | func NewTrie(data []byte, payload interface{}) *TrieNode { 30 | return &TrieNode{ 31 | suffix: data, 32 | children: []*TrieNode{}, 33 | longestPath: len(data), 34 | PathLen: len(data), 35 | Payload: payload, 36 | } 37 | } 38 | 39 | func (node *TrieNode) addNewChild(data []byte, payload interface{}, pathLen int) { 40 | if node.longestPath < pathLen { 41 | node.longestPath = pathLen 42 | } 43 | 44 | idx := len(node.children) 45 | node.children = append(node.children, nil) 46 | 47 | for idx > 0 { 48 | nextIdx := idx - 1 49 | if node.children[nextIdx].longestPath > pathLen { 50 | break 51 | } 52 | node.children[idx] = node.children[nextIdx] 53 | idx = nextIdx 54 | } 55 | 56 | node.children[idx] = &TrieNode{ 57 | suffix: data, 58 | children: []*TrieNode{}, 59 | longestPath: pathLen, 60 | PathLen: pathLen, 61 | Payload: payload, 62 | } 63 | } 64 | 65 | func (node *TrieNode) replace(oldChild *TrieNode, newChild *TrieNode) { 66 | for i, child := range node.children { 67 | if child == oldChild { 68 | node.children[i] = newChild 69 | return // ### return, replaced ### 70 | } 71 | } 72 | } 73 | 74 | // ForEach applies a function to each node in the tree including and below the 75 | // passed node. 76 | func (node *TrieNode) ForEach(callback func(*TrieNode)) { 77 | callback(node) 78 | for _, child := range node.children { 79 | child.ForEach(callback) 80 | } 81 | } 82 | 83 | // Add adds a new data path to the trie. 84 | // The TrieNode returned is the (new) root node so you should always reassign 85 | // the root with the return value of Add. 86 | func (node *TrieNode) Add(data []byte, payload interface{}) *TrieNode { 87 | return node.addPath(data, payload, len(data), nil) 88 | } 89 | 90 | func (node *TrieNode) addPath(data []byte, payload interface{}, pathLen int, parent *TrieNode) *TrieNode { 91 | dataLen := len(data) 92 | suffixLen := len(node.suffix) 93 | testLen := suffixLen 94 | if dataLen < suffixLen { 95 | testLen = dataLen 96 | } 97 | 98 | var splitIdx int 99 | for splitIdx = 0; splitIdx < testLen; splitIdx++ { 100 | if data[splitIdx] != node.suffix[splitIdx] { 101 | break // ### break, split found ### 102 | } 103 | } 104 | 105 | if splitIdx == suffixLen { 106 | // Continue down or stop here (full suffix match) 107 | 108 | if splitIdx == dataLen { 109 | node.Payload = payload // may overwrite 110 | return node // ### return, path already stored ### 111 | } 112 | 113 | data = data[splitIdx:] 114 | if suffixLen > 0 { 115 | for _, child := range node.children { 116 | if child.suffix[0] == data[0] { 117 | child.addPath(data, payload, pathLen, node) 118 | return node // ### return, continue on path ### 119 | } 120 | } 121 | } 122 | 123 | node.addNewChild(data, payload, pathLen) 124 | return node // ### return, new leaf ### 125 | } 126 | 127 | if splitIdx == dataLen { 128 | // Make current node a subpath of new data node (full data match) 129 | // This case implies that dataLen < suffixLen as splitIdx == suffixLen 130 | // did not match. 131 | 132 | node.suffix = node.suffix[splitIdx:] 133 | 134 | newParent := NewTrie(data, payload) 135 | newParent.PathLen = pathLen 136 | newParent.longestPath = node.longestPath 137 | newParent.children = []*TrieNode{node} 138 | 139 | if parent != nil { 140 | parent.replace(node, newParent) 141 | } 142 | return newParent // ### return, rotation ### 143 | } 144 | 145 | // New parent required with both nodes as children (partial match) 146 | 147 | node.suffix = node.suffix[splitIdx:] 148 | 149 | newParent := NewTrie(data[:splitIdx], nil) 150 | newParent.PathLen = 0 151 | newParent.longestPath = node.longestPath 152 | newParent.children = []*TrieNode{node} 153 | newParent.addNewChild(data[splitIdx:], payload, pathLen) 154 | 155 | if parent != nil { 156 | parent.replace(node, newParent) 157 | } 158 | return newParent // ### return, new parent ### 159 | } 160 | 161 | // Match compares the trie to the given data stream. 162 | // Match returns true if data can be completely matched to the trie. 163 | func (node *TrieNode) Match(data []byte) *TrieNode { 164 | dataLen := len(data) 165 | suffixLen := len(node.suffix) 166 | if dataLen < suffixLen { 167 | return nil // ### return, cannot be fully matched ### 168 | } 169 | 170 | for i := 0; i < suffixLen; i++ { 171 | if data[i] != node.suffix[i] { 172 | return nil // ### return, no match ### 173 | } 174 | } 175 | 176 | if dataLen == suffixLen { 177 | if node.PathLen > 0 { 178 | return node // ### return, full match ### 179 | } 180 | return nil // ### return, invalid match ### 181 | } 182 | 183 | data = data[suffixLen:] 184 | numChildren := len(node.children) 185 | for i := 0; i < numChildren; i++ { 186 | matchedNode := node.children[i].Match(data) 187 | if matchedNode != nil { 188 | return matchedNode // ### return, match found ### 189 | } 190 | } 191 | 192 | return nil // ### return, no valid path ### 193 | } 194 | 195 | // MatchStart compares the trie to the beginning of the given data stream. 196 | // MatchStart returns true if the beginning of data can be matched to the trie. 197 | func (node *TrieNode) MatchStart(data []byte) *TrieNode { 198 | dataLen := len(data) 199 | suffixLen := len(node.suffix) 200 | if dataLen < suffixLen { 201 | return nil // ### return, cannot be fully matched ### 202 | } 203 | 204 | for i := 0; i < suffixLen; i++ { 205 | if data[i] != node.suffix[i] { 206 | return nil // ### return, no match ### 207 | } 208 | } 209 | 210 | // Match longest path first 211 | 212 | data = data[suffixLen:] 213 | numChildren := len(node.children) 214 | for i := 0; i < numChildren; i++ { 215 | matchedNode := node.children[i].MatchStart(data) 216 | if matchedNode != nil { 217 | return matchedNode // ### return, match found ### 218 | } 219 | } 220 | 221 | // May be only a part of data but we have a valid match 222 | 223 | if node.PathLen > 0 { 224 | return node // ### return, full match ### 225 | } 226 | return nil // ### return, no valid path ### 227 | } 228 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/terrors/simpleerror.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package terrors 16 | 17 | // SimpleError provides a simple type for type based error handling. 18 | // This type can hold a message that is returned upon Error(). 19 | type SimpleError struct { 20 | Message string 21 | } 22 | 23 | func (s SimpleError) Error() string { 24 | return s.Message 25 | } 26 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tflag/flag.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tflag 16 | 17 | import ( 18 | "flag" 19 | "fmt" 20 | ) 21 | 22 | type flagKeyDescription struct { 23 | flagKey string 24 | usage string 25 | value string 26 | } 27 | 28 | var descriptions = []flagKeyDescription{} 29 | var tabWidth = []int{0, 0, 0} 30 | 31 | const ( 32 | tabKey = iota 33 | tabUsage = iota 34 | tabValue = iota 35 | ) 36 | 37 | func init() { 38 | flag.CommandLine.Usage = func() {} 39 | } 40 | 41 | func addKeyDescription(short string, long string, value interface{}, usage string) { 42 | desc := flagKeyDescription{ 43 | flagKey: fmt.Sprintf("-%s, -%s", short, long), 44 | usage: usage, 45 | value: fmt.Sprintf("%v", value), 46 | } 47 | 48 | if tabWidth[tabKey] < len(desc.flagKey) { 49 | tabWidth[tabKey] = len(desc.flagKey) 50 | } 51 | if tabWidth[tabValue] < len(desc.value) { 52 | tabWidth[tabValue] = len(desc.value) 53 | } 54 | if tabWidth[tabUsage] < len(desc.usage) { 55 | tabWidth[tabUsage] = len(desc.usage) 56 | } 57 | 58 | descriptions = append(descriptions, desc) 59 | } 60 | 61 | // SwitchVar adds a boolean flag that is meant to be used without value. If it 62 | // is given the value is true, otherwise false. 63 | func SwitchVar(flagVar *bool, short string, long string, usage string) *bool { 64 | return BoolVar(flagVar, short, long, false, usage) 65 | } 66 | 67 | // BoolVar adds a boolean flag to the parameters list. This is using the golang 68 | // flag package internally. 69 | func BoolVar(flagVar *bool, short string, long string, value bool, usage string) *bool { 70 | flag.BoolVar(flagVar, short, value, usage) 71 | flag.BoolVar(flagVar, long, value, usage) 72 | addKeyDescription(short, long, value, usage) 73 | return flagVar 74 | } 75 | 76 | // IntVar adds an integer flag to the parameters list. This is using the golang 77 | // flag package internally. 78 | func IntVar(flagVar *int, short string, long string, value int, usage string) *int { 79 | flag.IntVar(flagVar, short, value, usage) 80 | flag.IntVar(flagVar, long, value, usage) 81 | addKeyDescription(short, long, value, usage) 82 | return flagVar 83 | } 84 | 85 | // Int64Var adds am int64 flag to the parameters list. This is using the golang 86 | // flag package internally. 87 | func Int64Var(flagVar *int64, short string, long string, value int64, usage string) *int64 { 88 | flag.Int64Var(flagVar, short, value, usage) 89 | flag.Int64Var(flagVar, long, value, usage) 90 | addKeyDescription(short, long, value, usage) 91 | return flagVar 92 | } 93 | 94 | // Float64Var adds a float flag to the parameters list. This is using the golang 95 | // flag package internally. 96 | func Float64Var(flagVar *float64, short string, long string, value float64, usage string) *float64 { 97 | flag.Float64Var(flagVar, short, value, usage) 98 | flag.Float64Var(flagVar, long, value, usage) 99 | addKeyDescription(short, long, value, usage) 100 | return flagVar 101 | } 102 | 103 | // StringVar adds a string flag to the parameters list. This is using the golang 104 | // flag package internally. 105 | func StringVar(flagVar *string, short string, long string, value string, usage string) *string { 106 | flag.StringVar(flagVar, short, value, usage) 107 | flag.StringVar(flagVar, long, value, usage) 108 | addKeyDescription(short, long, value, usage) 109 | return flagVar 110 | } 111 | 112 | // Switch is a convenience wrapper for SwitchVar 113 | func Switch(short string, long string, usage string) *bool { 114 | var flagVar bool 115 | return SwitchVar(&flagVar, short, long, usage) 116 | } 117 | 118 | // Bool is a convenience wrapper for BoolVar 119 | func Bool(short string, long string, value bool, usage string) *bool { 120 | flagVar := value 121 | return BoolVar(&flagVar, short, long, value, usage) 122 | } 123 | 124 | // Int is a convenience wrapper for IntVar 125 | func Int(short string, long string, value int, usage string) *int { 126 | flagVar := value 127 | return IntVar(&flagVar, short, long, value, usage) 128 | } 129 | 130 | // Int64 is a convenience wrapper for Int64Var 131 | func Int64(short string, long string, value int64, usage string) *int64 { 132 | flagVar := value 133 | return Int64Var(&flagVar, short, long, value, usage) 134 | } 135 | 136 | // Float64 is a convenience wrapper for Float64Var 137 | func Float64(short string, long string, value float64, usage string) *float64 { 138 | flagVar := value 139 | return Float64Var(&flagVar, short, long, value, usage) 140 | } 141 | 142 | // String is a convenience wrapper for StringVar 143 | func String(short string, long string, value string, usage string) *string { 144 | flagVar := value 145 | return StringVar(&flagVar, short, long, value, usage) 146 | } 147 | 148 | // Parse is a wrapper to golang's flag.parse 149 | func Parse() { 150 | flag.Parse() 151 | } 152 | 153 | // PrintFlags prints information about the flags set for this application 154 | func PrintFlags(header string) { 155 | fmt.Println(header) 156 | valueFmt := fmt.Sprintf("%%-%ds\tdefault: %%-%ds\t%%-%ds\n", tabWidth[tabKey], tabWidth[tabValue], tabWidth[tabUsage]) 157 | 158 | for _, desc := range descriptions { 159 | fmt.Printf(valueFmt, desc.flagKey, desc.value, desc.usage) 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tfmt/bgcolor.go: -------------------------------------------------------------------------------- 1 | package tfmt 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | // BackgroundColor provides ANSI background color support. As Color implements Stringer 8 | // you can simply add a color to your print commands like 9 | // fmt.Print(BgRed,"Hello world"). 10 | type BackgroundColor int 11 | 12 | const ( 13 | // NoBackground is used to not change the current background color 14 | NoBackground = BackgroundColor(0) 15 | // BlackBackground color 16 | BlackBackground = BackgroundColor(40) 17 | // RedBackground color 18 | RedBackground = BackgroundColor(41) 19 | // GreenBackground color 20 | GreenBackground = BackgroundColor(42) 21 | // YellowBackground color 22 | YellowBackground = BackgroundColor(43) 23 | // BlueBackground color 24 | BlueBackground = BackgroundColor(44) 25 | // PurpleBackground color 26 | PurpleBackground = BackgroundColor(45) 27 | // CyanBackground color 28 | CyanBackground = BackgroundColor(46) 29 | // GrayBackground color 30 | GrayBackground = BackgroundColor(47) 31 | ) 32 | 33 | // String implements the stringer interface for color 34 | func (c BackgroundColor) String() string { 35 | if c == NoBackground { 36 | return "" 37 | } 38 | return "\x1b[" + strconv.Itoa(int(c)) + "m" 39 | } 40 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tfmt/color.go: -------------------------------------------------------------------------------- 1 | package tfmt 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | // Color provides ANSI color support. As Color implements Stringer you can 9 | // simply add a color to your print commands like fmt.Print(Red,"Hello world"). 10 | type Color int 11 | 12 | const ( 13 | // ResetColors resets foreground and background color 14 | ResetColors = Color(0) 15 | // NoColor is used to not change the current color 16 | NoColor = Color(-1) 17 | // Black color 18 | Black = Color(30) 19 | // Red color 20 | Red = Color(31) 21 | // Green color 22 | Green = Color(32) 23 | // Yellow color 24 | Yellow = Color(33) 25 | // Blue color 26 | Blue = Color(34) 27 | // Purple color 28 | Purple = Color(35) 29 | // Cyan color 30 | Cyan = Color(36) 31 | // Gray color 32 | Gray = Color(37) 33 | // DarkGray color (bold Black) 34 | DarkGray = Color(-30) 35 | // BrightRed color (bold Red) 36 | BrightRed = Color(-31) 37 | // BrightGreen color (bold Green) 38 | BrightGreen = Color(-32) 39 | // BrightYellow color (bold Yellow) 40 | BrightYellow = Color(-33) 41 | // BrightBlue color (bold Blue) 42 | BrightBlue = Color(-34) 43 | // BrightPurple color (bold Purple) 44 | BrightPurple = Color(-35) 45 | // BrightCyan color (bold Cyan) 46 | BrightCyan = Color(-36) 47 | // White color (bold Gray) 48 | White = Color(-37) 49 | ) 50 | 51 | // Colorize returns a colored string with the given colors. 52 | func Colorize(c Color, b BackgroundColor, text string) string { 53 | return fmt.Sprintf("%s%s%s%s%s", ResetColors, c, b, text, ResetColors.String()) 54 | } 55 | 56 | // Colorizef returns a colored, formatted string with the given colors. 57 | func Colorizef(c Color, b BackgroundColor, format string, args ...interface{}) string { 58 | args = append([]interface{}{ResetColors, c, b}, args...) 59 | args = append(args, ResetColors) 60 | return fmt.Sprintf("%s%s%s"+format+"%s", args...) 61 | } 62 | 63 | // String implements the stringer interface for color 64 | func (c Color) String() string { 65 | if c == NoColor { 66 | return "" 67 | } 68 | 69 | if int(c) < 0 { 70 | return "\x1b[1m\x1b[" + strconv.Itoa(int(-c)) + "m\x1b[22m" 71 | } 72 | return "\x1b[" + strconv.Itoa(int(c)) + "m" 73 | } 74 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tfmt/cursor.go: -------------------------------------------------------------------------------- 1 | package tfmt 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | // CursorUp provides ANSI cursor movement support. As CursorUp implements 8 | // Stringer you can simply add a it to your print commands like 9 | // fmt.Print(CursorUp(10),"Hello world"). 10 | type CursorUp int 11 | 12 | // CursorDown provides ANSI cursor movement support. As CursorDown implements 13 | // Stringer you can simply add a it to your print commands like 14 | // fmt.Print(CursorDown(10),"Hello world"). 15 | type CursorDown int 16 | 17 | // CursorRight provides ANSI cursor movement support. As CursorRight implements 18 | // Stringer you can simply add a it to your print commands like 19 | // fmt.Print(CursorRight(10),"Hello world"). 20 | type CursorRight int 21 | 22 | // CursorLeft provides ANSI cursor movement support. As CursorLeft implements 23 | // Stringer you can simply add a it to your print commands like 24 | // fmt.Print(CursorLeft(10),"Hello world"). 25 | type CursorLeft int 26 | 27 | // CursorPosition provides ANSI cursor movement support. As CursorPosition 28 | // implements Stringer you can simply add a it to your print commands like 29 | // fmt.Print(CursorPosition{x:0,y:0},"Hello world"). 30 | type CursorPosition struct { 31 | X int 32 | Y int 33 | } 34 | 35 | // CursorAction provides ANSI cursor management support. As CursorAction 36 | // implements Stringer you can simply add a it to your print commands. 37 | type CursorAction int 38 | 39 | const ( 40 | // CursorSave stores the current cursor position 41 | CursorSave = CursorAction(0) 42 | // CursorRestore restores the cursor position saved by CursorSave 43 | CursorRestore = CursorAction(1) 44 | // CursorClearLine clears the rest of the line 45 | CursorClearLine = CursorAction(2) 46 | // CursorClearScreen clears the screen and positions the cursor at 0,0 47 | CursorClearScreen = CursorAction(3) 48 | ) 49 | 50 | // String implements the stringer interface for CursorUp 51 | func (c CursorUp) String() string { 52 | return "\x1b[" + strconv.Itoa(int(c)) + "A" 53 | } 54 | 55 | // String implements the stringer interface for CursorDown 56 | func (c CursorDown) String() string { 57 | return "\x1b[" + strconv.Itoa(int(c)) + "B" 58 | } 59 | 60 | // String implements the stringer interface for CursorRight 61 | func (c CursorRight) String() string { 62 | return "\x1b[" + strconv.Itoa(int(c)) + "C" 63 | } 64 | 65 | // String implements the stringer interface for CursorLeft 66 | func (c CursorLeft) String() string { 67 | return "\x1b[" + strconv.Itoa(int(c)) + "D" 68 | } 69 | 70 | // String implements the stringer interface for CursorPosition 71 | func (c CursorPosition) String() string { 72 | return "\x1b[" + strconv.Itoa(c.X) + ";" + strconv.Itoa(c.Y) + "H" 73 | } 74 | 75 | // String implements the stringer interface for CursorAction 76 | func (c CursorAction) String() string { 77 | switch c { 78 | case CursorSave: 79 | return "\x1b[s" 80 | case CursorRestore: 81 | return "\x1b[u" 82 | case CursorClearLine: 83 | return "\x1b[K" 84 | case CursorClearScreen: 85 | return "\x1b[2J" 86 | } 87 | return "" 88 | } 89 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/thealthcheck/healthcheckserver.go: -------------------------------------------------------------------------------- 1 | // This package provides a simple health check server. 2 | // 3 | // The package lets you register arbitrary health check endpoints by 4 | // providing a name (== URL path) and a callback function. 5 | // 6 | // The health check server listens for HTTP requests on a given port; 7 | // probes are executed by issuing requests to the endpoints' named 8 | // URL paths. 9 | // 10 | // GETing the "/" path provides a list of registered endpoints, one 11 | // per line. GETing "/_ALL_" probes each registered endpoint sequentially, 12 | // returning each endpoint's path, HTTP status code and body per line. 13 | // 14 | // The package works as a "singleton" with just one server in order to 15 | // avoid cluttering the main program by passing handles around. 16 | package thealthcheck 17 | 18 | import ( 19 | "net/http" 20 | "fmt" 21 | "bytes" 22 | ) 23 | 24 | // 25 | const ( 26 | StatusOK = http.StatusOK 27 | StatusServiceUnavailable = http.StatusServiceUnavailable 28 | ) 29 | 30 | // Code wishing to get probed by the health-checker needs to provide this callback 31 | type CallbackFunc func () (code int, body string) 32 | 33 | // The HTTP server 34 | var server *http.Server 35 | 36 | // The HTTP request multiplexer 37 | var serveMux *http.ServeMux 38 | 39 | // List of endpoints known by the server 40 | var endpoints map[string]CallbackFunc 41 | 42 | // Init 43 | func init() { 44 | // Create the request multiplexer 45 | serveMux = http.NewServeMux() 46 | // Initialize the endpoint list 47 | endpoints = make(map[string]CallbackFunc) 48 | } 49 | 50 | // Configures the health check server 51 | // 52 | // listenAddr: an address understood by http.ListenAndServe(), e.g. ":8008" 53 | func Configure(listenAddr string) { 54 | // Create the HTTP server 55 | server = &http.Server{ 56 | Addr: listenAddr, 57 | Handler: serveMux, 58 | } 59 | 60 | // Add default wildcard handler 61 | serveMux.HandleFunc("/", 62 | func(responseWriter http.ResponseWriter, httpRequest *http.Request) { 63 | path := httpRequest.URL.Path 64 | 65 | // Handle "/": list all our registered endpoints 66 | if path == "/" { 67 | fmt.Fprintf(responseWriter, "/_ALL_\n") 68 | 69 | // TBD: Collate 70 | for endpointPath, _ := range endpoints { 71 | fmt.Fprintf(responseWriter, "%s\n", endpointPath) 72 | } 73 | return 74 | } 75 | 76 | // Default action: 404 77 | responseWriter.WriteHeader(http.StatusNotFound) 78 | fmt.Fprintf(responseWriter, "Path not found\n") 79 | return 80 | }, 81 | ) 82 | 83 | // Add magical "/_ALL" handler 84 | serveMux.HandleFunc("/_ALL_", 85 | func(responseWriter http.ResponseWriter, httpRequest *http.Request) { 86 | // Response code needs to be set before writing the response body, 87 | // so we need to pool the body temporarily into resultBody 88 | var resultCode = StatusOK 89 | var resultBody bytes.Buffer 90 | 91 | // Call all endpoints sequentially 92 | // (TBD: if there were _a lot_ of these we could do them in parallel) 93 | // TBD: Collate (output at least) 94 | for endpointPath, callback := range endpoints { 95 | // Call the callback 96 | code, body := callback() 97 | // Append path, code, body to response body 98 | fmt.Fprintf(&resultBody, 99 | "%s %d %s\n", 100 | endpointPath, 101 | code, 102 | body, 103 | ) 104 | 105 | if code != StatusOK { 106 | resultCode = StatusServiceUnavailable 107 | } 108 | } 109 | 110 | // Set HTTP response code 111 | responseWriter.WriteHeader(resultCode) 112 | 113 | // Write HTTP response body 114 | // (TBD: more efficient way, buffer -> writer?) 115 | fmt.Fprintf(responseWriter, resultBody.String()) 116 | }, 117 | ) 118 | } 119 | 120 | // Registers an endpoint with the health checker. 121 | // 122 | // The urlPath must be unique. The callback must return an HTTP response code 123 | // and body text. 124 | // 125 | // Boilerplate: 126 | // 127 | // healthcheck.AddEndpoint("/my/arbitrary/path" func()(code int, body string) { 128 | // return 200, "Foobar Plugin is OK" 129 | // }) 130 | // 131 | func AddEndpoint(urlPath string, callback CallbackFunc){ 132 | // Check parameters 133 | // -syntax 134 | if len(urlPath) == 0 || urlPath[:1] != "/" || urlPath[len(urlPath)-1:] == "/" { 135 | panic(fmt.Sprintf( 136 | "ERROR: Health check endpoint must begin and may not end with a slash: \"%s\"", 137 | urlPath)) 138 | } 139 | // - reserved paths 140 | for _, path := range []string{"/", "/_ALL_"} { 141 | if urlPath == path { 142 | panic(fmt.Sprintf( 143 | "ERROR: Health check path \"%s\" is reserved", path)) 144 | } 145 | } 146 | // - registered paths 147 | _, exists := endpoints[urlPath] 148 | if exists { 149 | panic(fmt.Sprintf( 150 | "ERROR: Health check endpoint \"%s\" already registered", urlPath)) 151 | } 152 | 153 | // Register the HTTP route & handler 154 | serveMux.HandleFunc( 155 | urlPath, 156 | func(responseWriter http.ResponseWriter, httpRequest *http.Request){ 157 | // Call the callback 158 | code, body := callback() 159 | // Set HTTP response code 160 | responseWriter.WriteHeader(code) 161 | // Write HTTP response body 162 | fmt.Fprintf(responseWriter, body) 163 | fmt.Fprintf(responseWriter, "\n") 164 | }, 165 | ) 166 | 167 | // Store the endpoint 168 | endpoints[urlPath] = callback 169 | } 170 | 171 | // Registers an endpoint with the health checker. 172 | // 173 | // This is a convenience version of AddEndpoint() that takes 174 | // the urlPath's components as a list of strings and catenates 175 | // them. 176 | func AddEndpointPathArray(urlPath []string, callback CallbackFunc) { 177 | // Catenate path 178 | var cat bytes.Buffer 179 | for _, pathComponent := range urlPath { 180 | fmt.Fprintf(&cat, "/%s", pathComponent) 181 | } 182 | // Call it 183 | AddEndpoint(cat.String(), callback) 184 | } 185 | 186 | // Starts the HTTP server 187 | // 188 | // Call this after Configure() and AddEndpoint() calls. 189 | // 190 | // TBD: is it possible to AddEndpoint() after Start()ing? 191 | func Start(){ 192 | err := server.ListenAndServe() 193 | 194 | if err != nil { 195 | panic(err.Error()) 196 | } 197 | } 198 | 199 | // TBD: Cleanup? 200 | func Stop(){ 201 | 202 | 203 | } 204 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tio/bytestream.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tio 16 | 17 | import "io" 18 | 19 | // ByteStream is a more lightweight variant of bytes.Buffer. 20 | // The managed byte array is increased to the exact required size and never 21 | // shrinks. Writing moves an internal offset (for appends) but reading always 22 | // starts at offset 0. 23 | type ByteStream struct { 24 | data []byte 25 | writeOffset int 26 | readOffset int 27 | } 28 | 29 | // NewByteStream creates a new byte stream of the desired capacity 30 | func NewByteStream(capacity int) ByteStream { 31 | return ByteStream{ 32 | data: make([]byte, capacity), 33 | writeOffset: 0, 34 | } 35 | } 36 | 37 | // NewByteStreamFrom creates a new byte stream that starts with the given 38 | // byte array. 39 | func NewByteStreamFrom(data []byte) ByteStream { 40 | return ByteStream{ 41 | data: data, 42 | writeOffset: len(data), 43 | } 44 | } 45 | 46 | // SetCapacity assures that capacity bytes are available in the buffer, growing 47 | // the managed byte array if needed. The buffer will grow by at least 64 bytes 48 | // if growing is required. 49 | func (stream *ByteStream) SetCapacity(capacity int) { 50 | if stream.Cap() < capacity { 51 | current := stream.data 52 | stream.data = make([]byte, capacity) 53 | if stream.writeOffset > 0 { 54 | copy(stream.data, current[:stream.writeOffset]) 55 | } 56 | } 57 | } 58 | 59 | // Reset sets the internal write offset to 0 and calls ResetRead 60 | func (stream *ByteStream) Reset() { 61 | stream.writeOffset = 0 62 | stream.ResetRead() 63 | } 64 | 65 | // ResetRead sets the internal read offset to 0. The read offset is only used 66 | // within the Read function to assure io.Reader compatibility 67 | func (stream *ByteStream) ResetRead() { 68 | stream.readOffset = 0 69 | } 70 | 71 | // Len returns the length of the underlying array. 72 | // This is equal to len(stream.Bytes()). 73 | func (stream ByteStream) Len() int { 74 | return stream.writeOffset 75 | } 76 | 77 | // Cap returns the capacity of the underlying array. 78 | // This is equal to cap(stream.Bytes()). 79 | func (stream ByteStream) Cap() int { 80 | return len(stream.data) 81 | } 82 | 83 | // Bytes returns a slice of the underlying byte array containing all written 84 | // data up to this point. 85 | func (stream ByteStream) Bytes() []byte { 86 | return stream.data[:stream.writeOffset] 87 | } 88 | 89 | // String returns a string of the underlying byte array containing all written 90 | // data up to this point. 91 | func (stream ByteStream) String() string { 92 | return string(stream.data[:stream.writeOffset]) 93 | } 94 | 95 | // Write implements the io.Writer interface. 96 | // This function assures that the capacity of the underlying byte array is 97 | // enough to store the incoming amount of data. Subsequent writes will allways 98 | // append to the end of the stream until Reset() is called. 99 | func (stream *ByteStream) Write(source []byte) (int, error) { 100 | sourceLen := len(source) 101 | if sourceLen == 0 { 102 | return 0, nil 103 | } 104 | 105 | stream.SetCapacity(stream.writeOffset + sourceLen) 106 | copy(stream.data[stream.writeOffset:], source[:sourceLen]) 107 | stream.writeOffset += sourceLen 108 | 109 | return sourceLen, nil 110 | } 111 | 112 | // WriteString is a convenience wrapper for Write([]byte(source)) 113 | func (stream *ByteStream) WriteString(source string) (int, error) { 114 | return stream.Write([]byte(source)) 115 | } 116 | 117 | // WriteByte writes a single byte to the stream. Capacity will be ensured. 118 | func (stream *ByteStream) WriteByte(source byte) error { 119 | stream.SetCapacity(stream.writeOffset + 1) 120 | stream.data[stream.writeOffset] = source 121 | stream.writeOffset++ 122 | return nil 123 | } 124 | 125 | // Read implements the io.Reader interface. 126 | // The underlying array is copied to target and io.EOF is returned once no data 127 | // is left to be read. Please note that ResetRead can rewind the internal read 128 | // index for this operation. 129 | func (stream *ByteStream) Read(target []byte) (int, error) { 130 | if stream.readOffset >= stream.writeOffset { 131 | return 0, io.EOF // ### return, no new data to read ### 132 | } 133 | 134 | bytesCopied := copy(target, stream.data[stream.readOffset:stream.writeOffset]) 135 | stream.readOffset += bytesCopied 136 | 137 | if stream.readOffset >= stream.writeOffset { 138 | return bytesCopied, io.EOF 139 | } 140 | return bytesCopied, nil 141 | } 142 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tio/bytewriter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tio 16 | 17 | import ( 18 | "io" 19 | ) 20 | 21 | // ByteWriter wraps a slice so that it can be passed to io.Writer compatible 22 | // functions. The referenced slice will be resized. 23 | type ByteWriter struct { 24 | buffer *[]byte 25 | } 26 | 27 | // NewByteWriter creates a new writer on the referenced byte slice. 28 | func NewByteWriter(buffer *[]byte) ByteWriter { 29 | return ByteWriter{ 30 | buffer: buffer, 31 | } 32 | } 33 | 34 | // Reset sets the length of the slize to 0. 35 | func (b ByteWriter) Reset() { 36 | *b.buffer = (*b.buffer)[:0] 37 | } 38 | 39 | // Write writes data to the wrapped slice if there is enough space available. 40 | // If not, data is written until the wrapped slice has reached its capacity 41 | // and io.EOF is returned. 42 | func (b ByteWriter) Write(data []byte) (int, error) { 43 | start := len(*b.buffer) 44 | size := len(data) 45 | capacity := cap(*b.buffer) 46 | 47 | if start+size > capacity { 48 | size := capacity - start 49 | *b.buffer = (*b.buffer)[:capacity] 50 | copy((*b.buffer)[start:], data[:size]) 51 | return size, io.EOF 52 | } 53 | 54 | *b.buffer = (*b.buffer)[:start+size] 55 | copy((*b.buffer)[start:], data) 56 | 57 | return size, nil 58 | } 59 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tio/files.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tio 16 | 17 | import ( 18 | "hash/crc32" 19 | "io/ioutil" 20 | "os" 21 | "path/filepath" 22 | "regexp" 23 | "sort" 24 | "strings" 25 | ) 26 | 27 | // FilesByDate implements the Sort interface by Date for os.FileInfo arrays 28 | type FilesByDate []os.FileInfo 29 | 30 | // Len returns the number of files in the array 31 | func (files FilesByDate) Len() int { 32 | return len(files) 33 | } 34 | 35 | // Swap exchanges the values stored at indexes a and b 36 | func (files FilesByDate) Swap(a, b int) { 37 | files[a], files[b] = files[b], files[a] 38 | } 39 | 40 | // Less compares the date of the files stored at a and b as in 41 | // "[a] modified < [b] modified". If both files were created in the same second 42 | // the file names are compared by using a lexicographic string compare. 43 | func (files FilesByDate) Less(a, b int) bool { 44 | timeA, timeB := files[a].ModTime().UnixNano(), files[b].ModTime().UnixNano() 45 | if timeA == timeB { 46 | return files[a].Name() < files[b].Name() 47 | } 48 | return timeA < timeB 49 | } 50 | 51 | // ListFilesByDateMatching gets all files from a directory that match a given 52 | // regular expression pattern and orders them by modification date (ascending). 53 | // Directories and symlinks are excluded from the returned list. 54 | func ListFilesByDateMatching(directory string, pattern string) ([]os.FileInfo, error) { 55 | filteredFiles := []os.FileInfo{} 56 | filter, err := regexp.Compile(pattern) 57 | if err != nil { 58 | return filteredFiles, err 59 | } 60 | 61 | files, err := ioutil.ReadDir(directory) 62 | if err != nil { 63 | return filteredFiles, err 64 | } 65 | 66 | sort.Sort(FilesByDate(files)) 67 | 68 | for _, file := range files { 69 | if file.IsDir() || file.Mode()&os.ModeSymlink == os.ModeSymlink { 70 | continue // ### continue, skip symlinks and directories ### 71 | } 72 | if filter.MatchString(file.Name()) { 73 | filteredFiles = append(filteredFiles, file) 74 | } 75 | } 76 | 77 | return filteredFiles, nil 78 | } 79 | 80 | // SplitPath separates a file path into directory, filename (without extension) 81 | // and file extension (with dot). If no directory could be derived "." is 82 | // returned as a directory. If no file extension could be derived "" is returned 83 | // as a file extension. 84 | func SplitPath(filePath string) (dir string, base string, ext string) { 85 | dir = filepath.Dir(filePath) 86 | ext = filepath.Ext(filePath) 87 | base = filepath.Base(filePath) 88 | base = base[:len(base)-len(ext)] 89 | return dir, base, ext 90 | } 91 | 92 | // FileExists does a proper check on wether a file exists or not. 93 | func FileExists(filePath string) bool { 94 | _, err := os.Lstat(filePath) 95 | return !os.IsNotExist(err) 96 | } 97 | 98 | // IsDirectory returns true if a given path points to a directory. 99 | func IsDirectory(filePath string) bool { 100 | stat, err := os.Stat(filePath) 101 | if err != nil { 102 | return false 103 | } 104 | return stat.IsDir() 105 | } 106 | 107 | // CommonPath returns the longest common path of both paths given. 108 | func CommonPath(path1, path2 string) string { 109 | parts1 := strings.Split(path1, "/") 110 | parts2 := strings.Split(path2, "/") 111 | maxIdx := len(parts1) 112 | if len(parts2) < maxIdx { 113 | maxIdx = len(parts2) 114 | } 115 | 116 | common := make([]string, 0, maxIdx) 117 | for i := 0; i < maxIdx; i++ { 118 | if parts1[i] == parts2[i] { 119 | common = append(common, parts1[i]) 120 | } 121 | } 122 | 123 | return strings.Join(common, "/") 124 | } 125 | 126 | // FileCRC32 returns the checksum of a given file 127 | func FileCRC32(path string) (uint32, error) { 128 | data, err := ioutil.ReadFile(path) 129 | if err != nil { 130 | return 0, err 131 | } 132 | return crc32.ChecksumIEEE(data), nil 133 | } 134 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tio/syscall.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build !windows 16 | 17 | package tio 18 | 19 | import ( 20 | "fmt" 21 | "os" 22 | "syscall" 23 | ) 24 | 25 | // OpenNamedPipe opens or attaches to a named pipe given by name. If the pipe 26 | // is created, perm will be used as file permissions. 27 | // This function is not available on windows. 28 | func OpenNamedPipe(name string, perm uint32) (*os.File, error) { 29 | if err := syscall.Mkfifo(name, perm); !os.IsExist(err) { 30 | return nil, fmt.Errorf("Failed to create named pipe %s: %s", name, err) 31 | } 32 | 33 | pipe, err := os.OpenFile(name, os.O_CREATE, os.ModeNamedPipe) 34 | if err != nil { 35 | return nil, fmt.Errorf("Could not open named pipe %s: %s", name, err) 36 | } 37 | 38 | stats, _ := pipe.Stat() 39 | if stats.Mode()&os.ModeNamedPipe == 0 { 40 | return nil, fmt.Errorf("File is not a named pipe %s", name) 41 | } 42 | 43 | return pipe, nil 44 | } 45 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tio/syscall_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tio 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | ) 21 | 22 | // OpenNamedPipe opens or attaches to a named pipe given by name. If the pipe 23 | // is created, perm will be used as file permissions. 24 | // This function is not available on windows. 25 | func OpenNamedPipe(name string, perm uint32) (*os.File, error) { 26 | return nil, fmt.Errorf("Named pipes are not supported on windows") 27 | } 28 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tlog/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tlog 16 | 17 | import ( 18 | "github.com/trivago/tgo/tfmt" 19 | "io" 20 | "log" 21 | "os" 22 | ) 23 | 24 | // Verbosity defines an enumeration for log verbosity 25 | type Verbosity byte 26 | 27 | const ( 28 | // VerbosityError shows only error messages 29 | VerbosityError = Verbosity(iota) 30 | // VerbosityWarning shows error and warning messages 31 | VerbosityWarning = Verbosity(iota) 32 | // VerbosityNote shows error, warning and note messages 33 | VerbosityNote = Verbosity(iota) 34 | // VerbosityDebug shows all messages 35 | VerbosityDebug = Verbosity(iota) 36 | ) 37 | 38 | var ( 39 | // Error is a predefined log channel for errors. This log is backed by consumer.Log 40 | Error = log.New(logDisabled, "", log.Lshortfile) 41 | 42 | // Warning is a predefined log channel for warnings. This log is backed by consumer.Log 43 | Warning = log.New(logDisabled, "", 0) 44 | 45 | // Note is a predefined log channel for notes. This log is backed by consumer.Log 46 | Note = log.New(logDisabled, "", 0) 47 | 48 | // Debug is a predefined log channel for debug messages. This log is backed by consumer.Log 49 | Debug = log.New(logDisabled, "", 0) 50 | ) 51 | 52 | var ( 53 | logEnabled = logReferrer{os.Stderr} 54 | logDisabled = logNull{} 55 | ) 56 | 57 | // LogScope allows to wrap the standard Error, Warning, Note and Debug loggers 58 | // into a scope, i.e. all messages written to this logger are prefixed. 59 | type LogScope struct { 60 | Error *log.Logger 61 | Warning *log.Logger 62 | Note *log.Logger 63 | Debug *log.Logger 64 | name string 65 | } 66 | 67 | // NewLogScope creates a new LogScope with the given prefix string. 68 | func NewLogScope(name string) LogScope { 69 | scopeMarker := tfmt.Colorizef(tfmt.DarkGray, tfmt.NoBackground, "[%s] ", name) 70 | 71 | return LogScope{ 72 | name: name, 73 | Error: log.New(logLogger{Error}, scopeMarker, 0), 74 | Warning: log.New(logLogger{Warning}, scopeMarker, 0), 75 | Note: log.New(logLogger{Note}, scopeMarker, 0), 76 | Debug: log.New(logLogger{Debug}, scopeMarker, 0), 77 | } 78 | } 79 | 80 | // NewSubScope creates a log scope inside an existing log scope. 81 | func (scope *LogScope) NewSubScope(name string) LogScope { 82 | scopeMarker := tfmt.Colorizef(tfmt.DarkGray, tfmt.NoBackground, "[%s.%s] ", scope.name, name) 83 | 84 | return LogScope{ 85 | name: name, 86 | Error: log.New(logLogger{Error}, scopeMarker, 0), 87 | Warning: log.New(logLogger{Warning}, scopeMarker, 0), 88 | Note: log.New(logLogger{Note}, scopeMarker, 0), 89 | Debug: log.New(logLogger{Debug}, scopeMarker, 0), 90 | } 91 | } 92 | 93 | func init() { 94 | log.SetFlags(0) 95 | log.SetOutput(logEnabled) 96 | SetVerbosity(VerbosityError) 97 | } 98 | 99 | // SetVerbosity defines the type of messages to be processed. 100 | // High level verobosities contain lower levels, i.e. log level warning will 101 | // contain error messages, too. 102 | func SetVerbosity(loglevel Verbosity) { 103 | Error = log.New(logDisabled, "", 0) 104 | Warning = log.New(logDisabled, "", 0) 105 | Note = log.New(logDisabled, "", 0) 106 | Debug = log.New(logDisabled, "", 0) 107 | 108 | switch loglevel { 109 | default: 110 | fallthrough 111 | 112 | case VerbosityDebug: 113 | Debug = log.New(&logEnabled, tfmt.Colorize(tfmt.Cyan, tfmt.NoBackground, "Debug: "), 0) 114 | fallthrough 115 | 116 | case VerbosityNote: 117 | Note = log.New(&logEnabled, "", 0) 118 | fallthrough 119 | 120 | case VerbosityWarning: 121 | Warning = log.New(&logEnabled, tfmt.Colorize(tfmt.Yellow, tfmt.NoBackground, "Warning: "), 0) 122 | fallthrough 123 | 124 | case VerbosityError: 125 | Error = log.New(&logEnabled, tfmt.Colorize(tfmt.Red, tfmt.NoBackground, "ERROR: "), log.Lshortfile) 126 | } 127 | } 128 | 129 | // SetCacheWriter will force all logs to be cached until another writer is set 130 | func SetCacheWriter() { 131 | if _, isCache := logEnabled.writer.(*logCache); !isCache { 132 | logEnabled.writer = new(logCache) 133 | } 134 | } 135 | 136 | // SetWriter forces (enabled) logs to be written to the given writer. 137 | func SetWriter(writer io.Writer) { 138 | oldWriter := logEnabled.writer 139 | logEnabled.writer = writer 140 | if cache, isCache := oldWriter.(*logCache); isCache { 141 | cache.Flush(logEnabled) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tlog/logcache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tlog 16 | 17 | import ( 18 | "io" 19 | "sync" 20 | ) 21 | 22 | type logCache struct { 23 | flushing sync.Mutex 24 | messages [][]byte 25 | } 26 | 27 | // Write caches the passed message. Blocked by flush function. 28 | func (log *logCache) Write(message []byte) (int, error) { 29 | log.flushing.Lock() 30 | defer log.flushing.Unlock() 31 | 32 | if len(message) > 0 { 33 | messageCopy := make([]byte, len(message)) 34 | copy(messageCopy, message) 35 | log.messages = append(log.messages, messageCopy) 36 | } 37 | return len(message), nil 38 | } 39 | 40 | // Flush writes all messages to the given writer and clears the list 41 | // of stored messages. Blocks Write function. 42 | func (log *logCache) Flush(writer io.Writer) { 43 | log.flushing.Lock() 44 | defer log.flushing.Unlock() 45 | 46 | for _, message := range log.messages { 47 | writer.Write(message) 48 | } 49 | log.messages = [][]byte{} 50 | } 51 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tlog/loglogger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tlog 16 | 17 | import ( 18 | "log" 19 | ) 20 | 21 | type logLogger struct { 22 | logger *log.Logger 23 | } 24 | 25 | // Write sends the message to the io.Writer passed to Configure 26 | func (lg logLogger) Write(message []byte) (int, error) { 27 | length := len(message) 28 | if length > 0 { 29 | lg.logger.Output(2, string(message)) 30 | } 31 | return length, nil 32 | } 33 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tlog/lognull.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tlog 16 | 17 | type logNull struct { 18 | } 19 | 20 | // Write Drops all messages 21 | func (log logNull) Write(message []byte) (int, error) { 22 | return len(message), nil 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tlog/logreferrer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tlog 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "io" 21 | "os" 22 | ) 23 | 24 | type logReferrer struct { 25 | writer io.Writer 26 | } 27 | 28 | // Write sends the message to the io.Writer passed to Configure 29 | func (log logReferrer) Write(message []byte) (int, error) { 30 | length := len(message) 31 | if length == 0 { 32 | return 0, nil 33 | } 34 | 35 | logMessage := bytes.TrimRight(message, "\r\n\t ") 36 | switch { 37 | case log.writer == nil: 38 | fmt.Println(string(logMessage)) 39 | return length, nil 40 | 41 | case log.writer == os.Stdout || log.writer == os.Stderr: 42 | fmt.Fprintln(log.writer, string(logMessage)) 43 | return length, nil 44 | 45 | default: 46 | return log.writer.Write(logMessage) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tmath/bits.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tmath 16 | 17 | // NextPowerOf2U16 returns the next power of 2 for a given 16bit number 18 | func NextPowerOf2U16(v uint16) uint16 { 19 | v-- 20 | v |= v >> 1 21 | v |= v >> 2 22 | v |= v >> 4 23 | v |= v >> 8 24 | v++ 25 | return v 26 | } 27 | 28 | // NextPowerOf2U32 returns the next power of 2 for a given 32bit number 29 | func NextPowerOf2U32(v uint32) uint32 { 30 | v-- 31 | v |= v >> 1 32 | v |= v >> 2 33 | v |= v >> 4 34 | v |= v >> 8 35 | v |= v >> 16 36 | v++ 37 | return v 38 | } 39 | 40 | // NextPowerOf2U64 returns the next power of 2 for a given 64bit number 41 | func NextPowerOf2U64(v uint64) uint64 { 42 | v-- 43 | v |= v >> 1 44 | v |= v >> 2 45 | v |= v >> 4 46 | v |= v >> 8 47 | v |= v >> 16 48 | v |= v >> 32 49 | v++ 50 | return v 51 | } 52 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tmath/math.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tmath 16 | 17 | // MaxI returns the maximum out of two integers 18 | func MaxI(a, b int) int { 19 | if a > b { 20 | return a 21 | } 22 | return b 23 | } 24 | 25 | // MaxInt64 returns the maximum out of two signed 64bit integers 26 | func MaxInt64(a, b int64) int64 { 27 | if a > b { 28 | return a 29 | } 30 | return b 31 | } 32 | 33 | // MaxUint64 returns the maximum out of two unsigned 64bit integers 34 | func MaxUint64(a, b uint64) uint64 { 35 | if a > b { 36 | return a 37 | } 38 | return b 39 | } 40 | 41 | // Max3I returns the maximum out of three integers 42 | func Max3I(a, b, c int) int { 43 | max := a 44 | if b > max { 45 | max = b 46 | } 47 | if c > max { 48 | max = c 49 | } 50 | return max 51 | } 52 | 53 | // Max3Int64 returns the maximum out of three signed 64-bit integers 54 | func Max3Int64(a, b, c int64) int64 { 55 | max := a 56 | if b > max { 57 | max = b 58 | } 59 | if c > max { 60 | max = c 61 | } 62 | return max 63 | } 64 | 65 | // Max3Uint64 returns the maximum out of three unsigned 64-bit integers 66 | func Max3Uint64(a, b, c uint64) uint64 { 67 | max := a 68 | if b > max { 69 | max = b 70 | } 71 | if c > max { 72 | max = c 73 | } 74 | return max 75 | } 76 | 77 | // MinI returns the minimum out of two integers 78 | func MinI(a, b int) int { 79 | if a < b { 80 | return a 81 | } 82 | return b 83 | } 84 | 85 | // MinInt64 returns the minimum out of two signed 64-bit integers 86 | func MinInt64(a, b int64) int64 { 87 | if a < b { 88 | return a 89 | } 90 | return b 91 | } 92 | 93 | // MinUint64 returns the minimum out of two unsigned 64-bit integers 94 | func MinUint64(a, b uint64) uint64 { 95 | if a < b { 96 | return a 97 | } 98 | return b 99 | } 100 | 101 | // Min3I returns the minimum out of three integers 102 | func Min3I(a, b, c int) int { 103 | min := a 104 | if b < min { 105 | min = b 106 | } 107 | if c < min { 108 | min = c 109 | } 110 | return min 111 | } 112 | 113 | // Min3Int64 returns the minimum out of three signed 64-bit integers 114 | func Min3Int64(a, b, c int64) int64 { 115 | min := a 116 | if b < min { 117 | min = b 118 | } 119 | if c < min { 120 | min = c 121 | } 122 | return min 123 | } 124 | 125 | // Min3Uint64 returns the minimum out of three unsigned 64-bit integers 126 | func Min3Uint64(a, b, c uint64) uint64 { 127 | min := a 128 | if b < min { 129 | min = b 130 | } 131 | if c < min { 132 | min = c 133 | } 134 | return min 135 | } 136 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tnet/network.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tnet 16 | 17 | import ( 18 | "io" 19 | "net" 20 | "strings" 21 | "syscall" 22 | ) 23 | 24 | // ParseAddress takes an address and tries to extract the protocol from it. 25 | // Protocols may be prepended by the "protocol://" notation. 26 | // If no protocol is given, defaultProtocol is returned. 27 | // The first parameter returned is the address, the second denotes the protocol. 28 | // The protocol is allways returned as lowercase string. 29 | func ParseAddress(addressString string, defaultProtocol string) (protocol, address string) { 30 | protocolIdx := strings.Index(addressString, "://") 31 | if protocolIdx == -1 { 32 | return strings.ToLower(defaultProtocol), addressString 33 | } 34 | 35 | return strings.ToLower(addressString[:protocolIdx]), addressString[protocolIdx+3:] 36 | } 37 | 38 | // SplitAddress splits an address of the form "protocol://host:port" into its 39 | // components. If no protocol is given, defaultProtocol is used. 40 | // This function uses net.SplitHostPort and ParseAddress. 41 | func SplitAddress(addressString string, defaultProtocol string) (protocol, host, port string, err error) { 42 | proto, address := ParseAddress(addressString, defaultProtocol) 43 | if proto == "unix" || proto == "unixgram" || proto == "unixpacket" { 44 | return proto, address, "", nil 45 | } 46 | 47 | host, port, err = net.SplitHostPort(address) 48 | return proto, host, port, err 49 | } 50 | 51 | // IsDisconnectedError returns true if the given error is related to a 52 | // disconnected socket. 53 | func IsDisconnectedError(err error) bool { 54 | if err == io.EOF { 55 | return true // ### return, closed stream ### 56 | } 57 | 58 | netErr, isNetErr := err.(*net.OpError) 59 | if isNetErr { 60 | errno, isErrno := netErr.Err.(syscall.Errno) 61 | if isErrno { 62 | switch errno { 63 | default: 64 | case syscall.ECONNRESET: 65 | return true // ### return, close connection ### 66 | } 67 | } 68 | } 69 | 70 | return false 71 | } 72 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tnet/stoplistener.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tnet 16 | 17 | import ( 18 | "net" 19 | ) 20 | 21 | // StopListener is a wrapper around net.TCPListener that allows to gracefully 22 | // shut down a connection loop. 23 | type StopListener struct { 24 | *net.TCPListener 25 | active bool 26 | } 27 | 28 | // StopRequestError is not an error but a note that the connection was requested 29 | // to be closed. 30 | type StopRequestError struct{} 31 | 32 | // NewStopListener creates a new, stoppable TCP server connection. 33 | // Address needs to be cmpliant to net.Listen. 34 | func NewStopListener(address string) (*StopListener, error) { 35 | listen, err := net.Listen("tcp", address) 36 | if err != nil { 37 | return nil, err // ### return, could not connect ### 38 | } 39 | 40 | return &StopListener{ 41 | TCPListener: listen.(*net.TCPListener), 42 | active: true, 43 | }, nil 44 | } 45 | 46 | // Error implements the standard error interface 47 | func (err StopRequestError) Error() string { 48 | return "Connection stop request" 49 | } 50 | 51 | // Accept is analogous to net.TCPListener.Accept but may return a connection 52 | // closed error if the connection was requested to shut down. 53 | func (listen *StopListener) Accept() (net.Conn, error) { 54 | conn, err := listen.TCPListener.Accept() 55 | if !listen.active { 56 | return nil, StopRequestError{} // ### return, stop requested ### 57 | } 58 | if err != nil { 59 | return nil, err // ### return, error ### 60 | } 61 | 62 | return conn, err 63 | } 64 | 65 | // Close requests a connection close on this listener 66 | func (listen *StopListener) Close() error { 67 | listen.active = false 68 | return listen.TCPListener.Close() 69 | } 70 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tnet/thttp/sockettransport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package thttp 16 | 17 | import ( 18 | "bufio" 19 | "net" 20 | "net/http" 21 | "time" 22 | ) 23 | 24 | // SocketRoundtripper defines an http.RoundTripper that uses a unix domain socket 25 | // as an endpoint. 26 | type SocketRoundtripper struct { 27 | path string 28 | } 29 | 30 | // NewSocketTransport creates an http transport that listens to the "unix" 31 | // protocol. I.e. requests must be of the form "unix://path". 32 | func NewSocketTransport(path string) *http.Transport { 33 | socket := NewSocketRoundTripper(path) 34 | transport := &http.Transport{} 35 | transport.RegisterProtocol("unix", socket) 36 | return transport 37 | } 38 | 39 | // NewSocketRoundTripper creates a http.RoundTripper using a unix domain socket 40 | func NewSocketRoundTripper(path string) http.RoundTripper { 41 | return SocketRoundtripper{ 42 | path: path, 43 | } 44 | } 45 | 46 | // RoundTrip dials the unix domain socket, sends the request and processes the 47 | // response. 48 | func (s SocketRoundtripper) RoundTrip(req *http.Request) (*http.Response, error) { 49 | if req.Body != nil { 50 | defer req.Body.Close() 51 | } 52 | socket, err := net.Dial("unix", s.path) 53 | if err != nil { 54 | return nil, err 55 | } 56 | defer socket.Close() 57 | 58 | socket.SetWriteDeadline(time.Now().Add(5 * time.Second)) 59 | if err := req.Write(socket); err != nil { 60 | return nil, err 61 | } 62 | 63 | socket.SetReadDeadline(time.Now().Add(5 * time.Second)) 64 | reader := bufio.NewReader(socket) 65 | return http.ReadResponse(reader, req) 66 | } 67 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/exit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tos 16 | 17 | import "syscall" 18 | 19 | const ( 20 | // ExitSuccess should be used if no error occured 21 | ExitSuccess = 0 22 | // ExitError should be used in general failure cases 23 | ExitError = 1 24 | // ExitNotAllowed should be used if a permission problem occured 25 | ExitNotAllowed = 2 26 | // ExitCannotExecute should be used if e.g. a subprocess failed to execute 27 | ExitCannotExecute = 126 28 | // ExitNotFound should be used if e.g. a subprocess executable was not found 29 | ExitNotFound = 127 30 | // ExitInvalidArgument should be used if argument parsing failed 31 | ExitInvalidArgument = 128 32 | // ExitSignal + n should be used if the signal with the number n terminated the process 33 | ExitSignal = 128 34 | // ExitSignalInt is shorthand for ExitSignal + syscall.SIGINT 35 | ExitSignalHup = ExitSignal + syscall.SIGHUP 36 | // ExitSignalInt is shorthand for ExitSignal + syscall.SIGINT 37 | ExitSignalInt = ExitSignal + syscall.SIGINT 38 | // ExitSignalKill is shorthand for ExitSignal + syscall.SIGKILL 39 | ExitSignalKill = ExitSignal + syscall.SIGKILL 40 | // ExitSignalTerm is shorthand for ExitSignal + syscall.SIGTERM 41 | ExitSignalTerm = ExitSignal + syscall.SIGTERM 42 | // ExitSignalPipe is shorthand for ExitSignal + syscall.SIGPIPE 43 | ExitSignalPipe = ExitSignal + syscall.SIGPIPE 44 | // ExitSignalAlarm is shorthand for ExitSignal + syscall.SIGALRM 45 | ExitSignalAlarm = ExitSignal + syscall.SIGALRM 46 | // ExitSignalTrap is shorthand for ExitSignal + syscall.SIGTRAP 47 | ExitSignalTrap = ExitSignal + syscall.SIGTRAP 48 | ) 49 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tos 16 | 17 | import ( 18 | "io" 19 | "io/ioutil" 20 | "os" 21 | ) 22 | 23 | // ChownByName is a wrapper around ChownId that allows changing user and group by name. 24 | func ChownByName(filePath, usr, grp string) error { 25 | var uid, gid int 26 | var err error 27 | 28 | if uid, err = GetUid(usr); err != nil { 29 | return err 30 | } 31 | 32 | if gid, err = GetGid(grp); err != nil { 33 | return err 34 | } 35 | 36 | return Chown(filePath, uid, gid) 37 | } 38 | 39 | // ChownId is a wrapper around os.Chown that allows changing user and group 40 | // recursively if given a directory. 41 | func Chown(filePath string, uid, gid int) error { 42 | stat, err := os.Lstat(filePath) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | if stat.IsDir() { 48 | files, err := ioutil.ReadDir(filePath) 49 | if err != nil { 50 | return err 51 | } 52 | for _, file := range files { 53 | if err := Chown(filePath+"/"+file.Name(), uid, gid); err != nil { 54 | return err 55 | } 56 | } 57 | } 58 | 59 | if stat.Mode()&os.ModeSymlink != 0 { 60 | // TODO: os.Chown fails on symlinks 61 | return nil 62 | } 63 | 64 | return os.Chown(filePath, uid, gid) 65 | } 66 | 67 | // Chmod is a wrapper around os.Chmod that allows changing rights recursively 68 | // if a directory is given. 69 | func Chmod(filePath string, mode os.FileMode) error { 70 | stat, err := os.Lstat(filePath) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | if stat.IsDir() { 76 | files, err := ioutil.ReadDir(filePath) 77 | if err != nil { 78 | return err 79 | } 80 | for _, file := range files { 81 | if err := Chmod(filePath+"/"+file.Name(), mode); err != nil { 82 | return err 83 | } 84 | } 85 | 86 | // Set executable rights for folders if read or write is allowed 87 | execRights := 0 88 | if mode&0600 != 0 { 89 | execRights |= 0100 90 | } 91 | if mode&0060 != 0 { 92 | execRights |= 0010 93 | } 94 | if mode&0006 != 0 { 95 | execRights |= 0001 96 | } 97 | 98 | return os.Chmod(filePath, mode|os.FileMode(execRights)) 99 | } 100 | 101 | if stat.Mode()&os.ModeSymlink != 0 { 102 | // TODO: os.Chmod fails on symlinks 103 | return nil 104 | } 105 | 106 | return os.Chmod(filePath, mode) 107 | } 108 | 109 | // IsSymlink returns true if a file is a symlink 110 | func IsSymlink(file string) (bool, error) { 111 | fileStat, err := os.Lstat(file) 112 | if err != nil { 113 | return false, err 114 | } 115 | 116 | return fileStat.Mode()&os.ModeSymlink != 0, nil 117 | } 118 | 119 | // Copy is a file copy helper. Files will be copied to their destination, 120 | // overwriting existing files. Already existing files that are not part of the 121 | // copy process will not be touched. If source is a directory it is walked 122 | // recursively. Non-existing folders in dest will be created. 123 | // Copy returns upon the first error encountered. In-between results will not 124 | // be rolled back. 125 | func Copy(dest, source string) error { 126 | srcStat, err := os.Lstat(source) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | if srcStat.IsDir() { 132 | files, err := ioutil.ReadDir(source) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | if err := os.MkdirAll(dest, srcStat.Mode()); err != nil && !os.IsExist(err) { 138 | return err 139 | } 140 | 141 | for _, file := range files { 142 | if err := Copy(dest+"/"+file.Name(), source+"/"+file.Name()); err != nil { 143 | return err 144 | } 145 | } 146 | return nil // ### return, copy done ### 147 | } 148 | 149 | // Symlink handling 150 | if srcStat.Mode()&os.ModeSymlink != 0 { 151 | target, err := os.Readlink(source) 152 | if err != nil { 153 | return err 154 | } 155 | return os.Symlink(target, dest) // ### return, copy done ### 156 | } 157 | 158 | srcFile, err := os.Open(source) 159 | if err != nil { 160 | return err 161 | } 162 | defer srcFile.Close() 163 | 164 | dstFile, err := os.Create(dest) 165 | if err != nil { 166 | return err 167 | } 168 | defer dstFile.Close() 169 | 170 | if _, err = io.Copy(dstFile, srcFile); err != nil { 171 | return err 172 | } 173 | 174 | return os.Chmod(dest, srcStat.Mode()) 175 | } 176 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/file_unix.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build cgo,go1.7 16 | 17 | package tos 18 | 19 | import ( 20 | "os" 21 | "os/user" 22 | "strconv" 23 | "syscall" 24 | ) 25 | 26 | // GetFileCredentials returns the user and group id of a given path. 27 | // This function is not supported on windows platforms. 28 | func GetFileCredentials(name string) (uid int, gid int, err error) { 29 | stat, err := os.Lstat(name) 30 | if err != nil { 31 | return 0, 0, err 32 | } 33 | 34 | nativeStat := stat.Sys().(*syscall.Stat_t) 35 | return int(nativeStat.Uid), int(nativeStat.Gid), nil 36 | } 37 | 38 | // GetFileCredentialsName returns the user and group name of a given path. 39 | // This function is not supported on windows platforms. 40 | func GetFileCredentialsName(name string) (usr string, grp string, err error) { 41 | uid, gid, err := GetFileCredentials(name) 42 | if err != nil { 43 | return "", "", err 44 | } 45 | 46 | usrData, err := user.LookupId(strconv.Itoa(uid)) 47 | if err != nil { 48 | return "", "", err 49 | } 50 | 51 | // Requires go1.7 52 | grpData, err := user.LookupGroupId(strconv.Itoa(gid)) 53 | if err != nil { 54 | return "", "", err 55 | } 56 | 57 | return usrData.Username, grpData.Name, nil 58 | } 59 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/file_unsupported.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build !go1.7 !cgo 16 | 17 | package tos 18 | 19 | import ( 20 | "fmt" 21 | ) 22 | 23 | // GetFileCredentials returns the user and group id of a given path. 24 | // This function is not supported on windows platforms. 25 | func GetFileCredentials(name string) (uid int, gid int, err error) { 26 | return 0, 0, fmt.Errorf("Not supported on this platform") 27 | } 28 | 29 | // GetFileCredentialsName returns the user and group name of a given path. 30 | // This function is not supported on windows platforms. 31 | func GetFileCredentialsName(name string) (usr string, grp string, err error) { 32 | return "", "", fmt.Errorf("Not supported on this platform") 33 | } 34 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/ids_darwin.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build darwin 16 | 17 | package tos 18 | 19 | const ( 20 | // NobodyUid represents the user id of the user nobody 21 | NobodyUid = 0xFFFFFFFE 22 | // NobodyUid represents the group id of the group nobody 23 | NobodyGid = 0xFFFFFFFE 24 | // RootUid represents the user id of the root user 25 | RootUid = 0 26 | // RootUid represents the group id of the root group 27 | RootGid = 0 28 | ) 29 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/ids_freebsd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build freebsd 16 | 17 | package tos 18 | 19 | const ( 20 | // NobodyUid represents the user id of the user nobody 21 | NobodyUid = 0xFFFE 22 | // NobodyUid represents the group id of the group nobody 23 | NobodyGid = 0xFFFE 24 | // RootUid represents the user id of the root user 25 | RootUid = 0 26 | // RootUid represents the group id of the root group 27 | RootGid = 0 28 | ) 29 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/ids_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build linux 16 | 17 | package tos 18 | 19 | const ( 20 | // NobodyUid represents the user id of the user nobody 21 | NobodyUid = 0xFFFE 22 | // NobodyUid represents the group id of the group nobody 23 | NobodyGid = 0xFFFE 24 | // RootUid represents the user id of the root user 25 | RootUid = 0 26 | // RootUid represents the group id of the root group 27 | RootGid = 0 28 | ) 29 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/ids_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build windows 16 | 17 | package tos 18 | 19 | // TODO 20 | /* 21 | const ( 22 | // NobodyUid represents the user id of the user nobody 23 | NobodyUid = 0 24 | // NobodyUid represents the group id of the group nobody 25 | NobodyGid = 0 26 | // RootUid represents the user id of the root user 27 | RootUid = 0 28 | // RootUid represents the group id of the root group 29 | RootGid = 0 30 | )*/ 31 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/pidfile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tos 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "io/ioutil" 21 | "os" 22 | "strconv" 23 | "syscall" 24 | "time" 25 | ) 26 | 27 | const ( 28 | InvalidPID = -1 29 | ) 30 | 31 | // WritePidFile writes this a process id into a file. 32 | // An error will be returned if the file already exists. 33 | func WritePidFile(pid int, filename string) error { 34 | pidString := strconv.Itoa(pid) 35 | pidFile, err := os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) 36 | if err != nil { 37 | return err 38 | } 39 | defer pidFile.Close() 40 | _, err = pidFile.WriteString(pidString) 41 | return err 42 | } 43 | 44 | // WritePidFileForced writes this a process id into a file. 45 | // An existing file will be overwritten. 46 | func WritePidFileForced(pid int, filename string) error { 47 | pidString := strconv.Itoa(pid) 48 | return ioutil.WriteFile(filename, []byte(pidString), 0644) 49 | } 50 | 51 | // GetPidFromFile tries loads the content of a pid file. 52 | // A pidfile is expected to contain only an integer with a valid process id. 53 | func GetPidFromFile(filename string) (int, error) { 54 | var ( 55 | pidBytes []byte 56 | pid int 57 | err error 58 | ) 59 | 60 | if pidBytes, err = ioutil.ReadFile(filename); err != nil { 61 | return InvalidPID, fmt.Errorf("Could not read pidfile %s: %s", filename, err) 62 | } 63 | 64 | pidString := string(bytes.Trim(pidBytes, "\r\n\t ")) 65 | if pid, err = strconv.Atoi(string(pidString)); err != nil { 66 | return InvalidPID, fmt.Errorf("Could not read pid from pidfile %s: %s", filename, err) 67 | } 68 | 69 | return pid, nil 70 | } 71 | 72 | // GetProcFromFile utilizes GetPidFromFile to create a os.Process handle for 73 | // the pid contained in the given pid file. 74 | func GetProcFromFile(filename string) (*os.Process, error) { 75 | var ( 76 | pid int 77 | err error 78 | proc *os.Process 79 | ) 80 | 81 | if pid, err = GetPidFromFile(filename); err != nil { 82 | return nil, err 83 | } 84 | 85 | // FindProcess always returns a proc on unix 86 | if proc, err = os.FindProcess(pid); err != nil { 87 | return nil, err 88 | } 89 | 90 | // Try to signal the process to check if it is running 91 | if err = proc.Signal(syscall.Signal(0)); err != nil { 92 | return nil, err 93 | } 94 | 95 | return proc, nil 96 | } 97 | 98 | // Terminate tries to gracefully shutdown a process by sending SIGTERM. 99 | // If the process does not shut down after (at least) gracePeriod a SIGKILL will be sent. 100 | // 10 checks will be done during gracePeriod to check if the process is still 101 | // alive. If a gracePeriod of 0 is passed SIGKILL will be send immediately. 102 | func Terminate(proc *os.Process, gracePeriod time.Duration) error { 103 | if gracePeriod > 0 { 104 | err := proc.Signal(syscall.SIGTERM) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | // Try to gracefully shutdown the process by sending TERMINATE 110 | // first. After 5 seconds KILL will be sent. 111 | stepDuration := gracePeriod / 10 112 | for i := 0; i < 10; i++ { 113 | time.Sleep(stepDuration) 114 | if err := proc.Signal(syscall.Signal(0)); err != nil { 115 | return nil // ### return, success ### 116 | } 117 | } 118 | } 119 | 120 | return proc.Kill() 121 | } 122 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tos/user.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tos 16 | 17 | import ( 18 | "os/user" 19 | "strconv" 20 | ) 21 | 22 | func GetUid(name string) (int, error) { 23 | switch name { 24 | case "nobody": 25 | return NobodyUid, nil 26 | 27 | case "root": 28 | return RootUid, nil 29 | 30 | default: 31 | userInfo, err := user.Lookup(name) 32 | if err != nil { 33 | return 0, err 34 | } 35 | 36 | return strconv.Atoi(userInfo.Uid) 37 | } 38 | } 39 | 40 | func GetGid(name string) (int, error) { 41 | switch name { 42 | case "nobody": 43 | return NobodyGid, nil 44 | 45 | case "root": 46 | return RootGid, nil 47 | 48 | default: 49 | groupInfo, err := user.LookupGroup(name) 50 | if err != nil { 51 | return 0, err 52 | } 53 | 54 | return strconv.Atoi(groupInfo.Gid) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/treflect/typeregistry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package treflect 16 | 17 | import ( 18 | "fmt" 19 | "reflect" 20 | "strings" 21 | ) 22 | 23 | // TypeRegistry is a name to type registry used to create objects by name. 24 | type TypeRegistry struct { 25 | namedType map[string]reflect.Type 26 | } 27 | 28 | // NewTypeRegistry creates a new TypeRegistry. Note that there is a global type 29 | // registry available in the main tgo package (tgo.TypeRegistry). 30 | func NewTypeRegistry() TypeRegistry { 31 | return TypeRegistry{ 32 | namedType: make(map[string]reflect.Type), 33 | } 34 | } 35 | 36 | // Register a plugin to the TypeRegistry by passing an uninitialized object. 37 | // Example: var MyConsumerClassID = shared.Plugin.Register(MyConsumer{}) 38 | func (registry TypeRegistry) Register(typeInstance interface{}) { 39 | structType := reflect.TypeOf(typeInstance) 40 | packageName := structType.PkgPath() 41 | typeName := structType.Name() 42 | 43 | pathTokens := strings.Split(packageName, "/") 44 | maxDepth := 3 45 | if len(pathTokens) < maxDepth { 46 | maxDepth = len(pathTokens) 47 | } 48 | 49 | for n := 1; n <= maxDepth; n++ { 50 | shortTypeName := strings.Join(pathTokens[len(pathTokens)-n:], ".") + "." + typeName 51 | registry.namedType[shortTypeName] = structType 52 | } 53 | } 54 | 55 | // New creates an uninitialized object by class name. 56 | // The class name has to be "package.class" or "package/subpackage.class". 57 | // The gollum package is omitted from the package path. 58 | func (registry TypeRegistry) New(typeName string) (interface{}, error) { 59 | structType, exists := registry.namedType[typeName] 60 | if exists { 61 | return reflect.New(structType).Interface(), nil 62 | } 63 | return nil, fmt.Errorf("Unknown class: %s", typeName) 64 | } 65 | 66 | // GetTypeOf returns only the type asscociated with the given name. 67 | // If the name is not registered, nil is returned. 68 | // The type returned will be a pointer type. 69 | func (registry TypeRegistry) GetTypeOf(typeName string) reflect.Type { 70 | if structType, exists := registry.namedType[typeName]; exists { 71 | return reflect.PtrTo(structType) 72 | } 73 | return nil 74 | } 75 | 76 | // IsTypeRegistered returns true if a type is registered to this registry. 77 | // Note that GetTypeOf can do the same thing by checking for nil but also 78 | // returns the type, so in many cases you will want to call this function. 79 | func (registry TypeRegistry) IsTypeRegistered(typeName string) bool { 80 | _, exists := registry.namedType[typeName] 81 | return exists 82 | } 83 | 84 | // GetRegistered returns the names of all registered types for a given package 85 | func (registry TypeRegistry) GetRegistered(packageName string) []string { 86 | var result []string 87 | for key := range registry.namedType { 88 | if strings.HasPrefix(key, packageName) { 89 | result = append(result, key) 90 | } 91 | } 92 | return result 93 | } 94 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tstrings/parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tstrings 16 | 17 | import ( 18 | "fmt" 19 | "github.com/trivago/tgo/tcontainer" 20 | "strings" 21 | ) 22 | 23 | // ParserFlag is an enum type for flags used in parser transitions. 24 | type ParserFlag int 25 | 26 | // ParserStateID is used as an integer-based reference to a specific parser state. 27 | // You can use any number (e.g. a hash) as a parser state representative. 28 | type ParserStateID uint32 29 | 30 | // ParsedFunc defines a function that a token has been matched 31 | type ParsedFunc func([]byte, ParserStateID) 32 | 33 | const ( 34 | // ParserFlagContinue continues parsing at the position of the match. 35 | // By default the matched token will be skipped. This flag prevents the 36 | // default behavior. In addition to that the parser will add the parsed 37 | // token to the value of the next match. 38 | ParserFlagContinue = ParserFlag(1 << iota) 39 | 40 | // ParserFlagAppend causes the parser to keep the current value for the next 41 | // match. By default a value will be restarted after each match. This flag 42 | // prevents the default behavior. 43 | ParserFlagAppend = ParserFlag(1 << iota) 44 | 45 | // ParserFlagInclude includes the matched token in the read value. 46 | // By default the value does not contain the matched token. 47 | ParserFlagInclude = ParserFlag(1 << iota) 48 | 49 | // ParserFlagPush pushes the current state on the stack before making the 50 | // transition. 51 | ParserFlagPush = ParserFlag(1 << iota) 52 | 53 | // ParserFlagPop pops the stack and uses the returned state as the next 54 | // state. If no state is on the stack the regular next state is used. 55 | ParserFlagPop = ParserFlag(1 << iota) 56 | 57 | // ParserStateStop defines a state that causes the parsing to stop when 58 | // transitioned into. 59 | ParserStateStop = ParserStateID(0xFFFFFFFF) 60 | ) 61 | 62 | // Transition defines a token based state change 63 | type Transition struct { 64 | nextState ParserStateID 65 | flags ParserFlag 66 | callback ParsedFunc 67 | } 68 | 69 | // TransitionDirective contains a transition description that can be passed to 70 | // the AddDirectives functions. 71 | type TransitionDirective struct { 72 | State string 73 | Token string 74 | NextState string 75 | Flags ParserFlag 76 | Callback ParsedFunc 77 | } 78 | 79 | // TransitionParser defines the behavior of a parser by storing transitions from 80 | // one state to another. 81 | type TransitionParser struct { 82 | lookup []string 83 | tokens []*tcontainer.TrieNode 84 | stack []ParserStateID 85 | } 86 | 87 | // NewTransition creates a new transition to a given state. 88 | func NewTransition(nextState ParserStateID, flags ParserFlag, callback ParsedFunc) Transition { 89 | return Transition{ 90 | nextState: nextState, 91 | flags: flags, 92 | callback: callback, 93 | } 94 | } 95 | 96 | // NewTransitionParser creates a new transition based parser 97 | func NewTransitionParser() TransitionParser { 98 | return TransitionParser{ 99 | lookup: []string{}, 100 | tokens: []*tcontainer.TrieNode{}, 101 | stack: []ParserStateID{}, 102 | } 103 | } 104 | 105 | // ParseTransitionDirective creates a transition directive from a string. 106 | // This string must be of the following format: 107 | // 108 | // State:Token:NextState:Flag,Flag,...:Function 109 | // 110 | // Spaces will be stripped from all fields but Token. If a fields requires a 111 | // colon it has to be escaped with a backslash. Other escape characters 112 | // supported are \n, \r and \t. 113 | // Flag can be a set of the 114 | // following flags (input will be converted to lowercase): 115 | // 116 | // * continue -> ParserFlagContinue 117 | // * append -> ParserFlagAppend 118 | // * include -> ParserFlagInclude 119 | // * push -> ParserFlagPush 120 | // * pop -> ParserFlagPop 121 | // 122 | // The name passed to the function token must be in the callbacks map. If it is 123 | // not the data of the token will not be written. I.e. in empty string equals 124 | // "do not write" here. 125 | // An empty NextState will be translated to the "Stop" state as ususal. 126 | func ParseTransitionDirective(config string, callbacks map[string]ParsedFunc) (TransitionDirective, error) { 127 | escape := strings.NewReplacer("\\n", "\n", "\\r", "\r", "\\t", "\t", "\\:", "\b") 128 | restore := strings.NewReplacer("\b", ":") 129 | 130 | tokens := strings.Split(escape.Replace(config), ":") 131 | if len(tokens) != 5 { 132 | return TransitionDirective{}, fmt.Errorf("Parser directive requires 5 tokens.") 133 | } 134 | 135 | // Restore colons 136 | for i, token := range tokens { 137 | tokens[i] = restore.Replace(token) 138 | } 139 | 140 | flags := ParserFlag(0) 141 | flagStrings := strings.Split(tokens[3], ",") 142 | 143 | for _, flagName := range flagStrings { 144 | switch strings.ToLower(strings.TrimSpace(flagName)) { 145 | case "continue": 146 | flags |= ParserFlagContinue 147 | case "append": 148 | flags |= ParserFlagAppend 149 | case "include": 150 | flags |= ParserFlagInclude 151 | case "push": 152 | flags |= ParserFlagPush 153 | case "pop": 154 | flags |= ParserFlagPop 155 | } 156 | } 157 | 158 | callback, _ := callbacks[strings.TrimSpace(tokens[4])] 159 | dir := TransitionDirective{ 160 | State: strings.TrimSpace(tokens[0]), 161 | Token: tokens[1], 162 | NextState: strings.TrimSpace(tokens[2]), 163 | Flags: flags, 164 | Callback: callback, 165 | } 166 | 167 | return dir, nil 168 | } 169 | 170 | // GetStateID creates a hash from the given state name. 171 | // Empty state names will be translated to ParserStateStop. 172 | func (parser *TransitionParser) GetStateID(stateName string) ParserStateID { 173 | if len(stateName) == 0 { 174 | return ParserStateStop 175 | } 176 | 177 | for id, name := range parser.lookup { 178 | if name == stateName { 179 | return ParserStateID(id) // ### return, found ### 180 | } 181 | } 182 | 183 | id := ParserStateID(len(parser.lookup)) 184 | parser.lookup = append(parser.lookup, stateName) 185 | parser.tokens = append(parser.tokens, nil) 186 | return id 187 | } 188 | 189 | // GetStateName returns the name for the given state id or an empty string if 190 | // the id could not be found. 191 | func (parser *TransitionParser) GetStateName(id ParserStateID) string { 192 | if id < ParserStateID(len(parser.lookup)) { 193 | return parser.lookup[id] 194 | } 195 | return "" 196 | } 197 | 198 | // AddDirectives is a convenience function to add multiple transitions in as a 199 | // batch. 200 | func (parser *TransitionParser) AddDirectives(directives []TransitionDirective) { 201 | for _, dir := range directives { 202 | parser.Add(dir.State, dir.Token, dir.NextState, dir.Flags, dir.Callback) 203 | } 204 | } 205 | 206 | // Add adds a new transition to a given parser state. 207 | func (parser *TransitionParser) Add(stateName string, token string, nextStateName string, flags ParserFlag, callback ParsedFunc) { 208 | nextStateID := parser.GetStateID(nextStateName) 209 | parser.AddTransition(stateName, NewTransition(nextStateID, flags, callback), token) 210 | } 211 | 212 | // Stop adds a stop transition to a given parser state. 213 | func (parser *TransitionParser) Stop(stateName string, token string, flags ParserFlag, callback ParsedFunc) { 214 | parser.AddTransition(stateName, NewTransition(ParserStateStop, flags, callback), token) 215 | } 216 | 217 | // AddTransition adds a transition from a given state to the map 218 | func (parser *TransitionParser) AddTransition(stateName string, newTrans Transition, token string) { 219 | stateID := parser.GetStateID(stateName) 220 | 221 | if state := parser.tokens[stateID]; state == nil { 222 | parser.tokens[stateID] = tcontainer.NewTrie([]byte(token), newTrans) 223 | } else { 224 | parser.tokens[stateID] = state.Add([]byte(token), newTrans) 225 | } 226 | } 227 | 228 | // Parse starts parsing at a given stateID. 229 | // This function returns the remaining parts of data that did not match a 230 | // transition as well as the last state the parser has been set to. 231 | func (parser *TransitionParser) Parse(data []byte, state string) ([]byte, ParserStateID) { 232 | currentStateID := parser.GetStateID(state) 233 | 234 | if currentStateID == ParserStateStop { 235 | return nil, currentStateID // ### return, immediate stop ### 236 | } 237 | 238 | currentState := parser.tokens[currentStateID] 239 | dataLen := len(data) 240 | readStartIdx := 0 241 | continueIdx := 0 242 | 243 | for parseIdx := 0; parseIdx < dataLen; parseIdx++ { 244 | node := currentState.MatchStart(data[parseIdx:]) 245 | if node == nil { 246 | continue // ### continue, no match ### 247 | } 248 | 249 | t := node.Payload.(Transition) 250 | if t.callback != nil { 251 | if t.flags&ParserFlagInclude != 0 { 252 | t.callback(data[readStartIdx:parseIdx+node.PathLen], currentStateID) 253 | } else { 254 | t.callback(data[readStartIdx:parseIdx], currentStateID) 255 | } 256 | } 257 | 258 | // Move the reader 259 | if t.flags&ParserFlagContinue == 0 { 260 | parseIdx += node.PathLen - 1 261 | } else { 262 | parseIdx-- 263 | } 264 | continueIdx = parseIdx + 1 265 | 266 | if t.flags&ParserFlagAppend == 0 { 267 | readStartIdx = continueIdx 268 | } 269 | 270 | nextStateID := t.nextState 271 | 272 | // Pop before push to allow both at the same time 273 | if t.flags&ParserFlagPop != 0 { 274 | stackLen := len(parser.stack) 275 | if stackLen > 0 { 276 | nextStateID = parser.stack[stackLen-1] 277 | parser.stack = parser.stack[:stackLen-1] 278 | } 279 | } 280 | 281 | if t.flags&ParserFlagPush != 0 { 282 | parser.stack = append(parser.stack, currentStateID) 283 | } 284 | 285 | // Transition to next 286 | currentStateID = nextStateID 287 | if currentStateID == ParserStateStop { 288 | break // ### break, stop state ### 289 | } 290 | 291 | currentState = parser.tokens[currentStateID] 292 | } 293 | 294 | if readStartIdx == dataLen { 295 | return nil, currentStateID // ### return, everything parsed ### 296 | } 297 | 298 | return data[readStartIdx:], currentStateID 299 | } 300 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tstrings/strings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tstrings 16 | 17 | import ( 18 | "encoding/json" 19 | "fmt" 20 | "math" 21 | "reflect" 22 | "strconv" 23 | "strings" 24 | "unsafe" 25 | ) 26 | 27 | var simpleEscapeChars = strings.NewReplacer("\\n", "\n", "\\r", "\r", "\\t", "\t", "\\f", "\f", "\\b", "\b") 28 | var jsonEscapeChars = strings.NewReplacer("\\", "\\\\", "\"", "\\\"", "\\n", "\n", "\\r", "\r", "\\t", "\t", "\\f", "\f", "\\b", "\b") 29 | 30 | // Unescape replaces occurrences of \\n, \\r and \\t with real escape codes. 31 | func Unescape(text string) string { 32 | return simpleEscapeChars.Replace(text) 33 | } 34 | 35 | // EscapeJSON replaces occurrences of \ and " with escaped versions. 36 | func EscapeJSON(text string) string { 37 | return jsonEscapeChars.Replace(text) 38 | } 39 | 40 | // ItoLen returns the length of an unsingned integer when converted to a string 41 | func ItoLen(number uint64) int { 42 | switch { 43 | case number < 10: 44 | return 1 45 | default: 46 | return int(math.Log10(float64(number)) + 1) 47 | } 48 | } 49 | 50 | // Itob writes an unsigned integer to the start of a given byte buffer. 51 | func Itob(number uint64, buffer []byte) error { 52 | numberLen := ItoLen(number) 53 | bufferLen := len(buffer) 54 | 55 | if numberLen > bufferLen { 56 | return fmt.Errorf("Number too large for buffer") 57 | } 58 | 59 | for i := numberLen - 1; i >= 0; i-- { 60 | buffer[i] = '0' + byte(number%10) 61 | number /= 10 62 | } 63 | 64 | return nil 65 | } 66 | 67 | // Itobe writes an unsigned integer to the end of a given byte buffer. 68 | func Itobe(number uint64, buffer []byte) error { 69 | for i := len(buffer) - 1; i >= 0; i-- { 70 | buffer[i] = '0' + byte(number%10) 71 | number /= 10 72 | 73 | // Check here because the number 0 has to be written, too 74 | if number == 0 { 75 | return nil 76 | } 77 | } 78 | 79 | return fmt.Errorf("Number too large for buffer") 80 | } 81 | 82 | // Btoi is a fast byte buffer to unsigned int parser that reads until the first 83 | // non-number character is found. It returns the number value as well as the 84 | // length of the number string encountered. 85 | // If a number could not be parsed the returned length will be 0 86 | func Btoi(buffer []byte) (uint64, int) { 87 | number := uint64(0) 88 | index := 0 89 | bufferLen := len(buffer) 90 | 91 | for index < bufferLen && buffer[index] >= '0' && buffer[index] <= '9' { 92 | parsed := uint64(buffer[index] - '0') 93 | number = number*10 + parsed 94 | index++ 95 | } 96 | 97 | return number, index 98 | } 99 | 100 | // IndexN returns the nth occurrences of sep in s or -1 if there is no nth 101 | // occurrence of sep. 102 | func IndexN(s, sep string, n int) int { 103 | sepIdx := 0 104 | for i := 0; i < n; i++ { 105 | nextIdx := strings.Index(s[sepIdx:], sep) 106 | if nextIdx == -1 { 107 | return -1 // ### return, not found ### 108 | } 109 | sepIdx += nextIdx + 1 110 | } 111 | return sepIdx - 1 112 | } 113 | 114 | // LastIndexN returns the nth occurrence of sep in s or -1 if there is no nth 115 | // occurrence of sep. Searching is going from the end of the string to the start. 116 | func LastIndexN(s, sep string, n int) int { 117 | if n == 0 { 118 | return -1 // ### return, nonsense ### 119 | } 120 | sepIdx := len(s) 121 | for i := 0; i < n; i++ { 122 | sepIdx = strings.LastIndex(s[:sepIdx], sep) 123 | if sepIdx == -1 { 124 | return -1 // ### return, not found ### 125 | } 126 | } 127 | return sepIdx 128 | } 129 | 130 | // IsInt returns true if the given string can be converted to an integer. 131 | // The string must contain no whitespaces. 132 | func IsInt(s string) bool { 133 | for i, c := range s { 134 | if (c < '0' || c > '9') && (c != '-' || i != 0) { 135 | return false 136 | } 137 | } 138 | return true 139 | } 140 | 141 | // IsJSON returns true if the given byte slice contains valid JSON data. 142 | // You can access the results by utilizing the RawMessage returned. 143 | func IsJSON(data []byte) (bool, error, *json.RawMessage) { 144 | delayedResult := new(json.RawMessage) 145 | err := json.Unmarshal(data, delayedResult) 146 | return err == nil, err, delayedResult 147 | 148 | } 149 | 150 | // ByteRef is a standard byte slice that is referencing a string. 151 | // This type is considered unsafe and should only be used for fast conversion 152 | // in case of read-only string access via []byte. 153 | type ByteRef []byte 154 | 155 | // NewByteRef creates an unsafe byte slice reference to the given string. 156 | // The referenced data is not guaranteed to stay valid if no reference to the 157 | // original string is held anymore. The returned ByteRef does not count as 158 | // reference. 159 | // Writing to the returned ByteRef will result in a segfault. 160 | func NewByteRef(a string) ByteRef { 161 | stringHeader := (*reflect.StringHeader)(unsafe.Pointer(&a)) 162 | sliceHeader := &reflect.SliceHeader{ 163 | Data: stringHeader.Data, 164 | Len: stringHeader.Len, 165 | Cap: stringHeader.Len, 166 | } 167 | return *(*ByteRef)(unsafe.Pointer(sliceHeader)) 168 | } 169 | 170 | // StringRef is a standard string that is referencing the data of a byte slice. 171 | // The string changes if the referenced byte slice changes. If the referenced 172 | // byte slice changes size this data might get invalid and result in segfaults. 173 | // As of that, this type is to be considered unsafe and should only be used 174 | // in special, performance critical situations. 175 | type StringRef string 176 | 177 | // NewStringRef creates a unsafe string reference to to the given byte slice. 178 | // The referenced data is not guaranteed to stay valid if no reference to the 179 | // original byte slice is held anymore. The returned StringRef does not count as 180 | // reference. 181 | // Changing the size of the underlying byte slice will result in a segfault. 182 | func NewStringRef(a []byte) StringRef { 183 | sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&a)) 184 | stringHeader := &reflect.StringHeader{ 185 | Data: sliceHeader.Data, 186 | Len: sliceHeader.Len, 187 | } 188 | return *(*StringRef)(unsafe.Pointer(stringHeader)) 189 | } 190 | 191 | // JoinStringers works like strings.Join but takes an array of fmt.Stringer. 192 | func JoinStringers(a []fmt.Stringer, sep string) string { 193 | if len(a) == 0 { 194 | return "" 195 | } 196 | if len(a) == 1 { 197 | return a[0].String() 198 | } 199 | 200 | str := make([]string, len(a)) 201 | for i, s := range a { 202 | str[i] = s.String() 203 | } 204 | 205 | return strings.Join(str, sep) 206 | } 207 | 208 | // TrimToNumber removes all characters from the left and right of the string 209 | // that are not in the set [\-0-9] on the left or [0-9] on the right 210 | func TrimToNumber(text string) string { 211 | leftTrimmed := strings.TrimLeftFunc(text, func(r rune) bool { 212 | return (r < '0' || r > '9') && r != '-' 213 | }) 214 | 215 | return strings.TrimRightFunc(leftTrimmed, func(r rune) bool { 216 | return r < '0' || r > '9' 217 | }) 218 | } 219 | 220 | // GetNumBase scans the given string for its numeric base. If num starts with 221 | // '0x' base 16 is assumed. Just '0' assumes base 8. 222 | func GetNumBase(num string) (string, int) { 223 | switch { 224 | case len(num) == 0: 225 | return num, 10 226 | 227 | case len(num) > 1 && num[0] == '0': 228 | if num[1] == 'x' { 229 | return num[2:], 16 230 | } 231 | return num[1:], 8 232 | } 233 | return num, 10 234 | } 235 | 236 | // AtoI64 converts a numeric string to an int64, using GetNumBase to detect 237 | // the numeric base for conversion. 238 | func AtoI64(num string) (int64, error) { 239 | if len(num) == 0 { 240 | return 0, nil 241 | } 242 | n, b := GetNumBase(num) 243 | return strconv.ParseInt(n, b, 64) 244 | } 245 | 246 | // AtoU64 converts a numeric string to an uint64, using GetNumBase to detect 247 | // the numeric base for conversion. 248 | func AtoU64(num string) (uint64, error) { 249 | if len(num) == 0 { 250 | return 0, nil 251 | } 252 | n, b := GetNumBase(num) 253 | return strconv.ParseUint(n, b, 64) 254 | } 255 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "github.com/trivago/tgo/terrors" 19 | ) 20 | 21 | // LockedError is returned when an item has been encountered as locked 22 | type LockedError terrors.SimpleError 23 | 24 | func (err LockedError) Error() string { 25 | return err.Error() 26 | } 27 | 28 | // TimeoutError is returned when a function returned because of a timeout 29 | type TimeoutError terrors.SimpleError 30 | 31 | func (err TimeoutError) Error() string { 32 | return err.Error() 33 | } 34 | 35 | // LimitError is returned when a datastructure reached its limit 36 | type LimitError terrors.SimpleError 37 | 38 | func (err LimitError) Error() string { 39 | return err.Error() 40 | } 41 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/fuse.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "sync" 19 | "sync/atomic" 20 | ) 21 | 22 | // Fuse is a local circuit breaker implementation that is ment to be used to 23 | // manage the state of a given resource between different threads of execution 24 | // (consumer/producer). If the resource is not available the fuse is "burned". 25 | // Components may now wait on that fuse and are woken as soon as the resource 26 | // becomes available again (the fuse is "activated" again). 27 | type Fuse struct { 28 | signal *sync.Cond 29 | burned *int32 30 | } 31 | 32 | // NewFuse creates a new Fuse and returns it. 33 | // A new fuse is always active. 34 | func NewFuse() *Fuse { 35 | return &Fuse{ 36 | signal: sync.NewCond(new(sync.Mutex)), 37 | burned: new(int32), 38 | } 39 | } 40 | 41 | // IsBurned returns true if the fuse in the "inactive" state 42 | func (fuse Fuse) IsBurned() bool { 43 | return atomic.LoadInt32(fuse.burned) == 1 44 | } 45 | 46 | // Burn sets the fuse back to the "inactive" state. 47 | // An already burned fuse cannot be burned again (call is ignored). 48 | func (fuse *Fuse) Burn() { 49 | atomic.StoreInt32(fuse.burned, 1) 50 | } 51 | 52 | // Activate sets the fuse back to the "running" state. 53 | // An already active fuse cannot be activated again (call is ignored). 54 | func (fuse *Fuse) Activate() { 55 | if atomic.CompareAndSwapInt32(fuse.burned, 1, 0) { 56 | fuse.signal.Broadcast() 57 | } 58 | } 59 | 60 | // Wait blocks until the fuse enters active state. 61 | // Multiple go routines may wait on the same fuse. 62 | func (fuse Fuse) Wait() { 63 | fuse.signal.L.Lock() 64 | defer fuse.signal.L.Unlock() 65 | if fuse.IsBurned() { 66 | fuse.signal.Wait() 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/mutex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "sync/atomic" 19 | ) 20 | 21 | // Mutex is a lightweight, spinner based mutex implementation, extending the 22 | // standard go mutex by the possibility to query the mutexe's state. This state 23 | // is very volatile but can come in handy sometimes. 24 | type Mutex struct { 25 | state *int32 26 | priority SpinPriority 27 | } 28 | 29 | // NewMutex creates a new mutex with the given spin priority used during Lock. 30 | func NewMutex(priority SpinPriority) *Mutex { 31 | return &Mutex{ 32 | state: new(int32), 33 | priority: priority, 34 | } 35 | } 36 | 37 | // Lock blocks (spins) until the lock becomes available 38 | func (m *Mutex) Lock() { 39 | spin := NewSpinner(m.priority) 40 | for !atomic.CompareAndSwapInt32(m.state, 0, 1) { 41 | spin.Yield() 42 | } 43 | } 44 | 45 | // Unlock unblocks one routine waiting on lock. 46 | func (m *Mutex) Unlock() { 47 | atomic.StoreInt32(m.state, 0) 48 | } 49 | 50 | // IsLocked returns the state of this mutex. The result of this function might 51 | // change directly after call so it should only be used in situations where 52 | // this fact is not considered problematic. 53 | func (m *Mutex) IsLocked() bool { 54 | return atomic.LoadInt32(m.state) == 1 55 | } 56 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/queue.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "sync/atomic" 19 | ) 20 | 21 | // Queue implements a multi-producer, multi-consumer, fair, lockfree queue. 22 | // Please note that while this queue is faster than a chan interface{} it 23 | // is not faster than a channel of a specific type due to the overhead 24 | // that interface{} brings. 25 | // 26 | // How it works: 27 | // Given a queue of capacity 7, we have two write indexes "p" and "n" for 28 | // process and next. 29 | // 30 | // reader n/p 31 | // v 32 | // [ |1|2|3| | | | ] 33 | // ^ ^ 34 | // p n 35 | // 36 | // Here threads 1,2 and 3 are writing to slots 1,2,3. When 3 is done 37 | // writing it has to wait for p to move to slot 3. Same for slot 2. 38 | // When 1 is done writing it immediatly moves p to slot 2, making it 39 | // possible for 2 to continue. Same for slot 3. 40 | // While p is still on 1 the "reader n" can be fetched but Pop will 41 | // wait until p has moved on. If "reader n" is done, "reader p" is 42 | // moved in similar fashion as p for writes. 43 | // This implements a FIFO queue for writes and reads and makes sure 44 | // that no incomplete reads or overwrites occur. 45 | type Queue struct { 46 | write queueAccess 47 | read queueAccess 48 | capacity uint64 49 | locked *int32 50 | spin Spinner 51 | items []interface{} 52 | } 53 | 54 | // NewQueue creates a new queue with medium spinning priority 55 | func NewQueue(capacity uint32) Queue { 56 | return NewQueueWithSpinner(capacity, NewSpinner(SpinPriorityMedium)) 57 | } 58 | 59 | // NewQueueWithSpinner allows to set the spinning priority of the queue to 60 | // be created. 61 | func NewQueueWithSpinner(capacity uint32, spinner Spinner) Queue { 62 | return Queue{ 63 | items: make([]interface{}, capacity), 64 | read: newQueueAccess(), 65 | write: newQueueAccess(), 66 | locked: new(int32), 67 | capacity: uint64(capacity), 68 | spin: spinner, 69 | } 70 | } 71 | 72 | // Push adds an item to the queue. This call may block if the queue is full. 73 | // An error is returned when the queue is locked. 74 | func (q *Queue) Push(item interface{}) error { 75 | if atomic.LoadInt32(q.locked) == 1 { 76 | return LockedError{"Queue is locked"} // ### return, closed ### 77 | } 78 | 79 | // Get ticket and slot 80 | ticket := atomic.AddUint64(q.write.next, 1) - 1 81 | slot := ticket % q.capacity 82 | spin := q.spin 83 | 84 | // Wait for pending reads on slot 85 | for ticket-atomic.LoadUint64(q.read.processing) >= q.capacity { 86 | spin.Yield() 87 | } 88 | 89 | q.items[slot] = item 90 | 91 | // Wait for previous writers to finish writing 92 | for ticket != atomic.LoadUint64(q.write.processing) { 93 | spin.Yield() 94 | } 95 | atomic.AddUint64(q.write.processing, 1) 96 | return nil 97 | } 98 | 99 | // Pop removes an item from the queue. This call may block if the queue is 100 | // empty. If the queue is drained Pop() will not block and return nil. 101 | func (q *Queue) Pop() interface{} { 102 | // Drained? 103 | if atomic.LoadInt32(q.locked) == 1 && 104 | atomic.LoadUint64(q.write.processing) == atomic.LoadUint64(q.read.processing) { 105 | return nil // ### return, closed and no items ### 106 | } 107 | 108 | // Get ticket andd slot 109 | ticket := atomic.AddUint64(q.read.next, 1) - 1 110 | slot := ticket % q.capacity 111 | spin := q.spin 112 | 113 | // Wait for slot to be written to 114 | for ticket >= atomic.LoadUint64(q.write.processing) { 115 | spin.Yield() 116 | // Drained? 117 | if atomic.LoadInt32(q.locked) == 1 && 118 | atomic.LoadUint64(q.write.processing) == atomic.LoadUint64(q.read.processing) { 119 | return nil // ### return, closed while spinning ### 120 | } 121 | } 122 | 123 | item := q.items[slot] 124 | 125 | // Wait for other reads to finish 126 | for ticket != atomic.LoadUint64(q.read.processing) { 127 | spin.Yield() 128 | } 129 | atomic.AddUint64(q.read.processing, 1) 130 | return item 131 | } 132 | 133 | // Close blocks the queue from write access. It also allows Pop() to return 134 | // false as a second return value 135 | func (q *Queue) Close() { 136 | atomic.StoreInt32(q.locked, 1) 137 | } 138 | 139 | // Reopen unblocks the queue to allow write access again. 140 | func (q *Queue) Reopen() { 141 | atomic.StoreInt32(q.locked, 0) 142 | } 143 | 144 | // IsClosed returns true if Close() has been called. 145 | func (q *Queue) IsClosed() bool { 146 | return atomic.LoadInt32(q.locked) == 1 147 | } 148 | 149 | // IsEmpty returns true if there is no item in the queue to be processed. 150 | // Please note that this state is extremely volatile unless IsClosed 151 | // returned true. 152 | func (q *Queue) IsEmpty() bool { 153 | return atomic.LoadUint64(q.write.processing) == atomic.LoadUint64(q.read.processing) 154 | } 155 | 156 | // IsDrained combines IsClosed and IsEmpty. 157 | func (q *Queue) IsDrained() bool { 158 | return q.IsClosed() && q.IsEmpty() 159 | } 160 | 161 | // Queue access encapsulates the two-index-access pattern for this queue. 162 | // If one or both indices overflow there will be errors. This happens after 163 | // 18 * 10^18 writes aka never if you are not doing more than 10^11 writes 164 | // per second (overflow after ~694 days). 165 | // Just to put this into perspective - an Intel Core i7 5960X at 3.5 Ghz can 166 | // do 336 * 10^9 ops per second (336ops/ns). A call to push allone costs about 167 | // 600-700µs so you won't come anywhere close to this. 168 | type queueAccess struct { 169 | processing *uint64 170 | next *uint64 171 | } 172 | 173 | func newQueueAccess() queueAccess { 174 | return queueAccess{ 175 | processing: new(uint64), 176 | next: new(uint64), 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/run.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "time" 19 | ) 20 | 21 | func AbortAfter(t time.Duration, routine func()) bool { 22 | cmd := make(chan struct{}) 23 | go func() { 24 | routine() 25 | close(cmd) 26 | }() 27 | 28 | select { 29 | case <-cmd: 30 | return true 31 | case <-time.After(t): 32 | return false 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/spinner.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "runtime" 19 | "time" 20 | ) 21 | 22 | // Spinner is a helper struct for spinning loops. 23 | type Spinner struct { 24 | count uint32 25 | suspendFor time.Duration 26 | } 27 | 28 | // SpinPriority is used for Spinner priority enum values 29 | type SpinPriority uint32 30 | 31 | const ( 32 | // SpinPrioritySuspend should be used for spinning loops that are expected 33 | // to wait for very long periods of time. The loop will sleep for 1 second 34 | // after 100 iterations. 35 | SpinPrioritySuspend = SpinPriority(iota) 36 | 37 | // SpinPriorityLow should be used for spinning loops that are expected to 38 | // spin for a rather long time before being able to exit. 39 | // After 100 loops the caller waits for 100 milliseconds. 40 | SpinPriorityLow = SpinPriority(iota) 41 | 42 | // SpinPriorityMedium should be used for spinning loops that are expected to 43 | // spin for a short amount of time before being able to exit. 44 | // After 100 loops the caller waits for 1 millisecond. 45 | SpinPriorityMedium = SpinPriority(iota) 46 | 47 | // SpinPriorityHigh should be used for spinning loops that are expected to 48 | // almost never spin. 49 | // After 100 loops the caller waits for 10 microseconds. 50 | SpinPriorityHigh = SpinPriority(iota) 51 | 52 | // SpinPriorityRealtime should be used for loops that need to run as fast 53 | // as possible. After 100 loops the go scheduler is triggered. 54 | SpinPriorityRealtime = SpinPriority(iota) 55 | ) 56 | 57 | var ( 58 | spinDelay = []time.Duration{ 59 | time.Second, // SpinPrioritySuspend 60 | 100 * time.Millisecond, // SpinPriorityLow 61 | time.Millisecond, // SpinPriorityMedium 62 | 10 * time.Microsecond, // SpinPriorityHigh 63 | time.Duration(0), // SpinPriorityRealtime 64 | } 65 | ) 66 | 67 | // NewSpinner creates a new helper for spinning loops 68 | func NewSpinner(priority SpinPriority) Spinner { 69 | return Spinner{ 70 | count: 0, 71 | suspendFor: spinDelay[priority], 72 | } 73 | } 74 | 75 | // NewCustomSpinner creates a new spinner with a custom delay. 76 | func NewCustomSpinner(suspendFor time.Duration) Spinner { 77 | return Spinner{ 78 | count: 0, 79 | suspendFor: suspendFor, 80 | } 81 | } 82 | 83 | // Yield should be called in spinning loops and will assure correct 84 | // spin/wait/schedule behavior according to the set priority. 85 | func (spin *Spinner) Yield() { 86 | if spin.count >= 100 { 87 | spin.count = 0 88 | // Always call Gosched if suspending is disabled to prevent stuck go 89 | // routines with GOMAXPROCS=1 and to be nice to the scheduler 90 | if spin.suspendFor == 0 { 91 | runtime.Gosched() 92 | } else { 93 | time.Sleep(spin.suspendFor) 94 | } 95 | return // ### return, suspended ### 96 | } 97 | 98 | spin.count++ 99 | 100 | } 101 | 102 | // Reset sets the internal counter back to 0 103 | func (spin *Spinner) Reset() { 104 | spin.count = 0 105 | } 106 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/stack.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "github.com/trivago/tgo/tmath" 19 | "sync/atomic" 20 | ) 21 | 22 | // Stack implements a simple, growing, lockfree stack. 23 | // The main idea is to use the sign bit of the head index as a mutex. 24 | // If the index is negative, the stack is locked so we need to spin. 25 | // If the index is non-negative the stack is unlocked and we can write or read. 26 | type Stack struct { 27 | data []interface{} 28 | growBy int 29 | head *uint32 30 | spin Spinner 31 | } 32 | 33 | const unlockMask = uint32(0x7FFFFFFF) 34 | const lockMask = uint32(0x80000000) 35 | 36 | // NewStack creates a new stack with the given initial size. 37 | // The given size will also be used as grow size. 38 | // SpinPriorityMedium is used to initialize the spinner. 39 | func NewStack(size int) Stack { 40 | return NewStackWithSpinnerAndGrowSize(size, size, NewSpinner(SpinPriorityMedium)) 41 | } 42 | 43 | // NewStackWithGrowSize allows to pass a custom grow size to the stack. 44 | // SpinPriorityMedium is used to initialize the spinner. 45 | func NewStackWithGrowSize(size, grow int) Stack { 46 | return NewStackWithSpinnerAndGrowSize(size, grow, NewSpinner(SpinPriorityMedium)) 47 | } 48 | 49 | // NewStackWithSpinner allows to pass a custom spinner to the stack. 50 | // The given size will also be used as grow size. 51 | func NewStackWithSpinner(size int, spin Spinner) Stack { 52 | return NewStackWithSpinnerAndGrowSize(size, size, spin) 53 | } 54 | 55 | // NewStackWithSpinnerAndGrowSize allows to fully configure the new stack. 56 | func NewStackWithSpinnerAndGrowSize(size, grow int, spin Spinner) Stack { 57 | return Stack{ 58 | data: make([]interface{}, tmath.MinI(size, 1)), 59 | growBy: tmath.MinI(grow, 1), 60 | head: new(uint32), 61 | spin: spin, 62 | } 63 | } 64 | 65 | // Len returns the number of elements on the stack. 66 | // Please note that this value can be highly unreliable in multithreaded 67 | // environments as this is only a snapshot of the state at calltime. 68 | func (s *Stack) Len() int { 69 | return int(atomic.LoadUint32(s.head) & unlockMask) 70 | } 71 | 72 | // Pop retrieves the topmost element from the stack. 73 | // A LimitError is returned when the stack is empty. 74 | func (s *Stack) Pop() (interface{}, error) { 75 | spin := s.spin 76 | for { 77 | head := atomic.LoadUint32(s.head) 78 | unlockedHead := head & unlockMask 79 | lockedHead := head | lockMask 80 | 81 | // Always work with unlocked head as head might be locked 82 | if unlockedHead == 0 { 83 | return nil, LimitError{"Stack is empty"} 84 | } 85 | 86 | if atomic.CompareAndSwapUint32(s.head, unlockedHead, lockedHead) { 87 | data := s.data[unlockedHead-1] // copy data 88 | atomic.StoreUint32(s.head, unlockedHead-1) // unlock 89 | return data, nil // ### return ### 90 | } 91 | 92 | spin.Yield() 93 | } 94 | } 95 | 96 | // Push adds an element to the top of the stack. 97 | // When the stack's capacity is reached the storage grows as defined during 98 | // construction. If the stack reaches 2^31 elements it is considered full 99 | // and will return an LimitError. 100 | func (s *Stack) Push(v interface{}) error { 101 | spin := s.spin 102 | for { 103 | head := atomic.LoadUint32(s.head) 104 | unlockedHead := head & unlockMask 105 | lockedHead := head | lockMask 106 | 107 | // Always work with unlocked head as head might be locked 108 | if unlockedHead == unlockMask { 109 | return LimitError{"Stack is full"} 110 | } 111 | 112 | if atomic.CompareAndSwapUint32(s.head, unlockedHead, lockedHead) { 113 | if unlockedHead == uint32(len(s.data)) { 114 | // Grow stack 115 | old := s.data 116 | s.data = make([]interface{}, len(s.data)+s.growBy) 117 | copy(s.data, old) 118 | } 119 | 120 | s.data[unlockedHead] = v // write to new head 121 | atomic.StoreUint32(s.head, unlockedHead+1) // unlock 122 | return nil // ### return ### 123 | } 124 | 125 | spin.Yield() 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /vendor/github.com/trivago/tgo/tsync/waitgroup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 trivago GmbH 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tsync 16 | 17 | import ( 18 | "sync/atomic" 19 | "time" 20 | ) 21 | 22 | // WaitGroup is a replacement for sync/waitgroup that allows the internal counter 23 | // to go negative. This version allows a missed Done to be recovered but will 24 | // make it a lot harder to detect missing Done or Add calls. 25 | // Use only where needed. 26 | type WaitGroup struct { 27 | counter int32 28 | } 29 | 30 | // Active returns true if the counter is > 0 31 | func (wg *WaitGroup) Active() bool { 32 | return atomic.LoadInt32(&wg.counter) > 0 33 | } 34 | 35 | // Inc is the shorthand version for Add(1) 36 | func (wg *WaitGroup) Inc() { 37 | atomic.AddInt32(&wg.counter, 1) 38 | } 39 | 40 | // Add increments the waitgroup counter by the given value. 41 | // Delta may be negative. 42 | func (wg *WaitGroup) Add(delta int) { 43 | atomic.AddInt32(&wg.counter, int32(delta)) 44 | } 45 | 46 | // Done is the shorthand version for Add(-1) 47 | func (wg *WaitGroup) Done() { 48 | atomic.AddInt32(&wg.counter, -1) 49 | } 50 | 51 | // Reset sets the counter to 0 52 | func (wg *WaitGroup) Reset() { 53 | atomic.StoreInt32(&wg.counter, 0) 54 | } 55 | 56 | // IncWhenDone wait until the counter is exactly 0 and triggeres an increment 57 | // if this is found to be true 58 | func (wg *WaitGroup) IncWhenDone() { 59 | spin := NewSpinner(SpinPriorityHigh) 60 | for !atomic.CompareAndSwapInt32(&wg.counter, 0, 1) { 61 | spin.Yield() 62 | } 63 | } 64 | 65 | // Wait blocks until the counter is 0 or less. 66 | func (wg *WaitGroup) Wait() { 67 | spin := NewSpinner(SpinPriorityHigh) 68 | for wg.Active() { 69 | spin.Yield() 70 | } 71 | } 72 | 73 | // WaitFor blocks until the counter is 0 or less. If the block takes longer than 74 | // the given timeout, WaitFor will return false. If duration is 0, Wait is called. 75 | func (wg *WaitGroup) WaitFor(timeout time.Duration) bool { 76 | if timeout == time.Duration(0) { 77 | wg.Wait() 78 | return true // ### return, always true ### 79 | } 80 | 81 | start := time.Now() 82 | spin := NewSpinner(SpinPriorityHigh) 83 | for wg.Active() { 84 | if time.Since(start) > timeout { 85 | return false // ### return, timed out ### 86 | } 87 | spin.Yield() 88 | } 89 | return true 90 | } 91 | --------------------------------------------------------------------------------