├── .gitignore ├── LICENSE.txt ├── README.md ├── command ├── arguments.go ├── command.go ├── connstats.go ├── debug.go ├── factory.go ├── filter.go ├── info.go ├── nodebug.go ├── query.go └── restart.go ├── internal ├── bitmask.go ├── dates.go ├── errors.go ├── regex.go ├── runes.go ├── runes_test.go └── strings.go ├── mgotools.go ├── mongo ├── json.go ├── json_test.go ├── pattern.go ├── pattern_test.go ├── sorter │ ├── key.go │ ├── key_test.go │ └── logical.go └── types.go ├── parser ├── common.go ├── db_24.go ├── db_26.go ├── db_30.go ├── db_32.go ├── db_34.go ├── db_36.go ├── db_40.go ├── db_42.go ├── executor │ ├── executor.go │ └── executor_test.go ├── helpers.go ├── helpers_test.go ├── message │ ├── methods.go │ └── types.go ├── mongod.go ├── mongod_test.go ├── mongos.go ├── record │ ├── base.go │ ├── constants.go │ ├── constants_test.go │ └── entry.go ├── s_24.go ├── s_26.go ├── s_30.go ├── s_32.go ├── s_34.go ├── s_36.go ├── s_40.go ├── s_42.go ├── source │ ├── accumulator.go │ ├── accumulator_test.go │ ├── factory.go │ ├── log.go │ └── log_test.go └── version │ ├── debug.go │ ├── instance.go │ ├── manager.go │ ├── types.go │ └── types_test.go └── target ├── formatting ├── pattern.go └── summary.go └── stdout.go /.gitignore: -------------------------------------------------------------------------------- 1 | mgotools 2 | *.log 3 | *.js 4 | *.pl 5 | *.out 6 | *.a 7 | .idea 8 | Logs 9 | *.txt 10 | !LICENSE.txt 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mgotools 2 | 3 | This tool suite is the spiritual successor to 4 | [mtools](https://github.com/rueckstiess/mtools), a series of python 5 | libraries designed to quickly parse and summarize 6 | [MongoDB](https://www.mongodb.com/) log files. 7 | 8 | MongoDB log files are full of useful information but also complicated 9 | to parse through quickly. These tools are designed to easily filter 10 | through large files quickly. They're designed to be extendable so 11 | other projects can incorporate the binaries. 12 | 13 | ## Usage 14 | There is one binary currently generated. The plan is to build drop-in 15 | binary replacements for mloginfo and mlogfilter. Until then, 16 | sub-commands are available. 17 | 18 | Each tool may take input from _stdin_ or log files. 19 | ``` 20 | > cat mongod.log | mgotools query 21 | > mgotools query mongod.log 22 | ``` 23 | 24 | Additionally, some command line arguments may be passed multiple times to apply 25 | to multiple log files. For example, `mgotools filter --from 2019-01-01 --from 2018-01-01 mongod1.log mongod2.log` 26 | 27 | In this example, the first `from` argument applies to `mongod1.log` and the 28 | second `from` argument applies to `mongod2.log`. 29 | 30 | ### filter 31 | `./mgotools filter --help` 32 | 33 | ### info 34 | `./mgotools info --help` 35 | 36 | ### queries 37 | `./mgotools query --help` 38 | 39 | The `query` command aggregates the canonicalized version 40 | 41 | ### connstats 42 | `./mgotools connstats --help` 43 | 44 | ### restart 45 | `./mgotools restart --help` 46 | 47 | ## Build 48 | The build process should be straightforward. Running the following commands 49 | should work on properly configured Go environments: 50 | ```bash 51 | > cd $GOPATH/src 52 | > git clone https://github.com/jtv4k/mgotools 53 | > cd mgotools 54 | > go get 55 | > go build 56 | ``` 57 | 58 | A binary named `mgotools` will be generated that can be executed using `./mgotools`. 59 | -------------------------------------------------------------------------------- /command/arguments.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type Flag int 9 | 10 | const ( 11 | Bool Flag = iota 12 | Int 13 | IntSourceSlice 14 | String 15 | StringSourceSlice 16 | ) 17 | 18 | type Argument struct { 19 | Name string 20 | ShortName string 21 | Usage string 22 | Type Flag 23 | } 24 | 25 | type ArgumentCollection struct { 26 | Booleans map[string]bool 27 | Integers map[string]int 28 | Strings map[string]string 29 | } 30 | 31 | func MakeCommandArgumentCollection(index int, args map[string]interface{}, cmd Definition) (ArgumentCollection, error) { 32 | var ( 33 | argsBool = make(map[string]bool) 34 | argsInt = make(map[string]int) 35 | argsString = make(map[string]string) 36 | ) 37 | 38 | for _, argument := range cmd.Flags { 39 | input, ok := args[argument.Name] 40 | if ok { 41 | switch argument.Type { 42 | case Bool: 43 | argsBool[argument.Name] = input.(bool) 44 | case Int: 45 | argsInt[argument.Name] = input.(int) 46 | case IntSourceSlice: 47 | values := input.([]int) 48 | switch { 49 | case len(values) == 1: 50 | argsInt[argument.Name] = values[0] 51 | case index >= len(values): 52 | return ArgumentCollection{}, fmt.Errorf("--%s must appear for each file", argument.Name) 53 | default: 54 | argsInt[argument.Name] = values[index] 55 | } 56 | case String: 57 | argsString[argument.Name] = strings.Join(input.([]string), " ") 58 | case StringSourceSlice: 59 | // multiple strings apply to each log individually 60 | values := input.([]string) 61 | switch { 62 | case len(values) == 1: 63 | argsString[argument.Name] = values[0] 64 | case index >= len(values): 65 | return ArgumentCollection{}, fmt.Errorf("--%s must appear for each file", argument.Name) 66 | default: 67 | argsString[argument.Name] = values[index] 68 | } 69 | } 70 | } 71 | } 72 | 73 | return ArgumentCollection{argsBool, argsInt, argsString}, nil 74 | } 75 | -------------------------------------------------------------------------------- /command/command.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "sync" 9 | 10 | "mgotools/parser/record" 11 | "mgotools/parser/source" 12 | ) 13 | 14 | type commandSource <-chan record.Base 15 | type commandTarget chan<- string 16 | type commandError chan<- error 17 | 18 | type Input struct { 19 | Arguments ArgumentCollection 20 | Name string 21 | Length int64 22 | Reader source.Factory 23 | } 24 | 25 | type Output struct { 26 | Writer io.WriteCloser 27 | Error io.WriteCloser 28 | } 29 | 30 | type Command interface { 31 | Finish(int, commandTarget) error 32 | Prepare(string, int, ArgumentCollection) error 33 | Run(int, commandTarget, commandSource, commandError) error 34 | Terminate(commandTarget) error 35 | } 36 | 37 | // A method for preparing all the bytes and pieces to pass along to the next step. 38 | func RunCommand(f Command, in []Input, out Output) error { 39 | var ( 40 | // Keep a count of all contexts provided as input. 41 | count = len(in) 42 | 43 | // A multi-directional fan-out channel for catching and passing along errors. 44 | errorChannel = make(chan error) 45 | 46 | // An output channel that will facilitate moving data from commands to the output handle. 47 | outputChannel = make(chan string) 48 | 49 | // A way to synchronize multiple goroutines. 50 | processSync sync.WaitGroup 51 | 52 | // A sync for multiple output handles (out, err) 53 | outputSync sync.WaitGroup 54 | 55 | // Create a helper to write to the output handle. 56 | outputWriter = bufio.NewWriter(out.Writer) 57 | 58 | // Create a helper to write to the error handle. 59 | errorWriter = bufio.NewWriter(out.Error) 60 | ) 61 | 62 | // Always flush the output at the end of execution. 63 | defer outputWriter.Flush() 64 | defer errorWriter.Flush() 65 | 66 | if len(in) == 0 || out.Error == nil || out.Writer == nil { 67 | return errors.New("an input and output handler are required") 68 | } 69 | 70 | // Pass each file and its information to the command so it can prepare. 71 | for index, handle := range in { 72 | if err := f.Prepare(handle.Name, index, handle.Arguments); err != nil { 73 | return err 74 | } 75 | } 76 | 77 | // Synchronize the several goroutines created in this method. 78 | processSync.Add(count) 79 | 80 | // There are two output syncs to wait for: output and errors. 81 | outputSync.Add(2) 82 | 83 | // Initiate a goroutine to wait for a single error and signal all other input parsing routines. 84 | go func() { 85 | // Wait for errors (or for the channel to close). 86 | defer outputSync.Done() 87 | 88 | for recv := range errorChannel { 89 | errorWriter.WriteString(recv.Error() + "\n") 90 | errorWriter.Flush() 91 | } 92 | }() 93 | 94 | go func() { 95 | // Create another goroutine for outputs. Start checking for output from the several input goroutines. 96 | // Output all received values directly (this may need to change in the future, i.e. should sorting be needed). 97 | defer outputSync.Done() 98 | 99 | for line := range outputChannel { 100 | outputWriter.WriteString(line + "\n") 101 | } 102 | }() 103 | 104 | // Finally, a new goroutine is needed for each individual input file handle. 105 | for i := 0; i < count; i += 1 { 106 | go func(index int) { 107 | // Signal that this file is complete. 108 | defer processSync.Done() 109 | 110 | // Start a goroutine to wait each input file handle to finish processing. 111 | run(f, index, in[index].Reader, outputChannel, errorChannel) 112 | 113 | // Collect any final errors and send them along. 114 | if err := f.Finish(index, outputChannel); err != nil { 115 | errorChannel <- err 116 | } 117 | }(i) 118 | } 119 | 120 | // Wait for all input goroutines to finish. 121 | processSync.Wait() 122 | 123 | // Allow the command to finalize any pending actions. 124 | f.Terminate(outputChannel) 125 | 126 | // Finalize the output processes by closing the out channel. 127 | close(outputChannel) 128 | close(errorChannel) 129 | 130 | // Wait for all output goroutines to finish. 131 | outputSync.Wait() 132 | 133 | return nil 134 | } 135 | 136 | func run(f Command, index int, in source.Factory, outputChannel chan<- string, errorChannel chan<- error) { 137 | var inputChannel = make(chan record.Base, 1024) 138 | var inputWaitGroup sync.WaitGroup 139 | 140 | // Count the number of goroutines that must complete before returning. 141 | inputWaitGroup.Add(2) 142 | 143 | go func() { 144 | // Decrement the wait group. 145 | defer inputWaitGroup.Done() 146 | 147 | // Close channels that will no longer be used after this method 148 | // exists (and signal any pending goroutines). 149 | defer close(inputChannel) 150 | 151 | for in.Next() { 152 | base, err := in.Get() 153 | if err == io.EOF { 154 | panic("eof error received before channel close") 155 | } else if err != nil { 156 | errorChannel <- fmt.Errorf("line %d: %s", base.LineNumber, err.Error()) 157 | } else { 158 | inputChannel <- base 159 | } 160 | } 161 | }() 162 | 163 | // Delegate line parsing to the individual commands. 164 | go func() { 165 | defer inputWaitGroup.Done() 166 | 167 | // Close the input file handle. 168 | defer in.Close() 169 | 170 | // Begin running the command. 171 | if err := f.Run(index, outputChannel, inputChannel, errorChannel); err != nil { 172 | errorChannel <- err 173 | } 174 | }() 175 | 176 | // Wait for both goroutines to complete. 177 | inputWaitGroup.Wait() 178 | return 179 | } 180 | -------------------------------------------------------------------------------- /command/factory.go: -------------------------------------------------------------------------------- 1 | // A command factory is an easy way of adding new commands without modifying 2 | // the top level code. It uses registration in `init()` methods to add 3 | // commands to a singleton, keeping command setup and initialization coupled 4 | // to the command code. 5 | 6 | package command 7 | 8 | import ( 9 | "mgotools/internal" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Definition struct { 15 | Usage string 16 | Flags []Argument 17 | } 18 | 19 | type factory struct { 20 | registry map[string]factoryDefinition 21 | } 22 | type factoryDefinition struct { 23 | args Definition 24 | create func() (Command, error) 25 | command Command 26 | } 27 | 28 | var instance = &factory{ 29 | registry: make(map[string]factoryDefinition), 30 | } 31 | 32 | func GetFactory() *factory { 33 | return instance 34 | } 35 | 36 | func (c *factory) GetNames() []string { 37 | keys := make([]string, len(c.registry)) 38 | index := 0 39 | for key := range c.registry { 40 | keys[index] = key 41 | index += 1 42 | } 43 | return keys 44 | } 45 | func (c *factory) Get(name string) (Command, error) { 46 | name = internal.StringToLower(name) 47 | reg, ok := c.registry[name] 48 | if !ok { 49 | return nil, errors.New("command not registered") 50 | } 51 | if reg.command == nil { 52 | cmd, err := reg.create() 53 | if err != nil { 54 | return nil, err 55 | } 56 | reg.command = cmd 57 | } 58 | return reg.command, nil 59 | } 60 | func (c *factory) GetDefinition(name string) (Definition, bool) { 61 | name = internal.StringToLower(name) 62 | reg, ok := c.registry[name] 63 | if !ok { 64 | return Definition{}, false 65 | } 66 | return reg.args, ok 67 | } 68 | func (c *factory) Register(name string, args Definition, create func() (Command, error)) { 69 | if name == "" { 70 | panic("empty name registered in the command factory") 71 | } else if create == nil { 72 | panic("command registered without a create method") 73 | } 74 | name = internal.StringToLower(name) 75 | c.registry[name] = factoryDefinition{ 76 | args, 77 | create, 78 | nil, 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /command/info.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "mgotools/internal" 8 | "mgotools/parser/message" 9 | "mgotools/parser/record" 10 | "mgotools/parser/version" 11 | "mgotools/target/formatting" 12 | ) 13 | 14 | type info struct { 15 | outputErrors bool 16 | 17 | Instance map[int]*infoInstance 18 | } 19 | 20 | type infoInstance struct { 21 | context *version.Context 22 | output *bytes.Buffer 23 | Summary formatting.Summary 24 | } 25 | 26 | func init() { 27 | args := Definition{ 28 | Usage: "basic information about a mongodb log", 29 | Flags: []Argument{ 30 | {Name: "errors", ShortName: "v", Type: Bool, Usage: "output parsing errors to stderr"}, 31 | }, 32 | } 33 | 34 | GetFactory().Register("info", args, func() (Command, error) { 35 | return &info{ 36 | outputErrors: false, 37 | 38 | Instance: make(map[int]*infoInstance), 39 | }, nil 40 | }) 41 | } 42 | 43 | func (f *info) Finish(index int, out commandTarget) error { 44 | instance := f.Instance[index] 45 | 46 | if len(instance.Summary.Version) == 0 { 47 | versions := f.Instance[index].context.Versions() 48 | if len(versions) == 1 { 49 | instance.Summary.Version = versions 50 | } 51 | } 52 | 53 | summary := bytes.NewBuffer([]byte{}) 54 | if index > 0 { 55 | instance.Summary.Divider(summary) 56 | } 57 | 58 | instance.Summary.Print(summary) 59 | 60 | _, err := instance.output.WriteTo(summary) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | out <- summary.String() 66 | return nil 67 | } 68 | 69 | func (f *info) Prepare(name string, instance int, args ArgumentCollection) error { 70 | parsers := version.Factory.GetAll() 71 | 72 | f.Instance[instance] = &infoInstance{ 73 | context: version.New(parsers, internal.DefaultDateParser.Clone()), 74 | Summary: formatting.NewSummary(name), 75 | output: bytes.NewBuffer([]byte{}), 76 | } 77 | 78 | if args.Booleans["errors"] { 79 | f.outputErrors = true 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (f *info) Run(index int, _ commandTarget, in commandSource, errs commandError) error { 86 | var exit error 87 | 88 | // Hold a configuration object for future use. 89 | instance := f.Instance[index] 90 | summary := &instance.Summary 91 | 92 | // Keep a separate date parser for quick-and-easy entry handling. 93 | dateParser := internal.DefaultDateParser 94 | 95 | // Clean up context resources. 96 | defer instance.context.Finish() 97 | 98 | iw := newInfoWriter(instance.output) 99 | alert := func(b record.Entry, m string) { 100 | iw.WriteString(b.Date.Format(string(b.Format))) 101 | iw.WriteString(fmt.Sprintf("[line %d]", b.LineNumber)) 102 | iw.WriteString(m) 103 | 104 | if iw.Err() != nil { 105 | exit = iw.Err() 106 | } 107 | } 108 | 109 | for base := range in { 110 | if exit != nil { 111 | // Check for an exit signal (in a worryingly un-atomic way). 112 | return exit 113 | } 114 | 115 | if base.RawContext != "[initandlisten]" { 116 | // The only context we care about for updating the summary is 117 | // "initandlisten" so skipping all other entries will speed things 118 | // up significantly. The summary still needs to be updated since 119 | // it maintains a count. 120 | date, format, err := dateParser.Parse(base.RawDate) 121 | 122 | summary.Update(record.Entry{ 123 | Base: base, 124 | Message: nil, 125 | Date: date, 126 | DateValid: err == nil, 127 | Format: format, 128 | }) 129 | continue 130 | } 131 | 132 | // Grab an entry from the base record. 133 | entry, err := instance.context.NewEntry(base) 134 | 135 | // Directly output errors in the info module. 136 | if f.outputErrors && err != nil { 137 | errs <- err 138 | continue 139 | } 140 | 141 | if !summary.Update(entry) { 142 | continue 143 | } 144 | 145 | if t, ok := entry.Message.(message.StartupInfo); ok { 146 | if summary.Host == t.Hostname && summary.Port > 0 && summary.Port != t.Port { 147 | alert(entry, fmt.Sprintf( 148 | "The server restarted on a new port (%d -> %d).", 149 | summary.Port, t.Port)) 150 | } else if summary.Host != "" && summary.Host != t.Hostname && summary.Port == t.Port { 151 | alert(entry, fmt.Sprintf( 152 | "The server restarted with a new hostname (%s -> %s).", 153 | summary.Host, t.Hostname)) 154 | } else if summary.Host != "" && summary.Port > 0 && summary.Host != t.Hostname && summary.Port != t.Port { 155 | alert(entry, fmt.Sprintf( 156 | "The server restarted with a new hostname (%s -> %s) and port (%d -> %d).", 157 | summary.Host, t.Hostname, summary.Port, t.Port)) 158 | } 159 | } 160 | } 161 | 162 | if len(summary.Version) == 0 { 163 | summary.Guess(instance.context.Versions()) 164 | } 165 | 166 | return nil 167 | } 168 | 169 | func (f *info) Terminate(commandTarget) error { 170 | return nil 171 | } 172 | 173 | type infoWriter struct { 174 | buffer *bytes.Buffer 175 | err error 176 | length int 177 | } 178 | 179 | func newInfoWriter(b *bytes.Buffer) *infoWriter { 180 | return &infoWriter{b, nil, 0} 181 | } 182 | 183 | func (i *infoWriter) WriteString(a string) { 184 | if i.err != nil { 185 | return 186 | } 187 | 188 | if i.length > 0 { 189 | i.err = i.buffer.WriteByte(' ') 190 | if i.err != nil { 191 | return 192 | } 193 | } 194 | 195 | _, i.err = i.buffer.WriteString(a) 196 | return 197 | } 198 | 199 | func (i *infoWriter) Err() error { 200 | i.length = 0 201 | return i.err 202 | } 203 | -------------------------------------------------------------------------------- /command/nodebug.go: -------------------------------------------------------------------------------- 1 | // This file represents the opposite of debug.go by setting a DEBUG constant 2 | // to false when no debugging information is compiled. 3 | // 4 | // +build !debug 5 | 6 | package command 7 | 8 | const DEBUG = false 9 | -------------------------------------------------------------------------------- /command/restart.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "time" 7 | 8 | "mgotools/internal" 9 | "mgotools/parser/message" 10 | "mgotools/parser/version" 11 | "mgotools/target/formatting" 12 | ) 13 | 14 | type restart struct { 15 | instance map[int]*restartInstance 16 | } 17 | 18 | type restartInstance struct { 19 | summary formatting.Summary 20 | restarts []struct { 21 | Date time.Time 22 | Startup message.Version 23 | } 24 | } 25 | 26 | func init() { 27 | GetFactory().Register("restart", Definition{}, func() (Command, error) { 28 | return &restart{make(map[int]*restartInstance)}, nil 29 | }) 30 | } 31 | 32 | func (r *restart) Finish(index int, out commandTarget) error { 33 | instance := r.instance[index] 34 | writer := bytes.NewBuffer([]byte{}) 35 | 36 | instance.summary.Print(writer) 37 | 38 | if len(instance.restarts) == 0 { 39 | out <- " no restarts found" 40 | return nil 41 | } 42 | 43 | writer.WriteRune('\n') 44 | writer.WriteString("RESTARTS\n") 45 | 46 | for _, restart := range r.instance[index].restarts { 47 | writer.WriteString(fmt.Sprintf(" %s %s\n", restart.Date.Format(string(internal.DateFormatCtimenoms)), restart.Startup.String())) 48 | } 49 | 50 | out <- writer.String() 51 | 52 | return nil 53 | } 54 | 55 | func (r *restart) Prepare(name string, index int, _ ArgumentCollection) error { 56 | r.instance[index] = &restartInstance{summary: formatting.NewSummary(name)} 57 | 58 | return nil 59 | } 60 | 61 | func (r *restart) Run(index int, out commandTarget, in commandSource, errors commandError) error { 62 | instance := r.instance[index] 63 | summary := &instance.summary 64 | 65 | // Create a local context object to create record.Entry objects. 66 | context := version.New(version.Factory.GetAll(), internal.DefaultDateParser.Clone()) 67 | defer context.Finish() 68 | 69 | for base := range in { 70 | entry, err := context.NewEntry(base) 71 | if err != nil { 72 | errors <- err 73 | } 74 | 75 | summary.Update(entry) 76 | if entry.Message == nil { 77 | continue 78 | } else if restart, ok := getVersionFromMessage(entry.Message); !ok { 79 | continue 80 | } else { 81 | instance.restarts = append(instance.restarts, struct { 82 | Date time.Time 83 | Startup message.Version 84 | }{Date: entry.Date, Startup: restart}) 85 | } 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func (r *restart) Terminate(commandTarget) error { 92 | return nil 93 | } 94 | 95 | func getVersionFromMessage(msg message.Message) (message.Version, bool) { 96 | if version, ok := msg.(message.Version); ok { 97 | return version, true 98 | } else if version, ok := msg.(message.StartupInfoLegacy); ok { 99 | return version.Version, true 100 | } else { 101 | return message.Version{}, false 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /internal/bitmask.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "encoding/hex" 4 | 5 | type Bitmap [8]byte 6 | 7 | func (bits *Bitmap) IsSet(i int) bool { i -= 1; return bits[i/8]&(1< 0 { 82 | atomic.StoreInt64(&d.pos, int64(offset)) 83 | } 84 | 85 | break 86 | } 87 | } 88 | 89 | return date, picked, err 90 | } 91 | 92 | func IsDay(match string) bool { 93 | return ArrayBinaryMatchString(match, DATE_DAYS) 94 | } 95 | 96 | // Takes string and returns if the string *looks* like an ISO formatted date, 97 | // specifically the format generated by MongoDB (YYYY-MM-DDThh:mm:ss). This 98 | // method ignores any additional information like milliseconds or timezones. 99 | func IsIso8601String(date string) bool { 100 | parts := []rune(date) 101 | 102 | // Use a separate statement for the bound check because combining it 103 | // with the return statement below generates 19 bound checks. 104 | if len(parts) < 19 { 105 | return false 106 | } 107 | 108 | return unicode.IsNumber(parts[0]) && 109 | unicode.IsNumber(parts[1]) && 110 | unicode.IsNumber(parts[2]) && 111 | unicode.IsNumber(parts[3]) && 112 | parts[4] == '-' && 113 | unicode.IsNumber(parts[5]) && 114 | unicode.IsNumber(parts[6]) && 115 | parts[7] == '-' && 116 | unicode.IsNumber(parts[8]) && 117 | unicode.IsNumber(parts[9]) && 118 | parts[10] == 'T' && 119 | unicode.IsNumber(parts[11]) && 120 | unicode.IsNumber(parts[12]) && 121 | parts[13] == ':' && 122 | unicode.IsNumber(parts[14]) && 123 | unicode.IsNumber(parts[15]) && 124 | parts[16] == ':' && 125 | unicode.IsNumber(parts[17]) && 126 | unicode.IsNumber(parts[18]) 127 | } 128 | 129 | func IsMonth(match string) bool { 130 | return ArrayBinaryMatchString(match, DATE_MONTHS) 131 | } 132 | 133 | func IsTime(match string) bool { 134 | // 00:00:00.000 135 | check := []rune(match) 136 | 137 | if len(check) == 8 { 138 | return unicode.IsNumber(check[0]) && 139 | unicode.IsNumber(check[1]) && 140 | check[2] == ':' && 141 | unicode.IsNumber(check[3]) && 142 | unicode.IsNumber(check[4]) && 143 | check[5] == ':' && 144 | unicode.IsNumber(check[6]) && 145 | unicode.IsNumber(check[7]) 146 | } else if len(check) == 12 { 147 | return unicode.IsNumber(check[0]) && 148 | unicode.IsNumber(check[1]) && 149 | check[2] == ':' && 150 | unicode.IsNumber(check[3]) && 151 | unicode.IsNumber(check[4]) && 152 | check[5] == ':' && 153 | unicode.IsNumber(check[6]) && 154 | unicode.IsNumber(check[7]) && 155 | check[8] == '.' && 156 | unicode.IsNumber(check[9]) && 157 | unicode.IsNumber(check[10]) && 158 | unicode.IsNumber(check[11]) 159 | } else { 160 | return false 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /internal/errors.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "errors" 4 | 5 | /* 6 | * General purpose formatting errors. 7 | */ 8 | var CommandNotFound = errors.New("command not found") 9 | var CommandStructure = errors.New("command structure unexpected") 10 | var ComponentUnmatched = errors.New("component unmatched") 11 | var ControlUnrecognized = VersionUnmatched{Message: "unrecognized control message"} 12 | var CounterUnrecognized = VersionUnmatched{Message: "unrecognized counter"} 13 | var MetadataUnmatched = VersionUnmatched{"unexpected connection meta format"} 14 | var MisplacedWordException = VersionUnmatched{"unexpected or misplaced word"} 15 | var NetworkUnrecognized = VersionUnmatched{"unrecognized network message"} 16 | var NoNamespaceFound = errors.New("no namespace found") 17 | var NoPlanSummaryFound = errors.New("no plan summary found") 18 | var NoStartupArgumentsFound = errors.New("no startup arguments found") 19 | var OperationStructure = errors.New("operation structure unexpected") 20 | var Overflow = errors.New("type overflow") 21 | var StorageUnmatched = VersionUnmatched{"unrecognized storage option"} 22 | var UnexpectedExceptionFormat = errors.New("error parsing exception") 23 | var UnexpectedEOL = errors.New("unexpected end of line") 24 | var UnexpectedLength = errors.New("unexpected text length") 25 | var UnexpectedValue = errors.New("unexpected value") 26 | var UnexpectedVersionFormat = errors.New("unexpected version format") 27 | 28 | /* 29 | * Log parser errors 30 | */ 31 | var VersionDateUnmatched = errors.New("unmatched date string") 32 | var VersionMessageUnmatched = errors.New("unmatched or empty message string") 33 | 34 | type VersionUnmatched struct { 35 | Message string 36 | } 37 | 38 | func (e VersionUnmatched) Error() string { 39 | if e.Message != "" { 40 | return "Log message not recognized: " + e.Message 41 | } else { 42 | return "Log message not recognized" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/regex.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | "sync" 7 | ) 8 | 9 | type regexRegistry struct { 10 | Compiled sync.Map 11 | mutex sync.Mutex 12 | } 13 | 14 | var regexRegistryInstance = ®exRegistry{} 15 | 16 | func GetRegexRegistry() *regexRegistry { 17 | return regexRegistryInstance 18 | } 19 | 20 | func (r *regexRegistry) Compile(pattern string) (*regexp.Regexp, error) { 21 | var regex interface{} 22 | var ok bool 23 | 24 | if regex, ok = r.Compiled.Load(pattern); !ok { 25 | r.mutex.Lock() 26 | defer r.mutex.Unlock() 27 | 28 | if compiled, err := regexp.Compile(pattern); err != nil { 29 | return nil, err 30 | } else { 31 | r.Compiled.Store(pattern, compiled) 32 | regex = compiled 33 | } 34 | } 35 | return regex.(*regexp.Regexp), nil 36 | } 37 | 38 | func (r *regexRegistry) CompileAndMatch(pattern string, match string) ([]string, error) { 39 | compiled, err := r.Compile(pattern) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | results := compiled.FindStringSubmatch(match) 45 | if results == nil { 46 | return results, errors.New("empty match") 47 | } 48 | 49 | return results, nil 50 | } 51 | -------------------------------------------------------------------------------- /internal/strings.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "unicode" 11 | "unicode/utf8" 12 | //"golang.org/x/text/unicode/norm" 13 | ) 14 | 15 | var debugMutex sync.Mutex 16 | 17 | func Debug(format string, v ...interface{}) { 18 | debugMutex.Lock() 19 | defer debugMutex.Unlock() 20 | 21 | fmt.Fprintf(os.Stderr, format+"\n", v...) 22 | } 23 | func ArgumentSplit(a string) []string { 24 | return strings.FieldsFunc(a, argumentSplitRune) 25 | } 26 | func argumentSplitRune(r rune) bool { 27 | switch r { 28 | case ',', ';', ' ': 29 | return true 30 | default: 31 | return false 32 | } 33 | } 34 | func ArgumentMatchOptions(match []string, a string) bool { 35 | for _, value := range ArgumentSplit(a) { 36 | if !ArrayInsensitiveMatchString(match, value) { 37 | return false 38 | } 39 | } 40 | return true 41 | } 42 | func ArrayBinaryMatchString(a string, m []string) bool { 43 | p := sort.SearchStrings(m, a) 44 | if p >= len(m) { 45 | return false 46 | } 47 | return m[p] == a 48 | } 49 | func ArrayFilterString(a []string, match func(string) bool) []string { 50 | b := a[:0] 51 | for _, x := range a { 52 | if match(x) { 53 | b = append(b, x) 54 | } 55 | } 56 | return b 57 | } 58 | func ArrayInsensitiveMatchString(a []string, match string) bool { 59 | for _, x := range a { 60 | if StringInsensitiveMatch(x, match) { 61 | return true 62 | } 63 | } 64 | return false 65 | } 66 | func ArrayMatchString(a []string, match string) bool { 67 | for _, x := range a { 68 | if StringMatch(x, match) { 69 | return true 70 | } 71 | } 72 | return false 73 | } 74 | func StringDoubleSplit(s string, d rune) (string, string, bool) { 75 | pos := strings.IndexRune(s, d) 76 | if pos > -1 { 77 | return s[0:pos], s[pos+1:], true 78 | } 79 | 80 | return s, "", false 81 | } 82 | 83 | // String normalization is necessary for unicode processing of logs. Starting this project using basic UTF8 rune 84 | // counting, but this should eventually translate to normalization (http://godoc.org/golang.org/x/text/unicode/norm). 85 | // See also: https://blog.golang.org/normalization 86 | func StringLength(s string) (n int) { return utf8.RuneCountInString(s) } 87 | func StringToLower(s string) string { return strings.ToLower(s) } 88 | func StringToUpper(s string) string { return strings.ToUpper(s) } 89 | 90 | func StringMatch(a, b string) bool { 91 | if strings.Compare(a, b) == 0 { 92 | return true 93 | } 94 | return false 95 | } 96 | 97 | func StringInsensitiveMatch(s, t string) bool { return strings.EqualFold(s, t) } 98 | 99 | // Cuts an element from an array, modifying the original array and returning the value cut. 100 | func SliceCutString(a []string, index int) (string, []string) { 101 | switch index { 102 | case 0: 103 | return a[0], a[1:] 104 | 105 | case len(a) - 1: 106 | return a[index], a[:index-1] 107 | 108 | default: 109 | return a[index], append(a[:index-1], a[index+1:]...) 110 | } 111 | } 112 | 113 | func IsNumeric(a string) bool { 114 | if _, err := strconv.Atoi(a); err == nil { 115 | return true 116 | } 117 | 118 | return false 119 | } 120 | 121 | func IsNumericRune(r rune) bool { 122 | return unicode.IsNumber(r) 123 | } 124 | -------------------------------------------------------------------------------- /mgotools.go: -------------------------------------------------------------------------------- 1 | // 2 | // mgotools.go 3 | // 4 | // The main utility built with this suite of tools. It takes files as command 5 | // line arguments or stdin and outputs to stdout. 6 | // 7 | package main 8 | 9 | import ( 10 | "errors" 11 | "fmt" 12 | "os" 13 | "path/filepath" 14 | 15 | _ "mgotools/parser" 16 | 17 | "mgotools/command" 18 | "mgotools/internal" 19 | "mgotools/parser/source" 20 | 21 | "github.com/urfave/cli" 22 | ) 23 | 24 | func main() { 25 | app := cli.NewApp() 26 | 27 | app.Name = "mgotools" 28 | app.Description = "A collection of tools designed to help parse and understand MongoDB logs" 29 | app.Action = runCommand 30 | 31 | app.Commands = makeClientFlags() 32 | 33 | app.Flags = []cli.Flag{ 34 | //cli.BoolFlag{Name: "linear, e", Usage: "parse input files linearly in order they are supplied (disable concurrency)"}, 35 | cli.BoolFlag{Name: "verbose, v", Usage: "outputs additional information about the parser"}, 36 | } 37 | cli.VersionFlag = cli.BoolFlag{Name: "version, V"} 38 | if err := app.Run(os.Args); err != nil { 39 | fmt.Println(err) 40 | } 41 | } 42 | 43 | func checkClientCommands(context *cli.Context, count int, def command.Definition) error { 44 | var length = 0 45 | for _, flag := range def.Flags { 46 | switch flag.Type { 47 | case command.IntSourceSlice: 48 | length = len(context.IntSlice(flag.Name)) 49 | case command.StringSourceSlice: 50 | length = len(context.StringSlice(flag.Name)) 51 | } 52 | if length > count { 53 | return errors.New("there cannot be more arguments than files") 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | func makeClientFlags() []cli.Command { 60 | var c []cli.Command 61 | commandFactory := command.GetFactory() 62 | for _, commandName := range commandFactory.GetNames() { 63 | cmd, _ := commandFactory.GetDefinition(commandName) 64 | clientCommand := cli.Command{Name: commandName, Action: runCommand, Usage: cmd.Usage} 65 | for _, argument := range cmd.Flags { 66 | if argument.ShortName != "" { 67 | argument.Name = fmt.Sprintf("%s, %s", argument.Name, argument.ShortName) 68 | } 69 | switch argument.Type { 70 | case command.Bool: 71 | clientCommand.Flags = append(clientCommand.Flags, cli.BoolFlag{Name: argument.Name, Usage: argument.Usage}) 72 | case command.Int: 73 | clientCommand.Flags = append(clientCommand.Flags, cli.IntFlag{Name: argument.Name, Usage: argument.Usage}) 74 | case command.IntSourceSlice: 75 | clientCommand.Flags = append(clientCommand.Flags, cli.IntSliceFlag{Name: argument.Name, Usage: argument.Usage}) 76 | case command.StringSourceSlice, command.String: 77 | clientCommand.Flags = append(clientCommand.Flags, cli.StringSliceFlag{Name: argument.Name, Usage: argument.Usage}) 78 | } 79 | } 80 | c = append(c, clientCommand) 81 | } 82 | return c 83 | } 84 | 85 | func runCommand(c *cli.Context) error { 86 | // Pull arguments from the helper interpreter. 87 | var ( 88 | commandFactory = command.GetFactory() 89 | clientContext = c.Args() 90 | //start = time.Now() 91 | ) 92 | if c.Command.Name == "" { 93 | return errors.New("command required") 94 | } else if cmdDefinition, ok := commandFactory.GetDefinition(c.Command.Name); !ok { 95 | return fmt.Errorf("unrecognized command %s", c.Command.Name) 96 | } else { 97 | //util.Debug("Command: %s, starting: %s", c.Command.Name, time.Now()) 98 | 99 | cmd, err := commandFactory.Get(c.Command.Name) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | // Get argument count. 105 | argc := c.NArg() 106 | fileCount := 0 107 | 108 | input := make([]command.Input, 0) 109 | output := command.Output{Writer: os.Stdout, Error: os.Stderr} 110 | 111 | // Check for pipe usage. 112 | pipe, err := os.Stdin.Stat() 113 | if err != nil { 114 | panic(err) 115 | } else if (pipe.Mode() & os.ModeNamedPipe) != 0 { 116 | if argc > 0 { 117 | return errors.New("file arguments and input pipes cannot be used simultaneously") 118 | } 119 | 120 | // Add stdin to the list of input files. 121 | args, err := command.MakeCommandArgumentCollection(0, getArgumentMap(cmdDefinition, c), cmdDefinition) 122 | if err != nil { 123 | return err 124 | } 125 | 126 | fileCount = 1 127 | stdio, err := source.NewLog(os.Stdin) 128 | 129 | input = append(input, command.Input{ 130 | Arguments: args, 131 | Name: "stdin", 132 | Length: int64(0), 133 | Reader: source.NewAccumulator(stdio), 134 | }) 135 | } 136 | 137 | // Loop through each argument and add files to the command. 138 | for index := 0; index < argc; index += 1 { 139 | path := clientContext.Get(index) 140 | size := int64(0) 141 | 142 | if s, err := os.Stat(path); os.IsNotExist(err) { 143 | internal.Debug("%s skipped (%s)", path, err) 144 | continue 145 | } else { 146 | size = s.Size() 147 | } 148 | 149 | // Open the file and check for errors. 150 | file, err := os.OpenFile(path, os.O_RDONLY, 0) 151 | if err != nil { 152 | return err 153 | } 154 | 155 | args, err := command.MakeCommandArgumentCollection(index, getArgumentMap(cmdDefinition, c), cmdDefinition) 156 | if err != nil { 157 | return err 158 | } 159 | 160 | logfile, err := source.NewLog(file) 161 | if err != nil { 162 | return err 163 | } 164 | 165 | fileCount += 1 166 | input = append(input, command.Input{ 167 | Arguments: args, 168 | Name: filepath.Base(path), 169 | Length: size, 170 | Reader: source.NewAccumulator(logfile), 171 | }) 172 | } 173 | 174 | // Check for basic command sanity. 175 | if err := checkClientCommands(c, fileCount, cmdDefinition); err != nil { 176 | return err 177 | } 178 | 179 | // Run the actual command. 180 | if err := command.RunCommand(cmd, input, output); err != nil { 181 | return err 182 | } 183 | 184 | //util.Debug("Finished at %s (%s)", time.Now(), time.Since(start).String()) 185 | return nil 186 | } 187 | } 188 | 189 | func getArgumentMap(commandDefinition command.Definition, c *cli.Context) map[string]interface{} { 190 | out := make(map[string]interface{}) 191 | for _, arg := range commandDefinition.Flags { 192 | if c.IsSet(arg.Name) { 193 | switch arg.Type { 194 | case command.Bool: 195 | out[arg.Name] = c.Bool(arg.Name) 196 | case command.Int: 197 | out[arg.Name] = c.Int(arg.Name) 198 | case command.IntSourceSlice: 199 | out[arg.Name] = c.IntSlice(arg.Name) 200 | case command.String, command.StringSourceSlice: 201 | out[arg.Name] = c.StringSlice(arg.Name) 202 | } 203 | } 204 | } 205 | return out 206 | } 207 | -------------------------------------------------------------------------------- /mongo/pattern.go: -------------------------------------------------------------------------------- 1 | package mongo 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "sort" 7 | 8 | "mgotools/internal" 9 | "mgotools/mongo/sorter" 10 | "mgotools/parser/record" 11 | ) 12 | 13 | type Pattern struct { 14 | pattern map[string]interface{} 15 | initialized bool 16 | } 17 | 18 | type V struct{} 19 | 20 | func NewPattern(s map[string]interface{}) Pattern { 21 | return Pattern{createPattern(s, false), true} 22 | } 23 | func (p Pattern) IsEmpty() bool { 24 | return !p.initialized 25 | } 26 | func (p Pattern) Equals(object Pattern) bool { 27 | if len(p.pattern) == 0 || len(object.pattern) == 0 { 28 | return len(p.pattern) == len(object.pattern) 29 | } 30 | return deepEqual(p.pattern, object.pattern) 31 | } 32 | func (p Pattern) Pattern() map[string]interface{} { 33 | return p.pattern 34 | } 35 | func (p Pattern) String() string { 36 | return createString(p, false) 37 | } 38 | func (p Pattern) StringCompact() string { 39 | return createString(p, true) 40 | } 41 | 42 | func compress(c interface{}) interface{} { 43 | switch t := c.(type) { 44 | case map[string]interface{}: 45 | for key := range t { 46 | if !internal.ArrayInsensitiveMatchString(record.OPERATORS_COMPARISON, key) { 47 | return t 48 | } 49 | } 50 | 51 | case []interface{}: 52 | for _, value := range t { 53 | if _, ok := value.(V); !ok { 54 | return t 55 | } 56 | } 57 | 58 | default: 59 | panic("attempted to compress unexpected value") 60 | } 61 | 62 | return V{} 63 | } 64 | 65 | func createPattern(s map[string]interface{}, expr bool) map[string]interface{} { 66 | for key := range s { 67 | switch t := s[key].(type) { 68 | case map[string]interface{}: 69 | if !expr || internal.ArrayInsensitiveMatchString(record.OPERATORS_COMPARISON, key) { 70 | s[key] = compress(createPattern(t, true)) 71 | } else if internal.ArrayInsensitiveMatchString(record.OPERATORS_EXPRESSION, key) { 72 | s[key] = createPattern(t, false) 73 | } else if internal.ArrayInsensitiveMatchString(record.OPERATORS_LOGICAL, key) { 74 | s[key] = createPattern(t, false) 75 | } else { 76 | s[key] = V{} 77 | } 78 | 79 | case []interface{}: 80 | if internal.ArrayInsensitiveMatchString(record.OPERATORS_LOGICAL, key) { 81 | v := createArray(t, false) 82 | if isValueArray(v) { 83 | s[key] = v 84 | } else { 85 | r := sorter.Patternize(v) 86 | sort.Sort(r) 87 | s[key] = r.Interface() 88 | } 89 | } else if internal.ArrayInsensitiveMatchString(record.OPERATORS_EXPRESSION, key) { 90 | s[key] = compress(createArray(t, true)) 91 | } else { 92 | s[key] = V{} 93 | } 94 | 95 | default: 96 | s[key] = V{} 97 | } 98 | } 99 | return s 100 | } 101 | 102 | func createString(p Pattern, compact bool) string { 103 | if !p.initialized { 104 | return "" 105 | } 106 | var arr func([]interface{}) string 107 | var obj func(map[string]interface{}) string 108 | arr = func(array []interface{}) string { 109 | var buffer = bytes.NewBufferString("[") 110 | total := len(array) 111 | v := 0 112 | 113 | if !compact && total > 0 { 114 | buffer.WriteRune(' ') 115 | } 116 | 117 | for index := 0; index < total; index += 1 { 118 | r := array[index] 119 | switch t := r.(type) { 120 | case []interface{}: 121 | r := arr(t) 122 | if r == "1" { 123 | v += 1 124 | } 125 | 126 | buffer.WriteString(r) 127 | 128 | case map[string]interface{}: 129 | buffer.WriteString(obj(t)) 130 | 131 | case V: 132 | buffer.WriteRune('1') 133 | v += 1 134 | 135 | default: 136 | panic(fmt.Sprintf("unexpected type %T in pattern", r)) 137 | } 138 | if index < total-1 { 139 | buffer.WriteString(", ") 140 | } else if !compact { 141 | buffer.WriteRune(' ') 142 | } 143 | } 144 | if v == total { 145 | return "1" 146 | } 147 | buffer.WriteRune(']') 148 | return buffer.String() 149 | } 150 | obj = func(object map[string]interface{}) string { 151 | var buffer = bytes.NewBuffer([]byte{'{'}) 152 | total := len(object) 153 | count := 0 154 | 155 | if !compact && total > 0 { 156 | buffer.WriteRune(' ') 157 | } 158 | 159 | // Iterating over a map will happen in a randomized order. Keys must 160 | // be sorted and iterated in a specific order: 161 | // https://codereview.appspot.com/5285042/patch/9001/10003 162 | keys := make(sorter.Key, 0) 163 | for key := range object { 164 | keys = append(keys, key) 165 | } 166 | sort.Sort(keys) 167 | 168 | for _, key := range keys { 169 | count += 1 170 | buffer.WriteRune('"') 171 | buffer.WriteString(key) 172 | buffer.WriteRune('"') 173 | buffer.WriteString(": ") 174 | 175 | switch t := object[key].(type) { 176 | case []interface{}: 177 | buffer.WriteString(arr(t)) 178 | 179 | case map[string]interface{}: 180 | buffer.WriteString(obj(t)) 181 | 182 | case V: 183 | buffer.WriteRune('1') 184 | } 185 | 186 | if count < total { 187 | buffer.WriteString(", ") 188 | } else if !compact { 189 | buffer.WriteRune(' ') 190 | } 191 | } 192 | 193 | buffer.WriteRune('}') 194 | return buffer.String() 195 | } 196 | 197 | return obj(p.pattern) 198 | } 199 | 200 | func createArray(t []interface{}, expr bool) []interface{} { 201 | for i := 0; i < len(t); i += 1 { 202 | switch t2 := t[i].(type) { 203 | case map[string]interface{}: 204 | t[i] = createPattern(t2, true) 205 | case []interface{}: 206 | if !expr { 207 | return createArray(t2, true) 208 | } else { 209 | t[i] = V{} 210 | } 211 | default: 212 | t[i] = V{} 213 | } 214 | } 215 | return t 216 | } 217 | 218 | // Why create a new DeepEqual method? Why not use reflect.DeepEqual? The reflect package is scary. Not in 219 | // the "I don't know how to use this" way but in a "reflection is great, but unnecessary here" kind of way. There is 220 | // no reason to use a cannon to kill this particular mosquito since we're only doing checks against objects, arrays, 221 | // and a single empty struct V{}. See the benchmark in pattern_test.go for more justification. 222 | func deepEqual(ax, bx map[string]interface{}) bool { 223 | if len(ax) != len(bx) { 224 | return false 225 | } 226 | var f func(a, b interface{}) bool 227 | f = func(a, b interface{}) bool { 228 | switch t := a.(type) { 229 | case map[string]interface{}: 230 | if s, ok := b.(map[string]interface{}); !ok { 231 | return false 232 | } else if !deepEqual(t, s) { 233 | return false 234 | } 235 | return true 236 | case []interface{}: 237 | s, ok := b.([]interface{}) 238 | if !ok || len(s) != len(t) { 239 | return false 240 | } 241 | l := len(s) 242 | for i := 0; i < l; i += 1 { 243 | if !f(t[i], s[i]) { 244 | return false 245 | } 246 | } 247 | return true 248 | case V: 249 | if _, ok := b.(V); !ok { 250 | return false 251 | } 252 | return true 253 | default: 254 | panic(fmt.Sprintf("unexpected type %T in pattern", t)) 255 | } 256 | } 257 | for key := range ax { 258 | if _, ok := bx[key]; !ok || !f(ax[key], bx[key]) { 259 | return false 260 | } 261 | } 262 | return true // len(a) == len(b) == 0 263 | } 264 | 265 | func isValueArray(a []interface{}) bool { 266 | for _, v := range a { 267 | if _, ok := v.(V); !ok { 268 | return false 269 | } 270 | } 271 | return true 272 | } 273 | -------------------------------------------------------------------------------- /mongo/pattern_test.go: -------------------------------------------------------------------------------- 1 | package mongo 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | type ( 9 | O = map[string]interface{} 10 | A = []interface{} 11 | ) 12 | 13 | func TestPattern_NewPattern(t *testing.T) { 14 | s := []O{ 15 | {"a": 5}, 16 | {"a": 5, "b": "y"}, 17 | {"a": O{"$in": "y"}}, 18 | {"a": O{"$gt": 5}}, 19 | {"a": O{"$exists": true}}, 20 | {"a.b": "y"}, 21 | {"$or": A{O{"a": 5}, O{"b": 5}}}, // Logical operator without compression. 22 | {"$or": A{O{"b": 5}, O{"a": 5}}}, // Sort of keys in logical operator. 23 | {"$or": A{O{"b": 5, "a": 5}, O{"a": 5}}}, 24 | {"$and": A{O{"$or": A{O{"a": 5}, O{"b": 5}}}, O{"$or": A{O{"c": 5}, O{"d": 5}}}}}, 25 | {"_id": ObjectId{}}, 26 | {"a": O{"$in": A{5, 5, 5}}}, 27 | {"a": O{"$elemMatch": O{"b": 5, "c": O{"$gte": 5}}}}, 28 | {"a": O{"$geoWithin": O{"$center": A{A{5, 5}, 5}}}}, 29 | {"a": O{"$geoWithin": O{"$geometry": O{"a": "y", "b": A{5, 5}}}}}, 30 | } 31 | d := []O{ 32 | {"a": V{}}, 33 | {"a": V{}, "b": V{}}, 34 | {"a": V{}}, 35 | {"a": V{}}, 36 | {"a": V{}}, 37 | {"a.b": V{}}, 38 | {"$or": A{O{"a": V{}}, O{"b": V{}}}}, // Logical operator without compression. 39 | {"$or": A{O{"a": V{}}, O{"b": V{}}}}, // Sort of keys in logical operator. 40 | {"$or": A{O{"a": V{}}, O{"a": V{}, "b": V{}}}}, 41 | {"$and": A{O{"$or": A{O{"a": V{}}, O{"b": V{}}}}, O{"$or": A{O{"c": V{}}, O{"d": V{}}}}}}, 42 | {"_id": V{}}, 43 | {"a": V{}}, 44 | {"a": O{"$elemMatch": O{"b": V{}, "c": V{}}}}, 45 | {"a": O{"$geoWithin": O{"$center": V{}}}}, 46 | {"a": O{"$geoWithin": O{"$geometry": O{"a": V{}, "b": V{}}}}}, 47 | } 48 | if len(s) != len(d) { 49 | t.Fatalf("mismatch between array sizes, %d and %d", len(s), len(d)) 50 | } 51 | 52 | for i := range s { 53 | if p := NewPattern(s[i]); !deepEqual(p.pattern, d[i]) { 54 | t.Errorf("pattern mismatch at %d:\n\t\t%#v\n\t\t%#v", i+1, d[i], s[i]) 55 | } 56 | } 57 | } 58 | 59 | func TestPattern_Equals(t *testing.T) { 60 | s := []O{ 61 | {}, 62 | {"a": V{}}, 63 | {"a": V{}, "b": V{}}, 64 | {"a": O{"b": V{}}}, 65 | {"a": A{}}, 66 | {"a": A{V{}, V{}}}, 67 | {"a": O{}}, 68 | {"a": A{O{"a": V{}}}}, 69 | {"a": V{}}, 70 | } 71 | d := []O{ 72 | {"a": V{}}, 73 | {"b": V{}}, 74 | {"b": V{}, "a": V{}, "c": V{}}, 75 | {"a": O{"c": V{}}}, 76 | {"a": A{V{}}}, 77 | {"a": A{V{}}}, 78 | {"a": A{}}, 79 | {"a": A{O{"b": V{}}}}, 80 | {"a": O{}}, 81 | } 82 | 83 | if len(s) != len(d) { 84 | t.Fatalf("mismatch between array sizes, %d and %d", len(s), len(d)) 85 | } 86 | 87 | for i := range s { 88 | p := Pattern{s[i], true} 89 | if !p.Equals(Pattern{s[i], true}) { 90 | t.Errorf("equality mismatch at %d: %#v", i, s[i]) 91 | } 92 | } 93 | for i := range s { 94 | p := Pattern{s[i], true} 95 | r := Pattern{d[i], true} 96 | if p.Equals(r) { 97 | t.Errorf("equality match at %d:\n%#v\n%v", i, s[i], d[i]) 98 | } 99 | } 100 | } 101 | func TestPattern_IsEmpty(t *testing.T) { 102 | p := Pattern{} 103 | if !p.IsEmpty() { 104 | t.Errorf("unexpected initialized variable") 105 | } 106 | r := NewPattern(O{}) 107 | if r.IsEmpty() { 108 | t.Errorf("unexpected uninitialzied value") 109 | } 110 | } 111 | func TestPattern_Pattern(t *testing.T) { 112 | p := Pattern{} 113 | if p.Pattern() != nil { 114 | t.Errorf("pattern should be empty") 115 | } 116 | } 117 | func TestPattern_String(t *testing.T) { 118 | s := []Pattern{ 119 | {O{"a": V{}}, true}, 120 | {O{"a": V{}, "b": V{}}, true}, 121 | {O{"a": A{V{}, V{}}}, true}, 122 | {O{}, true}, 123 | {O{"a": A{}}, true}, 124 | {O{"a": A{O{"b": V{}}}}, true}, 125 | {O{"a": A{A{V{}}}}, true}, 126 | } 127 | d := []string{ 128 | `{ "a": 1 }`, 129 | `{ "a": 1, "b": 1 }`, 130 | `{ "a": 1 }`, 131 | `{}`, 132 | `{ "a": 1 }`, 133 | `{ "a": [ { "b": 1 } ] }`, 134 | `{ "a": 1 }`, 135 | } 136 | if len(s) != len(d) { 137 | t.Fatalf("mismatch between array sizes, %d and %d", len(s), len(d)) 138 | } 139 | for i := range s { 140 | if s[i].String() != d[i] { 141 | t.Errorf("pattern mismatch (%d), expected '%s', got '%s'", i, d[i], s[i].String()) 142 | } 143 | } 144 | } 145 | 146 | func TestPattern_mtools(t *testing.T) { 147 | oid1, _ := NewObjectId("1234564863acd10e5cbf5f6e") 148 | oid2, _ := NewObjectId("1234564863acd10e5cbf5f7e") 149 | oid3, _ := NewObjectId("528556616dde23324f233168") 150 | 151 | s := []O{ 152 | //`{"d": {"$gt": 2, "$lt": 4}, "b": {"$gte": 3}, "c": {"$nin": [1, "foo", "bar"]}, "$or": [{"a":1}, {"b":1}] }`, 153 | { 154 | "d": O{"$gt": 2, "$lt": 4}, "b": O{"$gte": 3}, "c": O{"$nin": A{1, "foo", "bar"}}, "$or": A{O{"a": 1}, O{"b": 1}}, 155 | }, 156 | 157 | //`{"a": {"$gt": 2, "$lt": 4}, "b": {"$nin": [1, 2, 3]}, "$or": [{"a":1}, {"b":1}] }`, 158 | { 159 | "a": O{"$gt": 2, "$lt": 4}, "b": O{"$nin": A{1, 2, 3}}, "$or": A{O{"a": 1}, O{"b": 1}}, 160 | }, 161 | 162 | //`{"a": {"$gt": 2, "$lt": 4}, "b": {"$in": [ ObjectId("1234564863acd10e5cbf5f6e"), ObjectId("1234564863acd10e5cbf5f7e") ] } }`, 163 | { 164 | "a": O{"$gt": 2, "$lt": 4}, "b": O{"$in": A{oid1, oid2}}, 165 | }, 166 | 167 | //`{ "sk": -1182239108, "_id": { "$in": [ ObjectId("1234564863acd10e5cbf5f6e"), ObjectId("1234564863acd10e5cbf5f7e") ] } }`, 168 | { 169 | "sk": -1182239108, "_id": O{"$in": A{oid1, oid1}}, 170 | }, 171 | 172 | //`{ "a": 1, "b": { "c": 2, "d": "text" }, "e": "more test" }`, 173 | { 174 | "a": 1, "b": O{"c": 2, "d": "text"}, "e": "more test", 175 | }, 176 | 177 | //`{ "_id": ObjectId("528556616dde23324f233168"), "config": { "_id": 2, "host": "localhost:27017" }, "ns": "local.oplog.rs" }`, 178 | { 179 | "_id": oid3, "config": O{"_id": 2, "host": "localhost:27017"}, "ns": "local.oplog.rs", 180 | }, 181 | } 182 | 183 | d := []string{ 184 | `{"$or": [{"a": 1}, {"b": 1}], "b": 1, "c": {"$nin": 1}, "d": 1}`, 185 | `{"$or": [{"a": 1}, {"b": 1}], "a": 1, "b": {"$nin": 1}}`, 186 | `{"a": 1, "b": 1}`, 187 | `{"_id": 1, "sk": 1}`, 188 | `{"a": 1, "b": {"c": 1, "d": 1}, "e": 1}`, 189 | `{"_id": 1, "config": {"_id": 1, "host": 1}, "ns": 1}`, 190 | } 191 | 192 | for i, m := range s { 193 | p := NewPattern(m) 194 | 195 | if p.StringCompact() != d[i] { 196 | t.Errorf("mismatch at %d, got '%s', expected '%s'", i, p.StringCompact(), d[i]) 197 | } 198 | } 199 | } 200 | 201 | var patterns = []Pattern{ 202 | NewPattern(O{}), 203 | NewPattern(O{"a": 1}), 204 | NewPattern(O{"a": 1, "b": 1}), 205 | NewPattern(O{"a": O{"b": 1}}), 206 | NewPattern(O{"a": O{"b": A{1, 1}}}), 207 | NewPattern(O{"a": O{"b": O{"c": O{"d": 1}}}}), 208 | NewPattern(O{"a": A{O{"a": 1}, O{"b": 1}, O{"c": 1}}}), 209 | NewPattern(O{"a": 1, "b": 1, "c": 1, "d": 1}), 210 | NewPattern(O{"a": A{O{"b": 1}}, "c": A{O{"d": 1}}}), 211 | } 212 | 213 | func BenchmarkPattern_Equals(b *testing.B) { 214 | for i := 0; i < b.N; i += 1 { 215 | for _, s := range patterns { 216 | s.Equals(s) 217 | } 218 | } 219 | } 220 | func BenchmarkPattern_String(b *testing.B) { 221 | for i := 0; i < b.N; i += 1 { 222 | for _, s := range patterns { 223 | s.String() 224 | } 225 | } 226 | } 227 | func BenchmarkPattern_MatchEquals(b *testing.B) { 228 | match := NewPattern(O{"a": A{O{"a": 1}, O{"b": 1}, O{"c": 1}}}) 229 | size := len(patterns) 230 | for i := 0; i < b.N; i += 1 { 231 | for j := 0; j < size; j += 1 { 232 | if patterns[j].Equals(match) { 233 | } 234 | } 235 | } 236 | } 237 | func BenchmarkPattern_MatchString(b *testing.B) { 238 | match := NewPattern(O{"a": A{O{"a": 1}, O{"b": 1}, O{"c": 1}}}) 239 | var strings = make([]string, len(patterns)) 240 | size := len(patterns) 241 | for i := 0; i < size; i += 1 { 242 | strings[i] = patterns[i].String() 243 | } 244 | for i := 0; i < b.N; i += 1 { 245 | s := match.String() 246 | for j := 0; j < size; j += 1 { 247 | if s == strings[j] { 248 | } 249 | } 250 | } 251 | } 252 | func BenchmarkReflection_DeepEqual(b *testing.B) { 253 | for i := 0; i < b.N; i += 1 { 254 | for _, s := range patterns { 255 | reflect.DeepEqual(s, s) 256 | } 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /mongo/sorter/key.go: -------------------------------------------------------------------------------- 1 | package sorter 2 | 3 | // A custom sorting algorithm to keep keys starting with _ before $, and $ 4 | // before everything else. 5 | type Key []string 6 | 7 | func (k Key) Len() int { 8 | return len(k) 9 | } 10 | 11 | func (k Key) Less(i, j int) bool { 12 | a := k[i] 13 | b := k[j] 14 | la, lb := len(a), len(b) 15 | if la == 0 || lb == 0 { 16 | if a < b { 17 | return true 18 | } else { 19 | return false 20 | } 21 | } 22 | if (a[0] == '_' && b[0] != '_') || (a[0] == '$' && b[0] != '$') { 23 | return true 24 | } else if (a[0] != '_' && b[0] == '_') || (a[0] != '$' && b[0] == '$') { 25 | return false 26 | } else if (a[0] == '_' && b[0] == '_') || (a[0] == '$' && b[0] == '$') { 27 | if a[1:] < b[1:] { 28 | return true 29 | } else { 30 | return false 31 | } 32 | } else if a < b { 33 | return true 34 | } 35 | return false 36 | } 37 | 38 | func (k Key) Swap(i, j int) { 39 | c := k[i] 40 | k[i] = k[j] 41 | k[j] = c 42 | } 43 | -------------------------------------------------------------------------------- /mongo/sorter/key_test.go: -------------------------------------------------------------------------------- 1 | package sorter 2 | 3 | import "testing" 4 | 5 | func TestKey_Less(t *testing.T) { 6 | k := Key{ 7 | "b", // 0 8 | "c", // 1 9 | "a", // 2 10 | "_", // 3 11 | "_a", // 4 12 | "_a", // 5 13 | "_b", // 6 14 | "_", // 7 15 | "__", // 8 16 | "", // 9 17 | "$", // 10 18 | "$a", // 11 19 | } 20 | 21 | expect := map[int]int{ 22 | 0: 1, 23 | 3: 2, 24 | 4: 6, 25 | 7: 8, 26 | 9: 2, 27 | } 28 | for i, j := range expect { 29 | if !k.Less(i, j) { 30 | t.Errorf("'%s' (%d) less than '%s' (%d) = false, should be true", k[i], i, k[j], j) 31 | } 32 | } 33 | 34 | fail := map[int]int{ 35 | 3: 2, 36 | 1: 9, 37 | 2: 3, 38 | } 39 | for i, j := range fail { 40 | if k.Less(i, j) { 41 | t.Errorf("'%s' (%d) less than '%s' (%d) = true, should be false", k[i], i, k[j], j) 42 | } 43 | } 44 | } 45 | 46 | func TestKey_Swap(t *testing.T) { 47 | k := Key{"a", "b", "c"} 48 | k.Swap(0, 1) 49 | 50 | if k[0] != "b" && k[1] != "a" { 51 | t.Error("swap failed to swap") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /mongo/sorter/logical.go: -------------------------------------------------------------------------------- 1 | package sorter 2 | 3 | import "sort" 4 | 5 | // A custom sort algorithm for logical operators so an array of objects 6 | // gets sorted by the keys of each sub-object. 7 | type Logical []map[string]interface{} 8 | 9 | func Patternize(a []interface{}) Logical { 10 | b := make([]map[string]interface{}, len(a)) 11 | for i := 0; i < len(a); i += 1 { 12 | b[i] = a[i].(map[string]interface{}) 13 | } 14 | return b 15 | } 16 | 17 | func (p Logical) Interface() []interface{} { 18 | b := make([]interface{}, len(p)) 19 | for i := 0; i < len(b); i += 1 { 20 | b[i] = func() map[string]interface{} { return p[i] }() 21 | } 22 | return b 23 | } 24 | 25 | func (p Logical) Len() int { 26 | return len(p) 27 | } 28 | 29 | func (p Logical) Less(i, j int) bool { 30 | a := make(Key, 0, len(p[i])) 31 | b := make(Key, 0, len(p[j])) 32 | 33 | for k := range p[i] { 34 | a = append(a, k) 35 | } 36 | for k := range p[j] { 37 | b = append(b, k) 38 | } 39 | 40 | sort.Sort(a) 41 | sort.Sort(b) 42 | for i := 0; i < len(a) && i < len(b); i += 1 { 43 | if a[i] < b[i] { 44 | return true 45 | } else if a[i] > b[i] { 46 | return false 47 | } 48 | } 49 | 50 | // At this point, all keys are identical so the shorter wins. 51 | if len(a) < len(b) { 52 | return true 53 | } else { 54 | return false 55 | } 56 | } 57 | 58 | func (p Logical) Swap(i, j int) { 59 | c := p[i] 60 | p[i] = p[j] 61 | p[j] = c 62 | } 63 | -------------------------------------------------------------------------------- /mongo/types.go: -------------------------------------------------------------------------------- 1 | package mongo 2 | 3 | import ( 4 | "encoding/hex" 5 | "time" 6 | ) 7 | 8 | // Use type aliases (instead of type definitions) 9 | type Object = map[string]interface{} 10 | type Array = []interface{} 11 | 12 | type MaxKey struct{} 13 | type MinKey struct{} 14 | type Timestamp time.Time 15 | type Undefined struct{} 16 | 17 | type BinData struct { 18 | BinData []byte 19 | Type byte 20 | } 21 | 22 | type Regex struct { 23 | Regex string 24 | Options string 25 | } 26 | 27 | type Ref struct { 28 | Name string 29 | Id ObjectId 30 | } 31 | 32 | type ObjectId [12]byte 33 | 34 | func NewObjectId(s string) (ObjectId, bool) { 35 | var d = make([]byte, 12, 12) 36 | if len(s) != 24 { 37 | return ObjectId{}, false 38 | } else if l, err := hex.Decode(d, []byte(s)); l != 12 || err != nil { 39 | return ObjectId{}, false 40 | } else { 41 | var v ObjectId 42 | copy(v[:], d[:12]) 43 | return v, true 44 | } 45 | } 46 | 47 | func (o ObjectId) Slice() []byte { 48 | var s = make([]byte, 12, 12) 49 | copy(s, o[:12]) 50 | return s 51 | } 52 | 53 | func (o ObjectId) Equals(a ObjectId) bool { 54 | return o == a 55 | } 56 | -------------------------------------------------------------------------------- /parser/common.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "net" 6 | "strconv" 7 | "strings" 8 | 9 | "mgotools/internal" 10 | "mgotools/mongo" 11 | "mgotools/parser/message" 12 | "mgotools/parser/record" 13 | ) 14 | 15 | func commonParseAuthenticatedPrincipal(r *internal.RuneReader) (message.Message, error) { 16 | // Ignore the first four words and retrieve principal user 17 | r.SkipWords(4) 18 | user, ok := r.SlurpWord() 19 | if !ok { 20 | return nil, internal.UnexpectedEOL 21 | } 22 | 23 | // SERVER-39820 24 | ip, _ := r.SlurpWord() 25 | return message.Authentication{Principal: user, IP: ip}, nil 26 | } 27 | 28 | func commonParseBuildInfo(r *internal.RuneReader) (message.Message, error) { 29 | return message.BuildInfo{BuildInfo: r.SkipWords(2).Remainder()}, nil 30 | } 31 | 32 | func commonParseConnectionAccepted(r *internal.RuneReader) (message.Message, error) { 33 | if addr, port, conn, ok := connectionInit(r.SkipWords(3)); ok { 34 | return message.Connection{Address: addr, Port: port, Conn: conn, Opened: true}, nil 35 | } 36 | return nil, internal.NetworkUnrecognized 37 | } 38 | 39 | func commonParseClientMetadata(r *internal.RuneReader) (message.Message, error) { 40 | // Skip "received client metadata" and grab connection information. 41 | addr, port, conn, ok := connectionInit(r.SkipWords(4)) 42 | if !ok { 43 | return nil, internal.MetadataUnmatched 44 | } 45 | 46 | meta, err := mongo.ParseJsonRunes(r, false) 47 | if err == nil { 48 | return nil, err 49 | } 50 | 51 | return message.ConnectionMeta{ 52 | Connection: message.Connection{ 53 | Address: addr, 54 | Conn: conn, 55 | Port: port, 56 | Opened: true}, 57 | Meta: meta}, nil 58 | } 59 | 60 | func commonParseConnectionEnded(entry record.Entry, r *internal.RuneReader) (message.Message, error) { 61 | if addr, port, ok := connectionTerminate(r.SkipWords(2)); ok { 62 | return message.Connection{Address: addr, Port: port, Conn: entry.Connection, Opened: false}, nil 63 | } 64 | return nil, internal.UnexpectedValue 65 | } 66 | 67 | func commonParseSignalProcessing(r *internal.RuneReader) (message.Message, error) { 68 | return message.Signal{String: r.String()}, nil 69 | } 70 | 71 | func commonParseWaitingForConnections(_ *internal.RuneReader) (message.Message, error) { 72 | return message.Listening{}, nil 73 | } 74 | 75 | func commonParseWiredtigerOpen(r *internal.RuneReader) (message.Message, error) { 76 | return message.WiredTigerConfig{String: r.SkipWords(2).Remainder()}, nil 77 | } 78 | 79 | func connectionInit(msg *internal.RuneReader) (ip net.IP, port uint16, conn int, success bool) { 80 | ip, port, success = parseAddress(msg) 81 | if !success { 82 | return 83 | } 84 | 85 | if msgNumber, ok := msg.SlurpWord(); ok { 86 | if msgNumber[0] == '#' { 87 | conn, _ = strconv.Atoi(msgNumber[1:]) 88 | } else if len(msgNumber) > 4 && strings.HasPrefix(msgNumber, "conn") { 89 | if strings.HasSuffix(msgNumber, ":") { 90 | msgNumber = msgNumber[:len(msgNumber)-1] 91 | } 92 | conn, _ = strconv.Atoi(msgNumber[4:]) 93 | } 94 | } 95 | 96 | return 97 | } 98 | 99 | func connectionTerminate(msg *internal.RuneReader) (ip net.IP, port uint16, success bool) { 100 | ip, port, success = parseAddress(msg) 101 | return 102 | } 103 | 104 | func makeVersion(msg string, binary string) (message.Version, error) { 105 | msg = strings.TrimLeft(msg, "v") 106 | version := message.Version{Binary: binary} 107 | 108 | if parts := strings.Split(msg, "."); len(parts) >= 2 { 109 | version.Major, _ = strconv.Atoi(parts[0]) 110 | version.Minor, _ = strconv.Atoi(parts[1]) 111 | 112 | if len(parts) >= 3 { 113 | version.Revision, _ = strconv.Atoi(parts[2]) 114 | } 115 | } 116 | 117 | if msg == "" { 118 | return version, internal.UnexpectedVersionFormat 119 | } 120 | 121 | return version, nil 122 | } 123 | 124 | func parseAddress(r *internal.RuneReader) (ip net.IP, port uint16, ok bool) { 125 | addr, ok := r.SlurpWord() 126 | if !ok { 127 | return nil, 0, false 128 | } 129 | 130 | pos := bytes.IndexRune([]byte(addr), ':') 131 | if pos < 1 { 132 | return nil, 0, false 133 | 134 | } 135 | 136 | if value, err := strconv.Atoi(addr[pos+1:]); err != nil { 137 | return nil, 0, false 138 | } else { 139 | port = uint16(value) 140 | } 141 | if pos >= len(addr) { 142 | return nil, 0, false 143 | } 144 | 145 | ip = net.ParseIP(addr[:pos]) 146 | ok = true 147 | return 148 | } 149 | 150 | func startupInfo(msg string) (message.StartupInfo, error) { 151 | if optionsRegex, err := internal.GetRegexRegistry().Compile(`([^=\s]+)=([^\s]+)`); err == nil { 152 | matches := optionsRegex.FindAllStringSubmatch(msg, -1) 153 | startupInfo := message.StartupInfo{} 154 | 155 | for _, match := range matches { 156 | switch match[1] { 157 | case "dbpath": 158 | startupInfo.DbPath = match[2] 159 | case "host": 160 | startupInfo.Hostname = match[2] 161 | case "pid": 162 | startupInfo.Pid, _ = strconv.Atoi(match[2]) 163 | case "port": 164 | startupInfo.Port, _ = strconv.Atoi(match[2]) 165 | } 166 | } 167 | return startupInfo, nil 168 | } 169 | return message.StartupInfo{}, internal.NoStartupArgumentsFound 170 | } 171 | 172 | func startupOptions(msg string) (message.StartupOptions, error) { 173 | opt, err := mongo.ParseJson(msg, false) 174 | if err != nil { 175 | return message.StartupOptions{}, err 176 | } 177 | return message.StartupOptions{String: msg, Options: opt}, nil 178 | } 179 | -------------------------------------------------------------------------------- /parser/db_24.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | 7 | "mgotools/internal" 8 | "mgotools/mongo" 9 | "mgotools/parser/executor" 10 | "mgotools/parser/message" 11 | "mgotools/parser/record" 12 | "mgotools/parser/version" 13 | ) 14 | 15 | type Version24Parser struct { 16 | context *executor.Executor 17 | counters []string 18 | } 19 | 20 | var errorVersion24Unmatched = internal.VersionUnmatched{Message: "version 2.4"} 21 | 22 | func init() { 23 | version.Factory.Register(func() version.Parser { 24 | context := executor.New() 25 | 26 | // initandlisten 27 | context.RegisterForReader("build info", mongodBuildInfo) 28 | context.RegisterForReader("db version", mongodDbVersion) 29 | context.RegisterForReader("journal dir=", mongodJournal) 30 | context.RegisterForReader("options", mongodOptions) 31 | 32 | context.RegisterForEntry("MongoDB starting", mongodStartupInfo) 33 | 34 | // signalProcessingThread 35 | context.RegisterForReader("dbexit", mongodParseShutdown) 36 | 37 | // connection related 38 | context.RegisterForReader("connection accepted", commonParseConnectionAccepted) 39 | context.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 40 | 41 | context.RegisterForEntry("end connection", commonParseConnectionEnded) 42 | 43 | return &Version24Parser{ 44 | context: context, 45 | 46 | // A binary searchable (i.e. sorted) list of counters. 47 | counters: []string{ 48 | "cursorid", 49 | "exhaust", 50 | "idhack", 51 | "keyUpdates", 52 | "fastmod", 53 | "fastmodinsert", 54 | "ndeleted", 55 | "ninserted", 56 | "nmoved", 57 | "nscanned", 58 | "ntoreturn", 59 | "ntoskip", 60 | "numYields", 61 | "nupdated", 62 | "scanAndOrder", 63 | "upsert", 64 | }, 65 | } 66 | }) 67 | } 68 | 69 | func (v *Version24Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 70 | r := internal.NewRuneReader(entry.RawMessage) 71 | switch { 72 | 73 | case entry.Connection > 0: 74 | // Check for connection related messages, which is almost everything *not* related to startup messages. 75 | switch { 76 | case r.ExpectString("query"), 77 | r.ExpectString("update"), 78 | r.ExpectString("remove"): 79 | // Commands or queries! 80 | op, err := v.parse24WithPayload(r, false) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | if crud, ok := Crud(op.Operation, op.Counters, op.Payload); ok { 86 | if op.Operation == "query" { 87 | // Standardize with newer versions to queries appear as finds. 88 | op.Operation = "find" 89 | } 90 | 91 | crud.Message = op 92 | return crud, nil 93 | } 94 | 95 | return op, nil 96 | 97 | case r.ExpectString("command"): 98 | // Commands in 2.4 don't include anything that should be converted 99 | // into operations (e.g. find, update, remove, etc). 100 | if op, err := v.parse24WithPayload(r, true); err != nil { 101 | return nil, err 102 | } else { 103 | cmd := message.MakeCommandLegacy() 104 | cmd.Command = op.Operation 105 | cmd.Duration = op.Duration 106 | cmd.Namespace = NamespaceReplace(op.Operation, op.Payload, op.Namespace) 107 | cmd.Locks = op.Locks 108 | cmd.Payload = op.Payload 109 | 110 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 111 | } 112 | 113 | case r.ExpectString("insert"): 114 | // Inserts! 115 | return v.parse24WithoutPayload(r) 116 | 117 | case r.ExpectString("getmore"): 118 | op, err := v.parse24WithoutPayload(r) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 124 | 125 | } 126 | 127 | // Cases that don't match a command or operation should be handled by 128 | // standard matching code. 129 | fallthrough 130 | 131 | default: 132 | // Check for other generic messages. 133 | return v.context.Run(entry, r, errorVersion24Unmatched) 134 | } 135 | } 136 | 137 | func (v *Version24Parser) Check(base record.Base) bool { 138 | return base.Component == record.ComponentNone && 139 | base.Severity == record.SeverityNone && 140 | base.CString 141 | } 142 | 143 | func (v *Version24Parser) Version() version.Definition { 144 | return version.Definition{Major: 2, Minor: 4, Binary: record.BinaryMongod} 145 | } 146 | 147 | func (v Version24Parser) parse24WithPayload(r *internal.RuneReader, command bool) (message.OperationLegacy, error) { 148 | // command test.$cmd command: { getlasterror: 1.0, w: 1.0 } ntoreturn:1 keyUpdates:0 reslen:67 0ms 149 | // query test.foo query: { b: 1.0 } ntoreturn:0 ntoskip:0 nscanned:10 keyUpdates:0 locks(micros) r:146 nreturned:1 reslen:64 0ms 150 | // update vcm_audit.payload.files query: { _id: ObjectId('000000000000000000000000') } update: { _id: ObjectId('000000000000000000000000') } idhack:1 nupdated:1 upsert:1 keyUpdates:0 locks(micros) w:33688 194ms 151 | op := message.MakeOperationLegacy() 152 | op.Operation, _ = r.SlurpWord() 153 | op.Namespace, _ = r.SlurpWord() 154 | 155 | if cmd := r.PreviewWord(1); cmd == "" { 156 | return message.OperationLegacy{}, internal.UnexpectedEOL 157 | } else if cmd != "command:" && command { 158 | return message.OperationLegacy{}, internal.CommandStructure 159 | } else if cmd != "query:" && !command { 160 | return message.OperationLegacy{}, internal.OperationStructure 161 | } 162 | 163 | // Define the target for key:value finds (start with counters and end with 164 | // locks) since both counters and locks look the same in this version. 165 | var target = op.Counters 166 | 167 | // Iterate through each word in the line. 168 | ParamLoop: 169 | for param, ok := r.SlurpWord(); ok; param, ok = r.SlurpWord() { 170 | if param[len(param)-1] == ':' { 171 | param = param[:len(param)-1] 172 | if op.Operation == "" { 173 | op.Operation = param 174 | } 175 | if r.ExpectRune('{') { 176 | if command { 177 | // Commands place the operation name before the JSON object. 178 | r.SlurpWord() 179 | word := r.PreviewWord(1) 180 | op.Operation = word[:len(word)-1] 181 | 182 | r.RewindSlurpWord() 183 | } 184 | 185 | if payload, err := mongo.ParseJsonRunes(r, false); err != nil { 186 | if !command { 187 | // An issue parsing runes could be caused by any number 188 | // of problems. But there is a subset of cases that can be 189 | // ignored. We only care about ignoring this subset if 190 | // a query already exists since at least part of the line 191 | // may be useful. 192 | 193 | if _, ok := op.Payload["query"]; ok { 194 | // A query exists so continue forward until we 195 | // find something that looks like a key:value. 196 | for ; ok; param, ok = r.SlurpWord() { 197 | if key, _, ok := internal.StringDoubleSplit(param, ':'); ok { 198 | if internal.ArrayBinaryMatchString(key, v.counters) { 199 | // I dislike using labels, but it's 200 | // quick, easy, and perfectly fine here. 201 | continue ParamLoop 202 | } 203 | } 204 | } 205 | } 206 | } 207 | 208 | // Otherwise, there's a problem. 209 | return op, err 210 | } else if command { 211 | op.Payload = payload 212 | } else { 213 | op.Payload[param] = payload 214 | } 215 | } 216 | // For whatever reason, numYields has a space between it and the 217 | // number (e.g. "numYields: 5"). 218 | if !internal.IsNumericRune(r.NextRune()) { 219 | continue 220 | } else { 221 | // Re-add so the parseIntegerKeyValueErratic will succeed. 222 | param = param + ":" 223 | } 224 | } 225 | v.parseIntegerKeyValueErratic(param, target, r) 226 | if param == "locks(micros)" { 227 | target = op.Locks 228 | } else if strings.HasSuffix(param, "ms") { 229 | op.Duration, _ = strconv.ParseInt(param[0:len(param)-2], 10, 64) 230 | } 231 | } 232 | 233 | return op, nil 234 | } 235 | func (Version24Parser) parse24WithoutPayload(r *internal.RuneReader) (message.OperationLegacy, error) { 236 | // insert test.system.indexes ninserted:1 keyUpdates:0 locks(micros) w:10527 10ms 237 | op := message.MakeOperationLegacy() 238 | op.Operation, _ = r.SlurpWord() 239 | op.Namespace, _ = r.SlurpWord() 240 | target := op.Counters 241 | for param, ok := r.SlurpWord(); ok; param, ok = r.SlurpWord() { 242 | if param == "locks(micros)" { 243 | target = op.Locks 244 | continue 245 | } else if param == "locks:{" { 246 | // Wrong version, so exit. 247 | return message.OperationLegacy{}, internal.VersionUnmatched{} 248 | } 249 | IntegerKeyValue(param, target, record.COUNTERS) 250 | } 251 | if dur, ok := r.SlurpWord(); ok && strings.HasSuffix(dur, "ms") { 252 | op.Duration, _ = strconv.ParseInt(dur[0:len(dur)-3], 10, 64) 253 | } 254 | return op, nil 255 | } 256 | 257 | func (Version24Parser) parseIntegerKeyValueErratic(param string, target map[string]int64, r *internal.RuneReader) { 258 | if !IntegerKeyValue(param, target, record.COUNTERS) && param[len(param)-1] == ':' { 259 | param = param[0 : len(param)-1] 260 | if num, err := strconv.ParseInt(r.PreviewWord(1), 10, 64); err == nil { 261 | if _, ok := record.COUNTERS[param]; ok { 262 | target[record.COUNTERS[param]] = num 263 | r.SlurpWord() 264 | } else { 265 | panic("unexpected counter type " + param + " found") 266 | } 267 | } 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /parser/db_26.go: -------------------------------------------------------------------------------- 1 | // mongo/src/mongo/db/client.cpp 2 | 3 | package parser 4 | 5 | import ( 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "mgotools/internal" 11 | "mgotools/mongo" 12 | "mgotools/parser/executor" 13 | "mgotools/parser/message" 14 | "mgotools/parser/record" 15 | "mgotools/parser/version" 16 | ) 17 | 18 | type Version26Parser struct { 19 | crud *executor.Executor 20 | contexts *executor.Executor 21 | } 22 | 23 | var errorVersion26Unmatched = internal.VersionUnmatched{Message: "version 2.6"} 24 | 25 | func init() { 26 | version.Factory.Register(func() version.Parser { 27 | v := &Version26Parser{ 28 | crud: &executor.Executor{}, 29 | contexts: &executor.Executor{}, 30 | } 31 | 32 | messages := v.crud 33 | messages.RegisterForReader("command", v.commandCrud) 34 | 35 | messages.RegisterForReader("query", v.operationCrud) 36 | messages.RegisterForReader("getmore", v.operationCrud) 37 | messages.RegisterForReader("geonear", v.operationCrud) 38 | messages.RegisterForReader("insert", v.operationCrud) 39 | messages.RegisterForReader("update", v.operationCrud) 40 | messages.RegisterForReader("remove", v.operationCrud) 41 | 42 | context := v.contexts 43 | 44 | // initandlisten 45 | context.RegisterForReader("build info", mongodBuildInfo) 46 | context.RegisterForReader("db version", mongodDbVersion) 47 | context.RegisterForReader("journal dir=", mongodJournal) 48 | context.RegisterForReader("options", mongodOptions) 49 | 50 | context.RegisterForEntry("MongoDB starting", mongodStartupInfo) 51 | 52 | // signalProcessingThread 53 | context.RegisterForReader("dbexit", mongodParseShutdown) 54 | 55 | // connection related 56 | context.RegisterForReader("connection accepted", commonParseConnectionAccepted) 57 | context.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 58 | context.RegisterForReader("successfully authenticated as principal", commonParseAuthenticatedPrincipal) 59 | 60 | context.RegisterForEntry("end connection", commonParseConnectionEnded) 61 | 62 | return v 63 | }) 64 | } 65 | 66 | func (v *Version26Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 67 | // Retrieve a value-based RuneReader because most functions don't take a 68 | // reference. This makes sense here because the RuneReader should be "reset" 69 | // on failure to parse. What better way to reset a RuneReader than to never 70 | // modify it in the first place, right? 71 | r := *internal.NewRuneReader(entry.RawMessage) 72 | switch { 73 | 74 | case entry.Connection > 0: 75 | msg, err := v.crud.Run(entry, &r, errorVersion26Unmatched) 76 | 77 | if err != errorVersion26Unmatched { 78 | return msg, err 79 | } 80 | 81 | // Check for network status changes, which have a context that 82 | // matches operations and commands. 83 | fallthrough 84 | 85 | default: 86 | msg, err := v.contexts.Run(entry, &r, errorVersion26Unmatched) 87 | 88 | if err != errorVersion26Unmatched { 89 | return msg, err 90 | } else { 91 | return nil, errorVersion26Unmatched 92 | } 93 | 94 | } 95 | } 96 | 97 | func (v Version26Parser) commandCrud(r *internal.RuneReader) (message.Message, error) { 98 | c, err := v.command(r) 99 | if err != nil { 100 | return c, err 101 | } 102 | 103 | return CrudOrMessage(c, c.Command, c.Counters, c.Payload), nil 104 | } 105 | 106 | func (Version26Parser) command(r *internal.RuneReader) (message.CommandLegacy, error) { 107 | var err error 108 | cmd := message.MakeCommandLegacy() 109 | 110 | if c, n, o, err := Preamble(r); err != nil { 111 | return message.CommandLegacy{}, err 112 | } else if c != "command" || o != "command" { 113 | return message.CommandLegacy{}, internal.CommandStructure 114 | } else { 115 | cmd.Command = c 116 | cmd.Namespace = n 117 | 118 | if word, ok := r.SlurpWord(); !ok { 119 | return message.CommandLegacy{}, internal.CommandStructure 120 | } else if cmd.Payload, err = Payload(r); err != nil { 121 | return message.CommandLegacy{}, err 122 | } else { 123 | cmd.Command = word 124 | } 125 | } 126 | 127 | cmd.Namespace = NamespaceReplace(cmd.Command, cmd.Payload, cmd.Namespace) 128 | counters := false 129 | for { 130 | param, ok := r.SlurpWord() 131 | if !ok { 132 | break 133 | } 134 | 135 | if !counters { 136 | if ok, err := StringSections(param, &cmd.BaseCommand, cmd.Payload, r); err != nil { 137 | return message.CommandLegacy{}, nil 138 | } else if ok { 139 | continue 140 | } 141 | if param == "locks(micros)" { 142 | counters = true 143 | continue 144 | } 145 | } 146 | if strings.HasSuffix(param, "ms") { 147 | if cmd.Duration, err = strconv.ParseInt(param[:len(param)-2], 10, 64); err != nil { 148 | return message.CommandLegacy{}, err 149 | } 150 | break 151 | } else if strings.ContainsRune(param, ':') { 152 | if !IntegerKeyValue(param, cmd.Counters, record.COUNTERS) && 153 | !IntegerKeyValue(param, cmd.Locks, map[string]string{"r": "r", "R": "R", "w": "w", "W": "W"}) { 154 | return message.CommandLegacy{}, internal.CounterUnrecognized 155 | } 156 | } 157 | } 158 | 159 | return cmd, nil 160 | } 161 | 162 | func (Version26Parser) Check(base record.Base) bool { 163 | return base.Severity == record.SeverityNone && 164 | base.Component == record.ComponentNone 165 | } 166 | 167 | func (Version26Parser) operation(r *internal.RuneReader) (message.OperationLegacy, error) { 168 | // getmore test.foo cursorid:30107363235 ntoreturn:3 keyUpdates:0 numYields:0 locks(micros) r:14 nreturned:3 reslen:119 0ms 169 | // insert test.foo query: { _id: ObjectId('5a331671de4f2a133f17884b'), a: 2.0 } ninserted:1 keyUpdates:0 numYields:0 locks(micros) w:10 0ms 170 | // remove test.foo query: { a: { $gte: 9.0 } } ndeleted:1 keyUpdates:0 numYields:0 locks(micros) w:63 0ms 171 | // update test.foo query: { a: { $gte: 8.0 } } update: { $set: { b: 1.0 } } nscanned:9 nscannedObjects:9 nMatched:1 nModified:1 keyUpdates:0 numYields:0 locks(micros) w:135 0ms 172 | op := message.MakeOperationLegacy() 173 | 174 | if c, n, _, err := Preamble(r); err != nil { 175 | return message.OperationLegacy{}, err 176 | } else { 177 | // Rewind to capture the "query" portion of the line (or counter if 178 | // query doesn't exist). 179 | r.RewindSlurpWord() 180 | 181 | op.Operation = c 182 | op.Namespace = n 183 | } 184 | 185 | for param, ok := r.SlurpWord(); ok; param, ok = r.SlurpWord() { 186 | if ok, err := StringSections(param, &op.BaseCommand, op.Payload, r); err != nil { 187 | return op, err 188 | } else if ok { 189 | continue 190 | } 191 | 192 | switch { 193 | case strings.HasPrefix(param, "locks"): 194 | if param != "locks(micros)" { 195 | // Wrong version. 196 | return message.OperationLegacy{}, internal.VersionUnmatched{} 197 | } 198 | continue 199 | 200 | case !strings.HasSuffix(param, ":") && strings.ContainsRune(param, ':'): 201 | // A counter (in the form of key:value) needs to be applied to the correct target. 202 | if !IntegerKeyValue(param, op.Locks, map[string]string{"r": "r", "R": "R", "w": "w", "W": "W"}) && 203 | !IntegerKeyValue(param, op.Counters, record.COUNTERS) { 204 | return message.OperationLegacy{}, internal.CounterUnrecognized 205 | } 206 | 207 | case strings.HasSuffix(param, "ms"): 208 | // Found a duration, which is also the last thing in the line. 209 | op.Duration, _ = strconv.ParseInt(param[0:len(param)-2], 10, 64) 210 | return op, nil 211 | 212 | default: 213 | if length := len(param); length > 1 && internal.ArrayBinaryMatchString(param[:length-1], record.OPERATIONS) { 214 | if r.EOL() { 215 | return op, internal.OperationStructure 216 | } 217 | 218 | // Parse JSON, found immediately after an operation. 219 | var err error 220 | if op.Payload[param[:length-1]], err = mongo.ParseJsonRunes(r, false); err != nil { 221 | return op, err 222 | } 223 | } else { 224 | // An unexpected value means that this parser either isn't the correct version or the line is invalid. 225 | return message.OperationLegacy{}, internal.VersionUnmatched{Message: fmt.Sprintf("encountered unexpected value '%s'", param)} 226 | } 227 | } 228 | } 229 | 230 | return op, internal.CommandStructure 231 | } 232 | 233 | func (v Version26Parser) operationCrud(r *internal.RuneReader) (message.Message, error) { 234 | m, err := v.operation(r) 235 | if err != nil { 236 | return m, err 237 | } 238 | 239 | if crud, ok := Crud(m.Operation, m.Counters, m.Payload); ok { 240 | if m.Operation == "query" { 241 | // Standardize operation with later versions. 242 | m.Operation = "find" 243 | } 244 | 245 | crud.Message = m 246 | return crud, nil 247 | } 248 | 249 | return m, nil 250 | } 251 | 252 | func (Version26Parser) Version() version.Definition { 253 | return version.Definition{Major: 2, Minor: 6, Binary: record.BinaryMongod} 254 | } 255 | -------------------------------------------------------------------------------- /parser/db_30.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version30Parser struct { 12 | executor *executor.Executor 13 | 14 | counters map[string]string 15 | versionFlag bool 16 | } 17 | 18 | var errorVersion30Unmatched = internal.VersionUnmatched{Message: "version 3.0"} 19 | 20 | func init() { 21 | version.Factory.Register(func() version.Parser { 22 | ex := executor.New() 23 | 24 | // CONTROL component 25 | ex.RegisterForEntry("MongoDB starting", mongodStartupInfo) 26 | ex.RegisterForReader("db version", mongodDbVersion) 27 | ex.RegisterForReader("build info", mongodBuildInfo) 28 | ex.RegisterForReader("options", mongodOptions) 29 | ex.RegisterForReader("journal dir=", mongodJournal) 30 | ex.RegisterForReader("dbexit", mongodParseShutdown) 31 | 32 | // NETWORK component 33 | ex.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 34 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 35 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 36 | 37 | return &Version30Parser{ 38 | executor: ex, 39 | 40 | counters: map[string]string{ 41 | "cursorid": "cursorid", 42 | "ntoreturn": "ntoreturn", 43 | "ntoskip": "ntoskip", 44 | "exhaust": "exhaust", 45 | "nscanned": "keysExamined", 46 | "nscannedObjects": "docsExamined", 47 | "idhack": "idhack", 48 | "scanAndOrder": "scanAndOrder", 49 | "nmoved": "nmoved", 50 | "nMatched": "nmatched", 51 | "nModified": "nmodified", 52 | "ninserted": "ninserted", 53 | "ndeleted": "ndeleted", 54 | "fastmod": "fastmod", 55 | "fastmodinsert": "fastmodinsert", 56 | "upsert": "upsert", 57 | "keyUpdates": "keyUpdates", 58 | "writeConflicts": "writeConflicts", 59 | "nreturned": "nreturned", 60 | "numYields": "numYields", 61 | "reslen": "reslend", 62 | }, 63 | 64 | versionFlag: true, 65 | } 66 | }) 67 | } 68 | 69 | func (v *Version30Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 70 | r := internal.NewRuneReader(entry.RawMessage) 71 | 72 | // MDB 3.0 outputs commands and operations in a format almost identical to 73 | // MDB 2.6, which means we can use the legacy parser to handle the parsing. 74 | // The major difference is we have a component, so there's no need to 75 | // specially parse preamble (query, remove, command, etc). 76 | switch entry.Component { 77 | case record.ComponentCommand: 78 | c := r.PreviewWord(1) 79 | if c == "query" || c == "getmore" { 80 | return v.crud(false, r) 81 | } else { 82 | return v.crud(true, r) 83 | } 84 | 85 | case record.ComponentWrite: 86 | return v.crud(false, r) 87 | 88 | default: 89 | msg, err := v.executor.Run(entry, r, errorVersion30Unmatched) 90 | if err == nil { 91 | return msg, err 92 | } 93 | 94 | return nil, errorVersion30Unmatched 95 | } 96 | } 97 | 98 | func (v *Version30Parser) Check(base record.Base) bool { 99 | return v.versionFlag && 100 | base.Severity != record.SeverityNone && 101 | base.Severity >= record.SeverityD1 && base.Severity < record.SeverityD5 && 102 | v.expectedComponents(base.Component) 103 | } 104 | 105 | func (v *Version30Parser) command(r *internal.RuneReader) (message.Command, error) { 106 | cmd, err := CommandPreamble(r) 107 | if err != nil { 108 | return message.Command{}, err 109 | } 110 | 111 | err = MidLoop(r, "locks:", &cmd.BaseCommand, cmd.Counters, cmd.Payload, v.counters) 112 | if err != nil { 113 | if err == internal.CounterUnrecognized { 114 | v.versionFlag = false 115 | err = internal.VersionUnmatched{Message: "counter unrecognized"} 116 | } 117 | return message.Command{}, err 118 | } 119 | 120 | cmd.Locks, err = Locks(r) 121 | if err != nil { 122 | return message.Command{}, err 123 | } 124 | 125 | cmd.Duration, err = Duration(r) 126 | if err != nil { 127 | r.RewindSlurpWord() 128 | if r.ExpectString("protocol:") { 129 | v.versionFlag = false 130 | } 131 | return message.Command{}, err 132 | } 133 | 134 | return cmd, nil 135 | } 136 | 137 | func (v Version30Parser) crud(command bool, r *internal.RuneReader) (message.Message, error) { 138 | if command { 139 | // This should be similar to handling in version 2.6. 140 | c, err := v.command(r) 141 | if err != nil { 142 | return nil, err 143 | } 144 | 145 | return CrudOrMessage(c, c.Command, c.Counters, c.Payload), nil 146 | 147 | } else { 148 | o, err := v.operation(r) 149 | if err != nil { 150 | return nil, err 151 | } 152 | 153 | return CrudOrMessage(o, o.Operation, o.Counters, o.Payload), nil 154 | } 155 | } 156 | 157 | func (v *Version30Parser) expectedComponents(c record.Component) bool { 158 | switch c { 159 | case record.ComponentAccess, 160 | record.ComponentAccessControl, 161 | record.ComponentBridge, 162 | record.ComponentCommand, 163 | record.ComponentControl, 164 | record.ComponentDefault, 165 | record.ComponentGeo, 166 | record.ComponentIndex, 167 | record.ComponentJournal, 168 | record.ComponentNetwork, 169 | record.ComponentQuery, 170 | record.ComponentRepl, 171 | record.ComponentReplication, 172 | record.ComponentSharding, 173 | record.ComponentStorage, 174 | record.ComponentTotal, 175 | record.ComponentWrite, 176 | record.ComponentUnknown: 177 | return true 178 | default: 179 | return false 180 | } 181 | } 182 | 183 | func (v Version30Parser) operation(r *internal.RuneReader) (message.Operation, error) { 184 | op, err := OperationPreamble(r) 185 | if err != nil { 186 | return op, err 187 | } 188 | 189 | err = MidLoop(r, "locks:", &op.BaseCommand, op.Counters, op.Payload, v.counters) 190 | if err != nil { 191 | return message.Operation{}, err 192 | } else if !internal.ArrayBinaryMatchString(op.Operation, record.OPERATIONS) { 193 | return message.Operation{}, internal.OperationStructure 194 | } 195 | 196 | op.Locks, err = Locks(r) 197 | if err != nil { 198 | return message.Operation{}, err 199 | } 200 | 201 | op.Duration, err = Duration(r) 202 | if err != nil { 203 | return message.Operation{}, err 204 | } 205 | 206 | return op, nil 207 | } 208 | 209 | func (v *Version30Parser) Version() version.Definition { 210 | return version.Definition{Major: 3, Minor: 0, Binary: record.BinaryMongod} 211 | } 212 | -------------------------------------------------------------------------------- /parser/db_32.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | 6 | "mgotools/internal" 7 | "mgotools/parser/executor" 8 | "mgotools/parser/message" 9 | "mgotools/parser/record" 10 | "mgotools/parser/version" 11 | ) 12 | 13 | type Version32Parser struct { 14 | counters map[string]string 15 | executor *executor.Executor 16 | versionFlag bool 17 | } 18 | 19 | var errorVersion32Unmatched = internal.VersionUnmatched{Message: "version 3.2"} 20 | 21 | func init() { 22 | version.Factory.Register(func() version.Parser { 23 | ex := executor.New() 24 | 25 | // CONTROL components 26 | ex.RegisterForReader("wiredtiger_open config", commonParseWiredtigerOpen) // 3.2+ 27 | ex.RegisterForReader("db version", mongodDbVersion) 28 | ex.RegisterForReader("options", mongodOptions) 29 | ex.RegisterForReader("journal dir=", mongodJournal) 30 | ex.RegisterForReader("dbexit", mongodParseShutdown) 31 | 32 | // NETWORK component 33 | ex.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 34 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 35 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 36 | 37 | return &Version32Parser{ 38 | counters: map[string]string{ 39 | "cursorid": "cursorid", 40 | "ntoreturn": "ntoreturn", 41 | "ntoskip": "notoskip", 42 | "exhaust": "exhaust", 43 | "keysExamined": "keysExamined", 44 | "docsExamined": "docsExamined", 45 | "idhack": "idhack", 46 | "hasSortStage": "hasSortStage", 47 | "fromMultiPlanner": "fromMultiPlanner", 48 | "nmoved": "nmoved", 49 | "nMatched": "nmatched", 50 | "nModified": "nmodified", 51 | "ninserted": "ninserted", 52 | "ndeleted": "ndeleted", 53 | "numYields": "numYields", 54 | "nreturned": "nreturned", 55 | "fastmod": "fastmod", 56 | "fastmodinsert": "fastmodinsert", 57 | "upsert": "upsert", 58 | "cursorExhausted": "cursorExhausted", 59 | "keyUpdates": "keyUpdates", 60 | "writeConflicts": "writeConflicts", 61 | "reslen": "reslen", 62 | }, 63 | 64 | executor: ex, 65 | versionFlag: true, 66 | } 67 | }) 68 | } 69 | 70 | func (v *Version32Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 71 | r := *internal.NewRuneReader(entry.RawMessage) 72 | 73 | switch entry.Component { 74 | case record.ComponentCommand: 75 | // query, getmore, insert, update = COMMAND 76 | cmd, err := v.command(r) 77 | if err != nil { 78 | return cmd, err 79 | } 80 | 81 | return CrudOrMessage(cmd, cmd.Command, cmd.Counters, cmd.Payload), nil 82 | 83 | case record.ComponentWrite: 84 | // insert, remove, update = WRITE 85 | op, err := v.operation(r) 86 | if err != nil { 87 | return op, err 88 | } 89 | 90 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 91 | 92 | case record.ComponentNetwork: 93 | if entry.RawContext == "command" { 94 | if msg, err := v.command(r); err != nil { 95 | return msg, nil 96 | } 97 | } 98 | 99 | fallthrough 100 | 101 | default: 102 | return v.executor.Run(entry, &r, errorVersion32Unmatched) 103 | } 104 | } 105 | 106 | func (v *Version32Parser) Check(base record.Base) bool { 107 | return v.versionFlag && 108 | base.Severity != record.SeverityNone && 109 | base.Component != record.ComponentNone && v.expectedComponents(base.Component) 110 | } 111 | 112 | func (v Version32Parser) command(reader internal.RuneReader) (message.Command, error) { 113 | r := &reader 114 | 115 | cmd, err := CommandPreamble(r) 116 | if err != nil { 117 | return message.Command{}, err 118 | } else if cmd.Agent != "" { 119 | // version 3.2 does not provide an agent string. 120 | v.versionFlag = false 121 | return message.Command{}, errorVersion32Unmatched 122 | } 123 | 124 | err = MidLoop(r, "locks:", &cmd.BaseCommand, cmd.Counters, cmd.Payload, v.counters) 125 | if err != nil { 126 | v.versionFlag, err = CheckCounterVersionError(err, errorVersion32Unmatched) 127 | return message.Command{}, err 128 | } 129 | 130 | cmd.Locks, err = Locks(r) 131 | if err != nil { 132 | return message.Command{}, err 133 | } 134 | 135 | cmd.Protocol, err = Protocol(r) 136 | if err != nil { 137 | return message.Command{}, err 138 | } else if cmd.Protocol != "op_query" && cmd.Protocol != "op_command" { 139 | v.versionFlag = false 140 | return message.Command{}, internal.VersionUnmatched{Message: fmt.Sprintf("unexpected protocol %s", cmd.Protocol)} 141 | } 142 | 143 | cmd.Duration, err = Duration(r) 144 | if err != nil { 145 | return message.Command{}, err 146 | } 147 | 148 | return cmd, nil 149 | } 150 | 151 | func (v *Version32Parser) expectedComponents(c record.Component) bool { 152 | switch c { 153 | case record.ComponentAccess, 154 | record.ComponentAccessControl, 155 | record.ComponentASIO, 156 | record.ComponentBridge, 157 | record.ComponentCommand, 158 | record.ComponentControl, 159 | record.ComponentDefault, 160 | record.ComponentExecutor, 161 | record.ComponentFTDC, 162 | record.ComponentGeo, 163 | record.ComponentIndex, 164 | record.ComponentJournal, 165 | record.ComponentNetwork, 166 | record.ComponentQuery, 167 | record.ComponentRepl, 168 | record.ComponentReplication, 169 | record.ComponentSharding, 170 | record.ComponentStorage, 171 | record.ComponentTotal, 172 | record.ComponentWrite, 173 | record.ComponentUnknown: 174 | return true 175 | default: 176 | return false 177 | } 178 | } 179 | 180 | func (v *Version32Parser) operation(reader internal.RuneReader) (message.Operation, error) { 181 | r := &reader 182 | 183 | op, err := OperationPreamble(r) 184 | if err != nil { 185 | return op, err 186 | } 187 | 188 | err = MidLoop(r, "locks:", &op.BaseCommand, op.Counters, op.Payload, v.counters) 189 | if err != nil { 190 | v.versionFlag, err = CheckCounterVersionError(err, errorVersion32Unmatched) 191 | return message.Operation{}, err 192 | } 193 | 194 | op.Locks, err = Locks(r) 195 | if err != nil { 196 | return message.Operation{}, err 197 | } 198 | 199 | op.Duration, err = Duration(r) 200 | if err != nil { 201 | return message.Operation{}, err 202 | } 203 | 204 | return op, nil 205 | } 206 | 207 | func (v *Version32Parser) Version() version.Definition { 208 | return version.Definition{Major: 3, Minor: 2, Binary: record.BinaryMongod} 209 | } 210 | -------------------------------------------------------------------------------- /parser/db_34.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/mongo" 6 | "mgotools/parser/executor" 7 | "mgotools/parser/message" 8 | "mgotools/parser/record" 9 | "mgotools/parser/version" 10 | ) 11 | 12 | type Version34Parser struct { 13 | executor *executor.Executor 14 | counters map[string]string 15 | versionFlag bool 16 | } 17 | 18 | var errorVersion34Unmatched = internal.VersionUnmatched{Message: "version 3.2"} 19 | 20 | func init() { 21 | version.Factory.Register(func() version.Parser { 22 | ex := executor.New() 23 | 24 | // CONTROL components 25 | ex.RegisterForReader("build info", mongodBuildInfo) 26 | ex.RegisterForReader("dbexit", mongodParseShutdown) 27 | ex.RegisterForReader("db version", mongodDbVersion) 28 | ex.RegisterForReader("journal dir=", mongodJournal) 29 | ex.RegisterForReader("options", mongodOptions) 30 | ex.RegisterForReader("wiredtiger_open config", commonParseWiredtigerOpen) 31 | 32 | // NETWORK components 33 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 34 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 35 | ex.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 36 | ex.RegisterForReader("received client metadata from", commonParseClientMetadata) // 3.4+ 37 | 38 | return &Version34Parser{ 39 | counters: map[string]string{ 40 | "cursorid": "cursorid", 41 | "notoreturn": "ntoreturn", 42 | "ntoskip": "ntoskip", 43 | "exhaust": "exhaust", 44 | "keysExamined": "keysExamined", 45 | "docsExamined": "docsExamined", 46 | "hasSortStage": "hasSortStage", 47 | "fromMultiPlanner": "fromMultiPlanner", 48 | "replanned": "replanned", 49 | "nMatched": "nmatched", 50 | "nModified": "nmodified", 51 | "ninserted": "ninserted", 52 | "ndeleted": "ndeleted", 53 | "fastmodinsert": "fastmodinsert", 54 | "upsert": "upsert", 55 | "cursorExhausted": "cursorExhausted", 56 | "nmoved": "nmoved", 57 | "keysInserted": "keysInserted", 58 | "keysDeleted": "keysDeleted", 59 | "writeConflicts": "writeConflicts", 60 | "numYields": "numYields", 61 | "reslen": "reslen", 62 | "nreturned": "nreturned", 63 | }, 64 | 65 | executor: ex, 66 | versionFlag: true, 67 | } 68 | }) 69 | } 70 | 71 | func (v *Version34Parser) Check(base record.Base) bool { 72 | return v.versionFlag && 73 | base.Severity != record.SeverityNone && 74 | base.Component != record.ComponentNone 75 | } 76 | 77 | func (v *Version34Parser) command(reader internal.RuneReader) (message.Command, error) { 78 | r := &reader 79 | 80 | // Trivia: version 3.4 was the first to introduce app name metadata. 81 | cmd, err := CommandPreamble(r) 82 | if err != nil { 83 | return message.Command{}, err 84 | } 85 | 86 | if r.ExpectString("originatingCommand:") { 87 | r.SkipWords(1) 88 | cmd.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 89 | 90 | if err != nil { 91 | return message.Command{}, err 92 | } 93 | } 94 | 95 | // Commands cannot have a "collation:" section, so this should be identical 96 | // to earlier versions (e.g. 3.2.x). 97 | err = MidLoop(r, "locks:", &cmd.BaseCommand, cmd.Counters, cmd.Payload, v.counters) 98 | if err != nil { 99 | v.versionFlag, err = CheckCounterVersionError(err, errorVersion34Unmatched) 100 | return message.Command{}, err 101 | } 102 | 103 | cmd.Locks, err = Locks(r) 104 | if err != nil { 105 | return message.Command{}, err 106 | } 107 | 108 | cmd.Protocol, err = Protocol(r) 109 | if err != nil { 110 | return message.Command{}, err 111 | } else if cmd.Protocol != "op_query" && cmd.Protocol != "op_command" { 112 | v.versionFlag = false 113 | return message.Command{}, errorVersion34Unmatched 114 | } 115 | 116 | cmd.Duration, err = Duration(r) 117 | if err != nil { 118 | return message.Command{}, err 119 | } 120 | 121 | return cmd, nil 122 | } 123 | 124 | func (v *Version34Parser) expectedComponents(c record.Component) bool { 125 | switch c { 126 | case record.ComponentAccess, 127 | record.ComponentAccessControl, 128 | record.ComponentASIO, 129 | record.ComponentBridge, 130 | record.ComponentCommand, 131 | record.ComponentControl, 132 | record.ComponentDefault, 133 | record.ComponentExecutor, 134 | record.ComponentFTDC, 135 | record.ComponentGeo, 136 | record.ComponentIndex, 137 | record.ComponentJournal, 138 | record.ComponentNetwork, 139 | record.ComponentQuery, 140 | record.ComponentRepl, 141 | record.ComponentReplication, 142 | record.ComponentSharding, 143 | record.ComponentStorage, 144 | record.ComponentTotal, 145 | record.ComponentTracking, 146 | record.ComponentWrite, 147 | record.ComponentUnknown: 148 | return true 149 | default: 150 | return false 151 | } 152 | } 153 | 154 | func (v *Version34Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 155 | r := internal.NewRuneReader(entry.RawMessage) 156 | switch entry.Component { 157 | case record.ComponentCommand: 158 | cmd, err := v.command(*r) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | return CrudOrMessage(cmd, cmd.Command, cmd.Counters, cmd.Payload), nil 164 | 165 | case record.ComponentWrite: 166 | op, err := v.operation(*r) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 172 | 173 | default: 174 | return v.executor.Run(entry, r, errorVersion34Unmatched) 175 | } 176 | } 177 | 178 | func (v *Version34Parser) operation(reader internal.RuneReader) (message.Operation, error) { 179 | r := &reader 180 | 181 | op, err := OperationPreamble(r) 182 | if err != nil { 183 | return message.Operation{}, err 184 | } 185 | 186 | if !internal.ArrayBinaryMatchString(op.Operation, []string{"command", "commandReply", "compressed", "getmore", "insert", "killcursors", "msg", "none", "query", "remove", "reply", "update"}) { 187 | v.versionFlag = false 188 | return message.Operation{}, errorVersion34Unmatched 189 | } 190 | 191 | for { 192 | // Collation appears in this version for the first time and doesn't 193 | // appear in any subsequent versions. It also only appears on WRITE 194 | // operations. 195 | err = MidLoop(r, "collation:", &op.BaseCommand, op.Counters, op.Payload, v.counters) 196 | if err != nil { 197 | v.versionFlag, err = CheckCounterVersionError(err, errorVersion34Unmatched) 198 | return message.Operation{}, err 199 | } else if r.ExpectString("collation:") { 200 | r.SkipWords(1) 201 | op.Payload["collation"], err = mongo.ParseJsonRunes(r, false) 202 | if err != nil { 203 | return message.Operation{}, err 204 | } 205 | } else { 206 | // This condition occurs after reaching "locks:". 207 | break 208 | } 209 | } 210 | 211 | op.Locks, err = Locks(r) 212 | if err != nil { 213 | return message.Operation{}, err 214 | } 215 | 216 | op.Duration, err = Duration(r) 217 | if err != nil { 218 | return message.Operation{}, err 219 | } 220 | 221 | return op, nil 222 | } 223 | 224 | func (v *Version34Parser) Version() version.Definition { 225 | return version.Definition{Major: 3, Minor: 4, Binary: record.BinaryMongod} 226 | } 227 | -------------------------------------------------------------------------------- /parser/db_36.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/mongo" 6 | "mgotools/parser/executor" 7 | "mgotools/parser/message" 8 | "mgotools/parser/record" 9 | "mgotools/parser/version" 10 | ) 11 | 12 | type Version36Parser struct { 13 | counters map[string]string 14 | executor *executor.Executor 15 | } 16 | 17 | var errorVersion36Unmatched = internal.VersionUnmatched{Message: "version 3.6"} 18 | 19 | func init() { 20 | version.Factory.Register(func() version.Parser { 21 | ex := executor.New() 22 | 23 | // CONTROL components 24 | ex.RegisterForReader("dbexit", mongodParseShutdown) 25 | ex.RegisterForReader("db version", mongodDbVersion) 26 | ex.RegisterForReader("journal dir=", mongodJournal) 27 | ex.RegisterForReader("options", mongodOptions) 28 | ex.RegisterForReader("wiredtiger_open config", commonParseWiredtigerOpen) 29 | 30 | // NETWORK components 31 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 32 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 33 | ex.RegisterForReader("waiting for connection", commonParseWaitingForConnections) 34 | ex.RegisterForReader("received client metadata from", commonParseClientMetadata) 35 | 36 | return &Version36Parser{ 37 | counters: map[string]string{ 38 | "cursorid": "cursorid", 39 | "notoreturn": "ntoreturn", 40 | "ntoskip": "ntoskip", 41 | "exhaust": "exhaust", 42 | "keysExamined": "keysExamined", 43 | "docsExamined": "docsExamined", 44 | "hasSortStage": "hasSortStage", 45 | "fromMultiPlanner": "fromMultiPlanner", 46 | "replanned": "replanned", 47 | "nMatched": "nmatched", 48 | "nModified": "nmodified", 49 | "ninserted": "ninserted", 50 | "ndeleted": "ndeleted", 51 | "nreturned": "nreturned", 52 | "fastmodinsert": "fastmodinsert", 53 | "upsert": "upsert", 54 | "cursorExhausted": "cursorExhausted", 55 | "nmoved": "nmoved", 56 | "keysInserted": "keysInserted", 57 | "keysDeleted": "keysDeleted", 58 | "writeConflicts": "writeConflicts", 59 | "numYields": "numYields", 60 | "reslen": "reslen", 61 | }, 62 | 63 | executor: ex, 64 | } 65 | }) 66 | } 67 | 68 | func (v *Version36Parser) Check(base record.Base) bool { 69 | return base.Severity != record.SeverityNone && 70 | base.Component != record.ComponentNone 71 | } 72 | 73 | func (v *Version36Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 74 | r := internal.NewRuneReader(entry.RawMessage) 75 | switch entry.Component { 76 | case record.ComponentCommand: 77 | cmd, err := v.command(*r) 78 | if err != nil { 79 | return nil, err 80 | } 81 | return CrudOrMessage(cmd, cmd.Command, cmd.Counters, cmd.Payload), nil 82 | 83 | case record.ComponentWrite: 84 | op, err := v.operation(*r) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 89 | 90 | default: 91 | return v.executor.Run(entry, r, errorVersion36Unmatched) 92 | } 93 | } 94 | 95 | func (v *Version36Parser) command(reader internal.RuneReader) (message.Command, error) { 96 | r := &reader 97 | 98 | cmd, err := CommandPreamble(r) 99 | if err != nil { 100 | return message.Command{}, err 101 | } 102 | 103 | if r.ExpectString("originatingCommand") { 104 | r.SkipWords(1) 105 | cmd.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 106 | 107 | if err != nil { 108 | return message.Command{}, err 109 | } 110 | } 111 | 112 | if r.ExpectString("planSummary:") { 113 | r.Skip(12).ChompWS() 114 | 115 | cmd.PlanSummary, err = PlanSummary(r) 116 | if err != nil { 117 | return message.Command{}, err 118 | } 119 | } 120 | 121 | for { 122 | param, ok := r.SlurpWord() 123 | if !ok { 124 | break 125 | } else if param == "exception:" { 126 | cmd.Exception, ok = Exception(r) 127 | if !ok { 128 | return message.Command{}, internal.UnexpectedExceptionFormat 129 | } 130 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 131 | r.RewindSlurpWord() 132 | break 133 | } else if !IntegerKeyValue(param, cmd.Counters, v.counters) { 134 | return message.Command{}, internal.CounterUnrecognized 135 | } 136 | } 137 | 138 | cmd.Locks, err = Locks(r) 139 | if err != nil { 140 | return message.Command{}, err 141 | } 142 | 143 | cmd.Protocol, err = Protocol(r) 144 | if err != nil { 145 | return message.Command{}, err 146 | } else if cmd.Protocol != "op_msg" && cmd.Protocol != "op_query" && cmd.Protocol != "op_command" { 147 | return message.Command{}, errorVersion36Unmatched 148 | } 149 | 150 | cmd.Duration, err = Duration(r) 151 | if err != nil { 152 | return message.Command{}, err 153 | } 154 | 155 | return cmd, nil 156 | } 157 | 158 | func (v *Version36Parser) operation(reader internal.RuneReader) (message.Operation, error) { 159 | r := &reader 160 | 161 | op, err := OperationPreamble(r) 162 | if err != nil { 163 | return message.Operation{}, err 164 | } 165 | 166 | // Check against the expected list of operations. Anything not in this list 167 | // is either very broken or a different version. 168 | if !internal.ArrayBinaryMatchString(op.Operation, []string{"command", "commandReply", "compressed", "getmore", "insert", "killcursors", "msg", "none", "query", "remove", "reply", "update"}) { 169 | return message.Operation{}, errorVersion36Unmatched 170 | } 171 | 172 | // The next word should always be "command:" 173 | if c, ok := r.SlurpWord(); !ok { 174 | return message.Operation{}, internal.UnexpectedEOL 175 | } else if c != "command:" { 176 | return message.Operation{}, errorVersion36Unmatched 177 | } 178 | 179 | // There is no bareword like a command (even though the last word was 180 | // "command:") so the only available option is a JSON document. 181 | if !r.ExpectRune('{') { 182 | return message.Operation{}, internal.OperationStructure 183 | } 184 | 185 | op.Payload, err = mongo.ParseJsonRunes(r, false) 186 | if err != nil { 187 | return message.Operation{}, err 188 | } 189 | 190 | if r.ExpectString("originatingCommand:") { 191 | r.Skip(19).ChompWS() 192 | 193 | op.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 194 | if err != nil { 195 | return message.Operation{}, err 196 | } 197 | } 198 | 199 | if r.ExpectString("planSummary:") { 200 | r.Skip(12).ChompWS() 201 | 202 | op.PlanSummary, err = PlanSummary(r) 203 | if err != nil { 204 | return message.Operation{}, err 205 | } 206 | } 207 | 208 | for { 209 | param, ok := r.SlurpWord() 210 | if !ok { 211 | break 212 | } else if param == "exception:" { 213 | op.Exception, ok = Exception(r) 214 | if !ok { 215 | return message.Operation{}, internal.UnexpectedExceptionFormat 216 | } 217 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 218 | r.RewindSlurpWord() 219 | break 220 | } else if !IntegerKeyValue(param, op.Counters, v.counters) { 221 | return message.Operation{}, internal.CounterUnrecognized 222 | } 223 | } 224 | 225 | // Skip "locks:" and resume with JSON. 226 | r.Skip(6) 227 | 228 | op.Locks, err = mongo.ParseJsonRunes(r, false) 229 | if err != nil { 230 | return message.Operation{}, err 231 | } 232 | 233 | op.Duration, err = Duration(r) 234 | if err != nil { 235 | return message.Operation{}, err 236 | } 237 | 238 | return op, nil 239 | } 240 | 241 | func (v *Version36Parser) Version() version.Definition { 242 | return version.Definition{Major: 3, Minor: 6, Binary: record.BinaryMongod} 243 | } 244 | 245 | func (v *Version36Parser) expectedComponents(c record.Component) bool { 246 | switch c { 247 | case record.ComponentAccess, 248 | record.ComponentAccessControl, 249 | record.ComponentASIO, 250 | record.ComponentBridge, 251 | record.ComponentCommand, 252 | record.ComponentControl, 253 | record.ComponentDefault, 254 | record.ComponentExecutor, 255 | record.ComponentFTDC, 256 | record.ComponentGeo, 257 | record.ComponentHeartbeats, 258 | record.ComponentIndex, 259 | record.ComponentJournal, 260 | record.ComponentNetwork, 261 | record.ComponentQuery, 262 | record.ComponentRepl, 263 | record.ComponentReplHB, 264 | record.ComponentReplication, 265 | record.ComponentRollback, 266 | record.ComponentSharding, 267 | record.ComponentStorage, 268 | record.ComponentTotal, 269 | record.ComponentTracking, 270 | record.ComponentWrite, 271 | record.ComponentUnknown: 272 | return true 273 | default: 274 | return false 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /parser/db_40.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // COMPONENT:RECOVERY 4 | 5 | import ( 6 | "mgotools/internal" 7 | "mgotools/mongo" 8 | "mgotools/parser/executor" 9 | "mgotools/parser/message" 10 | "mgotools/parser/record" 11 | "mgotools/parser/version" 12 | ) 13 | 14 | type Version40Parser struct { 15 | counters map[string]string 16 | executor *executor.Executor 17 | } 18 | 19 | var errorVersion40Unmatched = internal.VersionUnmatched{Message: "version 4.0"} 20 | 21 | func init() { 22 | ex := executor.New() 23 | 24 | // CONTROL components 25 | ex.RegisterForReader("dbexit", mongodParseShutdown) 26 | ex.RegisterForReader("db version", mongodDbVersion) 27 | ex.RegisterForReader("journal dir=", mongodJournal) 28 | ex.RegisterForReader("options", mongodOptions) 29 | ex.RegisterForReader("wiredtiger_open config", commonParseWiredtigerOpen) 30 | 31 | // NETWORK components 32 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 33 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 34 | ex.RegisterForReader("waiting for connection", commonParseWaitingForConnections) 35 | ex.RegisterForReader("received client metadata from", commonParseClientMetadata) 36 | 37 | version.Factory.Register(func() version.Parser { 38 | return &Version40Parser{ 39 | counters: map[string]string{ 40 | "cursorid": "cursorid", 41 | "notoreturn": "ntoreturn", 42 | "ntoskip": "ntoskip", 43 | "exhaust": "exhaust", 44 | "keysExamined": "keysExamined", 45 | "docsExamined": "docsExamined", 46 | "hasSortStage": "hasSortStage", 47 | "fromMultiPlanner": "fromMultiPlanner", 48 | "replanned": "replanned", 49 | "nMatched": "nmatched", 50 | "nModified": "nmodified", 51 | "ninserted": "ninserted", 52 | "ndeleted": "ndeleted", 53 | "nreturned": "nreturned", 54 | "fastmodinsert": "fastmodinsert", 55 | "upsert": "upsert", 56 | "cursorExhausted": "cursorExhausted", 57 | "nmoved": "nmoved", 58 | "keysInserted": "keysInserted", 59 | "keysDeleted": "keysDeleted", 60 | "writeConflicts": "writeConflicts", 61 | "numYields": "numYields", 62 | "reslen": "reslen", 63 | }, 64 | 65 | executor: ex, 66 | } 67 | }) 68 | } 69 | 70 | func (v *Version40Parser) Check(base record.Base) bool { 71 | return base.Severity != record.SeverityNone && 72 | base.Component != record.ComponentNone 73 | } 74 | 75 | func (v *Version40Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 76 | r := internal.NewRuneReader(entry.RawMessage) 77 | switch entry.Component { 78 | case record.ComponentCommand: 79 | cmd, err := v.command(*r) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return CrudOrMessage(cmd, cmd.Command, cmd.Counters, cmd.Payload), nil 84 | 85 | case record.ComponentWrite: 86 | op, err := v.operation(*r) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 91 | 92 | default: 93 | return v.executor.Run(entry, r, errorVersion40Unmatched) 94 | } 95 | } 96 | 97 | func (v *Version40Parser) command(reader internal.RuneReader) (message.Command, error) { 98 | r := &reader 99 | 100 | cmd, err := CommandPreamble(r) 101 | if err != nil { 102 | return message.Command{}, err 103 | } 104 | 105 | if r.ExpectString("originatingCommand") { 106 | r.SkipWords(1) 107 | cmd.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 108 | 109 | if err != nil { 110 | return message.Command{}, err 111 | } 112 | } 113 | 114 | if r.ExpectString("planSummary:") { 115 | r.Skip(12).ChompWS() 116 | 117 | cmd.PlanSummary, err = PlanSummary(r) 118 | if err != nil { 119 | return message.Command{}, err 120 | } 121 | } 122 | 123 | for { 124 | param, ok := r.SlurpWord() 125 | if !ok { 126 | break 127 | } else if param == "exception:" { 128 | cmd.Exception, ok = Exception(r) 129 | if !ok { 130 | return message.Command{}, internal.UnexpectedExceptionFormat 131 | } 132 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 133 | r.RewindSlurpWord() 134 | break 135 | } else if !IntegerKeyValue(param, cmd.Counters, v.counters) { 136 | return message.Command{}, internal.CounterUnrecognized 137 | } 138 | } 139 | 140 | cmd.Locks, err = Locks(r) 141 | if err != nil { 142 | return message.Command{}, err 143 | } 144 | 145 | // Storage (may) exist between locks and protocols. 146 | cmd.Storage, err = Storage(r) 147 | if err != nil { 148 | return message.Command{}, errorVersion40Unmatched 149 | } 150 | 151 | // Grab the protocol string. 152 | cmd.Protocol, err = Protocol(r) 153 | if err != nil { 154 | return message.Command{}, err 155 | } else if cmd.Protocol != "op_msg" && cmd.Protocol != "op_query" && cmd.Protocol != "op_command" { 156 | return message.Command{}, errorVersion40Unmatched 157 | } 158 | 159 | // Grab the duration at the end of line. 160 | cmd.Duration, err = Duration(r) 161 | if err != nil { 162 | return message.Command{}, err 163 | } 164 | 165 | return cmd, nil 166 | } 167 | 168 | func (v *Version40Parser) operation(reader internal.RuneReader) (message.Operation, error) { 169 | r := &reader 170 | 171 | op, err := OperationPreamble(r) 172 | if err != nil { 173 | return message.Operation{}, err 174 | } 175 | 176 | // Check against the expected list of operations. Anything not in this list 177 | // is either very broken or a different version. 178 | if !internal.ArrayBinaryMatchString(op.Operation, []string{"command", "commandReply", "compressed", "getmore", "insert", "killcursors", "msg", "none", "query", "remove", "reply", "update"}) { 179 | return message.Operation{}, errorVersion40Unmatched 180 | } 181 | 182 | // The next word should always be "command:" 183 | if c, ok := r.SlurpWord(); !ok { 184 | return message.Operation{}, internal.UnexpectedEOL 185 | } else if c != "command:" { 186 | return message.Operation{}, errorVersion40Unmatched 187 | } 188 | 189 | // There is no bareword like a command (even though the last word was 190 | // "command:") so the only available option is a JSON document. 191 | if !r.ExpectRune('{') { 192 | return message.Operation{}, internal.OperationStructure 193 | } 194 | 195 | op.Payload, err = mongo.ParseJsonRunes(r, false) 196 | if err != nil { 197 | return message.Operation{}, err 198 | } 199 | 200 | if r.ExpectString("originatingCommand:") { 201 | r.Skip(19).ChompWS() 202 | 203 | op.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 204 | if err != nil { 205 | return message.Operation{}, err 206 | } 207 | } 208 | 209 | if r.ExpectString("planSummary:") { 210 | r.Skip(12).ChompWS() 211 | 212 | op.PlanSummary, err = PlanSummary(r) 213 | if err != nil { 214 | return message.Operation{}, err 215 | } 216 | } 217 | 218 | for { 219 | param, ok := r.SlurpWord() 220 | if !ok { 221 | break 222 | } else if param == "exception:" { 223 | op.Exception, ok = Exception(r) 224 | if !ok { 225 | return message.Operation{}, internal.UnexpectedExceptionFormat 226 | } 227 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 228 | r.RewindSlurpWord() 229 | break 230 | } else if !IntegerKeyValue(param, op.Counters, v.counters) { 231 | return message.Operation{}, internal.CounterUnrecognized 232 | } 233 | } 234 | 235 | // Skip "locks:" and resume with JSON. 236 | r.Skip(6) 237 | 238 | op.Locks, err = mongo.ParseJsonRunes(r, false) 239 | if err != nil { 240 | return message.Operation{}, err 241 | } 242 | 243 | // Storage seems to come before duration. 244 | op.Storage, err = Storage(r) 245 | if err != nil { 246 | return message.Operation{}, err 247 | } 248 | 249 | op.Duration, err = Duration(r) 250 | if err != nil { 251 | return message.Operation{}, err 252 | } 253 | 254 | return op, nil 255 | } 256 | 257 | func (v *Version40Parser) Version() version.Definition { 258 | return version.Definition{Major: 4, Minor: 0, Binary: record.BinaryMongod} 259 | } 260 | 261 | func (v *Version40Parser) expectedComponents(c record.Component) bool { 262 | switch c { 263 | case record.ComponentAccess, 264 | record.ComponentAccessControl, 265 | record.ComponentASIO, 266 | record.ComponentBridge, 267 | record.ComponentCommand, 268 | record.ComponentConnPool, 269 | record.ComponentControl, 270 | record.ComponentDefault, 271 | record.ComponentExecutor, 272 | record.ComponentFTDC, 273 | record.ComponentGeo, 274 | record.ComponentHeartbeats, 275 | record.ComponentIndex, 276 | record.ComponentJournal, 277 | record.ComponentNetwork, 278 | record.ComponentQuery, 279 | record.ComponentRepl, 280 | record.ComponentReplHB, 281 | record.ComponentReplication, 282 | record.ComponentRollback, 283 | record.ComponentSharding, 284 | record.ComponentStorage, 285 | record.ComponentTotal, 286 | record.ComponentTracking, 287 | record.ComponentWrite, 288 | record.ComponentUnknown: 289 | return true 290 | default: 291 | return false 292 | } 293 | } 294 | -------------------------------------------------------------------------------- /parser/db_42.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/mongo" 6 | "mgotools/parser/executor" 7 | "mgotools/parser/message" 8 | "mgotools/parser/record" 9 | "mgotools/parser/version" 10 | ) 11 | 12 | // Components:INITSYNC,ELECTION 13 | type Version42Parser struct { 14 | counters map[string]string 15 | executor *executor.Executor 16 | } 17 | 18 | var errorVersion42Unmatched = internal.VersionUnmatched{Message: "version 4.0"} 19 | 20 | func init() { 21 | ex := executor.New() 22 | 23 | // CONTROL components 24 | ex.RegisterForReader("build info", mongodBuildInfo) 25 | ex.RegisterForReader("dbexit", mongodParseShutdown) 26 | ex.RegisterForReader("db version", mongodDbVersion) 27 | ex.RegisterForReader("journal dir=", mongodJournal) 28 | ex.RegisterForReader("options", mongodOptions) 29 | ex.RegisterForReader("wiredtiger_open config", commonParseWiredtigerOpen) 30 | 31 | // NETWORK components 32 | ex.RegisterForReader("connection accepted", commonParseConnectionAccepted) 33 | ex.RegisterForEntry("end connection", commonParseConnectionEnded) 34 | ex.RegisterForReader("waiting for connection", commonParseWaitingForConnections) 35 | ex.RegisterForReader("received client metadata from", commonParseClientMetadata) 36 | 37 | version.Factory.Register(func() version.Parser { 38 | return &Version42Parser{ 39 | counters: map[string]string{ 40 | "cursorid": "cursorid", 41 | "notoreturn": "ntoreturn", 42 | "ntoskip": "ntoskip", 43 | "exhaust": "exhaust", 44 | "keysExamined": "keysExamined", 45 | "docsExamined": "docsExamined", 46 | "hasSortStage": "hasSortStage", 47 | "fromMultiPlanner": "fromMultiPlanner", 48 | "replanned": "replanned", 49 | "nMatched": "nmatched", 50 | "nModified": "nmodified", 51 | "ninserted": "ninserted", 52 | "ndeleted": "ndeleted", 53 | "nreturned": "nreturned", 54 | "fastmodinsert": "fastmodinsert", 55 | "upsert": "upsert", 56 | "cursorExhausted": "cursorExhausted", 57 | "nmoved": "nmoved", 58 | "keysInserted": "keysInserted", 59 | "keysDeleted": "keysDeleted", 60 | "writeConflicts": "writeConflicts", 61 | "numYields": "numYields", 62 | "reslen": "reslen", 63 | }, 64 | 65 | executor: ex, 66 | } 67 | }) 68 | } 69 | 70 | func (v *Version42Parser) Check(base record.Base) bool { 71 | return base.Severity != record.SeverityNone && 72 | base.Component != record.ComponentNone 73 | } 74 | 75 | func (v *Version42Parser) NewLogMessage(entry record.Entry) (message.Message, error) { 76 | r := internal.NewRuneReader(entry.RawMessage) 77 | switch entry.Component { 78 | case record.ComponentCommand: 79 | cmd, err := v.command(*r) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return CrudOrMessage(cmd, cmd.Command, cmd.Counters, cmd.Payload), nil 84 | 85 | case record.ComponentWrite: 86 | op, err := v.operation(*r) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return CrudOrMessage(op, op.Operation, op.Counters, op.Payload), nil 91 | 92 | default: 93 | return v.executor.Run(entry, r, errorVersion42Unmatched) 94 | } 95 | } 96 | 97 | func (v *Version42Parser) command(reader internal.RuneReader) (message.Command, error) { 98 | r := &reader 99 | 100 | cmd, err := CommandPreamble(r) 101 | if err != nil { 102 | return message.Command{}, err 103 | } 104 | 105 | if r.ExpectString("originatingCommand") { 106 | r.SkipWords(1) 107 | cmd.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 108 | 109 | if err != nil { 110 | return message.Command{}, err 111 | } 112 | } 113 | 114 | if r.ExpectString("planSummary:") { 115 | r.Skip(12).ChompWS() 116 | 117 | cmd.PlanSummary, err = PlanSummary(r) 118 | if err != nil { 119 | return message.Command{}, err 120 | } 121 | } 122 | 123 | for { 124 | param, ok := r.SlurpWord() 125 | if !ok { 126 | break 127 | } else if param == "exception:" { 128 | cmd.Exception, ok = Exception(r) 129 | if !ok { 130 | return message.Command{}, internal.UnexpectedExceptionFormat 131 | } 132 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 133 | r.RewindSlurpWord() 134 | break 135 | } else if !IntegerKeyValue(param, cmd.Counters, v.counters) { 136 | return message.Command{}, internal.CounterUnrecognized 137 | } 138 | } 139 | 140 | cmd.Locks, err = Locks(r) 141 | if err != nil { 142 | return message.Command{}, err 143 | } 144 | 145 | // Storage (may) exist between locks and protocols. 146 | cmd.Storage, err = Storage(r) 147 | if err != nil { 148 | return message.Command{}, errorVersion42Unmatched 149 | } 150 | 151 | // Grab the protocol string. 152 | cmd.Protocol, err = Protocol(r) 153 | if err != nil { 154 | return message.Command{}, err 155 | } else if cmd.Protocol != "op_msg" && cmd.Protocol != "op_query" && cmd.Protocol != "op_command" { 156 | return message.Command{}, errorVersion42Unmatched 157 | } 158 | 159 | // Grab the duration at the end of line. 160 | cmd.Duration, err = Duration(r) 161 | if err != nil { 162 | return message.Command{}, err 163 | } 164 | 165 | return cmd, nil 166 | } 167 | 168 | func (v *Version42Parser) operation(reader internal.RuneReader) (message.Operation, error) { 169 | r := &reader 170 | 171 | op, err := OperationPreamble(r) 172 | if err != nil { 173 | return message.Operation{}, err 174 | } 175 | 176 | // Check against the expected list of operations. Anything not in this list 177 | // is either very broken or a different version. 178 | if !internal.ArrayBinaryMatchString(op.Operation, []string{"command", "commandReply", "compressed", "getmore", "insert", "killcursors", "msg", "none", "query", "remove", "reply", "update"}) { 179 | return message.Operation{}, errorVersion42Unmatched 180 | } 181 | 182 | // The next word should always be "command:" 183 | if c, ok := r.SlurpWord(); !ok { 184 | return message.Operation{}, internal.UnexpectedEOL 185 | } else if c != "command:" { 186 | return message.Operation{}, errorVersion42Unmatched 187 | } 188 | 189 | // There is no bareword like a command (even though the last word was 190 | // "command:") so the only available option is a JSON document. 191 | if !r.ExpectRune('{') { 192 | return message.Operation{}, internal.OperationStructure 193 | } 194 | 195 | op.Payload, err = mongo.ParseJsonRunes(r, false) 196 | if err != nil { 197 | return message.Operation{}, err 198 | } 199 | 200 | if r.ExpectString("originatingCommand:") { 201 | r.Skip(19).ChompWS() 202 | 203 | op.Payload["originatingCommand"], err = mongo.ParseJsonRunes(r, false) 204 | if err != nil { 205 | return message.Operation{}, err 206 | } 207 | } 208 | 209 | if r.ExpectString("planSummary:") { 210 | r.Skip(12).ChompWS() 211 | 212 | op.PlanSummary, err = PlanSummary(r) 213 | if err != nil { 214 | return message.Operation{}, err 215 | } 216 | } 217 | 218 | for { 219 | param, ok := r.SlurpWord() 220 | if !ok { 221 | break 222 | } else if param == "exception:" { 223 | op.Exception, ok = Exception(r) 224 | if !ok { 225 | return message.Operation{}, internal.UnexpectedExceptionFormat 226 | } 227 | } else if l := len(param); l > 6 && param[:6] == "locks:" { 228 | r.RewindSlurpWord() 229 | break 230 | } else if !IntegerKeyValue(param, op.Counters, v.counters) { 231 | return message.Operation{}, internal.CounterUnrecognized 232 | } 233 | } 234 | 235 | // Skip "locks:" and resume with JSON. 236 | r.Skip(6) 237 | 238 | op.Locks, err = mongo.ParseJsonRunes(r, false) 239 | if err != nil { 240 | return message.Operation{}, err 241 | } 242 | 243 | // Storage seems to come before duration. 244 | op.Storage, err = Storage(r) 245 | if err != nil { 246 | return message.Operation{}, err 247 | } 248 | 249 | op.Duration, err = Duration(r) 250 | if err != nil { 251 | return message.Operation{}, err 252 | } 253 | 254 | return op, nil 255 | } 256 | 257 | func (v *Version42Parser) Version() version.Definition { 258 | return version.Definition{Major: 4, Minor: 2, Binary: record.BinaryMongod} 259 | } 260 | 261 | func (v *Version42Parser) expectedComponents(c record.Component) bool { 262 | switch c { 263 | case record.ComponentAccess, 264 | record.ComponentAccessControl, 265 | record.ComponentASIO, 266 | record.ComponentBridge, 267 | record.ComponentCommand, 268 | record.ComponentConnPool, 269 | record.ComponentControl, 270 | record.ComponentDefault, 271 | record.ComponentElection, 272 | record.ComponentExecutor, 273 | record.ComponentFTDC, 274 | record.ComponentGeo, 275 | record.ComponentHeartbeats, 276 | record.ComponentIndex, 277 | record.ComponentInitialSync, 278 | record.ComponentJournal, 279 | record.ComponentNetwork, 280 | record.ComponentQuery, 281 | record.ComponentRepl, 282 | record.ComponentReplHB, 283 | record.ComponentReplication, 284 | record.ComponentRollback, 285 | record.ComponentSharding, 286 | record.ComponentStorage, 287 | record.ComponentTotal, 288 | record.ComponentTracking, 289 | record.ComponentWrite, 290 | record.ComponentUnknown: 291 | return true 292 | default: 293 | return false 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /parser/executor/executor.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "sort" 5 | 6 | "mgotools/internal" 7 | "mgotools/parser/message" 8 | "mgotools/parser/record" 9 | ) 10 | 11 | type Type int 12 | 13 | const ( 14 | ReaderType = Type(iota) 15 | EntryType 16 | ) 17 | 18 | type Reader func(*internal.RuneReader) (message.Message, error) 19 | type Entry func(entry record.Entry, r *internal.RuneReader) (message.Message, error) 20 | 21 | type callback struct { 22 | Name string 23 | Type Type 24 | 25 | reader Reader 26 | entry Entry 27 | } 28 | 29 | type Executor struct { 30 | executor []callback 31 | peek int 32 | } 33 | 34 | func New() *Executor { 35 | return &Executor{ 36 | executor: make([]callback, 0), 37 | } 38 | } 39 | 40 | func (e *Executor) RegisterForReader(key string, f Reader) bool { 41 | if e.isKeyUsed(key) { 42 | return false 43 | } 44 | 45 | callback := callback{ 46 | Name: key, 47 | Type: ReaderType, 48 | reader: f, 49 | } 50 | 51 | e.appendKey(callback) 52 | return true 53 | } 54 | 55 | func (e *Executor) RegisterForEntry(key string, f Entry) bool { 56 | if e.isKeyUsed(key) { 57 | return false 58 | } 59 | 60 | callback := callback{ 61 | Name: key, 62 | Type: EntryType, 63 | entry: f, 64 | } 65 | 66 | e.appendKey(callback) 67 | return true 68 | } 69 | 70 | func (e *Executor) Run(entry record.Entry, r *internal.RuneReader, unmatched error) (message.Message, error) { 71 | preview := r.Peek(e.peek) 72 | 73 | pos, found := e.findPosition(preview) 74 | if !found { 75 | return nil, unmatched 76 | } 77 | 78 | callback := e.executor[pos] 79 | 80 | switch callback.Type { 81 | case ReaderType: 82 | return callback.reader(r) 83 | case EntryType: 84 | return callback.entry(entry, r) 85 | default: 86 | panic("unknown type encountered in executor") 87 | } 88 | } 89 | 90 | func (e *Executor) appendKey(f callback) { 91 | key := f.Name 92 | 93 | if len(key) == 0 { 94 | panic("attempted to register with an empty key") 95 | } 96 | 97 | if len(key) > e.peek { 98 | e.peek = len(key) 99 | } 100 | 101 | index, ok := e.findPosition(key) 102 | if ok { 103 | panic("attempt to register a duplicate key") 104 | } 105 | 106 | e.executor = append(e.executor, callback{}) 107 | copy(e.executor[index+1:], e.executor[index:]) 108 | e.executor[index] = f 109 | } 110 | 111 | func (e *Executor) findPosition(key string) (pos int, ok bool) { 112 | if e.executor == nil { 113 | e.executor = make([]callback, 0) 114 | } 115 | 116 | pos = sort.Search(len(e.executor), func(index int) bool { 117 | if len(e.executor[index].Name) <= len(key) && e.executor[index].Name >= key[:len(e.executor[index].Name)] { 118 | return true 119 | } 120 | 121 | return e.executor[index].Name >= key 122 | }) 123 | 124 | ok = pos < len(e.executor) && len(key) >= len(e.executor[pos].Name) && key[:len(e.executor[pos].Name)] == e.executor[pos].Name 125 | return 126 | } 127 | 128 | func (e *Executor) isKeyUsed(key string) bool { 129 | _, ok := e.findPosition(key) 130 | return ok 131 | } 132 | -------------------------------------------------------------------------------- /parser/executor/executor_test.go: -------------------------------------------------------------------------------- 1 | package executor 2 | 3 | import ( 4 | "testing" 5 | 6 | "mgotools/internal" 7 | "mgotools/parser/message" 8 | "mgotools/parser/record" 9 | 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | func TestExecutor_appendKey(t *testing.T) { 14 | e := Executor{} 15 | e.appendKey(callback{Name: "bcd"}) 16 | 17 | if len(e.executor) != 1 || e.executor[0].Name != "bcd" { 18 | t.Errorf("append to zero slice failed") 19 | } 20 | 21 | e.appendKey(callback{Name: "xyz"}) 22 | if len(e.executor) != 2 || e.executor[1].Name != "xyz" { 23 | t.Errorf("append to end of slice failed") 24 | } 25 | 26 | e.appendKey(callback{Name: "def"}) 27 | if len(e.executor) != 3 || e.executor[1].Name != "def" { 28 | t.Errorf("append to middle of slice failed") 29 | } 30 | 31 | e.appendKey(callback{Name: "abc"}) 32 | if len(e.executor) != 4 || e.executor[0].Name != "abc" { 33 | t.Errorf("append to beginning of slice failed") 34 | } 35 | } 36 | 37 | func TestExecutor_Run(t *testing.T) { 38 | e := Executor{} 39 | unmatched := errors.New("unmatched") 40 | 41 | r := func(_ *internal.RuneReader) (message.Message, error) { 42 | return nil, nil 43 | } 44 | 45 | e.RegisterForReader("abc def", r) 46 | e.RegisterForReader("def ghi", r) 47 | e.RegisterForEntry("z", func(_ record.Entry, _ *internal.RuneReader) (message.Message, error) { 48 | return nil, nil 49 | }) 50 | 51 | if _, err := e.Run(record.Entry{}, internal.NewRuneReader("abc def ghi jkl"), unmatched); err == unmatched { 52 | t.Error("failed to run 'abc def'") 53 | } 54 | 55 | if _, err := e.Run(record.Entry{}, internal.NewRuneReader("def ghi jkl"), unmatched); err == unmatched { 56 | t.Error("failed to run 'def ghi'") 57 | } 58 | 59 | if _, err := e.Run(record.Entry{}, internal.NewRuneReader("zyx abc"), unmatched); err == unmatched { 60 | t.Error("failed to run 'z'") 61 | } 62 | 63 | if _, err := e.Run(record.Entry{}, internal.NewRuneReader("ghi abc def"), unmatched); err != unmatched { 64 | t.Error("incorrect result for 'ghi abc def'") 65 | } 66 | 67 | if _, err := e.Run(record.Entry{}, internal.NewRuneReader("abc"), unmatched); err != unmatched { 68 | t.Error("incorrect result for 'abc'") 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /parser/helpers_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | 8 | "mgotools/internal" 9 | "mgotools/parser/message" 10 | ) 11 | 12 | func TestCheckCounterVersionError(t *testing.T) { 13 | type Tuple struct { 14 | E error 15 | V error 16 | } 17 | 18 | type Result struct { 19 | C bool 20 | E error 21 | } 22 | 23 | v := internal.VersionUnmatched{""} 24 | s := map[Tuple]Result{ 25 | {internal.CounterUnrecognized, v}: {true, v}, 26 | 27 | {internal.UnexpectedEOL, v}: {false, internal.UnexpectedEOL}, 28 | 29 | {nil, v}: {false, nil}, 30 | } 31 | 32 | for m, r := range s { 33 | c, e := CheckCounterVersionError(m.E, m.V) 34 | if c != r.C || e != r.E { 35 | t.Errorf("Expected (%v, %s), got (%v, %s)", r.C, r.E, c, e) 36 | } 37 | } 38 | } 39 | 40 | func TestCommandPreamble(t *testing.T) { 41 | type PreambleResult struct { 42 | Cmd string 43 | Ns string 44 | Agent string 45 | Payload message.Payload 46 | Err error 47 | } 48 | 49 | s := map[string]PreambleResult{ 50 | `command test.$cmd appName: "MongoDB Shell" command: isMaster { isMaster: 1, forShell: 1 }`: {"isMaster", "test.$cmd", "MongoDB Shell", message.Payload{"isMaster": 1, "forShell": 1}, nil}, 51 | 52 | `command test.$cmd command: isMaster { isMaster: 1, forShell: 1 }`: {"isMaster", "test.$cmd", "", message.Payload{"isMaster": 1, "forShell": 1}, nil}, 53 | 54 | `command test.$cmd planSummary: IXSCAN { a: 1 }`: {"", "test.$cmd", "", message.Payload{}, nil}, 55 | 56 | `command test.$cmd command: drop { drop: "$cmd" }`: {"drop", "test.$cmd", "", message.Payload{"drop": "$cmd"}, nil}, 57 | 58 | `command test command: dropDatabase { dropDatabase: 1 }`: {"dropDatabase", "test", "", message.Payload{"dropDatabase": 1}, nil}, 59 | 60 | `command test.$cmd command: { a: 1 }`: {"command", "test.$cmd", "", message.Payload{"a": 1}, nil}, 61 | 62 | `command test.$cmd`: {"", "", "", nil, internal.UnexpectedEOL}, 63 | 64 | `command test.$cmd appName: "...`: {"", "", "", nil, fmt.Errorf("unexpected end of string looking for quote (\")")}, 65 | 66 | `query test.$cmd query: { a: 1 }`: {"", "", "", nil, internal.CommandStructure}, 67 | 68 | `command test.$cmd command:`: {"", "", "", nil, internal.CommandStructure}, 69 | } 70 | 71 | for m, r := range s { 72 | cmd, err := CommandPreamble(internal.NewRuneReader(m)) 73 | if (err != nil && r.Err == nil) || (err == nil && r.Err != nil) || err != nil && r.Err != nil && err.Error() != r.Err.Error() { 74 | t.Errorf("Error mismatch: expected '%s', got '%s'", r.Err, err) 75 | } 76 | 77 | if cmd.Command != r.Cmd { 78 | t.Errorf("Command mismatch: expected '%s', got '%s'", r.Cmd, cmd.Command) 79 | } 80 | 81 | if cmd.Namespace != r.Ns { 82 | t.Errorf("Namespace mismatch: expected '%s', got '%s'", r.Ns, cmd.Namespace) 83 | } 84 | 85 | if cmd.Agent != r.Agent { 86 | t.Errorf("Agent mismatch: expected '%s', got '%s'", r.Agent, cmd.Agent) 87 | } 88 | 89 | if !reflect.DeepEqual(cmd.Payload, r.Payload) { 90 | t.Errorf("Payloads differ: \t%#v\n\t%#v", r.Payload, cmd.Payload) 91 | } 92 | } 93 | } 94 | 95 | func TestDuration(t *testing.T) { 96 | type R struct { 97 | N int64 98 | E error 99 | } 100 | s := map[string]R{ 101 | `10ms`: {10, nil}, 102 | `0ms`: {0, nil}, 103 | `-1ms`: {0, nil}, 104 | ``: {0, internal.UnexpectedEOL}, 105 | `ok`: {0, internal.MisplacedWordException}, 106 | } 107 | for m, r := range s { 108 | n, e := Duration(internal.NewRuneReader(m)) 109 | if n != r.N || e != r.E { 110 | t.Errorf("Expected (%v, %s), got (%v, %s)", r.N, r.E, n, e) 111 | } 112 | } 113 | } 114 | 115 | func TestPreamble(t *testing.T) { 116 | cmd, ns, op, err := Preamble(internal.NewRuneReader("command test.$cmd command:")) 117 | if cmd != "command" || ns != "test.$cmd" || op != "command" || err != nil { 118 | t.Errorf("Values differ (%s, %s, %s, %s)", cmd, ns, op, err) 119 | } 120 | 121 | cmd, ns, op, err = Preamble(internal.NewRuneReader("query test query:")) 122 | if cmd != "query" || ns != "test" || op != "query" || err != nil { 123 | t.Errorf("Expected 'test', got %s (err: %s)", ns, err) 124 | } 125 | 126 | cmd, ns, op, err = Preamble(internal.NewRuneReader("update test.$cmd query:")) 127 | if cmd != "update" || ns != "test.$cmd" || op != "query" || err != nil { 128 | t.Errorf("Values differ (%s, %s, %s, %s)", cmd, ns, op, err) 129 | } 130 | 131 | cmd, ns, op, err = Preamble(internal.NewRuneReader("command test.$cmd appName: \"agent\" command:")) 132 | if cmd != "command" || ns != "test.$cmd" || op != "appName" || err != nil { 133 | t.Errorf("Preamble failed, got (cmd: %s, ns: %s, op: %s, err: %s)", cmd, ns, op, err) 134 | } 135 | 136 | cmd, ns, op, err = Preamble(internal.NewRuneReader("command test.$cmd query")) 137 | if cmd != "command" || ns != "test.$cmd" || op != "query" || err != nil { 138 | t.Errorf("Preamble failed, got (cmd: %s, ns: %s, op: %s, err: %s)", cmd, ns, op, err) 139 | } 140 | 141 | cmd, ns, op, err = Preamble(internal.NewRuneReader("")) 142 | if err != internal.UnexpectedEOL { 143 | t.Errorf("Expected UnexpectedEOL error, got %s", err) 144 | } 145 | } 146 | 147 | func TestOperationPreamble(t *testing.T) { 148 | op, err := OperationPreamble(internal.NewRuneReader("insert test.$cmd query: { a: 1 }")) 149 | if op.Operation != "insert" || op.Namespace != "test.$cmd" || err != nil { 150 | t.Errorf("Values differ (%s, %s, %s)", op.Operation, op.Namespace, err) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /parser/message/methods.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | ) 7 | 8 | func (m Version) String() string { 9 | out := bytes.NewBuffer([]byte{}) 10 | 11 | if m.Binary != "" { 12 | out.WriteString(m.Binary) 13 | } 14 | if m.Major > 0 { 15 | if m.Binary != "" { 16 | out.WriteRune(' ') 17 | } 18 | 19 | out.WriteString(strconv.Itoa(m.Major)) 20 | out.WriteRune('.') 21 | out.WriteString(strconv.Itoa(m.Minor)) 22 | 23 | if m.Revision > 0 { 24 | out.WriteRune('.') 25 | out.WriteString(strconv.Itoa(m.Revision)) 26 | } 27 | } 28 | 29 | return out.String() 30 | } 31 | 32 | func BaseFromMessage(msg Message) (*BaseCommand, bool) { 33 | if msg == nil { 34 | return &BaseCommand{}, false 35 | } 36 | switch t := msg.(type) { 37 | case BaseCommand: 38 | return &t, true 39 | case Command: 40 | return &t.BaseCommand, true 41 | case CommandLegacy: 42 | return &t.BaseCommand, true 43 | case Operation: 44 | return &t.BaseCommand, true 45 | case OperationLegacy: 46 | return &t.BaseCommand, true 47 | case CRUD: 48 | return BaseFromMessage(t.Message) 49 | default: 50 | return &BaseCommand{}, false 51 | } 52 | } 53 | 54 | func PayloadFromMessage(msg Message) (*Payload, bool) { 55 | if msg == nil { 56 | return &Payload{}, false 57 | } 58 | switch t := msg.(type) { 59 | case Command: 60 | return &t.Payload, true 61 | case CommandLegacy: 62 | return &t.Payload, true 63 | case Operation: 64 | return &t.Payload, true 65 | case OperationLegacy: 66 | return &t.Payload, true 67 | default: 68 | return &Payload{}, false 69 | } 70 | } 71 | 72 | func MakeCommand() Command { 73 | return Command{ 74 | BaseCommand: BaseCommand{ 75 | Counters: make(map[string]int64), 76 | }, 77 | Command: "", 78 | Payload: make(Payload), 79 | Locks: make(map[string]interface{}), 80 | } 81 | } 82 | 83 | func MakeOperation() Operation { 84 | return Operation{ 85 | BaseCommand: BaseCommand{ 86 | Counters: make(map[string]int64), 87 | }, 88 | Operation: "", 89 | Payload: make(Payload), 90 | Locks: make(map[string]interface{}), 91 | } 92 | } 93 | 94 | func MakeCommandLegacy() CommandLegacy { 95 | return CommandLegacy{ 96 | BaseCommand: BaseCommand{ 97 | Counters: make(map[string]int64), 98 | }, 99 | Command: "", 100 | Payload: make(Payload), 101 | Locks: make(map[string]int64), 102 | } 103 | } 104 | 105 | func MakeOperationLegacy() OperationLegacy { 106 | return OperationLegacy{ 107 | BaseCommand: BaseCommand{ 108 | Counters: make(map[string]int64), 109 | }, 110 | Operation: "", 111 | Payload: make(Payload), 112 | Locks: make(map[string]int64), 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /parser/message/types.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import "net" 4 | 5 | type Message interface { 6 | } 7 | 8 | type Authentication struct { 9 | Principal string 10 | IP string 11 | } 12 | 13 | type BuildInfo struct { 14 | BuildInfo string 15 | } 16 | 17 | type Connection struct { 18 | Address net.IP 19 | Conn int 20 | Port uint16 21 | Opened bool 22 | 23 | Exception string 24 | } 25 | 26 | type ConnectionMeta struct { 27 | Connection 28 | Meta interface{} 29 | } 30 | 31 | type Empty struct{} 32 | 33 | type Journal string 34 | 35 | type Listening struct{} 36 | 37 | type OpenSSL struct { 38 | String string 39 | } 40 | 41 | type Shutdown struct { 42 | String string 43 | } 44 | 45 | type Signal struct { 46 | String string 47 | } 48 | 49 | type StartupInfoLegacy struct { 50 | StartupInfo 51 | Version 52 | } 53 | 54 | type StartupInfo struct { 55 | DbPath string 56 | Hostname string 57 | Pid int 58 | Port int 59 | } 60 | 61 | type StartupOptions struct { 62 | String string 63 | Options interface{} 64 | } 65 | 66 | type Version struct { 67 | Binary string 68 | Major int 69 | Minor int 70 | Revision int 71 | } 72 | 73 | type WiredTigerConfig struct { 74 | String string 75 | } 76 | 77 | // 78 | // Message Commands 79 | // 80 | 81 | type BaseCommand struct { 82 | Counters map[string]int64 83 | Duration int64 84 | Exception string 85 | Namespace string 86 | PlanSummary []PlanSummary 87 | } 88 | 89 | type Payload map[string]interface{} 90 | 91 | type Command struct { 92 | BaseCommand 93 | 94 | Agent string 95 | Command string 96 | Locks map[string]interface{} 97 | Payload Payload 98 | Protocol string 99 | Storage map[string]interface{} 100 | } 101 | 102 | // remove, update, query, insert 103 | type Operation struct { 104 | BaseCommand 105 | 106 | Agent string 107 | Locks map[string]interface{} 108 | Operation string 109 | Payload Payload 110 | Storage map[string]interface{} 111 | } 112 | 113 | type CommandLegacy struct { 114 | BaseCommand 115 | 116 | Command string 117 | Locks map[string]int64 118 | Payload Payload 119 | } 120 | 121 | type OperationLegacy struct { 122 | BaseCommand 123 | 124 | Locks map[string]int64 125 | Operation string 126 | Payload Payload 127 | } 128 | 129 | type Filter map[string]interface{} 130 | type Project map[string]interface{} 131 | type Sort map[string]interface{} 132 | type Update map[string]interface{} 133 | 134 | type PlanSummary struct { 135 | Type string 136 | Key interface{} 137 | } 138 | 139 | type CRUD struct { 140 | Message 141 | 142 | Comment string 143 | CursorId int64 144 | Filter Filter 145 | N int64 146 | Project Project 147 | Sort Sort 148 | Update Update 149 | } 150 | -------------------------------------------------------------------------------- /parser/mongod.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/message" 6 | "mgotools/parser/record" 7 | ) 8 | 9 | func mongodBuildInfo(r *internal.RuneReader) (message.Message, error) { 10 | return message.BuildInfo{BuildInfo: r.SkipWords(2).Remainder()}, nil 11 | } 12 | 13 | func mongodDbVersion(r *internal.RuneReader) (message.Message, error) { 14 | return makeVersion(r.SkipWords(2).Remainder(), "mongod") 15 | } 16 | 17 | func mongodJournal(r *internal.RuneReader) (message.Message, error) { 18 | path := r.Skip(12).Remainder() 19 | if path == "" { 20 | return nil, internal.UnexpectedEOL 21 | } 22 | 23 | // journal dir= 24 | return message.Journal(path), nil 25 | } 26 | 27 | func mongodOptions(r *internal.RuneReader) (message.Message, error) { 28 | r.SkipWords(1) 29 | return startupOptions(r.Remainder()) 30 | } 31 | 32 | func mongodParseShutdown(r *internal.RuneReader) (message.Message, error) { 33 | return message.Shutdown{String: r.Remainder()}, nil 34 | } 35 | 36 | func mongodStartupInfo(entry record.Entry, r *internal.RuneReader) (message.Message, error) { 37 | return startupInfo(entry.RawMessage) 38 | } 39 | -------------------------------------------------------------------------------- /parser/mongod_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "net" 5 | "reflect" 6 | "testing" 7 | 8 | "mgotools/internal" 9 | "mgotools/parser/message" 10 | ) 11 | 12 | func TestMongod_Network(t *testing.T) { 13 | valid := map[string]message.Message{ 14 | "connection accepted from 127.0.0.1:27017 #1": message.Connection{Address: net.IPv4(127, 0, 0, 1), Conn: 1, Port: 27017, Opened: true}, 15 | "waiting for connections": message.Listening{}, 16 | } 17 | 18 | mongod := mongod{} 19 | for value, expected := range valid { 20 | r := internal.NewRuneReader(value) 21 | got, err := mongod.Network(*r) 22 | if err != nil { 23 | t.Errorf("network parse failed, got: %s", err) 24 | } else if !reflect.DeepEqual(expected, got) { 25 | t.Errorf("network mismatch, expected (%v), got (%v)", expected, got) 26 | } 27 | } 28 | 29 | invalid := []string{ 30 | "connection accepted from 127.0.0.1", 31 | "connection accepted from :27017 #1", 32 | "connection accepted from #1", 33 | } 34 | 35 | for _, value := range invalid { 36 | r := internal.NewRuneReader(value) 37 | msg, err := mongod.Network(*r) 38 | if err == nil || msg != nil { 39 | t.Errorf("network should have failed on '%s' (%v)", value, msg) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /parser/mongos.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/message" 6 | ) 7 | 8 | func mongosParseStartupOptions(r *internal.RuneReader) (message.Message, error) { 9 | return startupOptions(r.SkipWords(1).Remainder()) 10 | } 11 | 12 | func mongosParseVersion(r *internal.RuneReader) (message.Message, error) { 13 | versionString, ok := r.SkipWords(2).SlurpWord() 14 | if !ok { 15 | return nil, internal.UnexpectedVersionFormat 16 | } 17 | 18 | if version, err := makeVersion(versionString, "mongos"); err != nil { 19 | return nil, err 20 | } else if err == nil && !r.ExpectString("starting:") { 21 | return version, nil 22 | } else if info, err := startupInfo(r.Remainder()); err != nil { 23 | return nil, err 24 | } else { 25 | return message.StartupInfoLegacy{ 26 | Version: version, 27 | StartupInfo: info, 28 | }, nil 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /parser/record/base.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import ( 4 | "mgotools/internal" 5 | ) 6 | 7 | type Component int 8 | 9 | const ( 10 | ComponentNone Component = 0 11 | ComponentAccess = 1 << iota 12 | ComponentAccessControl 13 | ComponentASIO 14 | ComponentBridge 15 | ComponentCommand 16 | ComponentConnPool 17 | ComponentControl 18 | ComponentDefault 19 | ComponentElection 20 | ComponentExecutor 21 | ComponentFTDC 22 | ComponentGeo 23 | ComponentHeartbeats 24 | ComponentIndex 25 | ComponentInitialSync 26 | ComponentJournal 27 | ComponentNetwork 28 | ComponentQuery 29 | ComponentRecovery 30 | ComponentRepl 31 | ComponentReplication 32 | ComponentReplHB 33 | ComponentRollback 34 | ComponentSharding 35 | ComponentShardingRefr 36 | ComponentStorage 37 | ComponentTotal 38 | ComponentTracking 39 | ComponentWrite 40 | ComponentUnknown 41 | ) 42 | 43 | func NewComponent(s string) (Component, bool) { 44 | switch s { 45 | case "ACCESS": // 3.0, 3.2, 3.4, 3.6 46 | return ComponentAccess, true 47 | case "ACCESSCONTROL": // 3.0, 3.2, 3.4, 3.6 48 | return ComponentAccessControl, true 49 | case "ASIO": // 3.2, 3.4, 3.6 50 | return ComponentASIO, true 51 | case "BRIDGE": // 3.0, 3.2, 3.4, 3.6 52 | return ComponentBridge, true 53 | case "COMMAND": // 3.0, 3.2, 3.4, 3.6 54 | return ComponentCommand, true 55 | case "CONNPOOL": // 4.0 56 | return ComponentConnPool, true 57 | case "CONTROL": // 3.0, 3.2, 3.4, 3.6 58 | return ComponentControl, true 59 | case "DEFAULT": // 3.0, 3.2, 3.4, 3.6 60 | return ComponentDefault, true 61 | case "ELECTION": // 4.2 62 | return ComponentElection, true 63 | case "EXECUTOR": // 3.2, 3.4, 3.6 64 | return ComponentExecutor, true 65 | case "FTDC": // 3.2, 3.4, 3.6 66 | return ComponentFTDC, true 67 | case "GEO": // 3.0, 3.2, 3.4, 3.6 68 | return ComponentGeo, true 69 | case "HEARTBEATS": // 3.6 70 | return ComponentHeartbeats, true 71 | case "INDEX": // 3.0, 3.2, 3.4, 3.6 72 | return ComponentIndex, true 73 | case "INITSYNC": // 4.2 74 | return ComponentInitialSync, true 75 | case "JOURNAL": // 3.0, 3.2, 3.4, 3.6 76 | return ComponentJournal, true 77 | case "NETWORK": // 3.0, 3.2, 3.4, 3.6 78 | return ComponentNetwork, true 79 | case "QUERY": // 3.0, 3.2, 3.4, 3.6 80 | return ComponentQuery, true 81 | case "RECOVERY": // 4.0 82 | return ComponentRecovery, true 83 | case "REPL": // 3.0, 3.2, 3.4, 3.6 84 | return ComponentRepl, true 85 | case "REPLICATION": // 3.0, 3.2, 3.4, 3.6 86 | return ComponentReplication, true 87 | case "REPL_HB": // 3.6 88 | return ComponentReplHB, true 89 | case "ROLLBACK": // 3.6 90 | return ComponentRollback, true 91 | case "SHARDING": // 3.0, 3.2, 3.4, 3.6 92 | return ComponentSharding, true 93 | case "SH_REFR": // 2.4 94 | return ComponentShardingRefr, true 95 | case "STORAGE": // 3.0, 3.2, 3.4, 3.6 96 | return ComponentStorage, true 97 | case "TOTAL": // 3.0, 3.2, 3.4, 3.6 98 | return ComponentTotal, true 99 | case "TRACKING": // 3.4, 3.6 100 | return ComponentTracking, true 101 | case "WRITE": // 3.0, 3.2, 3.4, 3.6 102 | return ComponentWrite, true 103 | case "-": 104 | return ComponentUnknown, true 105 | default: 106 | return ComponentNone, false 107 | } 108 | } 109 | 110 | func (c Component) String() string { 111 | switch c { 112 | case ComponentNone: 113 | return "" 114 | case ComponentAccess: 115 | return "ACCESS" 116 | case ComponentAccessControl: 117 | return "ACCESSCONTROL" 118 | case ComponentASIO: 119 | return "ASIO" 120 | case ComponentBridge: 121 | return "BRIDGE" 122 | case ComponentCommand: 123 | return "COMMAND" 124 | case ComponentConnPool: 125 | return "CONNPOOL" 126 | case ComponentControl: 127 | return "CONTROL" 128 | case ComponentDefault: 129 | return "DEFAULT" 130 | case ComponentElection: 131 | return "ELECTION" 132 | case ComponentExecutor: 133 | return "EXECUTOR" 134 | case ComponentFTDC: 135 | return "FTDC" 136 | case ComponentGeo: 137 | return "GEO" 138 | case ComponentHeartbeats: 139 | return "HEARTBEATS" 140 | case ComponentIndex: 141 | return "INDEX" 142 | case ComponentInitialSync: 143 | return "INITSYNC" 144 | case ComponentJournal: 145 | return "JOURNAL" 146 | case ComponentNetwork: 147 | return "NETWORK" 148 | case ComponentQuery: 149 | return "QUERY" 150 | case ComponentRecovery: 151 | return "RECOVERY" 152 | case ComponentRepl: 153 | return "REPL" 154 | case ComponentReplication: 155 | return "REPLICATION" 156 | case ComponentReplHB: 157 | return "REPL_HB" 158 | case ComponentRollback: 159 | return "ROLLBACK" 160 | case ComponentSharding: 161 | return "SHARDING" 162 | case ComponentShardingRefr: 163 | return "SH_REFR" 164 | case ComponentStorage: 165 | return "STORAGE" 166 | case ComponentTotal: 167 | return "TOTAL" 168 | case ComponentTracking: 169 | return "TRACKING" 170 | case ComponentUnknown: 171 | return "-" 172 | case ComponentWrite: 173 | return "WRITE" 174 | default: 175 | panic("unrecognized component") 176 | } 177 | } 178 | 179 | type Severity int 180 | 181 | const ( 182 | SeverityNone Severity = 0 183 | SeverityD = 1 << iota // Debug 184 | SeverityD1 // Debug 1 185 | SeverityD2 // Debug 2 186 | SeverityD3 // Debug 3 187 | SeverityD4 // Debug 4 188 | SeverityD5 // Debug 5 189 | SeverityE // Error 190 | SeverityF // Severe/Fatal 191 | SeverityI // Information/Log 192 | SeverityW // Warning 193 | ) 194 | 195 | type Binary uint32 196 | 197 | const ( 198 | BinaryAny Binary = iota 199 | BinaryMongod 200 | BinaryMongos 201 | ) 202 | 203 | type Base struct { 204 | *internal.RuneReader 205 | 206 | Component Component 207 | CString bool 208 | LineNumber uint 209 | RawDate string 210 | RawContext string 211 | RawMessage string 212 | Severity Severity 213 | } 214 | 215 | func NewSeverity(s string) (Severity, bool) { 216 | switch s { 217 | case "", "-": 218 | return SeverityNone, true 219 | case "D": 220 | return SeverityD, true 221 | case "D1": 222 | return SeverityD1, true 223 | case "D2": 224 | return SeverityD2, true 225 | case "D3": 226 | return SeverityD3, true 227 | case "D4": 228 | return SeverityD4, true 229 | case "D5": 230 | return SeverityD5, true 231 | case "E": 232 | return SeverityE, true 233 | case "F": 234 | return SeverityF, true 235 | case "I": 236 | return SeverityI, true 237 | case "W": 238 | return SeverityW, true 239 | default: 240 | return SeverityNone, false 241 | } 242 | } 243 | 244 | func (s Severity) String() string { 245 | switch s { 246 | case SeverityNone: 247 | return "-" 248 | case SeverityD: 249 | return "D" 250 | case SeverityD1: 251 | return "D1" 252 | case SeverityD2: 253 | return "D2" 254 | case SeverityD3: 255 | return "D3" 256 | case SeverityD4: 257 | return "D4" 258 | case SeverityD5: 259 | return "D5" 260 | case SeverityE: 261 | return "E" 262 | case SeverityF: 263 | return "F" 264 | case SeverityI: 265 | return "I" 266 | case SeverityW: 267 | return "W" 268 | default: 269 | panic("unrecognized severity") 270 | } 271 | } 272 | 273 | func (b Binary) String() string { 274 | switch b { 275 | case BinaryMongod: 276 | return "mongod" 277 | case BinaryMongos: 278 | return "mongos" 279 | default: 280 | return "unknown" 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /parser/record/constants.go: -------------------------------------------------------------------------------- 1 | // Package util provides general assistance to the parser and other areas 2 | // of code. The package is entirely self contained and should not reference 3 | // anything not already contained in the standard go release. 4 | package record 5 | 6 | import "mgotools/internal" 7 | 8 | // The constant arrays that follow should *ALWAYS* be in sorted binary order. A lot of methods expect these arrays 9 | // to be pre-sorted so they can be used with binary searches for fast array comparison. Failure to put these arrays 10 | // in proper binary order will because these methods to fail. 11 | 12 | // ref: /mongo/src/mongo/db/curop.cpp 13 | var COUNTERS = map[string]string{ 14 | "cursorExhausted": "cursorExhausted", 15 | "cursorid": "cursorid", 16 | "docsExamined": "docsExamined", 17 | "fastmod": "fastmod", 18 | "fastmodinsert": "fastmodinsert", 19 | "exhaust": "exhaust", 20 | "fromMultiPlanner": "fromMultiPlanner", 21 | "hasSortStage": "hasSortStage", 22 | "idhack": "idhack", 23 | "keysDeleted": "keysDeleted", 24 | "keysExamined": "keysExamined", 25 | "keysInserted": "keysInserted", 26 | "ndeleted": "ndeleted", 27 | "nDeleted": "ndeleted", 28 | "ninserted": "ninserted", 29 | "nInserted": "ninserted", 30 | "nmatched": "nmatched", 31 | "nMatched": "nmatched", 32 | "nmodified": "nmodified", 33 | "nModified": "nmodified", 34 | "nmoved": "nmoved", 35 | "nscanned": "keysExamined", 36 | "nscannedObjects": "docsExamined", 37 | "nreturned": "nreturned", 38 | "ntoreturn": "ntoreturn", 39 | "ntoskip": "notoskip", 40 | "planSummary": "planSummary", 41 | "numYields": "numYields", 42 | "keyUpdates": "keyUpdates", 43 | "replanned": "replanned", 44 | "reslen": "reslen", 45 | "scanAndOrder": "scanAndOrder", 46 | "upsert": "upsert", 47 | "writeConflicts": "writeConflicts", 48 | } 49 | 50 | var OPERATIONS = []string{ 51 | "aggregate", 52 | "command", 53 | "count", 54 | "distinct", 55 | "find", 56 | "geoNear", 57 | "geonear", 58 | "getMore", 59 | "getmore", 60 | "insert", 61 | "mapreduce", 62 | "query", 63 | "remove", 64 | "update", 65 | } 66 | 67 | var OPERATORS_COMPARISON = []string{ 68 | "$all", 69 | "$bitsAllClear", // 3.2 70 | "$bitsAllSet", // 3.2 71 | "$bitsAnyClear", // 3.2 72 | "$bitsAnySet", // 3.2 73 | "$eq", // 3.0 74 | "$exists", 75 | "$gt", 76 | "$gte", 77 | "$in", 78 | "$lt", 79 | "$lte", 80 | "$ne", 81 | "$size", 82 | "$type", 83 | } 84 | 85 | var OPERATORS_LOGICAL = []string{ 86 | "$and", 87 | "$nin", 88 | "$nor", 89 | "$not", 90 | "$or", 91 | } 92 | 93 | var OPERATORS_EXPRESSION = []string{ 94 | "$box", // $geoWithin 95 | "$center", // $geoWithin 96 | "$centerSphere", // $geoWithin 97 | "$comment", 98 | "$elemMatch", 99 | "$expr", 100 | "$geoIntersects", // 2.4 101 | "$geoWithin", // 2.4 102 | "$geometry", // $geoIntersects, $geoWithin 103 | "$jsonSchema", // 3.6 104 | "$mod", 105 | "$near", 106 | "$nearSphere", 107 | "$regex", 108 | "$text", 109 | "$where", 110 | } 111 | 112 | // IsContext checks for a bracketed string ([]) 113 | func IsContext(value string) bool { 114 | length := internal.StringLength(value) 115 | return length > 2 && value[0] == '[' && value[length-1] == ']' 116 | } 117 | -------------------------------------------------------------------------------- /parser/record/constants_test.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import "testing" 4 | 5 | func TestBinaryOrder(t *testing.T) { 6 | for name, a := range map[string][]string{ 7 | "OPERATIONS": OPERATIONS, 8 | "OPERATORS_COMPARISON": OPERATORS_COMPARISON, 9 | "OPERATORS_EXPRESSION": OPERATORS_EXPRESSION, 10 | "OPERATORS_LOGICAL": OPERATORS_LOGICAL, 11 | } { 12 | if index := testSortOrder(a); index > -1 { 13 | t.Errorf("%s not sorted; %s (%d) out of order", name, a[index], index) 14 | } 15 | } 16 | } 17 | 18 | func testSortOrder(a []string) int { 19 | for i := 1; i < len(a); i += 1 { 20 | if a[i-1] > a[i] { 21 | return i 22 | } 23 | } 24 | 25 | return -1 26 | } 27 | -------------------------------------------------------------------------------- /parser/record/entry.go: -------------------------------------------------------------------------------- 1 | package record 2 | 3 | import ( 4 | "bytes" 5 | "time" 6 | 7 | "mgotools/internal" 8 | "mgotools/parser/message" 9 | ) 10 | 11 | // Log examples: 12 | // Sat Jul 29 16:51:28.392 [initandlisten] db version v2.4.14 13 | // 2017-07-29T16:53:40.671-0700 [initandlisten] db version v2.6.12 14 | // 2017-07-29T16:55:33.242-0700 I CONTROL [initandlisten] db version v3.0.15 15 | // 2017-07-29T17:01:15.835-0700 I CONTROL [initandlisten] db version v3.2.12 16 | 17 | type Entry struct { 18 | Base 19 | Message message.Message 20 | 21 | Connection int 22 | Context string 23 | Date time.Time 24 | Format internal.DateFormat 25 | DateYearMissing bool 26 | DateRollover int 27 | DateValid bool 28 | Thread int 29 | 30 | Valid bool 31 | } 32 | 33 | func (r *Entry) String() string { 34 | var buffer = bytes.NewBuffer(make([]byte, 512)) 35 | if r.Format != "" { 36 | buffer.WriteString(string(r.Format)) 37 | } else { 38 | buffer.WriteString(string(internal.DateFormatIso8602Utc)) 39 | } 40 | 41 | buffer.WriteString(" ") 42 | buffer.WriteString(r.Severity.String()) 43 | buffer.WriteString(" ") 44 | buffer.WriteString(r.Component.String()) 45 | buffer.WriteString(" ") 46 | buffer.WriteString(r.RawContext) 47 | buffer.WriteString(" ") 48 | buffer.WriteString(r.RawMessage) 49 | return buffer.String() 50 | } 51 | -------------------------------------------------------------------------------- /parser/s_24.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version24SParser struct { 12 | executor.Executor 13 | } 14 | 15 | func init() { 16 | parser := Version24SParser{} 17 | 18 | version.Factory.Register(func() version.Parser { 19 | return &parser 20 | }) 21 | 22 | // Control (mongosMain) 23 | parser.RegisterForReader("build info", commonParseBuildInfo) 24 | parser.RegisterForReader("options:", mongosParseStartupOptions) 25 | parser.RegisterForReader("MongoS version", mongosParseVersion) 26 | 27 | // Network 28 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 29 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 30 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 31 | } 32 | 33 | var errorVersion24SUnmatched = internal.VersionUnmatched{"mongos 2.4"} 34 | 35 | func (v *Version24SParser) NewLogMessage(entry record.Entry) (msg message.Message, err error) { 36 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion24SUnmatched) 37 | } 38 | 39 | func (v *Version24SParser) Check(base record.Base) bool { 40 | return base.Severity == record.SeverityNone && 41 | base.Component == record.ComponentNone 42 | } 43 | 44 | func (v *Version24SParser) Version() version.Definition { 45 | return version.Definition{Major: 2, Minor: 4, Binary: record.BinaryMongos} 46 | } 47 | -------------------------------------------------------------------------------- /parser/s_26.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version26SParser struct{ executor.Executor } 12 | 13 | func init() { 14 | parser := Version24SParser{} 15 | version.Factory.Register(func() version.Parser { 16 | return &parser 17 | }) 18 | 19 | parser.RegisterForReader("build info", commonParseBuildInfo) 20 | parser.RegisterForReader("options:", mongosParseStartupOptions) 21 | parser.RegisterForReader("MongoS version", mongosParseVersion) 22 | 23 | // Network 24 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 25 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 26 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 27 | } 28 | 29 | var errorVersion26SUnmatched = internal.VersionUnmatched{Message: "mongos 2.6"} 30 | 31 | func (v *Version26SParser) Check(base record.Base) bool { 32 | return base.Severity == record.SeverityNone && 33 | base.Component == record.ComponentNone 34 | } 35 | 36 | func (v *Version26SParser) NewLogMessage(entry record.Entry) (msg message.Message, err error) { 37 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion26SUnmatched) 38 | } 39 | func (v *Version26SParser) Version() version.Definition { 40 | return version.Definition{Major: 2, Minor: 6, Binary: record.BinaryMongos} 41 | } 42 | -------------------------------------------------------------------------------- /parser/s_30.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version30SParser struct { 12 | executor.Executor 13 | } 14 | 15 | var errorVersion30SUnmatched = internal.VersionUnmatched{"mongos 3.0"} 16 | 17 | func init() { 18 | parser := Version30SParser{ 19 | executor.Executor{}, 20 | } 21 | 22 | version.Factory.Register(func() version.Parser { 23 | return &parser 24 | }) 25 | 26 | parser.RegisterForReader("build info", commonParseBuildInfo) 27 | parser.RegisterForReader("options:", mongosParseStartupOptions) 28 | parser.RegisterForReader("MongoS version", mongosParseVersion) 29 | 30 | // Network 31 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 32 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 33 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 34 | } 35 | 36 | func (v *Version30SParser) Check(base record.Base) bool { 37 | return base.Severity != record.SeverityNone && 38 | base.Component != record.ComponentNone 39 | } 40 | 41 | func (v *Version30SParser) NewLogMessage(entry record.Entry) (msg message.Message, err error) { 42 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion30SUnmatched) 43 | } 44 | func (v *Version30SParser) Version() version.Definition { 45 | return version.Definition{Major: 3, Minor: 0, Binary: record.BinaryMongos} 46 | } 47 | -------------------------------------------------------------------------------- /parser/s_32.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version32SParser struct{ executor.Executor } 12 | 13 | func init() { 14 | parser := Version32SParser{} 15 | version.Factory.Register(func() version.Parser { 16 | return &parser 17 | }) 18 | 19 | parser.RegisterForReader("options:", mongosParseStartupOptions) 20 | parser.RegisterForReader("MongoS version", mongosParseVersion) 21 | 22 | // Network 23 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 24 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 25 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 26 | } 27 | 28 | var errorVersion32SUnmatched = internal.VersionUnmatched{"mongos 3.2"} 29 | 30 | func (v *Version32SParser) Check(base record.Base) bool { 31 | return base.Severity != record.SeverityNone && 32 | base.Component != record.ComponentNone && 33 | base.Severity >= record.SeverityD1 && base.Severity < record.SeverityD5 34 | } 35 | 36 | func (v *Version32SParser) NewLogMessage(entry record.Entry) (message.Message, error) { 37 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion32SUnmatched) 38 | } 39 | 40 | func (v *Version32SParser) Version() version.Definition { 41 | return version.Definition{Major: 3, Minor: 2, Binary: record.BinaryMongos} 42 | } 43 | -------------------------------------------------------------------------------- /parser/s_34.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version34SParser struct{ executor.Executor } 12 | 13 | var errorVersion34SUnmatched = internal.VersionUnmatched{Message: "mongos 3.4"} 14 | 15 | func init() { 16 | parser := &Version34SParser{} 17 | 18 | version.Factory.Register(func() version.Parser { 19 | return parser 20 | }) 21 | 22 | parser.RegisterForReader("options:", mongosParseStartupOptions) 23 | parser.RegisterForReader("mongos version", mongosParseVersion) 24 | 25 | // Network 26 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 27 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 28 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 29 | } 30 | 31 | func (v *Version34SParser) Check(base record.Base) bool { 32 | return base.Severity != record.SeverityNone && 33 | base.Component != record.ComponentNone && 34 | base.Severity >= record.SeverityD1 && base.Severity < record.SeverityD5 35 | } 36 | 37 | func (v *Version34SParser) NewLogMessage(entry record.Entry) (message.Message, error) { 38 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion34SUnmatched) 39 | } 40 | 41 | func (v *Version34SParser) Version() version.Definition { 42 | return version.Definition{Major: 3, Minor: 4, Binary: record.BinaryMongos} 43 | } 44 | -------------------------------------------------------------------------------- /parser/s_36.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | type Version36SParser struct{ executor.Executor } 12 | 13 | func init() { 14 | parser := &Version36SParser{} 15 | 16 | version.Factory.Register(func() version.Parser { 17 | return parser 18 | }) 19 | 20 | parser.RegisterForReader("options:", mongosParseStartupOptions) 21 | parser.RegisterForReader("mongos version", mongosParseVersion) 22 | 23 | // Network 24 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 25 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 26 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 27 | } 28 | 29 | var errorVersion36SUnmatched = internal.VersionUnmatched{Message: "mongos 3.6"} 30 | 31 | func (v *Version36SParser) Check(base record.Base) bool { 32 | return base.Severity != record.SeverityNone && 33 | base.Component != record.ComponentNone && 34 | base.Severity >= record.SeverityD1 && base.Severity < record.SeverityD5 35 | } 36 | 37 | func (v *Version36SParser) NewLogMessage(entry record.Entry) (message.Message, error) { 38 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion36SUnmatched) 39 | } 40 | 41 | func (v *Version36SParser) Version() version.Definition { 42 | return version.Definition{Major: 3, Minor: 6, Binary: record.BinaryMongos} 43 | } 44 | -------------------------------------------------------------------------------- /parser/s_40.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | var errorVersion40SUnmatched = internal.VersionUnmatched{Message: "mongos 4.0"} 12 | 13 | type Version40SParser struct{ executor.Executor } 14 | 15 | func init() { 16 | parser := &Version40SParser{} 17 | 18 | version.Factory.Register(func() version.Parser { 19 | return parser 20 | }) 21 | 22 | parser.RegisterForReader("options:", mongosParseStartupOptions) 23 | parser.RegisterForReader("mongos version", mongosParseVersion) 24 | 25 | // Network 26 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 27 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 28 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 29 | } 30 | 31 | func (Version40SParser) Check(base record.Base) bool { 32 | return base.Severity != record.SeverityNone && 33 | base.Component != record.ComponentNone 34 | } 35 | 36 | func (v *Version40SParser) NewLogMessage(entry record.Entry) (message.Message, error) { 37 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion40SUnmatched) 38 | } 39 | 40 | func (Version40SParser) Version() version.Definition { 41 | return version.Definition{Major: 4, Minor: 0, Binary: record.BinaryMongos} 42 | } 43 | -------------------------------------------------------------------------------- /parser/s_42.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "mgotools/internal" 5 | "mgotools/parser/executor" 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | "mgotools/parser/version" 9 | ) 10 | 11 | var errorVersion42SUnmatched = internal.VersionUnmatched{Message: "mongos 4.0"} 12 | 13 | type Version42SParser struct{ executor.Executor } 14 | 15 | func init() { 16 | parser := &Version42SParser{} 17 | 18 | version.Factory.Register(func() version.Parser { 19 | return parser 20 | }) 21 | 22 | parser.RegisterForReader("options:", mongosParseStartupOptions) 23 | parser.RegisterForReader("mongos version", mongosParseVersion) 24 | 25 | // Network 26 | parser.RegisterForReader("connection accepted", commonParseConnectionAccepted) 27 | parser.RegisterForReader("waiting for connections", commonParseWaitingForConnections) 28 | parser.RegisterForEntry("end connection", commonParseConnectionEnded) 29 | } 30 | 31 | func (Version42SParser) Check(base record.Base) bool { 32 | return base.Severity != record.SeverityNone && 33 | base.Component != record.ComponentNone 34 | } 35 | 36 | func (v *Version42SParser) NewLogMessage(entry record.Entry) (message.Message, error) { 37 | return v.Run(entry, internal.NewRuneReader(entry.RawMessage), errorVersion40SUnmatched) 38 | } 39 | 40 | func (Version42SParser) Version() version.Definition { 41 | return version.Definition{Major: 4, Minor: 2, Binary: record.BinaryMongos} 42 | } 43 | -------------------------------------------------------------------------------- /parser/source/accumulator.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | 8 | "mgotools/parser/record" 9 | ) 10 | 11 | const OutputBuffer = 128 12 | const MaxBufferSize = 16777216 13 | 14 | type accumulatorReadCloser interface { 15 | io.ReadCloser 16 | NewBase(string, uint) (record.Base, error) 17 | } 18 | 19 | type accumulator struct { 20 | io.Closer 21 | 22 | eof bool 23 | next record.Base 24 | error error 25 | 26 | Log *bufio.Scanner 27 | Out chan accumulatorResult 28 | In chan string 29 | } 30 | 31 | var _ io.ReadCloser = (*accumulator)(nil) 32 | var _ Factory = (*accumulator)(nil) 33 | 34 | type accumulatorResult struct { 35 | Base record.Base 36 | Error error 37 | } 38 | 39 | func NewAccumulator(handle accumulatorReadCloser) *accumulator { 40 | r := &accumulator{ 41 | Closer: handle, 42 | eof: false, 43 | 44 | Log: bufio.NewScanner(handle), 45 | Out: make(chan accumulatorResult, OutputBuffer), 46 | In: make(chan string), 47 | } 48 | 49 | // Begin scanning the source and send it to the input channel. 50 | go func() { 51 | defer close(r.In) 52 | 53 | for r.Log.Scan() { 54 | r.In <- r.Log.Text() 55 | } 56 | }() 57 | 58 | go Accumulator(r.In, r.Out, handle.NewBase) 59 | return r 60 | } 61 | 62 | // The accumulator is designed to concat multi-line entries. The log format does 63 | // not properly escape quotes or newline entries, which causes problems when 64 | // lines are analyzed one at a time. 65 | // 66 | // Solving this problem requires a start marker, which is the date in all 67 | // versions of MongoDB. However, the date in older versions is a CString. 68 | // Thankfully, the record.Base object contains enough information to properly 69 | // parse multi-line input. 70 | func Accumulator(in <-chan string, out chan<- accumulatorResult, callback func(string, uint) (record.Base, error)) { 71 | defer func() { 72 | // Last defer called. 73 | close(out) 74 | }() 75 | 76 | type accumulatorCounter struct { 77 | count int 78 | last []accumulatorResult 79 | size int 80 | started bool 81 | } 82 | 83 | accumulate := func(a accumulatorCounter) string { 84 | // Create a buffer to construct a single line containing 85 | // every accumulated record.Base entry between the latest 86 | // line and the next line. Disregard any errors. 87 | accumulator := bytes.NewBuffer(make([]byte, 0, a.size+len(a.last))) 88 | for _, r := range a.last { 89 | accumulator.WriteString(r.Base.String()) 90 | accumulator.WriteRune('\n') 91 | } 92 | // Create a record.Base object with all the accumulated base 93 | // objects from previous lines. 94 | s := accumulator.String() 95 | s = s[:len(s)-1] 96 | // Remove the extraneous newline. 97 | return s 98 | } 99 | 100 | reset := func(a *accumulatorCounter) { 101 | a.last = a.last[:0] 102 | a.size = 0 103 | } 104 | 105 | flush := func(a *accumulatorCounter) { 106 | for _, r := range a.last { 107 | out <- r 108 | } 109 | reset(a) 110 | } 111 | 112 | a := accumulatorCounter{ 113 | count: 0, 114 | last: make([]accumulatorResult, 0), 115 | size: 0, 116 | started: false, 117 | } 118 | 119 | defer flush(&a) 120 | lineNumber := uint(0) 121 | 122 | for line := range in { 123 | lineNumber += 1 124 | base, err := callback(line, lineNumber) 125 | 126 | if base.RawDate != "" { 127 | // The current object has a valid date and thus starts a new log 128 | // line that _might_ span multiple lines. That means the previous 129 | // line containing a date does not span multiple lines. Check 130 | // whether a.last contains a value and output the value. 131 | if a.size > 0 { 132 | if len(a.last) == 1 { 133 | out <- a.last[0] 134 | reset(&a) 135 | } else { 136 | // Handle the actual accumulation and generate a string. The 137 | // string gets passed back to the callback method to create 138 | // a new object. 139 | s := accumulate(a) 140 | 141 | // Create a base object from the newly accumulated string. 142 | m, err := callback(s, a.last[0].Base.LineNumber) 143 | reset(&a) 144 | 145 | // Send the completed output and any errors. 146 | out <- accumulatorResult{ 147 | Base: m, 148 | Error: err, 149 | } 150 | } 151 | } 152 | 153 | // Started is not set until the first time a valid date is encountered. 154 | a.started = true 155 | 156 | // Keep the last entry and output nothing (for now). 157 | a.size += base.Length() 158 | a.last = append(a.last, accumulatorResult{ 159 | Base: base, 160 | Error: err, 161 | }) 162 | } else { 163 | if !a.started { 164 | // No date line has been discovered so the log is either invalid 165 | // or started in the middle of a multi-line string. Either case 166 | // demands simply outputting the erratic result. 167 | out <- accumulatorResult{ 168 | Base: base, 169 | Error: err, 170 | } 171 | } else if a.size > MaxBufferSize { 172 | // The buffer is too large so create a base object and try to do 173 | // something with it. The maximum object size is 16MB but logs 174 | // get truncated well before that, so this should be something 175 | // reasonable but less than or equal to 16MB 176 | flush(&a) 177 | } else { 178 | // Add each line to the accumulator array and keep track of how 179 | // many line bytes are being stored. 180 | a.last = append(a.last, accumulatorResult{base, err}) 181 | a.size += base.Length() 182 | } 183 | } 184 | } 185 | 186 | // It's possible that the log ended (for example, a stack trace that spans 187 | // multiple lines then terminates). A check is needed for pending lines 188 | // only when the first line is valid; otherwise the accumulated errors get 189 | // flushed directly. 190 | if len(a.last) > 1 && a.last[0].Error == nil { 191 | // Grab the accumulated string. 192 | s := accumulate(a) 193 | 194 | // Generate a base object based on the string. 195 | m, err := callback(s, a.last[0].Base.LineNumber) 196 | reset(&a) 197 | 198 | out <- accumulatorResult{ 199 | Base: m, 200 | Error: err, 201 | } 202 | } 203 | } 204 | 205 | // Implement a reader that returns a byte array of the most current accumulated 206 | // line. 207 | func (f *accumulator) Read(p []byte) (n int, err error) { 208 | if f.eof { 209 | return 0, f.error 210 | } 211 | s := []byte(f.next.String()) 212 | l := len(s) 213 | 214 | if l > cap(p) { 215 | panic("buffer too small for next set of results") 216 | } 217 | 218 | copy(p, s) 219 | return l, nil 220 | } 221 | 222 | func (f *accumulator) Next() bool { 223 | if f.eof { 224 | return false 225 | } 226 | 227 | b, ok := <-f.Out 228 | f.next = b.Base 229 | f.error = b.Error 230 | 231 | if !ok { 232 | f.eof = true 233 | return false 234 | } 235 | return true 236 | } 237 | 238 | func (f *accumulator) Get() (record.Base, error) { 239 | return f.next, f.error 240 | } 241 | 242 | func (f *accumulator) Close() error { 243 | return f.Closer.Close() 244 | } 245 | -------------------------------------------------------------------------------- /parser/source/accumulator_test.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | type accumulatorFile struct { 11 | Reader string 12 | Expected []string 13 | } 14 | 15 | func TestAccumulator_Run(tr *testing.T) { 16 | //tr.Parallel() 17 | 18 | tests := map[string]accumulatorFile{ 19 | "FileInvalid": fileInvalid, 20 | "FileIso1Valid": fileIso1Valid, 21 | "FileIso1Multiline": fileIso1MultiLine, 22 | } 23 | 24 | for name, reader := range tests { 25 | tr.Run(name, func(t *testing.T) { 26 | testAccumulator(reader, t) 27 | }) 28 | } 29 | } 30 | 31 | func testAccumulator(r accumulatorFile, t *testing.T) { 32 | scanner := bufio.NewScanner(bytes.NewBufferString(r.Reader)) 33 | 34 | in, out := make(chan string), make(chan accumulatorResult) 35 | go Accumulator(in, out, Log{}.NewBase) 36 | 37 | go func() { 38 | i := uint(0) 39 | for scanner.Scan() { 40 | i += 1 41 | in <- scanner.Text() 42 | } 43 | close(in) 44 | }() 45 | 46 | i := int(0) 47 | for m := range out { 48 | if m.Error == io.EOF { 49 | break 50 | } 51 | if i >= len(r.Expected) { 52 | t.Errorf("Too many results found") 53 | return 54 | } 55 | if m.Base.String() != r.Expected[i] { 56 | t.Errorf("Line %d does not match", i+1) 57 | t.Errorf("Got (%d): %s", m.Base.Length(), m.Base.String()) 58 | t.Errorf("Is (%d): %s", len(r.Expected[i]), r.Expected[i]) 59 | } 60 | i += 1 61 | } 62 | 63 | if i < len(r.Expected) { 64 | t.Errorf("Too few lines returned, %d of %d", i, len(r.Expected)) 65 | } 66 | } 67 | 68 | var fileIso1Valid = accumulatorFile{ 69 | Reader: `2018-01-16T15:00:44.732-0800 I STORAGE [signalProcessingThread] closeAllFiles() finished 70 | 2018-01-16T15:00:44.733-0800 I STORAGE [signalProcessingThread] shutdown: removing fs lock... 71 | 2018-01-16T15:00:44.733-0800 I CONTROL [signalProcessingThread] now exiting 72 | 2018-01-16T15:00:44.734-0800 I CONTROL [signalProcessingThread] shutting down with code:0`, 73 | 74 | Expected: []string{ 75 | "2018-01-16T15:00:44.732-0800 I STORAGE [signalProcessingThread] closeAllFiles() finished", 76 | "2018-01-16T15:00:44.733-0800 I STORAGE [signalProcessingThread] shutdown: removing fs lock...", 77 | "2018-01-16T15:00:44.733-0800 I CONTROL [signalProcessingThread] now exiting", 78 | "2018-01-16T15:00:44.734-0800 I CONTROL [signalProcessingThread] shutting down with code:0", 79 | }, 80 | } 81 | 82 | var fileIso1MultiLine = accumulatorFile{ 83 | Reader: `2018-01-16T15:00:44.569-0800 I COMMAND [conn1] command test.$cmd command: isMaster { isMaster: 1.0, forShell: 1.0 } numYields:0 reslen:174 locks:{} protocol:op_command 0ms 84 | 2018-01-16T15:00:44.571-0800 I COMMAND [conn1] command test.foo command: find { find: "foo", filter: { c: "this is a string "with quotes" and 85 | newlines 86 | 87 | done." } } planSummary: IXSCAN { c: 1 } keysExamined:0 docsExamined:0 cursorExhausted:1 numYields:0 nreturned:0 reslen:81 locks:{ Global: { acquireCount: { r: 2 } }, MMAPV1Journal: { acquireCount: { r: 1 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { R: 1 } } } protocol:op_command 0ms 88 | 2018-01-16T15:00:44.572-0800 I COMMAND [conn1] command test.$cmd command: isMaster { isMaster: 1.0, forShell: 1.0 } numYields:0 reslen:174 locks:{} protocol:op_command 0ms 89 | 2018-01-16T15:00:44.573-0800 I COMMAND [conn1] command test.foo command: explain { explain: { find: "foo", filter: { b: 1.0 } }, verbosity: "queryPlanner" } numYields:0 reslen:381 locks:{ Global: { acquireCount: { r: 2 } }, MMAPV1Journal: { acquireCount: { r: 1 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { R: 1 } } } protocol:op_command 0ms`, 90 | 91 | Expected: []string{ 92 | `2018-01-16T15:00:44.569-0800 I COMMAND [conn1] command test.$cmd command: isMaster { isMaster: 1.0, forShell: 1.0 } numYields:0 reslen:174 locks:{} protocol:op_command 0ms`, 93 | `2018-01-16T15:00:44.571-0800 I COMMAND [conn1] command test.foo command: find { find: "foo", filter: { c: "this is a string "with quotes" and 94 | newlines 95 | 96 | done." } } planSummary: IXSCAN { c: 1 } keysExamined:0 docsExamined:0 cursorExhausted:1 numYields:0 nreturned:0 reslen:81 locks:{ Global: { acquireCount: { r: 2 } }, MMAPV1Journal: { acquireCount: { r: 1 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { R: 1 } } } protocol:op_command 0ms`, 97 | `2018-01-16T15:00:44.572-0800 I COMMAND [conn1] command test.$cmd command: isMaster { isMaster: 1.0, forShell: 1.0 } numYields:0 reslen:174 locks:{} protocol:op_command 0ms`, 98 | `2018-01-16T15:00:44.573-0800 I COMMAND [conn1] command test.foo command: explain { explain: { find: "foo", filter: { b: 1.0 } }, verbosity: "queryPlanner" } numYields:0 reslen:381 locks:{ Global: { acquireCount: { r: 2 } }, MMAPV1Journal: { acquireCount: { r: 1 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { R: 1 } } } protocol:op_command 0ms`, 99 | }, 100 | } 101 | 102 | var fileInvalid = accumulatorFile{ 103 | Reader: `line 1 104 | line 2 105 | line 3 106 | line 4`, 107 | 108 | Expected: []string{ 109 | "line 1", 110 | "line 2", 111 | "line 3", 112 | "line 4", 113 | }, 114 | } 115 | -------------------------------------------------------------------------------- /parser/source/factory.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import "mgotools/parser/record" 4 | 5 | type Factory interface { 6 | Next() bool 7 | Get() (record.Base, error) 8 | Close() error 9 | } 10 | -------------------------------------------------------------------------------- /parser/source/log.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "bufio" 5 | "compress/gzip" 6 | "errors" 7 | "io" 8 | "strings" 9 | "sync" 10 | "unicode" 11 | 12 | "mgotools/internal" 13 | "mgotools/parser/record" 14 | ) 15 | 16 | var ErrorParsingDate = errors.New("unrecognized date format") 17 | var ErrorMissingContext = errors.New("missing context") 18 | 19 | type Log struct { 20 | io.Closer 21 | *bufio.Reader 22 | *bufio.Scanner 23 | 24 | next record.Base 25 | error error 26 | 27 | closed bool 28 | eof bool 29 | line uint 30 | mutex sync.RWMutex 31 | } 32 | 33 | // Enforce the interface at compile time. 34 | var _ Factory = (*Log)(nil) 35 | 36 | func NewLog(base io.ReadCloser) (*Log, error) { 37 | reader := bufio.NewReader(base) 38 | 39 | if scanner, err := makeScanner(reader); err != nil { 40 | return nil, err 41 | } else { 42 | return &Log{ 43 | Reader: reader, 44 | Closer: base, 45 | Scanner: scanner, 46 | 47 | // These are all defaults, but it doesn't hurts to be explicit. 48 | closed: false, 49 | eof: false, 50 | line: 0, 51 | mutex: sync.RWMutex{}, 52 | }, nil 53 | } 54 | } 55 | 56 | func makeScanner(reader *bufio.Reader) (*bufio.Scanner, error) { 57 | var scanner = bufio.NewScanner(reader) 58 | 59 | // Check for gzip magic headers. 60 | if peek, err := reader.Peek(2); err == nil { 61 | if peek[0] == 0x1f && peek[1] == 0x8b { 62 | if gzipReader, err := gzip.NewReader(reader); err == nil { 63 | scanner = bufio.NewScanner(gzipReader) 64 | } else { 65 | return nil, err 66 | } 67 | } 68 | } 69 | return scanner, nil 70 | } 71 | 72 | // Generate an Entry from a line of text. This method assumes the entry is *not* JSON. 73 | func (Log) NewBase(line string, num uint) (record.Base, error) { 74 | var ( 75 | base = record.Base{RuneReader: internal.NewRuneReader(line), LineNumber: num, Severity: record.SeverityNone} 76 | pos int 77 | ) 78 | 79 | // Check for a day in the first portion of the string, which represents version <= 2.4 80 | if day := base.PreviewWord(1); internal.IsDay(day) { 81 | base.RawDate = parseCDateString(&base) 82 | base.CString = true 83 | } else if internal.IsIso8601String(base.PreviewWord(1)) { 84 | base.RawDate, _ = base.SlurpWord() 85 | base.CString = false 86 | } 87 | 88 | if base.EOL() || base.RawDate == "" { 89 | return base, ErrorParsingDate 90 | } 91 | 92 | if base.ExpectRune('[') { 93 | // the context is first so assume the line remainder is the message 94 | if r, err := base.EnclosedString(']', false); err == nil { 95 | base.RawContext = r 96 | } 97 | 98 | for base.Expect(unicode.Space) { 99 | base.Next() 100 | } 101 | } else { 102 | // the context isn't first so there is likely more available to check 103 | for i := 0; i < 4; i += 1 { 104 | part, ok := base.SlurpWord() 105 | if !ok { 106 | // EOL 107 | break 108 | } 109 | 110 | if base.Severity == record.SeverityNone && 111 | base.Component == record.ComponentNone && 112 | base.RawContext == "" { 113 | severity, ok := record.NewSeverity(part) 114 | 115 | if ok { 116 | base.Severity = severity 117 | continue 118 | } 119 | } 120 | 121 | if base.Component == record.ComponentNone && 122 | base.RawContext == "" { 123 | component, ok := record.NewComponent(part) 124 | 125 | if ok { 126 | base.Component = component 127 | continue 128 | } 129 | } 130 | 131 | if base.RawContext == "" && part[0] == '[' { 132 | base.RewindSlurpWord() 133 | if r, err := base.EnclosedString(']', false); err == nil { 134 | base.RawContext = r 135 | continue 136 | } 137 | } 138 | 139 | base.RewindSlurpWord() 140 | break 141 | } 142 | } 143 | 144 | // All log entries for all supported versions have a context. 145 | if base.RawContext == "" { 146 | return base, ErrorMissingContext 147 | } 148 | 149 | pos = base.Pos() 150 | base.RawMessage = base.Remainder() 151 | base.Seek(pos, 0) 152 | 153 | return base, nil 154 | } 155 | 156 | func (f *Log) Close() error { 157 | f.mutex.Lock() 158 | defer f.mutex.Unlock() 159 | 160 | if !f.closed { 161 | f.closed = true 162 | return f.Closer.Close() 163 | } 164 | 165 | return nil 166 | } 167 | 168 | func (f *Log) isClosed() bool { 169 | f.mutex.RLock() 170 | defer f.mutex.RUnlock() 171 | 172 | return f.closed 173 | } 174 | 175 | func (f *Log) Get() (record.Base, error) { 176 | return f.next, f.error 177 | } 178 | 179 | func (f *Log) Next() bool { 180 | f.next, f.error = f.get() 181 | 182 | if f.error == io.EOF { 183 | return false 184 | } 185 | return true 186 | } 187 | 188 | func (f Log) get() (record.Base, error) { 189 | if !f.eof && !f.isClosed() && f.Scanner.Scan() { 190 | f.line += 1 191 | return f.NewBase(f.Scanner.Text(), f.line) 192 | } 193 | return record.Base{}, io.EOF 194 | } 195 | 196 | func isComponent(c string) bool { 197 | _, ok := record.NewComponent(c) 198 | return ok 199 | } 200 | 201 | // Take a parts array ([]string { "Sun", "Jan", "02", "15:04:05" }) and combined into a single element 202 | // ([]string { "Sun Jan 02 15:04:05" }) with all trailing elements appended to the array. 203 | func parseCDateString(r *record.Base) string { 204 | var ( 205 | ok = true 206 | target = make([]string, 4) 207 | ) 208 | start := r.Pos() 209 | for i := 0; i < 4 && ok; i++ { 210 | target[i], ok = r.SlurpWord() 211 | } 212 | 213 | switch { 214 | case !internal.IsDay(target[0]): 215 | case !internal.IsMonth(target[1]): 216 | case !internal.IsNumeric(target[2]): 217 | case !internal.IsTime(target[3]): 218 | r.Seek(start, 0) 219 | return "" 220 | } 221 | 222 | return strings.Join(target, " ") 223 | } 224 | -------------------------------------------------------------------------------- /parser/source/log_test.go: -------------------------------------------------------------------------------- 1 | package source 2 | 3 | import ( 4 | "testing" 5 | 6 | "mgotools/parser/record" 7 | ) 8 | 9 | func TestNewBase(tr *testing.T) { 10 | f := Log{} 11 | tr.Run("Base24", func(t *testing.T) { 12 | if b, err := f.NewBase("Tue Jan 16 15:00:40.105 [initandlisten] db version v2.4.14", 1); err != nil { 13 | t.Error("base (2.4) returned an error, should be successful") 14 | } else if b.RawDate == "" { 15 | t.Errorf("base.RawDate (2.4) is empty or returned with error (%s)", err) 16 | } else if b.RawDate != "Tue Jan 16 15:00:40.105" { 17 | t.Error("base.RawDate (2.4) is incorrect") 18 | } else if b.RawContext != "[initandlisten]" { 19 | t.Error("base.RawContext (2.4) is incorrect") 20 | } else if b.Component != record.ComponentNone { 21 | t.Error("base.Component (2.4) returned a component") 22 | } else if b.Severity != record.SeverityNone { 23 | t.Error("base.Severity (2.4) returned a severity") 24 | } else if b.RawMessage != "db version v2.4.14" { 25 | t.Error("base.RawMessage (2.4) is incorrect") 26 | } 27 | }) 28 | tr.Run("Base26", func(t *testing.T) { 29 | if b, err := f.NewBase("2018-01-16T15:00:41.014-0800 [initandlisten] db version v2.6.12", 1); err != nil { 30 | t.Error("base (2.6) returned an error, should be successful") 31 | } else if b.RawDate == "" { 32 | t.Errorf("base.RawDate (2.6) is empty or returned with error (%s)", err.Error()) 33 | } else if b.RawDate != "2018-01-16T15:00:41.014-0800" { 34 | t.Error("base.RawDate (2.6) is incorrect") 35 | } else if b.RawContext != "[initandlisten]" { 36 | t.Error("base.RawContext (2.6) is incorrect") 37 | } else if b.Component != record.ComponentNone { 38 | t.Error("base.Component (2.6) returned a component") 39 | } else if b.Severity != record.SeverityNone { 40 | t.Error("base.Severity (2.6) returned a severity") 41 | } else if b.RawMessage != "db version v2.6.12" { 42 | t.Error("base.RawMessage (2.6) is incorrect") 43 | } 44 | }) 45 | tr.Run("Base30", func(t *testing.T) { 46 | if b, err := f.NewBase("2018-01-16T15:00:41.759-0800 I CONTROL [initandlisten] db version v3.0.15", 1); err != nil { 47 | t.Error("base (3.x) returned an error, should be successful") 48 | } else if b.RawDate == "" { 49 | t.Errorf("base.RawDate (3.x) is empty or returned with error (%s)", err) 50 | } else if b.RawDate != "2018-01-16T15:00:41.759-0800" { 51 | t.Error("base.RawDate (3.x) is incorrect") 52 | } else if b.RawContext != "[initandlisten]" { 53 | t.Error("base.RawContext (3.x) is incorrect") 54 | } else if b.Component != record.ComponentControl { 55 | t.Error("base.Component (3.x) returned a component") 56 | } else if b.Severity != record.SeverityI { 57 | t.Error("base.Severity (3.x) returned a severity") 58 | } else if b.RawMessage != "db version v3.0.15" { 59 | t.Error("base.RawMessage (3.x) is incorrect") 60 | } 61 | }) 62 | tr.Run("InvalidPartial", func(t *testing.T) { 63 | if b, err := f.NewBase("line 1", 1); err == nil || b.RawDate != "" { 64 | t.Error("base.RawDate is not empty but should be") 65 | } 66 | if _, err := f.NewBase("Tue Jan 16 15:00:40.105", 1); err == nil { 67 | t.Error("base.RawDate (2.4) reported success, should be error") 68 | } 69 | if _, err := f.NewBase("2018-01-16 15:00:41.014-0800", 1); err == nil { 70 | t.Error("base.RawDate reported success, should be error") 71 | } 72 | if _, err := f.NewBase("2018-01-16T15:00:41.759-0800 I CONTROL [initandlisten] ", 1); err != nil { 73 | t.Error("base.RawMessage can be blank, but returned an error") 74 | } 75 | if _, err := f.NewBase("2018-01-16T15:00:41.759-0800 I INVALID [initandlisten] ", 1); err == nil { 76 | t.Error("base.Component cannot be invalid, but returned without error") 77 | } 78 | if _, err := f.NewBase("2018-01-16T15:00:41.759-0800 ! CONTROL [initandlisten] ", 1); err == nil { 79 | t.Error("base.Severity cannot be invalid, but returned without error") 80 | } 81 | if _, err := f.NewBase("2018-01-16T15:00:41.759-0800 I CONTROL ", 1); err == nil { 82 | t.Error("base.RawContext is empty, should be an error") 83 | } 84 | }) 85 | tr.Run("Invalid24Date", func(t *testing.T) { 86 | if _, err := f.NewBase("Xyz Jan 16 15:00:40.105 [initandlisten]", 1); err != nil && err != ErrorParsingDate { 87 | t.Error("base.RawDate is incorrect, without incorrect error type") 88 | } 89 | if _, err := f.NewBase("Tue Xyz 16 15:00:40.105 [initandlisten]", 1); err != nil && err != ErrorParsingDate { 90 | t.Error("base.RawDate is incorrect, without incorrect error type") 91 | } 92 | if _, err := f.NewBase("Tue Jan AB 15:00:40.105 [initandlisten]", 1); err != nil && err != ErrorParsingDate { 93 | t.Error("base.RawDate is incorrect, without incorrect error type") 94 | } 95 | if _, err := f.NewBase("Tue Jan 16 XX:00:40.105 [initandlisten]", 1); err != nil && err != ErrorParsingDate { 96 | t.Error("base.RawDate is incorrect, without incorrect error type") 97 | } 98 | if _, err := f.NewBase("Tue Jan 16 15:00:40.XXX [initandlisten]", 1); err != nil && err != ErrorParsingDate { 99 | t.Error("base.RawDate is incorrect, without incorrect error type") 100 | } 101 | if _, err := f.NewBase("Tue Jan 16 15:00:40 [initandlisten]", 1); err != nil { 102 | t.Error("base.RawDate is correct, but returned an error") 103 | } 104 | }) 105 | } 106 | -------------------------------------------------------------------------------- /parser/version/debug.go: -------------------------------------------------------------------------------- 1 | // This package exists to support enhanced debugging. It's a package wrapper 2 | // to expose an private static method globally. 3 | // 4 | // +build debug 5 | 6 | package version 7 | 8 | import ( 9 | "mgotools/parser/record" 10 | ) 11 | 12 | func (c *Context) Convert(base record.Base, parser Parser) (record.Entry, error) { 13 | return c.convert(base, parser) 14 | } 15 | -------------------------------------------------------------------------------- /parser/version/instance.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "mgotools/internal" 10 | "mgotools/parser/message" 11 | "mgotools/parser/record" 12 | ) 13 | 14 | type Context struct { 15 | parserFactory *manager 16 | versions []Definition 17 | 18 | Count int 19 | Errors int 20 | Lines int 21 | LastWinner Definition 22 | 23 | DatePreviousMonth time.Month 24 | DatePreviousYear int 25 | DateRollover int 26 | DateYearMissing bool 27 | 28 | dateParser *internal.DateParser 29 | day int 30 | month time.Month 31 | 32 | shutdown sync.Once 33 | } 34 | 35 | func New(parsers []Parser, date *internal.DateParser) *Context { 36 | context := Context{ 37 | Count: 0, 38 | Errors: 0, 39 | 40 | DateRollover: 0, 41 | DateYearMissing: false, 42 | 43 | dateParser: date, 44 | day: time.Now().Day(), 45 | month: time.Now().Month(), 46 | versions: make([]Definition, len(parsers)), 47 | } 48 | 49 | for index, version := range parsers { 50 | context.versions[index] = version.Version() 51 | } 52 | 53 | context.parserFactory = newManager(context.convert, parsers) 54 | return &context 55 | } 56 | 57 | func (c *Context) Versions() []Definition { 58 | versions := make([]Definition, 0) 59 | for _, check := range c.versions { 60 | if r, f := c.parserFactory.IsRejected(check); f && !r { 61 | versions = append(versions, check) 62 | } 63 | } 64 | 65 | return versions 66 | } 67 | 68 | func (c *Context) Finish() { 69 | c.shutdown.Do(c.parserFactory.Finish) 70 | } 71 | 72 | func (c *Context) NewEntry(base record.Base) (record.Entry, error) { 73 | manager := c.parserFactory 74 | 75 | // Attempt to retrieve a version from the base. 76 | entry, version, err := manager.Try(base) 77 | c.LastWinner = version 78 | 79 | if err == internal.VersionMessageUnmatched { 80 | return record.Entry{}, err 81 | } 82 | 83 | // Check for compatibility problems with old versions. 84 | if version.Major == 2 && version.Minor <= 4 { 85 | // Date rollover is necessary when the timestamp doesn't include the year. A year is automatically 86 | // appended to every log.Base entry that doesn't have one. It does this using the current year and 87 | // a rollover value. Rollover occurs ever time January is detected within the log. 88 | if currentMonth := entry.Date.Month(); currentMonth < c.DatePreviousMonth { 89 | // Reset the previous month and year, and update the date rollover. 90 | c.DateRollover += 1 91 | c.DatePreviousYear += 1 92 | } 93 | } 94 | 95 | // Handle situations where the date is missing (typically old versions). 96 | if entry.DateYearMissing || entry.Date.Year() == 0 { 97 | c.DateYearMissing = true 98 | 99 | year := time.Now().Year() 100 | if c.DateRollover == 0 && (entry.Date.Month() > c.month || (entry.Date.Month() == c.month && entry.Date.Day() > c.day)) { 101 | year -= 1 102 | } 103 | 104 | entry.Date = time.Date(year, entry.Date.Month(), entry.Date.Day(), entry.Date.Hour(), entry.Date.Minute(), entry.Date.Second(), entry.Date.Nanosecond(), entry.Date.Location()) 105 | } 106 | 107 | reject := func(msg message.Version) { 108 | switch msg.Binary { 109 | case "mongod": 110 | manager.Reject(func(version Definition) bool { 111 | return version.Major != msg.Major || version.Minor != msg.Minor || version.Binary != record.BinaryMongod 112 | }) 113 | case "mongos": 114 | manager.Reject(func(version Definition) bool { 115 | return version.Major != msg.Major || version.Minor != msg.Minor || version.Binary != record.BinaryMongos 116 | }) 117 | } 118 | } 119 | 120 | // Update index context if it is available. 121 | if entry.Message != nil && entry.Connection == 0 { 122 | switch msg := entry.Message.(type) { 123 | case message.StartupInfo, message.BuildInfo: 124 | // Reset all available versions since the server restarted. 125 | manager.Reset() 126 | 127 | case message.Version: 128 | // Reject all versions but the current version. 129 | manager.Reset() 130 | reject(msg) 131 | 132 | case message.StartupInfoLegacy: 133 | // Reject all versions but the current version, except with 134 | // a legacy mongos. 135 | manager.Reset() 136 | reject(msg.Version) 137 | 138 | case message.Listening: 139 | // noop 140 | } 141 | } 142 | 143 | c.Count += 1 144 | c.Lines += 1 145 | return entry, nil 146 | } 147 | 148 | func (c *Context) convert(base record.Base, factory Parser) (record.Entry, error) { 149 | var ( 150 | err error 151 | out = record.Entry{Base: base, DateValid: true, Valid: true} 152 | ) 153 | 154 | if out.Date, out.Format, err = c.dateParser.Parse(base.RawDate); err != nil { 155 | return record.Entry{Valid: false}, internal.VersionDateUnmatched 156 | } 157 | 158 | // No dates matched so mark the date invalid and reset the count. 159 | out.DateYearMissing = out.Date.Year() == 0 160 | if internal.StringLength(base.RawDate) > 11 { 161 | // Compensate for dates that do not append a zero to the date. 162 | if base.RawDate[9] == ' ' { 163 | base.RawDate = base.RawDate[:8] + "0" + base.RawDate[8:] 164 | } 165 | // Take a date in ctime format and add the year. 166 | base.RawDate = base.RawDate[:10] + " " + strconv.Itoa(internal.DATE_YEAR+c.DateRollover) + base.RawDate[10:] 167 | } 168 | 169 | if internal.StringLength(out.RawContext) > 2 && record.IsContext(out.RawContext) { 170 | out.Context = out.RawContext[1 : internal.StringLength(out.RawContext)-1] 171 | length := internal.StringLength(out.Context) 172 | 173 | if strings.HasPrefix(out.Context, "conn") && length > 4 { 174 | out.Connection, _ = strconv.Atoi(out.Context[4:]) 175 | } else if strings.HasPrefix(out.Context, "thread") && length > 6 { 176 | out.Thread, _ = strconv.Atoi(out.Context[6:]) 177 | } 178 | } 179 | 180 | // Check for the base message for validity and parse it. 181 | if out.RawMessage == "" { 182 | // No log message exists so it cannot be further analyzed. 183 | return out, internal.VersionMessageUnmatched 184 | } 185 | 186 | // Try parsing the remaining factories for a log message until one succeeds. 187 | out.Message, _ = factory.NewLogMessage(out) 188 | return out, err 189 | } 190 | -------------------------------------------------------------------------------- /parser/version/types.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "strconv" 5 | 6 | "mgotools/parser/message" 7 | "mgotools/parser/record" 8 | ) 9 | 10 | /* 11 | * Factory 12 | */ 13 | 14 | // Global Factory to register different version files. 15 | var Factory = &factory{factories: make([]Parser, 0, 64)} 16 | 17 | type Parser interface { 18 | Check(base record.Base) bool 19 | NewLogMessage(record.Entry) (message.Message, error) 20 | Version() Definition 21 | } 22 | 23 | type factory struct { 24 | factories []Parser 25 | } 26 | 27 | func (f *factory) GetAll() []Parser { 28 | return f.factories 29 | } 30 | 31 | func (f *factory) Register(init func() Parser) { 32 | f.factories = append(f.factories, init()) 33 | } 34 | 35 | /* 36 | * Definition 37 | */ 38 | type Definition struct { 39 | Major int 40 | Minor int 41 | Binary record.Binary 42 | } 43 | 44 | // Compares two versions. a < b == -1, a > b == 1, a = b == 0 45 | func (a *Definition) Compare(b Definition) int { 46 | switch { 47 | case a.Major == b.Major && a.Minor == b.Minor: 48 | return 0 49 | case a.Major < b.Major, 50 | a.Major == b.Major && a.Minor < b.Minor: 51 | return -1 52 | case a.Major > b.Major, 53 | a.Major == b.Major && a.Minor > b.Minor: 54 | return 1 55 | } 56 | panic("version comparison failed") 57 | } 58 | 59 | func (a *Definition) Equals(b Definition) bool { 60 | return a.Compare(b) == 0 && a.Binary == b.Binary 61 | } 62 | 63 | func (v Definition) String() string { 64 | var dst [12]byte 65 | offset := 0 66 | 67 | switch v.Binary { 68 | case record.BinaryMongod: 69 | dst = [12]byte{'m', 'o', 'n', 'g', 'o', 'd', ' ', 0, '.', '.'} 70 | case record.BinaryMongos: 71 | dst = [12]byte{'m', 'o', 'n', 'g', 'o', 's', ' ', 0, '.', '.'} 72 | case record.BinaryAny: 73 | dst = [12]byte{'m', 'o', 'n', 'g', 'o', '?', ' ', 0, '.', '.'} 74 | default: 75 | panic("unexpected binary " + strconv.Itoa(int(v.Binary))) 76 | } 77 | 78 | if v.Major < 10 { 79 | dst[7] = byte(v.Major) + 0x30 80 | } else if v.Major < 100 { 81 | offset = 1 82 | dst[7] = byte(v.Major/10) + 0x30 83 | dst[8] = byte(v.Major%10) + 0x30 84 | } else { 85 | panic("version too large") 86 | } 87 | 88 | if v.Minor < 10 { 89 | dst[9+offset] = byte(v.Minor) + 0x30 90 | } else if v.Minor < 100 { 91 | dst[9+offset] = byte(v.Minor/10) + 0x30 92 | dst[10+offset] = byte(v.Minor%10) + 0x30 93 | offset = 2 94 | } else { 95 | panic("version too large") 96 | } 97 | return string(dst[:12-2+offset]) 98 | } 99 | -------------------------------------------------------------------------------- /parser/version/types_test.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "mgotools/parser/record" 9 | ) 10 | 11 | func BenchmarkShort(b *testing.B) { 12 | t := Definition{2, 4, 1} 13 | for n, f := range map[string]func(Definition) string{ 14 | "VersionFmtString": VersionFmtString, 15 | "VersoinCompoundItoa": VersionCompoundItoa, 16 | "VersionBytesItoa": VersionBytesItoa, 17 | "VersionBytes": VersionBytes, 18 | } { 19 | b.Run(n+strconv.Itoa(t.Major)+"."+strconv.Itoa(t.Minor), func(c *testing.B) { 20 | for i := 0; i < c.N; i += 1 { 21 | _ = f(t) 22 | } 23 | }) 24 | } 25 | } 26 | 27 | func BenchmarkMediumShort(b *testing.B) { 28 | t := Definition{10, 2, 2} 29 | for n, f := range map[string]func(Definition) string{ 30 | "VersionFmtString": VersionFmtString, 31 | "VersoinCompoundItoa": VersionCompoundItoa, 32 | "VersionBytesItoa": VersionBytesItoa, 33 | "VersionBytes": VersionBytes, 34 | } { 35 | b.Run(n+strconv.Itoa(t.Major)+"."+strconv.Itoa(t.Minor), func(c *testing.B) { 36 | for i := 0; i < c.N; i += 1 { 37 | _ = f(t) 38 | } 39 | }) 40 | } 41 | 42 | } 43 | 44 | func BenchmarkShortMedium(b *testing.B) { 45 | t := Definition{2, 10, 2} 46 | for n, f := range map[string]func(Definition) string{ 47 | "VersionFmtString": VersionFmtString, 48 | "VersoinCompoundItoa": VersionCompoundItoa, 49 | "VersionBytesItoa": VersionBytesItoa, 50 | "VersionBytes": VersionBytes, 51 | } { 52 | b.Run(n+strconv.Itoa(t.Major)+"."+strconv.Itoa(t.Minor), func(c *testing.B) { 53 | for i := 0; i < c.N; i += 1 { 54 | _ = f(t) 55 | } 56 | }) 57 | } 58 | 59 | } 60 | 61 | func BenchmarkLong(b *testing.B) { 62 | t := Definition{10, 50, 1} 63 | for n, f := range map[string]func(Definition) string{ 64 | "VersionFmtString": VersionFmtString, 65 | "VersoinCompoundItoa": VersionCompoundItoa, 66 | "VersionBytesItoa": VersionBytesItoa, 67 | "VersionBytes": VersionBytes, 68 | } { 69 | b.Run(n+strconv.Itoa(t.Major)+"."+strconv.Itoa(t.Minor), func(c *testing.B) { 70 | for i := 0; i < c.N; i += 1 { 71 | _ = f(t) 72 | } 73 | }) 74 | } 75 | } 76 | 77 | func VersionFmtString(version Definition) string { 78 | switch version.Binary { 79 | case record.BinaryMongod: 80 | return fmt.Sprintf("mongod %d.%d", version.Major, version.Minor) 81 | case record.BinaryMongos: 82 | return fmt.Sprintf("mongos %d.%d", version.Major, version.Minor) 83 | default: 84 | panic("unexpected binary") 85 | } 86 | } 87 | 88 | func VersionCompoundItoa(version Definition) string { 89 | switch version.Binary { 90 | case record.BinaryMongod: 91 | return "mongod " + strconv.Itoa(version.Major) + "." + strconv.Itoa(version.Minor) 92 | case record.BinaryMongos: 93 | return "mongos " + strconv.Itoa(version.Major) + "." + strconv.Itoa(version.Minor) 94 | default: 95 | panic("unexpected binary") 96 | } 97 | } 98 | 99 | func VersionBytesItoa(version Definition) string { 100 | var dst []byte 101 | switch version.Binary { 102 | case record.BinaryMongod: 103 | dst = []byte{'m', 'o', 'n', 'g', 'o', 'd', ' '} 104 | case record.BinaryMongos: 105 | dst = []byte{'m', 'o', 'n', 'g', 'o', 's', ' '} 106 | default: 107 | panic("unexpected binary") 108 | } 109 | dst = strconv.AppendInt(dst, int64(version.Major), 10) 110 | dst = append(dst, '.') 111 | dst = strconv.AppendInt(dst, int64(version.Major), 10) 112 | return string(dst) 113 | } 114 | 115 | func VersionBytes(version Definition) string { 116 | var dst [12]byte 117 | offset := 0 118 | 119 | switch version.Binary { 120 | case record.BinaryMongod: 121 | dst = [12]byte{'m', 'o', 'n', 'g', 'o', 'd', ' ', 0, '.', '.'} 122 | case record.BinaryMongos: 123 | dst = [12]byte{'m', 'o', 'n', 'g', 'o', 's', ' ', 0, '.', '.'} 124 | default: 125 | panic("unexpected binary") 126 | } 127 | 128 | if version.Major < 10 { 129 | dst[7] = byte(version.Major) + 0x30 130 | } else if version.Major < 100 { 131 | dst[7] = byte(version.Major/10) + 0x30 132 | dst[8] = byte(version.Major%10) + 0x30 133 | offset = 1 134 | } else { 135 | panic("major version too large") 136 | } 137 | 138 | if version.Minor < 10 { 139 | dst[9+offset] = byte(version.Minor) + 0x30 140 | } else if version.Minor < 100 { 141 | dst[9+offset] = byte(version.Minor/10) + 0x30 142 | dst[10+offset] = byte(version.Minor%10) + 0x30 143 | offset = 2 144 | } else { 145 | panic("minor version too large") 146 | } 147 | return string(dst[:12-2+offset]) 148 | } 149 | -------------------------------------------------------------------------------- /target/formatting/pattern.go: -------------------------------------------------------------------------------- 1 | package formatting 2 | 3 | import ( 4 | "io" 5 | "math" 6 | "strconv" 7 | 8 | "github.com/olekukonko/tablewriter" 9 | ) 10 | 11 | type Table []Pattern 12 | 13 | type Pattern struct { 14 | Namespace string 15 | Pattern string 16 | Operation string 17 | Count int64 18 | Min int64 19 | Max int64 20 | N95Percentile float64 21 | Sum int64 22 | } 23 | 24 | func (patterns Table) Print(wrap bool, out io.Writer) { 25 | if len(patterns) == 0 { 26 | out.Write([]byte("no queries found.")) 27 | return 28 | } 29 | 30 | table := tablewriter.NewWriter(out) 31 | defer table.Render() 32 | 33 | table.Append([]string{"namespace", "operation", "pattern", "count", "min (ms)", "max (ms)", "mean (ms)", "95%-ile (ms)", "sum (ms)"}) 34 | table.SetAutoWrapText(wrap) 35 | table.SetBorder(false) 36 | table.SetRowLine(false) 37 | table.SetCenterSeparator(" ") 38 | table.SetColumnSeparator(" ") 39 | table.SetColWidth(60) 40 | 41 | for _, pattern := range patterns { 42 | if pattern.Count == 0 { 43 | table.Append([]string{ 44 | pattern.Namespace, 45 | pattern.Operation, 46 | pattern.Pattern, 47 | "0", 48 | "-", 49 | "-", 50 | "-", 51 | "-", 52 | "-", 53 | }) 54 | } else { 55 | var n95 = "-" 56 | if !math.IsNaN(pattern.N95Percentile) && pattern.Count > 1 { 57 | n95 = strconv.FormatFloat(pattern.N95Percentile, 'f', 1, 64) 58 | } 59 | 60 | table.Append([]string{ 61 | pattern.Namespace, 62 | pattern.Operation, 63 | pattern.Pattern, 64 | strconv.FormatInt(pattern.Count, 10), 65 | strconv.FormatInt(pattern.Min, 10), 66 | strconv.FormatInt(pattern.Max, 10), 67 | strconv.FormatFloat(float64(pattern.Sum/pattern.Count), 'f', 0, 64), 68 | n95, 69 | strconv.FormatInt(pattern.Sum, 10), 70 | }) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /target/formatting/summary.go: -------------------------------------------------------------------------------- 1 | package formatting 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "mgotools/internal" 13 | "mgotools/parser/message" 14 | "mgotools/parser/record" 15 | "mgotools/parser/version" 16 | ) 17 | 18 | type Summary struct { 19 | Source string 20 | Host string 21 | Port int 22 | Start time.Time 23 | End time.Time 24 | Format map[internal.DateFormat]int 25 | Length uint 26 | Version []version.Definition 27 | Storage string 28 | 29 | mutex sync.Mutex 30 | guessed bool 31 | } 32 | 33 | func NewSummary(name string) Summary { 34 | return Summary{ 35 | Source: name, 36 | Host: "", 37 | Port: 0, 38 | Start: time.Time{}, 39 | End: time.Time{}, 40 | Length: 0, 41 | Version: nil, 42 | Storage: "", 43 | Format: make(map[internal.DateFormat]int), 44 | } 45 | } 46 | 47 | func (s *Summary) Guess(versions []version.Definition) { 48 | s.mutex.Lock() 49 | defer s.mutex.Unlock() 50 | 51 | for _, v := range versions { 52 | s.Version = append(s.Version, v) 53 | } 54 | 55 | s.guessed = true 56 | } 57 | 58 | func (Summary) Divider(w io.Writer) { 59 | _, _ = w.Write([]byte("\n------------------------------------------\n")) 60 | } 61 | 62 | func (s Summary) Print(w io.Writer) { 63 | s.mutex.Lock() 64 | defer s.mutex.Unlock() 65 | 66 | host := s.Host 67 | if host != "" && s.Port > 0 { 68 | host = fmt.Sprintf("%s:%d", host, s.Port) 69 | } 70 | 71 | write := func(out io.Writer, name, value, empty string) { 72 | if value == "" && empty == "" { 73 | return 74 | } 75 | if internal.StringLength(name) < 11 { 76 | out.Write([]byte(strings.Repeat(" ", 11-internal.StringLength(name)))) 77 | } 78 | out.Write([]byte(name)) 79 | out.Write([]byte(": ")) 80 | 81 | if value != "" { 82 | out.Write([]byte(value)) 83 | } else { 84 | out.Write([]byte(empty)) 85 | } 86 | 87 | out.Write([]byte("\n")) 88 | } 89 | 90 | formatTable := func(histogram map[internal.DateFormat]int) string { 91 | formatString := func(format internal.DateFormat) string { 92 | switch format { 93 | case internal.DateFormatCtime, 94 | internal.DateFormatCtimenoms: 95 | return "cdate" 96 | case internal.DateFormatCtimeyear: 97 | return "cdate-year" 98 | case internal.DateFormatIso8602Local: 99 | return "iso8602-local" 100 | case internal.DateFormatIso8602Utc: 101 | return "iso8602" 102 | default: 103 | return "unknown" 104 | } 105 | } 106 | 107 | if len(histogram) < 2 { 108 | for key := range histogram { 109 | return formatString(key) 110 | } 111 | return "unknown" 112 | } else { 113 | buffer := bytes.NewBuffer([]byte{}) 114 | total := 0 115 | for _, count := range histogram { 116 | total += count 117 | } 118 | for format, count := range histogram { 119 | buffer.WriteString(formatString(format)) 120 | buffer.WriteString(" (") 121 | buffer.WriteString(strconv.FormatFloat(100*float64(count)/float64(total), 'f', 1, 64)) 122 | buffer.WriteString("%) ") 123 | } 124 | return buffer.String() 125 | } 126 | } 127 | 128 | write(w, "source", s.Source, "") 129 | write(w, "host", host, "unknown") 130 | write(w, "start", s.Start.Format("2006 Jan 02 15:04:05.000"), "") 131 | write(w, "end", s.End.Format("2006 Jan 02 15:04:05.000"), "") 132 | write(w, "date format", formatTable(s.Format), "") 133 | write(w, "length", strconv.FormatUint(uint64(s.Length), 10), "0") 134 | 135 | var versions = make([]string, 0, len(s.Version)) 136 | for _, v := range s.Version { 137 | if v.Major < 3 && s.Storage == "" { 138 | s.Storage = "MMAPv1" 139 | } 140 | 141 | if v.Major > 1 && (len(versions) == 0 || versions[len(versions)-1] != v.String()) { 142 | versions = append(versions, v.String()) 143 | } 144 | } 145 | 146 | if !s.guessed { 147 | version := strings.Join(versions, " -> ") 148 | write(w, "version", version, "unknown") 149 | } else { 150 | leastVersion := version.Definition{Major: 999, Minor: 999, Binary: record.Binary(999)} 151 | 152 | for _, version := range s.Version { 153 | if version.Major < leastVersion.Major && leastVersion.Major > 0 { 154 | leastVersion.Major = version.Major 155 | } 156 | if version.Major == leastVersion.Major && version.Minor < leastVersion.Minor { 157 | leastVersion.Minor = version.Minor 158 | } 159 | if version.Major == leastVersion.Major && version.Minor == leastVersion.Minor && version.Binary < leastVersion.Binary && version.Binary > record.BinaryAny { 160 | leastVersion.Binary = version.Binary 161 | } 162 | } 163 | 164 | if leastVersion.Major < 999 && leastVersion.Minor < 999 && int(leastVersion.Binary) < 999 { 165 | write(w, "version", fmt.Sprintf("(guess) >= %s", leastVersion.String()), "") 166 | } 167 | } 168 | 169 | write(w, "storage", s.Storage, "unknown") 170 | w.Write([]byte{'\n'}) 171 | } 172 | 173 | func (s *Summary) Update(entry record.Entry) bool { 174 | s.mutex.Lock() 175 | defer s.mutex.Unlock() 176 | 177 | if s.Start.IsZero() && entry.DateValid { 178 | // Only set the start date once (which should coincide with the 179 | // first line of each instance). 180 | s.Start = entry.Date 181 | } 182 | 183 | if !entry.Date.IsZero() && entry.DateValid { 184 | // Keep the most recent date for log summary purposes. 185 | s.End = entry.Date 186 | } 187 | 188 | if entry.DateValid { 189 | s.Format[entry.Format] += 1 190 | } 191 | 192 | // Track until the last parsable line number. 193 | if s.Length < entry.LineNumber { 194 | s.Length = entry.LineNumber 195 | } 196 | 197 | switch t := entry.Message.(type) { 198 | case nil: 199 | return false 200 | 201 | case message.StartupInfo: 202 | s.options(t.Port, t.Hostname) 203 | 204 | case message.StartupInfoLegacy: 205 | s.options(t.Port, t.Hostname) 206 | s.version(t.Version) 207 | 208 | case message.Version: 209 | s.version(t) 210 | 211 | case message.WiredTigerConfig: 212 | s.Storage = "WiredTiger" 213 | } 214 | 215 | return true 216 | } 217 | 218 | func (s *Summary) options(port int, hostname string) { 219 | // The server restarted. 220 | s.Port = port 221 | s.Host = hostname 222 | } 223 | 224 | func (s *Summary) version(msg message.Version) { 225 | if msg.Major == 2 { 226 | s.Storage = "MMAPv1" 227 | } else if (msg.Major == 4 && msg.Minor > 0) || msg.Major > 4 { 228 | s.Storage = "WiredTiger" 229 | } 230 | var binary record.Binary 231 | switch msg.Binary { 232 | case "mongod": 233 | binary = record.BinaryMongod 234 | 235 | case "mongos": 236 | binary = record.BinaryMongos 237 | 238 | default: 239 | binary = record.BinaryAny 240 | } 241 | s.Version = append(s.Version, version.Definition{ 242 | Major: msg.Major, 243 | Minor: msg.Minor, 244 | Binary: binary, 245 | }) 246 | if s.Storage == "" && msg.Major < 3 { 247 | s.Storage = "MMAPv1" 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /target/stdout.go: -------------------------------------------------------------------------------- 1 | package target 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type Stdout struct { 8 | } 9 | 10 | func (Stdout) String(in string) error { 11 | fmt.Println(in) 12 | return nil 13 | } 14 | --------------------------------------------------------------------------------