├── generate_msgtypes.sh ├── main.go ├── raft_info.go ├── formatters.go ├── msgpack.go ├── README.md ├── client_state.go ├── msgtypes.go ├── raft_logs.go ├── raft_state.go └── LICENSE /generate_msgtypes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | generate_file() { 4 | cat < msgtypes.go 24 | 25 | gofmt -w msgtypes.go 26 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/mitchellh/cli" 8 | ) 9 | 10 | func main() { 11 | commands := map[string]cli.CommandFactory{ 12 | "raft info": func() (cli.Command, error) { 13 | return &RaftInfoCommand{}, nil 14 | }, 15 | "raft logs": func() (cli.Command, error) { 16 | return &RaftLogsCommand{}, nil 17 | }, 18 | "raft state": func() (cli.Command, error) { 19 | return &RaftStateCommand{}, nil 20 | }, 21 | "client state": func() (cli.Command, error) { 22 | return &ClientStateCommand{}, nil 23 | }, 24 | } 25 | cli := &cli.CLI{ 26 | Name: "nomad-debug", 27 | Args: os.Args[1:], 28 | HelpWriter: os.Stdout, 29 | Commands: commands, 30 | } 31 | 32 | exitCode, err := cli.Run() 33 | if err != nil { 34 | fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error()) 35 | } 36 | 37 | os.Exit(exitCode) 38 | } 39 | -------------------------------------------------------------------------------- /raft_info.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | ) 9 | 10 | type RaftInfoCommand struct { 11 | } 12 | 13 | func (a *RaftInfoCommand) Help() string { 14 | helpText := ` 15 | Usage: nomad-debug info logs 16 | 17 | Emits some info about the raft logs. 18 | ` 19 | 20 | return strings.TrimSpace(helpText) 21 | } 22 | 23 | func (c *RaftInfoCommand) Name() string { return "raft info" } 24 | 25 | func (c *RaftInfoCommand) Synopsis() string { 26 | return "output info of raft log" 27 | } 28 | 29 | func (c *RaftInfoCommand) Run(args []string) int { 30 | if len(args) != 1 { 31 | return 1 32 | } 33 | 34 | p := filepath.Join(args[0], "server", "raft", "raft.db") 35 | 36 | store, firstIdx, lastIdx, err := raftState(p) 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "failed to open raft logs: %v\n", err) 39 | return 1 40 | } 41 | defer store.Close() 42 | 43 | fmt.Println("path: ", p) 44 | fmt.Println("length: ", lastIdx-firstIdx+1) 45 | fmt.Println("first index: ", firstIdx) 46 | fmt.Println("last index: ", lastIdx) 47 | 48 | return 0 49 | } 50 | -------------------------------------------------------------------------------- /formatters.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | type Formatter interface { 11 | io.Closer 12 | Write(v interface{}, fields ...string) error 13 | } 14 | 15 | type CSVFormatter struct { 16 | w *csv.Writer 17 | } 18 | 19 | func NewCSVFormatter(writer io.Writer, headers []string) (*CSVFormatter, error) { 20 | w := csv.NewWriter(writer) 21 | err := w.Write(headers) 22 | if err != nil { 23 | return nil, err 24 | } 25 | return &CSVFormatter{ 26 | w: csv.NewWriter(writer), 27 | }, nil 28 | } 29 | 30 | func (f *CSVFormatter) Write(v interface{}, fields ...string) error { 31 | b, err := json.Marshal(v) 32 | if err != nil { 33 | return fmt.Errorf("failed to serialize v: %v", err) 34 | } 35 | 36 | r := append(fields, string(b)) 37 | return f.w.Write(r) 38 | 39 | } 40 | 41 | func (f *CSVFormatter) Close() error { 42 | f.w.Flush() 43 | return nil 44 | } 45 | 46 | type JSONFormatter struct { 47 | w *json.Encoder 48 | } 49 | 50 | func NewJSONFormatter(writer io.Writer, headers []string) (*JSONFormatter, error) { 51 | return nil, fmt.Errorf("not supported") 52 | } 53 | 54 | func (f *JSONFormatter) Write(v interface{}, fields ...string) error { 55 | return fmt.Errorf("not supported") 56 | 57 | } 58 | 59 | func (f *JSONFormatter) Close() error { 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /msgpack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "reflect" 7 | "time" 8 | 9 | "github.com/hashicorp/go-msgpack/codec" 10 | ) 11 | 12 | var MsgpackHandle = func() *codec.MsgpackHandle { 13 | h := &codec.MsgpackHandle{} 14 | h.RawToString = true 15 | h.MapType = reflect.TypeOf(map[string]interface{}(nil)) 16 | h.SliceType = reflect.TypeOf([]interface{}(nil)) 17 | return h 18 | }() 19 | 20 | func computeTimeSize() int { 21 | var nb bytes.Buffer 22 | err := codec.NewEncoder(&nb, MsgpackHandle).Encode(time.Time{}) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | // remove one byte container len 28 | return nb.Len() - 1 29 | } 30 | 31 | var timeMsgPackSize = computeTimeSize() 32 | 33 | func maybeDecodeTime(v string) (*time.Time, error) { 34 | if len(v) != timeMsgPackSize { 35 | return nil, fmt.Errorf("bad length: %d", len(v)) 36 | } 37 | 38 | var nb bytes.Buffer 39 | err := codec.NewEncoder(&nb, MsgpackHandle).Encode(v) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | var tt time.Time 45 | err = codec.NewDecoder(&nb, MsgpackHandle).Decode(&tt) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | if tt.IsZero() { 51 | return nil, nil 52 | } 53 | 54 | return &tt, nil 55 | } 56 | 57 | func fixTime(v interface{}) { 58 | switch v2 := v.(type) { 59 | case map[string]interface{}: 60 | for ek, ev := range v2 { 61 | if s, ok := ev.(string); ok { 62 | if t, err := maybeDecodeTime(s); err == nil { 63 | v2[ek] = t 64 | } 65 | } else { 66 | fixTime(ev) 67 | } 68 | } 69 | case []interface{}: 70 | for _, e := range v2 { 71 | fixTime(e) 72 | } 73 | default: 74 | return 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Debug nomad state 2 | 3 | Have you needed to inspect nomad state? Investigate how a job/alloc got to its state? Investigate behavior of client on restore? Research nomad server Raft FSM changes by replaying raft transactions and inspecting state? If so, you are in luck! `nomad-debug` helps debugging such scenarios by dumping the state of nomad in json format, for further analysis by `jq`. 4 | 5 | The main commands are: 6 | 7 | ``` 8 | # dump all raft log entries as json array to stdout 9 | nomad-debug raft logs 10 | 11 | # dump the nomad server state store, by replaying raft log events 12 | nomad-debug raft state 13 | 14 | # dump the nomad client state 15 | nomad-debug client state 16 | ``` 17 | 18 | ## Caveats 19 | 20 | * The raft logs may not represent cluster state accurately at time of server shutting down. The raft log main contain: 21 | * some uncommitted log entries: these are transactions that haven't fully replicated to quorum of nodes, so actual state may lag behind the state found here 22 | * some spurious log entries: specially around leader election, some persisted logs might be some garbage to be overwriten later 23 | * some missing log entries: the raft logs of a follower might be lagging behind the leader 24 | 25 | * `client state` only works against Nomad 0.9 client. Client 0.8 and earlier are not supported. 26 | 27 | ## How to use 28 | 29 | Checkout this repository as a subdir of `nomad` and run `go install`: 30 | 31 | ``` 32 | $ cd ~/go/src/github.com/hashicorp/nomad/ 33 | $ git clone git@github.com:notnoop/nomad-debug.git 34 | $ cd nomad-debug 35 | $ go install . 36 | ``` 37 | 38 | ## TODO 39 | 40 | * [ ] Support nomad server raft snapshoted state 41 | * [ ] Export to a database (e.g. sqlite, postgresql) to ease querying against database 42 | * [ ] Vendor nomad and its dependencies to avoid needing to checkout as a subdirectory 43 | -------------------------------------------------------------------------------- /client_state.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/hashicorp/go-hclog" 12 | trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" 13 | "github.com/hashicorp/nomad/client/state" 14 | "github.com/hashicorp/nomad/nomad/structs" 15 | "github.com/hashicorp/nomad/plugins/base" 16 | ) 17 | 18 | type ClientStateCommand struct { 19 | } 20 | 21 | func (a *ClientStateCommand) Help() string { 22 | helpText := ` 23 | Usage: nomad-debug client state 24 | 25 | Emits a json representation of the stored client state in json form. 26 | ` 27 | 28 | return strings.TrimSpace(helpText) 29 | } 30 | 31 | func (c *ClientStateCommand) Name() string { return "raft logs" } 32 | 33 | func (c *ClientStateCommand) Synopsis() string { 34 | return "output content of client state" 35 | } 36 | 37 | func (c *ClientStateCommand) Run(args []string) int { 38 | if len(args) != 1 { 39 | return 1 40 | } 41 | 42 | logger := hclog.L() 43 | 44 | p := filepath.Join(args[0], "client") 45 | db, err := state.NewBoltStateDB(logger, p) 46 | if err != nil { 47 | fmt.Fprintf(os.Stderr, "failed to open client state: %v\n", err) 48 | return 1 49 | } 50 | defer db.Close() 51 | 52 | allocs, _, err := db.GetAllAllocations() 53 | if err != nil { 54 | fmt.Fprintf(os.Stderr, "failed to get allocations: %v\n", err) 55 | return 1 56 | } 57 | 58 | data := map[string]*clientStateAlloc{} 59 | for _, alloc := range allocs { 60 | allocID := alloc.ID 61 | deployState, err := db.GetDeploymentStatus(allocID) 62 | if err != nil { 63 | fmt.Fprintf(os.Stderr, "failed to get deployment status for %s: %v", allocID, err) 64 | return 1 65 | } 66 | 67 | tasks := map[string]*taskState{} 68 | tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) 69 | for _, jt := range tg.Tasks { 70 | ls, rs, err := db.GetTaskRunnerState(allocID, jt.Name) 71 | if err != nil { 72 | fmt.Fprintf(os.Stderr, "failed to get task runner state %s: %v", allocID, err) 73 | return 1 74 | } 75 | 76 | var ds interface{} 77 | err = ls.TaskHandle.GetDriverState(&ds) 78 | if err != nil { 79 | fmt.Fprintf(os.Stderr, "failed to parse driver state %s: %v", allocID, err) 80 | return 1 81 | } 82 | 83 | tasks[jt.Name] = &taskState{ 84 | LocalState: ls, 85 | RemoteState: rs, 86 | DriverState: ds, 87 | } 88 | } 89 | 90 | data[allocID] = &clientStateAlloc{ 91 | Alloc: alloc, 92 | DeployStatus: deployState, 93 | Tasks: tasks, 94 | } 95 | } 96 | 97 | enc := json.NewEncoder(os.Stdout) 98 | enc.SetIndent("", " ") 99 | if err := enc.Encode(data); err != nil { 100 | fmt.Fprintf(os.Stderr, "failed to encode output: %v\n", err) 101 | return 1 102 | } 103 | 104 | return 0 105 | } 106 | 107 | func unwrapDriverState(rawDriverConfig string) (interface{}, error) { 108 | if rawDriverConfig == "" { 109 | return nil, nil 110 | } 111 | 112 | b, err := base64.StdEncoding.DecodeString(rawDriverConfig) 113 | if err != nil { 114 | return "", err 115 | } 116 | var result interface{} 117 | err = base.MsgPackDecode(b, &result) 118 | if err != nil { 119 | return "", err 120 | } 121 | fixTime(result) 122 | 123 | return result, nil 124 | 125 | } 126 | 127 | type clientStateAlloc struct { 128 | Alloc *structs.Allocation 129 | DeployStatus *structs.AllocDeploymentStatus 130 | Tasks map[string]*taskState 131 | } 132 | 133 | type taskState struct { 134 | LocalState *trstate.LocalState 135 | RemoteState *structs.TaskState 136 | DriverState interface{} 137 | } 138 | -------------------------------------------------------------------------------- /msgtypes.go: -------------------------------------------------------------------------------- 1 | // Code generated by go generate; DO NOT EDIT. 2 | // This file was generated at 3 | // Wed Apr 29 16:31:41 UTC 2020 4 | package main 5 | 6 | import "github.com/hashicorp/nomad/nomad/structs" 7 | 8 | var msgTypeNames = map[structs.MessageType]string{ 9 | structs.NodeRegisterRequestType: "NodeRegisterRequestType", 10 | structs.NodeDeregisterRequestType: "NodeDeregisterRequestType", 11 | structs.NodeUpdateStatusRequestType: "NodeUpdateStatusRequestType", 12 | structs.NodeUpdateDrainRequestType: "NodeUpdateDrainRequestType", 13 | structs.JobRegisterRequestType: "JobRegisterRequestType", 14 | structs.JobDeregisterRequestType: "JobDeregisterRequestType", 15 | structs.EvalUpdateRequestType: "EvalUpdateRequestType", 16 | structs.EvalDeleteRequestType: "EvalDeleteRequestType", 17 | structs.AllocUpdateRequestType: "AllocUpdateRequestType", 18 | structs.AllocClientUpdateRequestType: "AllocClientUpdateRequestType", 19 | structs.ReconcileJobSummariesRequestType: "ReconcileJobSummariesRequestType", 20 | structs.VaultAccessorRegisterRequestType: "VaultAccessorRegisterRequestType", 21 | structs.VaultAccessorDeregisterRequestType: "VaultAccessorDeregisterRequestType", 22 | structs.ApplyPlanResultsRequestType: "ApplyPlanResultsRequestType", 23 | structs.DeploymentStatusUpdateRequestType: "DeploymentStatusUpdateRequestType", 24 | structs.DeploymentPromoteRequestType: "DeploymentPromoteRequestType", 25 | structs.DeploymentAllocHealthRequestType: "DeploymentAllocHealthRequestType", 26 | structs.DeploymentDeleteRequestType: "DeploymentDeleteRequestType", 27 | structs.JobStabilityRequestType: "JobStabilityRequestType", 28 | structs.ACLPolicyUpsertRequestType: "ACLPolicyUpsertRequestType", 29 | structs.ACLPolicyDeleteRequestType: "ACLPolicyDeleteRequestType", 30 | structs.ACLTokenUpsertRequestType: "ACLTokenUpsertRequestType", 31 | structs.ACLTokenDeleteRequestType: "ACLTokenDeleteRequestType", 32 | structs.ACLTokenBootstrapRequestType: "ACLTokenBootstrapRequestType", 33 | structs.AutopilotRequestType: "AutopilotRequestType", 34 | structs.UpsertNodeEventsType: "UpsertNodeEventsType", 35 | structs.JobBatchDeregisterRequestType: "JobBatchDeregisterRequestType", 36 | structs.AllocUpdateDesiredTransitionRequestType: "AllocUpdateDesiredTransitionRequestType", 37 | structs.NodeUpdateEligibilityRequestType: "NodeUpdateEligibilityRequestType", 38 | structs.BatchNodeUpdateDrainRequestType: "BatchNodeUpdateDrainRequestType", 39 | structs.SchedulerConfigRequestType: "SchedulerConfigRequestType", 40 | structs.NodeBatchDeregisterRequestType: "NodeBatchDeregisterRequestType", 41 | structs.ClusterMetadataRequestType: "ClusterMetadataRequestType", 42 | structs.ServiceIdentityAccessorRegisterRequestType: "ServiceIdentityAccessorRegisterRequestType", 43 | structs.ServiceIdentityAccessorDeregisterRequestType: "ServiceIdentityAccessorDeregisterRequestType", 44 | structs.CSIVolumeRegisterRequestType: "CSIVolumeRegisterRequestType", 45 | structs.CSIVolumeDeregisterRequestType: "CSIVolumeDeregisterRequestType", 46 | structs.CSIVolumeClaimRequestType: "CSIVolumeClaimRequestType", 47 | structs.ScalingEventRegisterRequestType: "ScalingEventRegisterRequestType", 48 | } 49 | -------------------------------------------------------------------------------- /raft_logs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate ./generate_msgtypes.sh 4 | 5 | import ( 6 | "bytes" 7 | "encoding/json" 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/hashicorp/go-msgpack/codec" 14 | "github.com/hashicorp/nomad/nomad/structs" 15 | "github.com/hashicorp/raft" 16 | raftboltdb "github.com/hashicorp/raft-boltdb" 17 | ) 18 | 19 | type RaftLogsCommand struct { 20 | } 21 | 22 | func (a *RaftLogsCommand) Help() string { 23 | helpText := ` 24 | Usage: nomad-debug raft logs 25 | 26 | Emits the raft logs content in json form. 27 | ` 28 | 29 | return strings.TrimSpace(helpText) 30 | } 31 | 32 | func (c *RaftLogsCommand) Name() string { return "raft logs" } 33 | 34 | func (c *RaftLogsCommand) Synopsis() string { 35 | return "output content of raft log" 36 | } 37 | 38 | func (c *RaftLogsCommand) Run(args []string) int { 39 | if len(args) != 1 { 40 | return 1 41 | } 42 | 43 | p := filepath.Join(args[0], "server", "raft", "raft.db") 44 | 45 | store, firstIdx, lastIdx, err := raftState(p) 46 | if err != nil { 47 | fmt.Fprintf(os.Stderr, "failed to open raft logs: %v\n", err) 48 | return 1 49 | } 50 | defer store.Close() 51 | 52 | arr := make([]*logMessage, 0, lastIdx-firstIdx+1) 53 | for i := firstIdx; i <= lastIdx; i++ { 54 | var e raft.Log 55 | err := store.GetLog(i, &e) 56 | if err != nil { 57 | fmt.Fprintf(os.Stderr, "failed to read log entry at index %d: %v\n", i, err) 58 | continue 59 | //return 1 60 | } 61 | 62 | m, err := decode(&e) 63 | if err != nil { 64 | fmt.Fprintf(os.Stderr, "failed to decode log entry at index %d: %v\n", i, err) 65 | continue 66 | //return 1 67 | } 68 | 69 | arr = append(arr, m) 70 | } 71 | 72 | enc := json.NewEncoder(os.Stdout) 73 | enc.SetIndent("", " ") 74 | if err := enc.Encode(arr); err != nil { 75 | fmt.Fprintf(os.Stderr, "failed to encode output: %v\n", err) 76 | return 1 77 | } 78 | 79 | return 0 80 | } 81 | 82 | func raftState(p string) (store *raftboltdb.BoltStore, firstIdx uint64, lastIdx uint64, err error) { 83 | s, err := raftboltdb.NewBoltStore(p) 84 | if err != nil { 85 | return nil, 0, 0, fmt.Errorf("failed to open raft logs: %v", err) 86 | } 87 | 88 | firstIdx, err = s.FirstIndex() 89 | if err != nil { 90 | return nil, 0, 0, fmt.Errorf("failed to fetch first index: %v", err) 91 | } 92 | 93 | lastIdx, err = s.LastIndex() 94 | if err != nil { 95 | return nil, 0, 0, fmt.Errorf("failed to fetch last index: %v", err) 96 | } 97 | 98 | return s, firstIdx, lastIdx, nil 99 | } 100 | 101 | type logMessage struct { 102 | LogType string 103 | Term uint64 104 | Index uint64 105 | 106 | CommandType string `json:",omitempty"` 107 | IgnoreUnknownTypeFlag bool `json:",omitempty"` 108 | Body interface{} `json:",omitempty"` 109 | } 110 | 111 | func decode(e *raft.Log) (*logMessage, error) { 112 | m := &logMessage{ 113 | LogType: logTypes[e.Type], 114 | Term: e.Term, 115 | Index: e.Index, 116 | } 117 | 118 | if m.LogType == "" { 119 | m.LogType = fmt.Sprintf("%d", e.Type) 120 | } 121 | 122 | var data []byte 123 | if e.Type == raft.LogCommand { 124 | if len(e.Data) == 0 { 125 | return nil, fmt.Errorf("command did not include data") 126 | } 127 | 128 | msgType := structs.MessageType(e.Data[0]) 129 | 130 | m.CommandType = msgTypeNames[msgType & ^structs.IgnoreUnknownTypeFlag] 131 | m.IgnoreUnknownTypeFlag = (msgType & structs.IgnoreUnknownTypeFlag) != 0 132 | 133 | data = e.Data[1:] 134 | } else { 135 | data = e.Data 136 | } 137 | 138 | if len(data) != 0 { 139 | decoder := codec.NewDecoder(bytes.NewReader(data), MsgpackHandle) 140 | 141 | var v interface{} 142 | var err error 143 | if m.CommandType == msgTypeNames[structs.JobBatchDeregisterRequestType] { 144 | var vr structs.JobBatchDeregisterRequest 145 | err = decoder.Decode(&vr) 146 | v = jsonifyJobBatchDeregisterRequest(&vr) 147 | } else { 148 | var vr interface{} 149 | err = decoder.Decode(&vr) 150 | v = vr 151 | } 152 | 153 | if err != nil { 154 | fmt.Fprintf(os.Stderr, "failed to decode log entry at index %d: failed to decode body of %v.%v %v\n", e.Index, e.Type, m.CommandType, err) 155 | v = "FAILED TO DECODE DATA" 156 | } 157 | fixTime(v) 158 | m.Body = v 159 | } 160 | 161 | return m, nil 162 | } 163 | 164 | func jsonifyJobBatchDeregisterRequest(v *structs.JobBatchDeregisterRequest) interface{} { 165 | var data struct { 166 | Jobs map[string]*structs.JobDeregisterOptions 167 | Evals []*structs.Evaluation 168 | structs.WriteRequest 169 | } 170 | data.Evals = v.Evals 171 | data.WriteRequest = v.WriteRequest 172 | 173 | data.Jobs = make(map[string]*structs.JobDeregisterOptions, len(v.Jobs)) 174 | if len(v.Jobs) != 0 { 175 | for k, v := range v.Jobs { 176 | data.Jobs[k.Namespace+"."+k.ID] = v 177 | } 178 | } 179 | return data 180 | } 181 | 182 | var logTypes = map[raft.LogType]string{ 183 | raft.LogCommand: "LogCommand", 184 | raft.LogNoop: "LogNoop", 185 | raft.LogAddPeerDeprecated: "LogAddPeerDeprecated", 186 | raft.LogRemovePeerDeprecated: "LogRemovePeerDeprecated", 187 | raft.LogBarrier: "LogBarrier", 188 | raft.LogConfiguration: "LogConfiguration", 189 | } 190 | -------------------------------------------------------------------------------- /raft_state.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/hashicorp/go-hclog" 12 | "github.com/hashicorp/go-memdb" 13 | "github.com/hashicorp/nomad/nomad" 14 | "github.com/hashicorp/raft" 15 | ) 16 | 17 | type RaftStateCommand struct { 18 | } 19 | 20 | func (a *RaftStateCommand) Help() string { 21 | helpText := ` 22 | Usage: nomad-debug raft state 23 | 24 | Emit the nomad server state obtained by replaying the events of the raft log, in json format. 25 | 26 | Options: 27 | 28 | --last-index= 29 | Set the last log index to be applied, to drop spurious log entries not 30 | properly commited. If passed last_index is zero or negative, it's perceived 31 | as an offset from the last index seen in raft. 32 | ` 33 | 34 | return strings.TrimSpace(helpText) 35 | } 36 | 37 | func (c *RaftStateCommand) Name() string { return "raft logs" } 38 | 39 | func (c *RaftStateCommand) Synopsis() string { 40 | return "output content of raft log" 41 | } 42 | 43 | func (c *RaftStateCommand) Run(args []string) int { 44 | r, err := c.run(args) 45 | if err != nil { 46 | fmt.Fprintln(os.Stderr, err) 47 | } 48 | return r 49 | } 50 | 51 | func (c *RaftStateCommand) run(args []string) (int, error) { 52 | var fLastIdx int64 53 | 54 | flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError) 55 | flags.Usage = func() { fmt.Println(c.Help()) } 56 | flags.Int64Var(&fLastIdx, "last-index", 0, "") 57 | 58 | if err := flags.Parse(args); err != nil { 59 | return 1, fmt.Errorf("failed to parse arguments: %v", err) 60 | } 61 | args = flags.Args() 62 | 63 | if len(args) != 1 { 64 | return 1, fmt.Errorf("expected one arg but got %d", len(args)) 65 | } 66 | 67 | p := filepath.Join(args[0], "server", "raft") 68 | 69 | store, firstIdx, lastIdx, err := raftState(filepath.Join(p, "raft.db")) 70 | if err != nil { 71 | return 1, fmt.Errorf("failed to open raft logs: %v", err) 72 | } 73 | defer store.Close() 74 | 75 | snaps, err := raft.NewFileSnapshotStore(p, 1000, os.Stderr) 76 | if err != nil { 77 | return 1, fmt.Errorf("failed to open snapshot dir: %v", err) 78 | } 79 | 80 | logger := hclog.L() 81 | 82 | // use dummy non-enabled FSM depedencies 83 | periodicDispatch := nomad.NewPeriodicDispatch(logger, nil) 84 | blockedEvals := nomad.NewBlockedEvals(nil, logger) 85 | evalBroker, err := nomad.NewEvalBroker(1, 1, 1, 1) 86 | if err != nil { 87 | return 1, err 88 | } 89 | fsmConfig := &nomad.FSMConfig{ 90 | EvalBroker: evalBroker, 91 | Periodic: periodicDispatch, 92 | Blocked: blockedEvals, 93 | Logger: logger, 94 | Region: "default", 95 | } 96 | 97 | fsm, err := nomad.NewFSM(fsmConfig) 98 | if err != nil { 99 | return 1, err 100 | } 101 | 102 | // restore from snapshot first 103 | sFirstIdx, err := restoreFromSnapshot(fsm, snaps) 104 | if err != nil { 105 | return 1, err 106 | } 107 | 108 | if sFirstIdx+1 < firstIdx { 109 | return 1, fmt.Errorf("missing logs after snapshot [%v,%v]", sFirstIdx+1, firstIdx-1) 110 | } else if sFirstIdx > 0 { 111 | firstIdx = sFirstIdx + 1 112 | } 113 | 114 | lastIdx = lastIndex(lastIdx, fLastIdx) 115 | 116 | for i := firstIdx; i <= lastIdx; i++ { 117 | var e raft.Log 118 | err := store.GetLog(i, &e) 119 | if err != nil { 120 | return 1, fmt.Errorf("failed to read log entry at index %d: %v", i, err) 121 | } 122 | 123 | if e.Type == raft.LogCommand { 124 | fsm.Apply(&e) 125 | } 126 | } 127 | 128 | state := fsm.State() 129 | result := map[string][]interface{}{ 130 | "ACLPolicies": toArray(state.ACLPolicies(nil)), 131 | "ACLTokens": toArray(state.ACLTokens(nil)), 132 | "Allocs": toArray(state.Allocs(nil)), 133 | "Deployments": toArray(state.Deployments(nil)), 134 | "Evals": toArray(state.Evals(nil)), 135 | "Indexes": toArray(state.Indexes()), 136 | "JobSummaries": toArray(state.JobSummaries(nil)), 137 | "JobVersions": toArray(state.JobVersions(nil)), 138 | "Jobs": toArray(state.Jobs(nil)), 139 | "Nodes": toArray(state.Nodes(nil)), 140 | "PeriodicLaunches": toArray(state.PeriodicLaunches(nil)), 141 | "VaultAccessors": toArray(state.VaultAccessors(nil)), 142 | } 143 | 144 | enc := json.NewEncoder(os.Stdout) 145 | enc.SetIndent("", " ") 146 | if err := enc.Encode(result); err != nil { 147 | return 1, fmt.Errorf("failed to encode output: %v", err) 148 | } 149 | 150 | return 0, nil 151 | } 152 | 153 | func restoreFromSnapshot(fsm raft.FSM, snaps raft.SnapshotStore) (uint64, error) { 154 | snapshots, err := snaps.List() 155 | if err != nil { 156 | return 0, err 157 | } 158 | 159 | for _, snapshot := range snapshots { 160 | _, source, err := snaps.Open(snapshot.ID) 161 | if err != nil { 162 | continue 163 | } 164 | 165 | err = fsm.Restore(source) 166 | source.Close() 167 | if err != nil { 168 | fmt.Fprintf(os.Stderr, "failed to restore source %v: %v", snapshot.ID, err) 169 | continue 170 | } 171 | 172 | return snapshot.Index, nil 173 | } 174 | 175 | return 0, nil 176 | } 177 | 178 | func lastIndex(raftLastIdx uint64, cliLastIdx int64) uint64 { 179 | switch { 180 | case cliLastIdx < 0: 181 | if raftLastIdx > uint64(-cliLastIdx) { 182 | return raftLastIdx - uint64(-cliLastIdx) 183 | } else { 184 | return 0 185 | } 186 | case cliLastIdx == 0: 187 | return raftLastIdx 188 | case uint64(cliLastIdx) < raftLastIdx: 189 | return uint64(cliLastIdx) 190 | default: 191 | return raftLastIdx 192 | } 193 | } 194 | 195 | func toArray(iter memdb.ResultIterator, err error) []interface{} { 196 | if err != nil { 197 | return []interface{}{err} 198 | } 199 | 200 | r := []interface{}{} 201 | 202 | item := iter.Next() 203 | for item != nil { 204 | r = append(r, item) 205 | item = iter.Next() 206 | } 207 | 208 | return r 209 | } 210 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License, version 2.0 2 | 3 | 1. Definitions 4 | 5 | 1.1. "Contributor" 6 | 7 | means each individual or legal entity that creates, contributes to the 8 | creation of, or owns Covered Software. 9 | 10 | 1.2. "Contributor Version" 11 | 12 | means the combination of the Contributions of others (if any) used by a 13 | Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | 17 | means Covered Software of a particular Contributor. 18 | 19 | 1.4. "Covered Software" 20 | 21 | means Source Code Form to which the initial Contributor has attached the 22 | notice in Exhibit A, the Executable Form of such Source Code Form, and 23 | Modifications of such Source Code Form, in each case including portions 24 | thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | a. that the initial Contributor has attached the notice described in 30 | Exhibit B to the Covered Software; or 31 | 32 | b. that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the terms of 34 | a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | 38 | means any form of the work other than Source Code Form. 39 | 40 | 1.7. "Larger Work" 41 | 42 | means a work that combines Covered Software with other material, in a 43 | separate file or files, that is not Covered Software. 44 | 45 | 1.8. "License" 46 | 47 | means this document. 48 | 49 | 1.9. "Licensable" 50 | 51 | means having the right to grant, to the maximum extent possible, whether 52 | at the time of the initial grant or subsequently, any and all of the 53 | rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | 57 | means any of the following: 58 | 59 | a. any file in Source Code Form that results from an addition to, 60 | deletion from, or modification of the contents of Covered Software; or 61 | 62 | b. any new file in Source Code Form that contains any Covered Software. 63 | 64 | 1.11. "Patent Claims" of a Contributor 65 | 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the License, 69 | by the making, using, selling, offering for sale, having made, import, 70 | or transfer of either its Contributions or its Contributor Version. 71 | 72 | 1.12. "Secondary License" 73 | 74 | means either the GNU General Public License, Version 2.0, the GNU Lesser 75 | General Public License, Version 2.1, the GNU Affero General Public 76 | License, Version 3.0, or any later versions of those licenses. 77 | 78 | 1.13. "Source Code Form" 79 | 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | 84 | means an individual or a legal entity exercising rights under this 85 | License. For legal entities, "You" includes any entity that controls, is 86 | controlled by, or is under common control with You. For purposes of this 87 | definition, "control" means (a) the power, direct or indirect, to cause 88 | the direction or management of such entity, whether by contract or 89 | otherwise, or (b) ownership of more than fifty percent (50%) of the 90 | outstanding shares or beneficial ownership of such entity. 91 | 92 | 93 | 2. License Grants and Conditions 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | a. under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | b. under Patent Claims of such Contributor to make, use, sell, offer for 107 | sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | a. for any code that a Contributor has removed from Covered Software; or 125 | 126 | b. for infringements caused by: (i) Your and any other third party's 127 | modifications of Covered Software, or (ii) the combination of its 128 | Contributions with other software (except as part of its Contributor 129 | Version); or 130 | 131 | c. under Patent Claims infringed by Covered Software in the absence of 132 | its Contributions. 133 | 134 | This License does not grant any rights in the trademarks, service marks, 135 | or logos of any Contributor (except as may be necessary to comply with 136 | the notice requirements in Section 3.4). 137 | 138 | 2.4. Subsequent Licenses 139 | 140 | No Contributor makes additional grants as a result of Your choice to 141 | distribute the Covered Software under a subsequent version of this 142 | License (see Section 10.2) or under the terms of a Secondary License (if 143 | permitted under the terms of Section 3.3). 144 | 145 | 2.5. Representation 146 | 147 | Each Contributor represents that the Contributor believes its 148 | Contributions are its original creation(s) or it has sufficient rights to 149 | grant the rights to its Contributions conveyed by this License. 150 | 151 | 2.6. Fair Use 152 | 153 | This License is not intended to limit any rights You have under 154 | applicable copyright doctrines of fair use, fair dealing, or other 155 | equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under 169 | the terms of this License. You must inform recipients that the Source 170 | Code Form of the Covered Software is governed by the terms of this 171 | License, and how they can obtain a copy of this License. You may not 172 | attempt to alter or restrict the recipients' rights in the Source Code 173 | Form. 174 | 175 | 3.2. Distribution of Executable Form 176 | 177 | If You distribute Covered Software in Executable Form then: 178 | 179 | a. such Covered Software must also be made available in Source Code Form, 180 | as described in Section 3.1, and You must inform recipients of the 181 | Executable Form how they can obtain a copy of such Source Code Form by 182 | reasonable means in a timely manner, at a charge no more than the cost 183 | of distribution to the recipient; and 184 | 185 | b. You may distribute such Executable Form under the terms of this 186 | License, or sublicense it under different terms, provided that the 187 | license for the Executable Form does not attempt to limit or alter the 188 | recipients' rights in the Source Code Form under this License. 189 | 190 | 3.3. Distribution of a Larger Work 191 | 192 | You may create and distribute a Larger Work under terms of Your choice, 193 | provided that You also comply with the requirements of this License for 194 | the Covered Software. If the Larger Work is a combination of Covered 195 | Software with a work governed by one or more Secondary Licenses, and the 196 | Covered Software is not Incompatible With Secondary Licenses, this 197 | License permits You to additionally distribute such Covered Software 198 | under the terms of such Secondary License(s), so that the recipient of 199 | the Larger Work may, at their option, further distribute the Covered 200 | Software under the terms of either this License or such Secondary 201 | License(s). 202 | 203 | 3.4. Notices 204 | 205 | You may not remove or alter the substance of any license notices 206 | (including copyright notices, patent notices, disclaimers of warranty, or 207 | limitations of liability) contained within the Source Code Form of the 208 | Covered Software, except that You may alter any license notices to the 209 | extent required to remedy known factual inaccuracies. 210 | 211 | 3.5. Application of Additional Terms 212 | 213 | You may choose to offer, and to charge a fee for, warranty, support, 214 | indemnity or liability obligations to one or more recipients of Covered 215 | Software. However, You may do so only on Your own behalf, and not on 216 | behalf of any Contributor. You must make it absolutely clear that any 217 | such warranty, support, indemnity, or liability obligation is offered by 218 | You alone, and You hereby agree to indemnify every Contributor for any 219 | liability incurred by such Contributor as a result of warranty, support, 220 | indemnity or liability terms You offer. You may include additional 221 | disclaimers of warranty and limitations of liability specific to any 222 | jurisdiction. 223 | 224 | 4. Inability to Comply Due to Statute or Regulation 225 | 226 | If it is impossible for You to comply with any of the terms of this License 227 | with respect to some or all of the Covered Software due to statute, 228 | judicial order, or regulation then You must: (a) comply with the terms of 229 | this License to the maximum extent possible; and (b) describe the 230 | limitations and the code they affect. Such description must be placed in a 231 | text file included with all distributions of the Covered Software under 232 | this License. Except to the extent prohibited by statute or regulation, 233 | such description must be sufficiently detailed for a recipient of ordinary 234 | skill to be able to understand it. 235 | 236 | 5. Termination 237 | 238 | 5.1. The rights granted under this License will terminate automatically if You 239 | fail to comply with any of its terms. However, if You become compliant, 240 | then the rights granted under this License from a particular Contributor 241 | are reinstated (a) provisionally, unless and until such Contributor 242 | explicitly and finally terminates Your grants, and (b) on an ongoing 243 | basis, if such Contributor fails to notify You of the non-compliance by 244 | some reasonable means prior to 60 days after You have come back into 245 | compliance. Moreover, Your grants from a particular Contributor are 246 | reinstated on an ongoing basis if such Contributor notifies You of the 247 | non-compliance by some reasonable means, this is the first time You have 248 | received notice of non-compliance with this License from such 249 | Contributor, and You become compliant prior to 30 days after Your receipt 250 | of the notice. 251 | 252 | 5.2. If You initiate litigation against any entity by asserting a patent 253 | infringement claim (excluding declaratory judgment actions, 254 | counter-claims, and cross-claims) alleging that a Contributor Version 255 | directly or indirectly infringes any patent, then the rights granted to 256 | You by any and all Contributors for the Covered Software under Section 257 | 2.1 of this License shall terminate. 258 | 259 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 260 | license agreements (excluding distributors and resellers) which have been 261 | validly granted by You or Your distributors under this License prior to 262 | termination shall survive termination. 263 | 264 | 6. Disclaimer of Warranty 265 | 266 | Covered Software is provided under this License on an "as is" basis, 267 | without warranty of any kind, either expressed, implied, or statutory, 268 | including, without limitation, warranties that the Covered Software is free 269 | of defects, merchantable, fit for a particular purpose or non-infringing. 270 | The entire risk as to the quality and performance of the Covered Software 271 | is with You. Should any Covered Software prove defective in any respect, 272 | You (not any Contributor) assume the cost of any necessary servicing, 273 | repair, or correction. This disclaimer of warranty constitutes an essential 274 | part of this License. No use of any Covered Software is authorized under 275 | this License except under this disclaimer. 276 | 277 | 7. Limitation of Liability 278 | 279 | Under no circumstances and under no legal theory, whether tort (including 280 | negligence), contract, or otherwise, shall any Contributor, or anyone who 281 | distributes Covered Software as permitted above, be liable to You for any 282 | direct, indirect, special, incidental, or consequential damages of any 283 | character including, without limitation, damages for lost profits, loss of 284 | goodwill, work stoppage, computer failure or malfunction, or any and all 285 | other commercial damages or losses, even if such party shall have been 286 | informed of the possibility of such damages. This limitation of liability 287 | shall not apply to liability for death or personal injury resulting from 288 | such party's negligence to the extent applicable law prohibits such 289 | limitation. Some jurisdictions do not allow the exclusion or limitation of 290 | incidental or consequential damages, so this exclusion and limitation may 291 | not apply to You. 292 | 293 | 8. Litigation 294 | 295 | Any litigation relating to this License may be brought only in the courts 296 | of a jurisdiction where the defendant maintains its principal place of 297 | business and such litigation shall be governed by laws of that 298 | jurisdiction, without reference to its conflict-of-law provisions. Nothing 299 | in this Section shall prevent a party's ability to bring cross-claims or 300 | counter-claims. 301 | 302 | 9. Miscellaneous 303 | 304 | This License represents the complete agreement concerning the subject 305 | matter hereof. If any provision of this License is held to be 306 | unenforceable, such provision shall be reformed only to the extent 307 | necessary to make it enforceable. Any law or regulation which provides that 308 | the language of a contract shall be construed against the drafter shall not 309 | be used to construe this License against a Contributor. 310 | 311 | 312 | 10. Versions of the License 313 | 314 | 10.1. New Versions 315 | 316 | Mozilla Foundation is the license steward. Except as provided in Section 317 | 10.3, no one other than the license steward has the right to modify or 318 | publish new versions of this License. Each version will be given a 319 | distinguishing version number. 320 | 321 | 10.2. Effect of New Versions 322 | 323 | You may distribute the Covered Software under the terms of the version 324 | of the License under which You originally received the Covered Software, 325 | or under the terms of any subsequent version published by the license 326 | steward. 327 | 328 | 10.3. Modified Versions 329 | 330 | If you create software not governed by this License, and you want to 331 | create a new license for such software, you may create and use a 332 | modified version of this License if you rename the license and remove 333 | any references to the name of the license steward (except to note that 334 | such modified license differs from this License). 335 | 336 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 337 | Licenses If You choose to distribute Source Code Form that is 338 | Incompatible With Secondary Licenses under the terms of this version of 339 | the License, the notice described in Exhibit B of this License must be 340 | attached. 341 | 342 | Exhibit A - Source Code Form License Notice 343 | 344 | This Source Code Form is subject to the 345 | terms of the Mozilla Public License, v. 346 | 2.0. If a copy of the MPL was not 347 | distributed with this file, You can 348 | obtain one at 349 | http://mozilla.org/MPL/2.0/. 350 | 351 | If it is not possible or desirable to put the notice in a particular file, 352 | then You may include the notice in a location (such as a LICENSE file in a 353 | relevant directory) where a recipient would be likely to look for such a 354 | notice. 355 | 356 | You may add additional accurate notices of copyright ownership. 357 | 358 | Exhibit B - "Incompatible With Secondary Licenses" Notice 359 | 360 | This Source Code Form is "Incompatible 361 | With Secondary Licenses", as defined by 362 | the Mozilla Public License, v. 2.0. 363 | 364 | --------------------------------------------------------------------------------