├── .dockerignore ├── Dockerfile ├── LICENSE ├── README.md ├── api └── api.go ├── backend ├── async.go ├── backend.go └── sqlite.go ├── engine └── engine.go ├── frontend ├── api.go ├── frontend.go ├── static │ └── ui │ │ ├── favicon.ico │ │ ├── index.html │ │ ├── index.js │ │ ├── logo-32.png │ │ └── logo-96.png ├── syslog.go ├── ui.go └── web.go ├── go.mod ├── go.sum ├── raftman.go ├── spi └── spi.go └── utils ├── retention.go └── utils.go /.dockerignore: -------------------------------------------------------------------------------- 1 | frontend/static.go 2 | vendor/ 3 | raftman -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine3.10 AS golang 2 | WORKDIR /src 3 | RUN apk --no-cache add build-base git \ 4 | && GO111MODULE=off go get github.com/mjibson/esc 5 | COPY . ./ 6 | RUN go generate && go build 7 | 8 | FROM alpine:3.10 9 | ENTRYPOINT ["/usr/local/bin/raftman"] 10 | RUN mkdir -p /var/lib/raftman 11 | COPY --from=golang /src/raftman /usr/local/bin/raftman 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Pierre-David Bélanger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # raftman 2 | 3 | ![raftman](https://raw.githubusercontent.com/pierredavidbelanger/raftman/master/frontend/static/ui/logo-96.png) 4 | 5 | A syslog server with integrated full text search via a JSON API and Web UI. 6 | 7 | - [getting started](#getting-started) 8 | - [configuration](#configuration) 9 | 10 | ## getting started 11 | 12 | ### store logs 13 | 14 | To get started quickly, just run the containerized version of raftman: 15 | 16 | ``` 17 | sudo docker run --rm --name raftman \ 18 | -v /tmp:/var/lib/raftman \ 19 | -p 514:514/udp \ 20 | -p 5514:5514 \ 21 | -p 8181:8181 \ 22 | -p 8282:8282 \ 23 | pierredavidbelanger/raftman 24 | ``` 25 | 26 | 27 | This will start raftman with all default options. It listen on port 514 (UDP) and 5514 (TCP) on the host for incoming RFC5424 syslog packets and store them into an SQLite database stored in `/tmp/logs.db` on the host. It also exposes the JSON API on http://localhost:8181/api/ and the Web UI on http://localhost:8282/. 28 | 29 | ### send logs 30 | 31 | Time to fill our database. The easyest way is to just start [logspout](https://github.com/gliderlabs/logspout) and tell it to point to raftman's syslog port: 32 | 33 | ``` 34 | docker run --rm --name logspout \ 35 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 36 | --link raftman \ 37 | gliderlabs/logspout \ 38 | syslog://raftman:514 39 | ``` 40 | 41 | 42 | This last container will grab other containers output lines and send them as syslog packet to the configured syslog server (ie: our linked raftman container). 43 | 44 | ### generate logs 45 | 46 | Now, we also need to generate some output. This will do the job for now: 47 | 48 | ``` 49 | docker run --rm --name test \ 50 | alpine \ 51 | echo 'Can you see me' 52 | ``` 53 | 54 | 55 | ### visualise logs 56 | 57 | Then we can visualize our logs: 58 | 59 | with the raftman API: 60 | 61 | ``` 62 | curl http://localhost:8181/api/list \ 63 | -d '{"Limit": 100, "Message": "see"}' 64 | ``` 65 | 66 | 67 | or pop the Web UI at http://localhost:8282/ 68 | 69 | ## configuration 70 | 71 | All raftman configuration options are set as arguments in the command line. 72 | 73 | For example, here is the what the command line would looks like if we set all the default values explicitly: 74 | 75 | ``` 76 | raftman \ 77 | -backend sqlite:///var/lib/raftman/logs.db?insertQueueSize=512&queryQueueSize=16&timeout=5s&batchSize=32&retention=INF \ 78 | -frontend syslog+udp://:514?format=RFC5424&queueSize=512&timeout=0s \ 79 | -frontend syslog+tcp://:5514?format=RFC5424&queueSize=512&timeout=0s \ 80 | -frontend api+http://:8181/api/ \ 81 | -frontend ui+http://:8282/ 82 | ``` 83 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type LogEntry struct { 8 | Timestamp time.Time 9 | Hostname string 10 | Application string 11 | Message string 12 | } 13 | 14 | type QueryRequest struct { 15 | FromTimestamp time.Time 16 | ToTimestamp time.Time 17 | Hostname string 18 | Application string 19 | Message string 20 | Limit int 21 | Offset int 22 | } 23 | 24 | type QueryStatResponse struct { 25 | Stat map[string]map[string]uint64 `json:",omitempty"` 26 | Error string `json:",omitempty"` 27 | } 28 | 29 | type QueryListResponse struct { 30 | Entries []*LogEntry `json:",omitempty"` 31 | Error string `json:",omitempty"` 32 | } 33 | 34 | type InsertRequest struct { 35 | Entry *LogEntry 36 | Entries []*LogEntry 37 | } 38 | 39 | type InsertResponse struct { 40 | Error string `json:",omitempty"` 41 | } 42 | -------------------------------------------------------------------------------- /backend/async.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/api" 6 | "github.com/pierredavidbelanger/raftman/utils" 7 | "net/url" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type queryStatM struct { 13 | req *api.QueryRequest 14 | res chan *api.QueryStatResponse 15 | } 16 | 17 | func newQueryStatM(req *api.QueryRequest) *queryStatM { 18 | return &queryStatM{req, make(chan *api.QueryStatResponse, 1)} 19 | } 20 | 21 | func (m *queryStatM) push(c chan *queryStatM) *queryStatM { 22 | c <- m 23 | return m 24 | } 25 | 26 | func (m *queryStatM) pollWithTimeout(d time.Duration) (*api.QueryStatResponse, error) { 27 | t := time.NewTimer(d) 28 | select { 29 | case v := <-m.res: 30 | return v, nil 31 | case <-t.C: 32 | return nil, fmt.Errorf("operation timed out after %s", d) 33 | } 34 | } 35 | 36 | type queryListM struct { 37 | req *api.QueryRequest 38 | res chan *api.QueryListResponse 39 | } 40 | 41 | func newQueryListM(req *api.QueryRequest) *queryListM { 42 | return &queryListM{req, make(chan *api.QueryListResponse, 1)} 43 | } 44 | 45 | func (m *queryListM) push(c chan *queryListM) *queryListM { 46 | c <- m 47 | return m 48 | } 49 | 50 | func (m *queryListM) pollWithTimeout(d time.Duration) (*api.QueryListResponse, error) { 51 | t := time.NewTimer(d) 52 | select { 53 | case v := <-m.res: 54 | return v, nil 55 | case <-t.C: 56 | return nil, fmt.Errorf("operation timed out after %s", d) 57 | } 58 | } 59 | 60 | type asyncBackend struct { 61 | insertQ chan *api.LogEntry 62 | queryStatQ chan *queryStatM 63 | queryListQ chan *queryListM 64 | stopQ chan *sync.Cond 65 | timeout time.Duration 66 | } 67 | 68 | func initAsyncBackend(backendURL *url.URL, b *asyncBackend) error { 69 | insertQueueSize, err := utils.GetIntQueryParam(backendURL, "insertQueueSize", 512) 70 | if err != nil { 71 | return err 72 | } 73 | queryQueueSize, err := utils.GetIntQueryParam(backendURL, "queryQueueSize", 16) 74 | if err != nil { 75 | return err 76 | } 77 | timeout, err := utils.GetDurationQueryParam(backendURL, "timeout", 5*time.Second) 78 | if err != nil { 79 | return err 80 | } 81 | b.insertQ = make(chan *api.LogEntry, insertQueueSize) 82 | b.queryStatQ = make(chan *queryStatM, queryQueueSize) 83 | b.queryListQ = make(chan *queryListM, queryQueueSize) 84 | b.stopQ = make(chan *sync.Cond, 1) 85 | b.timeout = timeout 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /backend/backend.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/spi" 6 | "net/url" 7 | ) 8 | 9 | func NewBackend(e spi.LogEngine, backendURL *url.URL) (spi.LogBackend, error) { 10 | switch backendURL.Scheme { 11 | case "sqlite": 12 | return newSQLiteBackend(backendURL) 13 | } 14 | return nil, fmt.Errorf("Invalid backend %s", backendURL.Scheme) 15 | } 16 | -------------------------------------------------------------------------------- /backend/sqlite.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "fmt" 7 | _ "github.com/mattn/go-sqlite3" 8 | "github.com/pierredavidbelanger/raftman/api" 9 | "github.com/pierredavidbelanger/raftman/utils" 10 | "log" 11 | "math" 12 | "net/url" 13 | "os" 14 | "path/filepath" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type sqliteBackend struct { 20 | asyncBackend 21 | batchSize int 22 | retention utils.Retention 23 | dbFilePath string 24 | db *sql.DB 25 | hStmt *sql.Stmt 26 | bStmt *sql.Stmt 27 | } 28 | 29 | func newSQLiteBackend(backendURL *url.URL) (*sqliteBackend, error) { 30 | 31 | b := sqliteBackend{} 32 | err := initAsyncBackend(backendURL, &b.asyncBackend) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | batchSize, err := utils.GetIntQueryParam(backendURL, "batchSize", 32) 38 | if err != nil { 39 | return nil, err 40 | } 41 | b.batchSize = batchSize 42 | 43 | retention, err := utils.GetRetentionQueryParam(backendURL, "retention", utils.INF) 44 | if err != nil { 45 | return nil, err 46 | } 47 | b.retention = retention 48 | 49 | dbFilePath := backendURL.Path 50 | if dbFilePath == "" { 51 | return nil, fmt.Errorf("Invalid SQLite database file path '%s'", dbFilePath) 52 | } 53 | b.dbFilePath = dbFilePath 54 | 55 | return &b, nil 56 | } 57 | 58 | func (b *sqliteBackend) Start() error { 59 | 60 | var err error 61 | 62 | dbDir := filepath.Dir(b.dbFilePath) 63 | err = os.MkdirAll(dbDir, os.ModePerm) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | db, err := sql.Open("sqlite3", b.dbFilePath) 69 | if err != nil { 70 | return err 71 | } 72 | b.db = db 73 | 74 | _, err = db.Exec("CREATE TABLE IF NOT EXISTS logh (ts DATETIME, host VARCHAR(255), app VARCHAR(255))") 75 | if err != nil { 76 | db.Close() 77 | return err 78 | } 79 | 80 | _, err = db.Exec("CREATE INDEX IF NOT EXISTS logh_idx ON logh (ts, host, app)") 81 | if err != nil { 82 | db.Close() 83 | return err 84 | } 85 | 86 | _, err = db.Exec("CREATE VIRTUAL TABLE IF NOT EXISTS logb USING FTS4(msg, tokenize=unicode61)") 87 | if err != nil { 88 | db.Close() 89 | return err 90 | } 91 | 92 | hStmt, err := db.Prepare("INSERT INTO logh (ts, host, app) VALUES (?, ?, ?)") 93 | if err != nil { 94 | db.Close() 95 | return err 96 | } 97 | b.hStmt = hStmt 98 | 99 | bStmt, err := db.Prepare("INSERT INTO logb (docid, msg) VALUES (LAST_INSERT_ROWID(), ?)") 100 | if err != nil { 101 | hStmt.Close() 102 | db.Close() 103 | return err 104 | } 105 | b.bStmt = bStmt 106 | 107 | go b.run() 108 | 109 | return nil 110 | } 111 | 112 | func (b *sqliteBackend) Close() error { 113 | 114 | cond := sync.NewCond(&sync.Mutex{}) 115 | cond.L.Lock() 116 | b.stopQ <- cond 117 | cond.Wait() 118 | cond.L.Unlock() 119 | 120 | if b.bStmt != nil { 121 | b.bStmt.Close() 122 | b.bStmt = nil 123 | } 124 | if b.hStmt != nil { 125 | b.hStmt.Close() 126 | b.hStmt = nil 127 | } 128 | if b.db != nil { 129 | b.db.Close() 130 | b.db = nil 131 | } 132 | 133 | return nil 134 | } 135 | 136 | func (b *sqliteBackend) Insert(req *api.InsertRequest) (*api.InsertResponse, error) { 137 | if req.Entry != nil { 138 | b.insertQ <- req.Entry 139 | } 140 | if len(req.Entries) > 0 { 141 | for _, e := range req.Entries { 142 | b.insertQ <- e 143 | } 144 | } 145 | return &api.InsertResponse{}, nil 146 | } 147 | 148 | func (b *sqliteBackend) QueryStat(req *api.QueryRequest) (*api.QueryStatResponse, error) { 149 | return newQueryStatM(req).push(b.queryStatQ).pollWithTimeout(b.timeout) 150 | } 151 | 152 | func (b *sqliteBackend) QueryList(req *api.QueryRequest) (*api.QueryListResponse, error) { 153 | return newQueryListM(req).push(b.queryListQ).pollWithTimeout(b.timeout) 154 | } 155 | 156 | func (b *sqliteBackend) run() { 157 | retentionTicker := time.NewTicker(1 * time.Hour) 158 | for { 159 | select { 160 | case e := <-b.insertQ: 161 | b.handleInsert(e) 162 | case m := <-b.queryStatQ: 163 | b.handleQueryStat(m) 164 | case m := <-b.queryListQ: 165 | b.handleQueryList(m) 166 | case now := <-retentionTicker.C: 167 | b.handleRetention(now) 168 | case cond := <-b.stopQ: 169 | cond.Broadcast() 170 | return 171 | } 172 | } 173 | } 174 | 175 | func (b *sqliteBackend) handleInsert(e *api.LogEntry) { 176 | 177 | var err error 178 | 179 | tx, err := b.db.Begin() 180 | if err != nil { 181 | log.Printf("Unable to begin transaction: %s", err) 182 | return 183 | } 184 | 185 | err = b.handleInsertBatch(tx, e) 186 | if err != nil { 187 | log.Printf("Unable to insert: %s", err) 188 | err = tx.Rollback() 189 | if err != nil { 190 | log.Printf("Unable to rollback: %s", err) 191 | } 192 | return 193 | } 194 | 195 | err = tx.Commit() 196 | if err != nil { 197 | log.Printf("Unable to commit transaction: %s", err) 198 | } 199 | } 200 | 201 | func (b *sqliteBackend) handleInsertBatch(tx *sql.Tx, e *api.LogEntry) error { 202 | 203 | var err error 204 | 205 | err = b.insertEntry(tx, e) 206 | if err != nil { 207 | return err 208 | } 209 | 210 | for i := 0; i < b.batchSize; i++ { 211 | select { 212 | case e = <-b.insertQ: 213 | err = b.insertEntry(tx, e) 214 | if err != nil { 215 | return err 216 | } 217 | default: 218 | return nil 219 | } 220 | } 221 | 222 | return nil 223 | } 224 | 225 | func (b *sqliteBackend) insertEntry(tx *sql.Tx, e *api.LogEntry) error { 226 | if _, err := tx.Stmt(b.hStmt).Exec(e.Timestamp, e.Hostname, e.Application); err != nil { 227 | return err 228 | } 229 | if _, err := tx.Stmt(b.bStmt).Exec(e.Message); err != nil { 230 | return err 231 | } 232 | return nil 233 | } 234 | 235 | func (b *sqliteBackend) buildQueryFromAndWhere(req *api.QueryRequest, sqlBuf *bytes.Buffer, args *[]interface{}) { 236 | fmt.Fprint(sqlBuf, "FROM logh AS h JOIN logb AS b ON b.docid = h.rowid ") 237 | fmt.Fprint(sqlBuf, "WHERE 1=1 ") 238 | if !req.FromTimestamp.IsZero() { 239 | fmt.Fprint(sqlBuf, "AND h.ts >= ? ") 240 | *args = append(*args, req.FromTimestamp) 241 | } 242 | if !req.ToTimestamp.IsZero() { 243 | fmt.Fprint(sqlBuf, "AND h.ts < ? ") 244 | *args = append(*args, req.ToTimestamp) 245 | } 246 | if req.Hostname != "" { 247 | fmt.Fprint(sqlBuf, "AND h.host = ? ") 248 | *args = append(*args, req.Hostname) 249 | if req.Application != "" { 250 | fmt.Fprint(sqlBuf, "AND h.app = ? ") 251 | *args = append(*args, req.Application) 252 | } 253 | } 254 | if req.Message != "" { 255 | fmt.Fprint(sqlBuf, "AND b.msg MATCH ? ") 256 | *args = append(*args, req.Message) 257 | } 258 | } 259 | 260 | func clamp(min, v, max int) int { 261 | if v < min { 262 | return min 263 | } 264 | if v > max { 265 | return max 266 | } 267 | return v 268 | } 269 | 270 | func (b *sqliteBackend) buildQueryLimit(req *api.QueryRequest, sqlBuf *bytes.Buffer, args *[]interface{}) { 271 | fmt.Fprint(sqlBuf, "LIMIT ? OFFSET ? ") 272 | *args = append(*args, clamp(0, req.Limit, 256)) 273 | *args = append(*args, clamp(0, req.Offset, math.MaxInt16)) 274 | } 275 | 276 | func (b *sqliteBackend) handleQueryStat(m *queryStatM) { 277 | 278 | args := []interface{}{} 279 | 280 | sqlBuf := &bytes.Buffer{} 281 | fmt.Fprint(sqlBuf, "SELECT h.host, h.app, COUNT(b.docid) ") 282 | b.buildQueryFromAndWhere(m.req, sqlBuf, &args) 283 | fmt.Fprint(sqlBuf, "GROUP BY h.host, h.app ") 284 | fmt.Fprint(sqlBuf, "ORDER BY h.host, h.app ") 285 | b.buildQueryLimit(m.req, sqlBuf, &args) 286 | 287 | res := api.QueryStatResponse{} 288 | 289 | rows, err := b.db.Query(sqlBuf.String(), args...) 290 | if err != nil { 291 | res.Error = err.Error() 292 | m.res <- &res 293 | return 294 | } 295 | defer rows.Close() 296 | 297 | stat := make(map[string]map[string]uint64) 298 | for rows.Next() { 299 | var app string 300 | var proc string 301 | var count uint64 302 | err = rows.Scan(&app, &proc, &count) 303 | if err != nil { 304 | res.Error = err.Error() 305 | m.res <- &res 306 | return 307 | } 308 | procs, ok := stat[app] 309 | if !ok { 310 | procs = make(map[string]uint64) 311 | stat[app] = procs 312 | } 313 | procs[proc] = count 314 | } 315 | 316 | err = rows.Err() 317 | if err != nil { 318 | res.Error = err.Error() 319 | m.res <- &res 320 | return 321 | } 322 | 323 | res.Stat = stat 324 | m.res <- &res 325 | } 326 | 327 | func (b *sqliteBackend) handleQueryList(m *queryListM) { 328 | 329 | args := []interface{}{} 330 | 331 | sqlBuf := &bytes.Buffer{} 332 | fmt.Fprint(sqlBuf, "SELECT h.ts, h.host, h.app, b.msg ") 333 | b.buildQueryFromAndWhere(m.req, sqlBuf, &args) 334 | fmt.Fprint(sqlBuf, "ORDER BY h.ts DESC ") 335 | b.buildQueryLimit(m.req, sqlBuf, &args) 336 | 337 | res := api.QueryListResponse{} 338 | 339 | rows, err := b.db.Query(sqlBuf.String(), args...) 340 | if err != nil { 341 | res.Error = err.Error() 342 | m.res <- &res 343 | return 344 | } 345 | defer rows.Close() 346 | 347 | entries := make([]*api.LogEntry, 0, clamp(0, m.req.Limit, 500)) 348 | for rows.Next() { 349 | entry := api.LogEntry{} 350 | err = rows.Scan(&entry.Timestamp, &entry.Hostname, &entry.Application, &entry.Message) 351 | if err != nil { 352 | res.Error = err.Error() 353 | m.res <- &res 354 | return 355 | } 356 | entries = append(entries, &entry) 357 | } 358 | 359 | err = rows.Err() 360 | if err != nil { 361 | res.Error = err.Error() 362 | m.res <- &res 363 | return 364 | } 365 | 366 | res.Entries = entries 367 | m.res <- &res 368 | } 369 | 370 | func (b *sqliteBackend) handleRetention(now time.Time) { 371 | 372 | if b.retention == utils.INF { 373 | // Keep all the things! 374 | return 375 | } 376 | 377 | upto := now.Add(-time.Duration(b.retention)) 378 | 379 | tx, err := b.db.Begin() 380 | if err != nil { 381 | log.Printf("Unable to begin transaction: %s", err) 382 | return 383 | } 384 | 385 | err = b.handleRetentionBatch(tx, upto) 386 | if err != nil { 387 | log.Printf("Unable to delete: %s", err) 388 | err = tx.Rollback() 389 | if err != nil { 390 | log.Printf("Unable to rollback: %s", err) 391 | } 392 | return 393 | } 394 | 395 | err = tx.Commit() 396 | if err != nil { 397 | log.Printf("Unable to commit transaction: %s", err) 398 | } 399 | } 400 | 401 | func (b *sqliteBackend) handleRetentionBatch(tx *sql.Tx, upto time.Time) error { 402 | 403 | var err error 404 | 405 | rows, err := tx.Query("SELECT rowid FROM logh AS h WHERE h.ts < ?", upto) 406 | if err != nil { 407 | return err 408 | } 409 | defer rows.Close() 410 | 411 | for rows.Next() { 412 | var rowid uint64 413 | err = rows.Scan(&rowid) 414 | if err != nil { 415 | return err 416 | } 417 | _, err = tx.Exec("DELETE FROM logh WHERE rowid = ?", rowid) 418 | if err != nil { 419 | return err 420 | } 421 | _, err = tx.Exec("DELETE FROM logb WHERE docid = ?", rowid) 422 | if err != nil { 423 | return err 424 | } 425 | } 426 | 427 | return nil 428 | } 429 | -------------------------------------------------------------------------------- /engine/engine.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/backend" 6 | "github.com/pierredavidbelanger/raftman/frontend" 7 | "github.com/pierredavidbelanger/raftman/spi" 8 | "log" 9 | "net/url" 10 | "os" 11 | "os/signal" 12 | ) 13 | 14 | type engine struct { 15 | backURL *url.URL 16 | frontURLs []*url.URL 17 | back spi.LogBackend 18 | fronts []spi.LogFrontend 19 | } 20 | 21 | func NewEngine(backendURL *url.URL, frontendURLs []*url.URL) (spi.LogEngine, error) { 22 | 23 | e := engine{} 24 | 25 | b, err := backend.NewBackend(&e, backendURL) 26 | if err != nil { 27 | return nil, fmt.Errorf("Unable to create backend '%s': %s", backendURL, err) 28 | } 29 | e.backURL = backendURL 30 | e.back = b 31 | 32 | for _, frontendURL := range frontendURLs { 33 | f, err := frontend.NewFrontend(&e, frontendURL) 34 | if err != nil { 35 | return nil, fmt.Errorf("Unable to create frontend '%s': %s", frontendURL, err) 36 | } 37 | e.frontURLs = append(e.frontURLs, frontendURL) 38 | e.fronts = append(e.fronts, f) 39 | } 40 | 41 | return &e, nil 42 | } 43 | 44 | func (e *engine) Start() error { 45 | 46 | log.Printf("Start backend '%s'", e.backURL) 47 | if err := e.back.Start(); err != nil { 48 | return fmt.Errorf("Unable to start backend '%s': %s", e.backURL, err) 49 | } 50 | 51 | for i, f := range e.fronts { 52 | log.Printf("Start frontend '%s'", e.frontURLs[i]) 53 | if err := f.Start(); err != nil { 54 | return fmt.Errorf("Unable to start frontend '%s': %s", e.frontURLs[i], err) 55 | } 56 | } 57 | 58 | return nil 59 | } 60 | 61 | func (e *engine) Close() error { 62 | 63 | for i, f := range e.fronts { 64 | if err := f.Close(); err != nil { 65 | fmt.Printf("Unable to close frontend '%s': %s", e.frontURLs[i], err) 66 | } 67 | } 68 | 69 | if err := e.back.Close(); err != nil { 70 | fmt.Printf("Unable to start backend '%s': %s", e.backURL, err) 71 | } 72 | 73 | return nil 74 | } 75 | 76 | func (e *engine) Wait() error { 77 | c := make(chan os.Signal, 1) 78 | signal.Notify(c, os.Interrupt) 79 | <-c 80 | return nil 81 | } 82 | 83 | func (e *engine) GetBackend() (*url.URL, spi.LogBackend) { 84 | return e.backURL, e.back 85 | } 86 | 87 | func (e *engine) GetFrontends() ([]*url.URL, []spi.LogFrontend) { 88 | return e.frontURLs, e.fronts 89 | } 90 | -------------------------------------------------------------------------------- /frontend/api.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/pierredavidbelanger/raftman/api" 6 | "github.com/pierredavidbelanger/raftman/spi" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | type apiFrontend struct { 12 | webFrontend 13 | } 14 | 15 | func newAPIFrontend(e spi.LogEngine, frontendURL *url.URL) (*apiFrontend, error) { 16 | f := apiFrontend{} 17 | if err := initWebFrontend(e, frontendURL, &f.webFrontend); err != nil { 18 | return nil, err 19 | } 20 | return &f, nil 21 | } 22 | 23 | func (f *apiFrontend) Start() error { 24 | mux := http.NewServeMux() 25 | mux.HandleFunc(f.path+"stat", f.handleStat) 26 | mux.HandleFunc(f.path+"list", f.handleList) 27 | return f.startHandler(mux) 28 | } 29 | 30 | func (f *apiFrontend) Close() error { 31 | return f.close() 32 | } 33 | 34 | func (f *apiFrontend) handleStat(w http.ResponseWriter, r *http.Request) { 35 | 36 | req := api.QueryRequest{} 37 | 38 | if r.Method == "POST" { 39 | defer r.Body.Close() 40 | err := json.NewDecoder(r.Body).Decode(&req) 41 | if err != nil { 42 | http.Error(w, err.Error(), 500) 43 | return 44 | } 45 | } 46 | 47 | res, err := f.b.QueryStat(&req) 48 | if err != nil { 49 | res = &api.QueryStatResponse{Error: err.Error()} 50 | w.WriteHeader(400) 51 | } 52 | 53 | err = json.NewEncoder(w).Encode(res) 54 | if err != nil { 55 | http.Error(w, err.Error(), 500) 56 | return 57 | } 58 | } 59 | 60 | func (f *apiFrontend) handleList(w http.ResponseWriter, r *http.Request) { 61 | 62 | req := api.QueryRequest{} 63 | 64 | if r.Method == "POST" { 65 | defer r.Body.Close() 66 | err := json.NewDecoder(r.Body).Decode(&req) 67 | if err != nil { 68 | http.Error(w, err.Error(), 500) 69 | return 70 | } 71 | } 72 | 73 | res, err := f.b.QueryList(&req) 74 | if err != nil { 75 | res = &api.QueryListResponse{Error: err.Error()} 76 | w.WriteHeader(400) 77 | } 78 | 79 | err = json.NewEncoder(w).Encode(res) 80 | if err != nil { 81 | http.Error(w, err.Error(), 500) 82 | return 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /frontend/frontend.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/spi" 6 | "net/url" 7 | ) 8 | 9 | func NewFrontend(e spi.LogEngine, frontendURL *url.URL) (spi.LogFrontend, error) { 10 | switch frontendURL.Scheme { 11 | case "syslog+tcp", "syslog+udp": 12 | return newSyslogServerFrontend(e, frontendURL) 13 | case "api+http": 14 | return newAPIFrontend(e, frontendURL) 15 | case "ui+http": 16 | return newUIFrontend(e, frontendURL) 17 | } 18 | return nil, fmt.Errorf("Invalid frontend %s", frontendURL.Scheme) 19 | } 20 | -------------------------------------------------------------------------------- /frontend/static/ui/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pierredavidbelanger/raftman/6a8f36a6af8b64211ce3e463c5c251258b9f41cc/frontend/static/ui/favicon.ico -------------------------------------------------------------------------------- /frontend/static/ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /frontend/static/ui/index.js: -------------------------------------------------------------------------------- 1 | webix.ready(function () { 2 | 3 | var tsFormat = webix.Date.dateToStr("%Y-%m-%d %H:%i:%s"); 4 | var tsFormatter = function (s) { 5 | return tsFormat(new Date(s)); 6 | }; 7 | 8 | webix.ui({ 9 | rows: [ 10 | { 11 | view: "toolbar", 12 | height: 40, 13 | cols: [ 14 | {view: "button", type: "image", id: "home", image: "logo-32.png", width: 50}, 15 | {view: "datepicker", id: "fromTimestamp", timepicker: true, width: 200}, 16 | {view: "datepicker", id: "toTimestamp", timepicker: true, width: 200}, 17 | {view: "text", id: "message", width: 300}, 18 | {view: "checkbox", id: "follow", label: "Follow", value: true, width: 100}, 19 | {view: "button", id: "prevPage", value: "<<", width: 50}, 20 | {view: "button", id: "nextPage", value: ">>", width: 50} 21 | ] 22 | }, 23 | { 24 | cols: [ 25 | { 26 | view: "datatable", 27 | id: "queryStat", 28 | //autoConfig: true, 29 | columns: [ 30 | {id: "Hostname", header: "Hostname", width: 150}, 31 | {id: "Application", header: "Application", width: 150}, 32 | {id: "Count", header: "Count", fillspace: true} 33 | ], 34 | select: "row", 35 | data: [], 36 | width: 300 37 | }, 38 | { 39 | view: "datatable", 40 | id: "queryList", 41 | //autoConfig: true, 42 | columns: [ 43 | {id: "Timestamp", header: "Timestamp", width: 175, format: tsFormatter}, 44 | {id: "Hostname", header: "Hostname", width: 150}, 45 | {id: "Application", header: "Application", width: 150}, 46 | {id: "Message", header: "Message", fillspace: true} 47 | ], 48 | data: [] 49 | } 50 | ] 51 | } 52 | ] 53 | }); 54 | 55 | function post(url, data) { 56 | return $.ajax(url, { 57 | method: "POST", 58 | contentType: "application/json", 59 | data: JSON.stringify(data), 60 | dataType: "json" 61 | }) 62 | } 63 | 64 | //var home = $$("home"); 65 | var fromTimestamp = $$("fromTimestamp"); 66 | var toTimestamp = $$("toTimestamp"); 67 | var message = $$("message"); 68 | var follow = $$("follow"); 69 | var prevPage = $$("prevPage"); 70 | var nextPage = $$("nextPage"); 71 | var queryStat = $$("queryStat"); 72 | var queryList = $$("queryList"); 73 | 74 | var queryStatRequest = { 75 | Limit: 500 76 | }; 77 | 78 | var queryListRequest = { 79 | Limit: 50 80 | }; 81 | 82 | var autoUpdateEnabled = true; 83 | 84 | var updateStat = function () { 85 | return post("api/stat", queryStatRequest).done(function (data) { 86 | var selectedId = queryStat.getSelectedId(); 87 | queryStat.clearAll(); 88 | queryStat.add({ 89 | id: "*-*", 90 | Hostname: "*", 91 | Application: "*" 92 | }); 93 | if (data.Stat) { 94 | $.each(data.Stat, function (hostname, applications) { 95 | queryStat.add({ 96 | id: hostname + "-*", 97 | Hostname: hostname, 98 | Application: "*" 99 | }); 100 | $.each(applications, function (application, count) { 101 | queryStat.add({ 102 | id: hostname + "-" + application, 103 | Hostname: hostname, 104 | Application: application, 105 | Count: count 106 | }); 107 | }); 108 | }); 109 | } 110 | queryStat.adjustColumn("Hostname"); 111 | queryStat.adjustColumn("Application"); 112 | queryStat.adjustColumn("Count"); 113 | if (selectedId) { 114 | try { 115 | queryStat.select(selectedId); 116 | } catch (e) { 117 | queryStat.select(queryStat.getIdByIndex(0)); 118 | } 119 | } 120 | }); 121 | }; 122 | 123 | var updateList = function () { 124 | return post("api/list", queryListRequest).done(function (data) { 125 | queryList.clearAll(); 126 | if (data.Entries) { 127 | data.Entries = data.Entries.reverse(); 128 | $(data.Entries).each(function (_, v) { 129 | queryList.add(v); 130 | }); 131 | } 132 | queryList.adjustColumn("Timestamp"); 133 | queryList.adjustColumn("Hostname"); 134 | queryList.adjustColumn("Application"); 135 | if (data.Entries) { 136 | queryList.showItemByIndex(data.Entries.length); 137 | } 138 | }); 139 | }; 140 | 141 | var autoUpdate = function () { 142 | setTimeout(function () { 143 | if (autoUpdateEnabled) { 144 | updateStat().always(autoUpdate); 145 | } else { 146 | autoUpdate(); 147 | } 148 | }, 5000); 149 | }; 150 | 151 | fromTimestamp.attachEvent("onChange", function (value) { 152 | queryStatRequest.FromTimestamp = queryListRequest.FromTimestamp = value; 153 | queryListRequest.Offset = 0; 154 | updateStat(); 155 | }); 156 | 157 | toTimestamp.attachEvent("onChange", function (value) { 158 | queryStatRequest.ToTimestamp = queryListRequest.ToTimestamp = value; 159 | queryListRequest.Offset = 0; 160 | updateStat(); 161 | }); 162 | 163 | message.attachEvent("onChange", function (value) { 164 | queryStatRequest.Message = queryListRequest.Message = value; 165 | queryListRequest.Offset = 0; 166 | updateStat(); 167 | }); 168 | 169 | follow.attachEvent("onChange", function (value) { 170 | autoUpdateEnabled = value; 171 | if (value === true) { 172 | queryListRequest.Offset = 0; 173 | updateStat(); 174 | } 175 | }); 176 | 177 | prevPage.attachEvent("onItemClick", function () { 178 | follow.setValue(false); 179 | if (!queryListRequest.Offset) { 180 | queryListRequest.Offset = 0; 181 | } 182 | queryListRequest.Offset += queryListRequest.Limit; 183 | updateList(); 184 | }); 185 | 186 | nextPage.attachEvent("onItemClick", function () { 187 | if (follow.getValue()) { 188 | follow.setValue(false); 189 | } 190 | if (!queryListRequest.Offset) { 191 | queryListRequest.Offset = 0; 192 | } 193 | queryListRequest.Offset -= queryListRequest.Limit; 194 | if (queryListRequest.Offset <= 0) { 195 | queryListRequest.Offset = 0; 196 | follow.setValue(true); 197 | } else { 198 | updateList(); 199 | } 200 | }); 201 | 202 | queryStat.attachEvent("onAfterSelect", function (data) { 203 | if (data && data.id) { 204 | var stat = queryStat.getItem(data.id); 205 | if (stat) { 206 | queryListRequest.Hostname = stat.Hostname !== "*" ? stat.Hostname : null; 207 | queryListRequest.Application = stat.Application !== "*" ? stat.Application : null; 208 | } 209 | } 210 | queryListRequest.Offset = 0; 211 | updateList(); 212 | }); 213 | 214 | updateStat().done(function () { 215 | queryStat.select(queryStat.getIdByIndex(0)); 216 | autoUpdate(); 217 | }); 218 | 219 | }); -------------------------------------------------------------------------------- /frontend/static/ui/logo-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pierredavidbelanger/raftman/6a8f36a6af8b64211ce3e463c5c251258b9f41cc/frontend/static/ui/logo-32.png -------------------------------------------------------------------------------- /frontend/static/ui/logo-96.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pierredavidbelanger/raftman/6a8f36a6af8b64211ce3e463c5c251258b9f41cc/frontend/static/ui/logo-96.png -------------------------------------------------------------------------------- /frontend/syslog.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/api" 6 | "github.com/pierredavidbelanger/raftman/spi" 7 | "github.com/pierredavidbelanger/raftman/utils" 8 | "gopkg.in/mcuadros/go-syslog.v2" 9 | "gopkg.in/mcuadros/go-syslog.v2/format" 10 | "net/url" 11 | "strings" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | type syslogServerFrontend struct { 17 | e spi.LogEngine 18 | b spi.LogBackend 19 | logsQ syslog.LogPartsChannel 20 | stopQ chan *sync.Cond 21 | format format.Format 22 | server *syslog.Server 23 | } 24 | 25 | func newSyslogServerFrontend(e spi.LogEngine, frontendURL *url.URL) (*syslogServerFrontend, error) { 26 | 27 | if frontendURL.Host == "" { 28 | return nil, fmt.Errorf("Empty host in frontend URL '%s'", frontendURL) 29 | } 30 | 31 | syslogFormat, err := utils.GetSyslogFormatQueryParam(frontendURL, "format", syslog.RFC5424) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | queueSize, err := utils.GetIntQueryParam(frontendURL, "queueSize", 512) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | timeout, err := utils.GetDurationQueryParam(frontendURL, "timeout", 0*time.Second) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | f := syslogServerFrontend{} 47 | f.e = e 48 | 49 | logsQ := make(syslog.LogPartsChannel, queueSize) 50 | f.logsQ = logsQ 51 | 52 | stopQ := make(chan *sync.Cond, 1) 53 | f.stopQ = stopQ 54 | 55 | f.format = syslogFormat 56 | 57 | server := syslog.NewServer() 58 | server.SetFormat(syslogFormat) 59 | server.SetTimeout(int64(timeout.Seconds() * 1000)) 60 | server.SetHandler(syslog.NewChannelHandler(logsQ)) 61 | switch strings.ToLower(frontendURL.Scheme) { 62 | case "syslog+tcp": 63 | err = server.ListenTCP(frontendURL.Host) 64 | case "syslog+udp": 65 | err = server.ListenUDP(frontendURL.Host) 66 | } 67 | if err != nil { 68 | return nil, err 69 | } 70 | f.server = server 71 | 72 | return &f, nil 73 | } 74 | 75 | func (f *syslogServerFrontend) Start() error { 76 | 77 | _, b := f.e.GetBackend() 78 | f.b = b 79 | 80 | err := f.server.Boot() 81 | if err != nil { 82 | return err 83 | } 84 | 85 | go f.run() 86 | 87 | return nil 88 | } 89 | 90 | func (f *syslogServerFrontend) Close() error { 91 | 92 | cond := sync.NewCond(&sync.Mutex{}) 93 | cond.L.Lock() 94 | f.stopQ <- cond 95 | cond.Wait() 96 | cond.L.Unlock() 97 | 98 | return f.server.Kill() 99 | } 100 | 101 | func (f *syslogServerFrontend) run() { 102 | for { 103 | select { 104 | case logParts := <-f.logsQ: 105 | f.b.Insert(&api.InsertRequest{Entry: f.toLogEntry(logParts)}) 106 | case cond := <-f.stopQ: 107 | cond.Broadcast() 108 | return 109 | } 110 | } 111 | } 112 | 113 | func (f *syslogServerFrontend) toLogEntry(logParts format.LogParts) *api.LogEntry { 114 | e := api.LogEntry{} 115 | switch f.format { 116 | case syslog.RFC3164: 117 | if val, ok := logParts["timestamp"].(time.Time); ok { 118 | e.Timestamp = val 119 | } else { 120 | e.Timestamp = time.Now() 121 | } 122 | if val, ok := logParts["hostname"].(string); ok { 123 | e.Hostname = val 124 | } 125 | if val, ok := logParts["tag"].(string); ok { 126 | e.Application = val 127 | } 128 | if val, ok := logParts["content"].(string); ok { 129 | e.Message = val 130 | } 131 | case syslog.RFC5424: 132 | if val, ok := logParts["timestamp"].(time.Time); ok { 133 | e.Timestamp = val 134 | } else { 135 | e.Timestamp = time.Now() 136 | } 137 | if val, ok := logParts["hostname"].(string); ok { 138 | e.Hostname = val 139 | } 140 | if val, ok := logParts["app_name"].(string); ok { 141 | e.Application = val 142 | } 143 | if val, ok := logParts["message"].(string); ok { 144 | e.Message = val 145 | } 146 | } 147 | return &e 148 | } 149 | -------------------------------------------------------------------------------- /frontend/ui.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "github.com/pierredavidbelanger/raftman/spi" 5 | "net/http" 6 | "net/url" 7 | "os" 8 | ) 9 | 10 | type uiFrontend struct { 11 | webFrontend 12 | api *apiFrontend 13 | } 14 | 15 | func newUIFrontend(e spi.LogEngine, frontendURL *url.URL) (*uiFrontend, error) { 16 | f := uiFrontend{} 17 | if err := initWebFrontend(e, frontendURL, &f.webFrontend); err != nil { 18 | return nil, err 19 | } 20 | f.api = &apiFrontend{} 21 | return &f, nil 22 | } 23 | 24 | func (f *uiFrontend) Start() error { 25 | _, b := f.e.GetBackend() 26 | f.api.b = b 27 | mux := http.NewServeMux() 28 | mux.HandleFunc(f.path+"api/stat", f.api.handleStat) 29 | mux.HandleFunc(f.path+"api/list", f.api.handleList) 30 | var useLocal bool 31 | if _, err := os.Stat("frontend/static/ui/index.html"); err == nil { 32 | useLocal = true 33 | } 34 | mux.Handle(f.path, http.FileServer(Dir(useLocal, "/frontend/static/ui"))) 35 | return f.startHandler(mux) 36 | } 37 | 38 | func (f *uiFrontend) Close() error { 39 | return f.close() 40 | } 41 | -------------------------------------------------------------------------------- /frontend/web.go: -------------------------------------------------------------------------------- 1 | package frontend 2 | 3 | import ( 4 | "fmt" 5 | "github.com/pierredavidbelanger/raftman/spi" 6 | "net" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | type webFrontend struct { 12 | e spi.LogEngine 13 | b spi.LogBackend 14 | addr string 15 | path string 16 | s *http.Server 17 | } 18 | 19 | func initWebFrontend(e spi.LogEngine, frontendURL *url.URL, f *webFrontend) error { 20 | f.e = e 21 | if frontendURL.Host == "" { 22 | return fmt.Errorf("Empty host in frontend URL '%s'", frontendURL) 23 | } 24 | f.addr = frontendURL.Host 25 | f.path = frontendURL.Path 26 | return nil 27 | } 28 | 29 | func (f *webFrontend) startHandler(h http.Handler) error { 30 | 31 | _, b := f.e.GetBackend() 32 | f.b = b 33 | 34 | f.s = &http.Server{Addr: f.addr, Handler: h} 35 | 36 | ln, err := net.Listen("tcp", f.addr) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | go f.s.Serve(ln) 42 | 43 | return nil 44 | } 45 | 46 | func (f *webFrontend) close() error { 47 | return f.s.Close() 48 | } 49 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/pierredavidbelanger/raftman 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/kr/pretty v0.1.0 // indirect 7 | github.com/mattn/go-sqlite3 v0.0.0-20170529145928-83772a7051f5 8 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect 9 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 10 | gopkg.in/mcuadros/go-syslog.v2 v2.2.1 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 2 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 3 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 4 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 5 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 6 | github.com/mattn/go-sqlite3 v0.0.0-20170529145928-83772a7051f5 h1:0lg46Uix1H9LcOKpoo0ris6lgHXyKB/QTptpmN5k0fE= 7 | github.com/mattn/go-sqlite3 v0.0.0-20170529145928-83772a7051f5/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 8 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 9 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 10 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 11 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 12 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 13 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 14 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/mcuadros/go-syslog.v2 v2.2.1 h1:60g8zx1BijSVSgLTzLCW9UC4/+i1Ih9jJ1DR5Tgp9vE= 16 | gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= 17 | -------------------------------------------------------------------------------- /raftman.go: -------------------------------------------------------------------------------- 1 | //go:generate $GOPATH/bin/esc -o frontend/static.go -pkg frontend frontend/static 2 | package main 3 | 4 | import ( 5 | "flag" 6 | "fmt" 7 | "github.com/pierredavidbelanger/raftman/engine" 8 | "log" 9 | "net/url" 10 | ) 11 | 12 | func main() { 13 | 14 | var frontendArgs URLValues 15 | var backendArgs URLValues 16 | 17 | flag.Var(&frontendArgs, "frontend", "Frontend URLs") 18 | flag.Var(&backendArgs, "backend", "Backend URL") 19 | 20 | flag.Parse() 21 | 22 | if len(backendArgs) == 0 { 23 | backendArgs = append(backendArgs, mustParseURL("sqlite:///var/lib/raftman/logs.db")) 24 | } else if len(backendArgs) > 1 { 25 | log.Fatal("At most one backend must be defined") 26 | } 27 | 28 | if len(frontendArgs) == 0 { 29 | frontendArgs = append(frontendArgs, mustParseURL("syslog+udp://:514")) 30 | frontendArgs = append(frontendArgs, mustParseURL("syslog+tcp://:5514")) 31 | frontendArgs = append(frontendArgs, mustParseURL("api+http://:8181/api/")) 32 | frontendArgs = append(frontendArgs, mustParseURL("ui+http://:8282/")) 33 | } 34 | 35 | e, err := engine.NewEngine(backendArgs[0], frontendArgs) 36 | if err != nil { 37 | log.Fatalf("Unable to create engine: %s", err) 38 | } 39 | 40 | if err = e.Start(); err != nil { 41 | log.Fatalf("Unable to start engine: %s", err) 42 | } 43 | defer e.Close() 44 | 45 | e.Wait() 46 | } 47 | 48 | type URLValues []*url.URL 49 | 50 | func (s *URLValues) String() string { 51 | return fmt.Sprintf("%+v", *s) 52 | } 53 | 54 | func (s *URLValues) Set(value string) error { 55 | parsed, err := url.Parse(value) 56 | if err != nil { 57 | return err 58 | } 59 | *s = append(*s, parsed) 60 | return nil 61 | } 62 | 63 | func mustParseURL(value string) *url.URL { 64 | parsed, err := url.Parse(value) 65 | if err != nil { 66 | log.Fatalf("Unable to parse URL: %s", err) 67 | } 68 | return parsed 69 | } 70 | -------------------------------------------------------------------------------- /spi/spi.go: -------------------------------------------------------------------------------- 1 | package spi 2 | 3 | import ( 4 | "github.com/pierredavidbelanger/raftman/api" 5 | "io" 6 | "net/url" 7 | ) 8 | 9 | type LogBackend interface { 10 | Start() error 11 | io.Closer 12 | Insert(*api.InsertRequest) (*api.InsertResponse, error) 13 | QueryStat(*api.QueryRequest) (*api.QueryStatResponse, error) 14 | QueryList(*api.QueryRequest) (*api.QueryListResponse, error) 15 | } 16 | 17 | type LogFrontend interface { 18 | Start() error 19 | io.Closer 20 | } 21 | 22 | type LogEngine interface { 23 | Start() error 24 | Wait() error 25 | io.Closer 26 | GetBackend() (*url.URL, LogBackend) 27 | GetFrontends() ([]*url.URL, []LogFrontend) 28 | } 29 | -------------------------------------------------------------------------------- /utils/retention.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | "time" 8 | ) 9 | 10 | type Retention time.Duration 11 | 12 | func (r Retention) String() string { 13 | if r < 0 { 14 | return "Infinite" 15 | } 16 | return time.Duration(r).String() 17 | } 18 | 19 | const INF Retention = Retention(-1) 20 | 21 | var retentionRE *regexp.Regexp = regexp.MustCompile(`^(INF)$|^(?:(\d+)(w))?(?:(\d+)(d))?(?:(\d+)(h))?(?:(\d+)(m))?$`) 22 | 23 | func ParseRetention(s string) (Retention, error) { 24 | if !retentionRE.MatchString(s) { 25 | return Retention(0), fmt.Errorf("invalid (INF|wdhm) duration '%s'", s) 26 | } 27 | sm := retentionRE.FindStringSubmatch(s) 28 | if sm[1] == "INF" { 29 | return INF, nil 30 | } 31 | var t Retention 32 | if sm[3] == "w" { 33 | n, _ := strconv.Atoi(sm[2]) 34 | t += Retention(n) * 7 * 24 * Retention(time.Hour) 35 | } 36 | if sm[5] == "d" { 37 | n, _ := strconv.Atoi(sm[4]) 38 | t += Retention(n) * 24 * Retention(time.Hour) 39 | } 40 | if sm[7] == "h" { 41 | n, _ := strconv.Atoi(sm[6]) 42 | t += Retention(n) * Retention(time.Hour) 43 | } 44 | if sm[9] == "m" { 45 | n, _ := strconv.Atoi(sm[8]) 46 | t += Retention(n) * Retention(time.Minute) 47 | } 48 | return t, nil 49 | } 50 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "gopkg.in/mcuadros/go-syslog.v2" 6 | "gopkg.in/mcuadros/go-syslog.v2/format" 7 | "net/url" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | func GetIntQueryParam(u *url.URL, name string, defaultValue int) (int, error) { 14 | s := u.Query().Get(name) 15 | if s == "" { 16 | return defaultValue, nil 17 | } 18 | return strconv.Atoi(s) 19 | } 20 | 21 | func GetDurationQueryParam(u *url.URL, name string, defaultValue time.Duration) (time.Duration, error) { 22 | s := u.Query().Get(name) 23 | if s == "" { 24 | return defaultValue, nil 25 | } 26 | return time.ParseDuration(s) 27 | } 28 | 29 | func GetRetentionQueryParam(u *url.URL, name string, defaultValue Retention) (Retention, error) { 30 | s := u.Query().Get(name) 31 | if s == "" { 32 | return defaultValue, nil 33 | } 34 | return ParseRetention(s) 35 | } 36 | 37 | func GetSyslogFormatQueryParam(u *url.URL, name string, defaultValue format.Format) (format.Format, error) { 38 | s := u.Query().Get(name) 39 | if s == "" { 40 | return defaultValue, nil 41 | } 42 | // TODO: must support them in syslog.toLogEntry 43 | switch strings.ToUpper(s) { 44 | case "RFC3164": 45 | return syslog.RFC3164, nil 46 | case "RFC5424": 47 | return syslog.RFC5424, nil 48 | //case "RFC6587": 49 | // return syslog.RFC6587, nil 50 | //case "AUTOMATIC": 51 | // return syslog.Automatic, nil 52 | } 53 | return nil, fmt.Errorf("Invalid syslog format %s", s) 54 | } 55 | --------------------------------------------------------------------------------