├── static ├── js │ ├── confirm.coffee │ ├── web_socket.coffee │ ├── view.coffee │ ├── ajax_search.coffee │ ├── init.js │ ├── live_logs.coffee │ └── hotkeys.js ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ ├── glyphicons-halflings-regular.woff2 │ └── glyphicons-halflings-regular.svg ├── bower.json ├── less │ ├── live_logs.less │ └── custom.less └── build │ ├── app.min.css │ └── app.min.js ├── screenshots ├── dashboard.png └── live-logs.png ├── .travis.yml ├── .gitignore ├── common ├── log_record.go ├── validation.go ├── cached_user.go ├── common.go ├── common_test.go ├── elastic_test.go ├── elastic.go └── user.go ├── web ├── routers │ ├── live │ │ └── live.go │ ├── projects │ │ ├── types.go │ │ └── projects.go │ ├── home │ │ ├── logs.go │ │ └── home.go │ ├── profile │ │ └── profile.go │ └── users │ │ ├── login.go │ │ └── register.go ├── templates │ ├── layouts │ │ ├── simple.tmpl │ │ └── main.tmpl │ ├── shared │ │ ├── error.tmpl │ │ └── head.tmpl │ ├── home │ │ ├── no_records.tmpl │ │ ├── empty_project.tmpl │ │ ├── index.tmpl │ │ └── table.tmpl │ ├── maintenance.tmpl │ ├── live │ │ └── index.tmpl │ ├── projects │ │ ├── types.tmpl │ │ ├── new.tmpl │ │ └── index.tmpl │ ├── profile │ │ └── index.tmpl │ └── users │ │ ├── register.tmpl │ │ └── login.tmpl ├── widgets │ ├── log_message.go │ ├── log_message_test.go │ └── pagination.go ├── middleware │ └── middleware.go ├── context │ └── context.go └── web.go ├── package.json ├── main.go ├── backend ├── tcp_server.go ├── http_server.go ├── per_second.go ├── backlog.go └── backend.go ├── LICENSE ├── Gruntfile.js ├── web_socket └── web_socket.go ├── commands └── commands.go └── readme.md /static/js/confirm.coffee: -------------------------------------------------------------------------------- 1 | $ -> 2 | $(".confirm").click -> 3 | confirm "Are you sure?" -------------------------------------------------------------------------------- /screenshots/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/screenshots/dashboard.png -------------------------------------------------------------------------------- /screenshots/live-logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/screenshots/live-logs.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.4 4 | script: 5 | - go get github.com/smartystreets/goconvey 6 | - go test -v ./... 7 | -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firstrow/logvoyage/HEAD/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | gin-bin 2 | node_modules 3 | static/bower_components 4 | backend/backend 5 | cmd/cmd 6 | client/client 7 | web/web 8 | web/tmp 9 | backend/tmp 10 | -------------------------------------------------------------------------------- /common/log_record.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type LogRecord struct { 8 | Datetime time.Time `json:"datetime"` 9 | Message string `json:"message"` 10 | } 11 | -------------------------------------------------------------------------------- /web/routers/live/live.go: -------------------------------------------------------------------------------- 1 | package live 2 | 3 | import ( 4 | "github.com/firstrow/logvoyage/web/context" 5 | ) 6 | 7 | func Index(ctx *context.Context) { 8 | ctx.HTML("live/index", context.ViewData{}, "layouts/simple") 9 | } 10 | -------------------------------------------------------------------------------- /web/templates/layouts/simple.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 || {{.Source.datetime | FormatTimeToHuman}} | 14 |15 | 16 | 17 | 18 | {{.Source | buildLogLine}} 19 | | 20 |
#{type}
" 39 | @scrollToBottom() if @autoScroll 40 | 41 | clear: -> 42 | @container.html '' 43 | @messages = [] 44 | 45 | scrollToBottom: -> 46 | @container.scrollTop @container.prop('scrollHeight') 47 | 48 | switchTheme: -> 49 | if @container.hasClass "dark" 50 | @setTheme "light" 51 | else 52 | @setTheme "dark" 53 | 54 | setTheme: (t) => 55 | if t == "dark" or t == "light" 56 | @container.removeClass().addClass(t) 57 | $.cookie("livelogstheme", t) 58 | 59 | _detectAutoScroll: (e) => 60 | @autoScroll = (@container.height() + @container.scrollTop()) == @container.prop('scrollHeight') 61 | 62 | _filter: (e) => 63 | wait = => 64 | @filter = $(e.target).val() 65 | @_filterAllMessages() 66 | setTimeout wait, 500 67 | 68 | _filterAllMessages: => 69 | @container.html '' 70 | for data in @messages 71 | @appendMessage data.type, data.message 72 | 73 | # Returns highlighted text if mached search 74 | # or false if not 75 | _filterMessage: (text) => 76 | re = new RegExp("(#{@filter})", 'ig') 77 | if text.match(re) 78 | return text.replace(re, '$1') 79 | false 80 | 81 | escapeHtml: (unsafe) => 82 | unsafe.replace(/&/g, "&") 83 | .replace(//g, ">") 85 | .replace(/"/g, """) 86 | .replace(/'/g, "'") 87 | -------------------------------------------------------------------------------- /common/elastic.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | 12 | "github.com/belogik/goes" 13 | ) 14 | 15 | const ( 16 | ES_HOST = "127.0.0.1" 17 | ES_PORT = "9200" 18 | ) 19 | 20 | var ( 21 | ErrSendingElasticSearchRequest = errors.New("Error sending request to ES.") 22 | ErrCreatingHttpRequest = errors.New("Could not create http.NewRequest") 23 | ErrReadResponse = errors.New("Could not read ES response") 24 | ErrDecodingJson = errors.New("Error decoding ES response") 25 | ) 26 | 27 | func GetConnection() *goes.Connection { 28 | return goes.NewConnection(ES_HOST, ES_PORT) 29 | } 30 | 31 | type IndexMapping map[string]map[string]map[string]interface{} 32 | 33 | // Retuns list of types available in search index 34 | func GetTypes(index string) ([]string, error) { 35 | var mapping IndexMapping 36 | result, err := SendToElastic(index+"/_mapping", "GET", []byte{}) 37 | if err != nil { 38 | return nil, ErrSendingElasticSearchRequest 39 | } 40 | err = json.Unmarshal([]byte(result), &mapping) 41 | if err != nil { 42 | return nil, ErrDecodingJson 43 | } 44 | keys := []string{} 45 | for k := range mapping[index]["mappings"] { 46 | keys = append(keys, k) 47 | } 48 | return keys, nil 49 | } 50 | 51 | // Count documents in collection 52 | func CountTypeDocs(index string, logType string) float64 { 53 | result, err := SendToElastic(fmt.Sprintf("%s/%s/_count", index, logType), "GET", nil) 54 | if err != nil { 55 | return 0 56 | } 57 | 58 | var m map[string]interface{} 59 | err = json.Unmarshal([]byte(result), &m) 60 | if err != nil { 61 | return 0 62 | } 63 | return m["count"].(float64) 64 | } 65 | 66 | func DeleteType(index string, logType string) { 67 | _, err := SendToElastic(fmt.Sprintf("%s/%s", index, logType), "DELETE", nil) 68 | if err != nil { 69 | log.Println(err.Error()) 70 | } 71 | } 72 | 73 | // Send raw bytes to elastic search server 74 | // TODO: Bulk processing 75 | func SendToElastic(url string, method string, b []byte) (string, error) { 76 | eurl := fmt.Sprintf("http://%s:%s/%s", ES_HOST, ES_PORT, url) 77 | 78 | req, err := http.NewRequest(method, eurl, bytes.NewBuffer(b)) 79 | if err != nil { 80 | return "", err 81 | } 82 | 83 | client := &http.Client{} 84 | resp, err := client.Do(req) 85 | if err != nil { 86 | return "", ErrSendingElasticSearchRequest 87 | } 88 | defer resp.Body.Close() 89 | 90 | // Read body to close connection 91 | // If dont read body golang will keep connection open 92 | r, err := ioutil.ReadAll(resp.Body) 93 | if err != nil { 94 | return "", ErrReadResponse 95 | } 96 | 97 | return string(r), nil 98 | } 99 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function(grunt) { 2 | 3 | grunt.initConfig({ 4 | pkg: grunt.file.readJSON('package.json'), 5 | concat: { 6 | options: { 7 | separator: "\n" 8 | }, 9 | javascripts: { 10 | src: [ 11 | 'static/bower_components/jquery/dist/jquery.min.js', 12 | 'static/bower_components/bootstrap/dist/js/bootstrap.min.js', 13 | 'static/bower_components/jquery-jsonview/dist/jquery.jsonview.js', 14 | 'static/bower_components/jquery-cookie/jquery.cookie.js', 15 | 'static/bower_components/datetimepicker/jquery.datetimepicker.js', 16 | 'static/bower_components/bootstrap-multiselect/dist/js/bootstrap-multiselect.js', 17 | 'static/bower_components/ladda-bootstrap/dist/spin.min.js', 18 | 'static/bower_components/ladda-bootstrap/dist/ladda.min.js', 19 | 'static/bower_components/jquery.hotkeys/jquery.hotkeys.js', 20 | 'static/bower_components/sockjs-client/dist/sockjs.js', 21 | 'static/bower_components/pubsub-js/src/pubsub.js', 22 | 'static/bower_components/bootstrap-select/dist/js/bootstrap-select.min.js', 23 | 'static/js/*.js', 24 | ], 25 | dest: 'static/build/all.min.js' 26 | }, 27 | css: { 28 | src: [ 29 | 'static/bower_components/bootstrap/dist/css/bootstrap.css', 30 | 'static/bower_components/jquery-jsonview/dist/jquery.jsonview.css', 31 | 'static/bower_components/datetimepicker/jquery.datetimepicker.css', 32 | 'static/bower_components/bootstrap-multiselect/dist/css/bootstrap-multiselect.css', 33 | 'static/bower_components/epoch/epoch.min.css', 34 | 'static/bower_components/chosen/chosen.min.css', 35 | 'static/bower_components/ladda-bootstrap/dist/ladda-themeless.min.css', 36 | 'static/bower_components/bootstrap-select/dist/css/bootstrap-select.min.css', 37 | ], 38 | dest: 'static/build/all.min.css' 39 | } 40 | }, 41 | less: { 42 | development: { 43 | files: { 44 | "static/build/app.min.css": "static/less/*.less" 45 | } 46 | } 47 | }, 48 | coffee: { 49 | compileJoined: { 50 | options: { 51 | join: true 52 | }, 53 | files: { 54 | 'static/build/app.min.js': ['static/js/*.coffee'] 55 | } 56 | } 57 | }, 58 | watch: { 59 | scripts: { 60 | files: ['static/js/*.*'], 61 | tasks: ['js'], 62 | options: { 63 | spawn: false 64 | } 65 | }, 66 | css: { 67 | files: ['static/less/*.less'], 68 | tasks: ['css'], 69 | options: { 70 | spawn: false 71 | } 72 | } 73 | } 74 | }); 75 | 76 | grunt.loadNpmTasks('grunt-contrib-concat'); 77 | grunt.loadNpmTasks('grunt-contrib-coffee'); 78 | grunt.loadNpmTasks('grunt-contrib-less'); 79 | grunt.loadNpmTasks('grunt-contrib-watch'); 80 | 81 | // Default task(s). 82 | grunt.registerTask('default', ['all']); 83 | grunt.registerTask('all', ['js', 'css']); 84 | grunt.registerTask('js', ['concat:javascripts', 'coffee']); 85 | grunt.registerTask('css', ['concat:css', 'less']); 86 | 87 | }; 88 | -------------------------------------------------------------------------------- /static/build/app.min.css: -------------------------------------------------------------------------------- 1 | .navbar-inverse { 2 | background-color: #3d4a57; 3 | border-color: #333; 4 | } 5 | .navbar-inverse .navbar-nav > li > a, 6 | .navbar .navbar-brand { 7 | color: #fbfbfb; 8 | text-decoration: none; 9 | } 10 | .main { 11 | font-size: 12px; 12 | } 13 | #logstable td { 14 | word-wrap: break-word; 15 | word-break: break-all; 16 | font-size: 12px; 17 | vertical-align: text-top; 18 | } 19 | #logstable tr:hover td { 20 | cursor: pointer; 21 | background: #E3F2FD; 22 | } 23 | #pagination { 24 | margin-left: 10px; 25 | } 26 | .error { 27 | color: red; 28 | } 29 | .no-padding { 30 | padding: 0; 31 | } 32 | #time_start, 33 | #time_stop { 34 | width: 130px; 35 | text-align: center; 36 | border: 1px; 37 | } 38 | .timebox { 39 | display: none; 40 | outline: none; 41 | font-size: 12px; 42 | } 43 | h2 .actions { 44 | padding-right: 15px; 45 | padding-top: 5px; 46 | font-size: 14px; 47 | } 48 | .search_info { 49 | padding: 8px 8px 8px 5px; 50 | background-color: #cfd8dc; 51 | } 52 | #searchBox { 53 | -webkit-box-shadow: none; 54 | -moz-box-shadow: none; 55 | box-shadow: none; 56 | border: 1px solid #3e3f3a; 57 | background: #2f302c; 58 | color: #e0d8c6; 59 | width: 500px; 60 | } 61 | #searchBox:focus { 62 | border-top: 2px solid #2765A1; 63 | border-left: 2px solid #2765A1; 64 | border-bottom: 2px solid #2765A1; 65 | } 66 | .jsonview { 67 | font-size: 13px; 68 | word-wrap: break-word; 69 | } 70 | a.view { 71 | text-decoration: none; 72 | } 73 | .list_actions { 74 | position: absolute; 75 | right: 40px; 76 | z-index: 100; 77 | display: none; 78 | } 79 | .list_actions a { 80 | text-decoration: none; 81 | } 82 | .well:hover .list_actions { 83 | display: block; 84 | } 85 | .top50px { 86 | margin-top: 50px; 87 | } 88 | /* Bootstrap overrides */ 89 | .modal.modal-wide .modal-dialog { 90 | width: 950px; 91 | } 92 | .modal-wide .modal-body { 93 | overflow-y: auto; 94 | } 95 | .modal-backdrop { 96 | height: 100%; 97 | z-index: 1030; 98 | } 99 | a#current-project { 100 | color: #c4c4c4; 101 | cursor: default; 102 | } 103 | 104 | #liveLogsContainer { 105 | width: 100%; 106 | overflow: auto; 107 | } 108 | #liveLogsContainer p { 109 | margin: 0 0 0 5px; 110 | padding: 0; 111 | font-size: 12px; 112 | word-wrap: break-word; 113 | } 114 | #liveLogsContainer span.highlight { 115 | background-color: #a57705; 116 | color: #000; 117 | } 118 | #liveLogsContainer .type { 119 | padding-right: 5px; 120 | } 121 | #liveLogsContainer.dark { 122 | background: #1d1d1d; 123 | color: #859900; 124 | } 125 | #liveLogsContainer.dark .type { 126 | color: #268bd2; 127 | } 128 | #liveLogsContainer.light { 129 | background: #fdf5db; 130 | color: #1f9283; 131 | } 132 | #liveLogsContainer.light .type { 133 | color: #268bd2; 134 | } 135 | #liveLogsSearch { 136 | position: fixed; 137 | float: auto; 138 | width: 300px; 139 | bottom: 0; 140 | width: 100%; 141 | background: #171717; 142 | padding: 3px; 143 | } 144 | #liveLogsSearch form input[type=text] { 145 | background: #222222; 146 | border: 0; 147 | outline: none; 148 | width: 500px; 149 | color: #999999; 150 | } 151 | #liveLogsSearch form button { 152 | background: transparent; 153 | color: #999999; 154 | outline: none; 155 | } 156 | #liveLogsSearch form button:hover { 157 | color: #fff; 158 | } 159 | -------------------------------------------------------------------------------- /web/widgets/pagination.go: -------------------------------------------------------------------------------- 1 | package widgets 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | _ "log" 7 | "math" 8 | "net/http" 9 | "net/url" 10 | "strconv" 11 | ) 12 | 13 | // Help to paginate elastic search 14 | type pagination struct { 15 | totalRecords uint64 16 | perPage int 17 | padding int // How many links display before and after current 18 | req *http.Request 19 | } 20 | 21 | func (this *pagination) SetTotalRecords(total uint64) { 22 | this.totalRecords = total 23 | } 24 | 25 | func (this *pagination) GetTotalRecords() uint64 { 26 | return this.totalRecords 27 | } 28 | 29 | func (this *pagination) GetTotalPages() int { 30 | r := float64(this.GetTotalRecords()) / float64(this.GetPerPage()) 31 | return int(math.Ceil(r)) 32 | } 33 | 34 | func (this *pagination) SetPerPage(limit int) { 35 | this.perPage = limit 36 | } 37 | 38 | func (this *pagination) GetPerPage() int { 39 | return this.perPage 40 | } 41 | 42 | // Detect `from` number for elastic 43 | func (this *pagination) DetectFrom() int { 44 | page, _ := strconv.Atoi(this.req.URL.Query().Get("p")) 45 | if page > 1 { 46 | return this.GetPerPage()*page - this.GetPerPage() 47 | } 48 | return 0 49 | } 50 | 51 | // Get current page number 52 | func (this *pagination) GetPageNumber() int { 53 | number, err := strconv.Atoi(this.req.URL.Query().Get("p")) 54 | if err != nil { 55 | return 1 56 | } else { 57 | return number 58 | } 59 | } 60 | 61 | // Checks if pagination has one or more pages 62 | func (this *pagination) HasPages() bool { 63 | return this.GetTotalPages() > 1 64 | } 65 | 66 | func (this *pagination) Render() template.HTML { 67 | currentPage := this.GetPageNumber() 68 | startPage := currentPage - this.padding 69 | 70 | if startPage < 1 { 71 | startPage = 1 72 | } 73 | stopPage := startPage + (this.padding * 2) - 1 74 | 75 | if stopPage > this.GetTotalPages() { 76 | stopPage = this.GetTotalPages() 77 | } 78 | 79 | prevPage := currentPage - 1 80 | nextPage := currentPage + 1 81 | var next string 82 | var prev string 83 | if prevPage < 1 { 84 | prevPage = 1 85 | prev = fmt.Sprintf("
15 | * 
16 |
17 |
18 |
19 |
20 | **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
21 |
22 | - [Installation](#installation)
23 | - [Pre-Requirements.](#pre-requirements)
24 | - [Installing](#installing)
25 | - [Usage](#usage)
26 | - [Sending data to storage](#sending-data-to-storage)
27 | - [Telnet](#telnet)
28 | - [Curl](#curl)
29 | - [Search data](#search-data)
30 | - [Third-party clients](#third-party-clients)
31 | - [Submitting a Pull Request](#submitting-a-pull-request)
32 | - [Front-end development](#front-end-development)
33 | - [Bower](#bower)
34 | - [Building](#building)
35 | - [Auto rebuild](#auto-rebuild)
36 | - [WebSocket messages](#websocket-messages)
37 | - [Roadmap v0.1](#roadmap-v01)
38 | - [License](#license)
39 |
40 |
41 |
42 | ## Installation
43 |
44 | ### Pre-Requirements.
45 | - [ElasticSearch](https://gist.github.com/firstrow/f57bc873cfd6839b6ea8)
46 | - [Redis](http://redis.io/topics/quickstart)
47 |
48 | ### Installing
49 | Installing LogVoyage is as easy as installing any other go package:
50 | ``` bash
51 | go get github.com/firstrow/logvoyage
52 | logvoyage create_users_index
53 | ```
54 |
55 | ## Usage
56 | Once you installed LogVoyage you need to start backend and web servers.
57 | ``` bash
58 | logvoyage start-all
59 | ```
60 | Or you can start/stop servers separately
61 | ``` bash
62 | logvoyage backend
63 | logvoyage web
64 | ```
65 | Once server started you can access it at [http://localhost:3000](http://localhost:3000).
66 | Execute `logvoyage help` for more info about available commands.
67 |
68 | ### Sending data to storage
69 | By default LogVoyage opens two backend ports accesible to the outsise world.
70 |
71 | 1. 27077 - TCP port
72 | 2. 27078 - HTTP port
73 |
74 | #### Telnet
75 |
76 | ```
77 | NOTE: Keep in mind to change `API_KEY` and `LOG_TYPE`.
78 | You can find your api key at http://localhost:3000/profile page.
79 | ```
80 |
81 | ``` bash
82 | telnet 127.0.0.1 27077
83 | API_KEY@LOG_TYPE {"message": "login", "user_id": 1}
84 | API_KEY@LOG_TYPE simple text message
85 | ```
86 |
87 | Now you can see your messages at http://localhost:3000 and try some queries
88 |
89 | #### Curl
90 |
91 | Or we can use curl POST request to send messages. Each message should be separated by new line.
92 |
93 | ``` bash
94 | echo 'This is simple text message' | curl -d @- http://localhost:27078/bulk\?apiKey\=API_KEY\&type\=LOG_TYPE
95 | echo '{"message": "JSON format also supported", "action":"test"}' | curl -d @- http://localhost:27078/bulk\?apiKey\=API_KEY\&type\=LOG_TYPE
96 | ```
97 |
98 | ## Search data
99 | Refer to [Query String Syntax](http://www.elastic.co/guide/en/elasticsearch/reference/1.x/query-dsl-query-string-query.html#query-string-syntax)
100 | for more info about text queries available.
101 |
102 | Examples:
103 |
104 | ``` bash
105 | user_id:1
106 | simple*
107 | amount:>10 and status:completed
108 | ```
109 |
110 | ## Third-party clients
111 | If you know any programming language, you can join our project and implement
112 | LogVoyage client.
113 |
114 | ## Submitting a Pull Request
115 |
116 | 1. Propose a change by opening an issue.
117 | 2. Fork the project.
118 | 3. Create a topic branch.
119 | 4. Implement your feature or bug fix.
120 | 5. Commit and push your changes.
121 | 6. Submit a pull request.
122 |
123 | ## Front-end development
124 | ### Bower
125 | To manage 3rd-party libraries simply add it to static/bower.json and run
126 | ```
127 | bower install
128 | ```
129 |
130 | ### Building
131 | We are using grunt to build project js and css files.
132 | Execute next commands to setup environment:
133 | ```
134 | npm install
135 | grunt
136 | ```
137 | After grunt is done, you can find result files in static/build directory.
138 |
139 | ### Auto rebuild
140 | To automatically rebuild js, css, coffee, less files simply run in console
141 | ```
142 | grunt watch
143 | ```
144 |
145 | ### WebSocket messages
146 | ``` coffee
147 | // Sample coffescript code
148 | PubSub.subscribe "log_message", (type, data) ->
149 | console.log data.message
150 | ```
151 |
152 | Sample messages:
153 |
154 | ``` json
155 | {
156 | "type": "log_message",
157 | "log_type": "nginx_access",
158 | "message": "test received log message goes here..."
159 | }
160 | ```
161 |
162 | ``` json
163 | {
164 | "type": "logs_per_second",
165 | "count": 5
166 | }
167 | ```
168 |
169 | ## Roadmap v0.1
170 | - Daemons
171 | - Zero-downtime deployment
172 | - Finish web ui
173 | - Docker image
174 | - Docs
175 |
176 | ## License
177 | LogVoyage is available without any costs under an MIT license. See LICENSE file
178 | for details.
179 |
--------------------------------------------------------------------------------
/web/routers/home/home.go:
--------------------------------------------------------------------------------
1 | package home
2 |
3 | import (
4 | "errors"
5 | "net/http"
6 | "net/url"
7 | "strconv"
8 | "time"
9 |
10 | "github.com/belogik/goes"
11 | "github.com/go-martini/martini"
12 |
13 | "github.com/firstrow/logvoyage/common"
14 | "github.com/firstrow/logvoyage/web/context"
15 | "github.com/firstrow/logvoyage/web/widgets"
16 | )
17 |
18 | const (
19 | timeLayout = "2006/01/02 15:04" // Users input time format
20 | perPage = 100
21 | )
22 |
23 | type DateTimeRange struct {
24 | Start string
25 | Stop string
26 | }
27 |
28 | func (this *DateTimeRange) IsValid() bool {
29 | return this.Start != "" || this.Stop != ""
30 | }
31 |
32 | // Represents search request to perform in ES
33 | type SearchRequest struct {
34 | Text string // test to search
35 | Indexes []string // ES indexeses to perform search
36 | Types []string // search types
37 | Size int // home much objects ES must return
38 | From int // how much objects should ES skip from first
39 | TimeRange DateTimeRange
40 | }
41 |
42 | func buildSearchRequest(text string, indexes []string, types []string, size int, from int, datetime DateTimeRange) SearchRequest {
43 | return SearchRequest{
44 | Text: text,
45 | Indexes: indexes,
46 | From: from,
47 | Types: types,
48 | Size: perPage,
49 | TimeRange: datetime,
50 | }
51 | }
52 |
53 | // Detects time range from request and returns
54 | // elastic compatible format string
55 | func buildTimeRange(req *http.Request) DateTimeRange {
56 | var timeRange DateTimeRange
57 |
58 | switch req.URL.Query().Get("time") {
59 | case "15m":
60 | timeRange.Start = "now-15m"
61 | case "30m":
62 | timeRange.Start = "now-30m"
63 | case "60m":
64 | timeRange.Start = "now-60m"
65 | case "12h":
66 | timeRange.Start = "now-12h"
67 | case "24h":
68 | timeRange.Start = "now-24h"
69 | case "week":
70 | timeRange.Start = "now-7d"
71 | case "custom":
72 | timeStart, err := time.Parse(timeLayout, req.URL.Query().Get("time_start"))
73 | if err == nil {
74 | timeRange.Start = timeStart.Format(time.RFC3339)
75 | }
76 | timeStop, err := time.Parse(timeLayout, req.URL.Query().Get("time_stop"))
77 | if err == nil {
78 | timeRange.Stop = timeStop.Format(time.RFC3339)
79 | }
80 | }
81 |
82 | return timeRange
83 | }
84 |
85 | // Search logs in elastic.
86 | func search(searchRequest SearchRequest) (goes.Response, error) {
87 | conn := common.GetConnection()
88 |
89 | var query = map[string]interface{}{
90 | "from": searchRequest.From,
91 | "size": searchRequest.Size,
92 | "sort": map[string]string{
93 | "datetime": "desc",
94 | },
95 | }
96 |
97 | if len(searchRequest.Text) > 0 {
98 | strconv.Quote(searchRequest.Text)
99 | query["query"] = map[string]interface{}{
100 | "query_string": map[string]string{
101 | "default_field": "message",
102 | "query": searchRequest.Text,
103 | },
104 | }
105 | }
106 |
107 | // Build time range query
108 | if searchRequest.TimeRange.IsValid() {
109 | datetime := make(map[string]string)
110 | if searchRequest.TimeRange.Start != "" {
111 | datetime["gte"] = searchRequest.TimeRange.Start
112 | }
113 | if searchRequest.TimeRange.Stop != "" {
114 | datetime["lte"] = searchRequest.TimeRange.Stop
115 | }
116 | query["filter"] = map[string]interface{}{
117 | "range": map[string]interface{}{
118 | "datetime": datetime,
119 | },
120 | }
121 | }
122 |
123 | extraArgs := make(url.Values, 0)
124 | searchResults, err := conn.Search(query, searchRequest.Indexes, searchRequest.Types, extraArgs)
125 |
126 | if err != nil {
127 | return goes.Response{}, errors.New("No records found.")
128 | } else {
129 | return *searchResults, nil
130 | }
131 | }
132 |
133 | // This function handles two routes "/" and "/project/:id"
134 | func ProjectSearch(ctx *context.Context, params martini.Params) {
135 | var types []string
136 | var project *common.Project
137 |
138 | query_text := ctx.Request.URL.Query().Get("q")
139 | selected_types := ctx.Request.URL.Query()["types"]
140 |
141 | // Project scope
142 | if _, err := params["id"]; err {
143 | project, err := ctx.User.GetProject(params["id"])
144 | if err != nil {
145 | ctx.HTML("shared/error", context.ViewData{
146 | "message": "Project not found",
147 | })
148 | return
149 | }
150 | if len(project.Types) == 0 {
151 | ctx.HTML("home/empty_project", context.ViewData{
152 | "project": project,
153 | })
154 | return
155 | }
156 | if len(selected_types) > 0 {
157 | types = selected_types
158 | } else {
159 | types = project.Types
160 | }
161 | }
162 |
163 | // Pagination
164 | pagination := widgets.NewPagination(ctx.Request)
165 | pagination.SetPerPage(perPage)
166 |
167 | // Load records
168 | searchRequest := buildSearchRequest(
169 | query_text,
170 | []string{ctx.User.GetIndexName()},
171 | types,
172 | pagination.GetPerPage(),
173 | pagination.DetectFrom(),
174 | buildTimeRange(ctx.Request),
175 | )
176 | // Search data in elastic
177 | data, _ := search(searchRequest)
178 |
179 | pagination.SetTotalRecords(data.Hits.Total)
180 |
181 | var viewName string
182 | viewData := context.ViewData{
183 | "project": project,
184 | "logs": data.Hits.Hits,
185 | "total": data.Hits.Total,
186 | "took": data.Took,
187 | "types": types,
188 | "time": ctx.Request.URL.Query().Get("time"),
189 | "time_start": ctx.Request.URL.Query().Get("time_start"),
190 | "time_stop": ctx.Request.URL.Query().Get("time_stop"),
191 | "query_text": query_text,
192 | "pagination": pagination,
193 | }
194 |
195 | if ctx.Request.Header.Get("X-Requested-With") == "XMLHttpRequest" {
196 | viewName = "home/table"
197 | } else {
198 | viewName = "home/index"
199 | }
200 |
201 | ctx.HTML(viewName, viewData)
202 | }
203 |
--------------------------------------------------------------------------------
/web/templates/layouts/main.tmpl:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | " + type + "
"); 98 | } 99 | if (this.autoScroll) { 100 | return this.scrollToBottom(); 101 | } 102 | }; 103 | 104 | LiveLogs.prototype.clear = function() { 105 | this.container.html(''); 106 | return this.messages = []; 107 | }; 108 | 109 | LiveLogs.prototype.scrollToBottom = function() { 110 | return this.container.scrollTop(this.container.prop('scrollHeight')); 111 | }; 112 | 113 | LiveLogs.prototype.switchTheme = function() { 114 | if (this.container.hasClass("dark")) { 115 | return this.setTheme("light"); 116 | } else { 117 | return this.setTheme("dark"); 118 | } 119 | }; 120 | 121 | LiveLogs.prototype.setTheme = function(t) { 122 | if (t === "dark" || t === "light") { 123 | this.container.removeClass().addClass(t); 124 | return $.cookie("livelogstheme", t); 125 | } 126 | }; 127 | 128 | LiveLogs.prototype._detectAutoScroll = function(e) { 129 | return this.autoScroll = (this.container.height() + this.container.scrollTop()) === this.container.prop('scrollHeight'); 130 | }; 131 | 132 | LiveLogs.prototype._filter = function(e) { 133 | var wait; 134 | wait = (function(_this) { 135 | return function() { 136 | _this.filter = $(e.target).val(); 137 | return _this._filterAllMessages(); 138 | }; 139 | })(this); 140 | return setTimeout(wait, 500); 141 | }; 142 | 143 | LiveLogs.prototype._filterAllMessages = function() { 144 | var data, _i, _len, _ref, _results; 145 | this.container.html(''); 146 | _ref = this.messages; 147 | _results = []; 148 | for (_i = 0, _len = _ref.length; _i < _len; _i++) { 149 | data = _ref[_i]; 150 | _results.push(this.appendMessage(data.type, data.message)); 151 | } 152 | return _results; 153 | }; 154 | 155 | LiveLogs.prototype._filterMessage = function(text) { 156 | var re; 157 | re = new RegExp("(" + this.filter + ")", 'ig'); 158 | if (text.match(re)) { 159 | return text.replace(re, '$1'); 160 | } 161 | return false; 162 | }; 163 | 164 | LiveLogs.prototype.escapeHtml = function(unsafe) { 165 | return unsafe.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """).replace(/'/g, "'"); 166 | }; 167 | 168 | return LiveLogs; 169 | 170 | })(); 171 | 172 | $(function() { 173 | return $("body").on("click", "a.view", function(e) { 174 | var el; 175 | e.preventDefault(); 176 | el = this; 177 | $("#recordViewLabel").html($(this).data("type")); 178 | $("#recordViewDateTime").html($(this).data("datetime")); 179 | $("#viewRecordModal .btn-danger").unbind("click").click(function() { 180 | if (confirm("Are you sure want to delete this event?")) { 181 | return $.ajax({ 182 | url: $(el).attr("href"), 183 | type: "DELETE", 184 | success: function() { 185 | $(".modal .close").click(); 186 | return $(el).parents("tr").css("opacity", "0.2"); 187 | }, 188 | error: function() { 189 | return alert("Error: Record not deleted."); 190 | } 191 | }); 192 | } else { 193 | return e.preventDefault(); 194 | } 195 | }); 196 | return $.getJSON($(this).attr("href"), function(data) { 197 | $(".modal-body").JSONView(data); 198 | return $("#viewRecordModal").modal(); 199 | }).fail(function() { 200 | $(".modal-body").html("Error: Record not found or wrong JSON structure."); 201 | return $("#viewRecordModal").modal(); 202 | }); 203 | }); 204 | }); 205 | 206 | WSocket = (function() { 207 | function WSocket(apiKey) { 208 | this.apiKey = apiKey; 209 | this.ws = new WebSocket("ws://" + window.location.hostname + ":12345/ws"); 210 | this.ws.onopen = ((function(_this) { 211 | return function() { 212 | return _this.register(); 213 | }; 214 | })(this)); 215 | this.ws.onmessage = ((function(_this) { 216 | return function() { 217 | return _this.onMessage(event); 218 | }; 219 | })(this)); 220 | } 221 | 222 | WSocket.prototype.register = function() { 223 | this.ws.send(this.apiKey); 224 | return console.log("registered user " + this.apiKey); 225 | }; 226 | 227 | WSocket.prototype.onMessage = function(event) { 228 | var data; 229 | data = JSON.parse(event.data); 230 | return PubSub.publish(data.type, data); 231 | }; 232 | 233 | return WSocket; 234 | 235 | })(); 236 | 237 | $(function() { 238 | return new WSocket(options.apiKey); 239 | }); 240 | 241 | }).call(this); 242 | -------------------------------------------------------------------------------- /static/fonts/glyphicons-halflings-regular.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | --------------------------------------------------------------------------------