├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── benchmark ├── blhistory ├── browser_like_history.go └── browser_like_history_test.go ├── clhistory └── clhistory.go ├── cmd ├── generate_syslog │ ├── README.md │ └── main.go └── nerdlog-tui │ ├── app.go │ ├── cmdline.go │ ├── column_edit_view.go │ ├── config.go │ ├── from_to_range.go │ ├── histogram.go │ ├── histogram_test.go │ ├── main.go │ ├── main_view.go │ ├── menu.go │ ├── message_view.go │ ├── my_text_view.go │ ├── options.go │ ├── query.go │ ├── query_edit_view.go │ ├── row_details_view.go │ ├── rune_buffer.go │ ├── select_query.go │ ├── select_query_test.go │ ├── time_or_dur.go │ ├── ui │ ├── dropdown.go │ ├── table_with_dropdown.go │ └── util.go │ ├── xmarks.go │ └── xmarks_test.go ├── core ├── config.go ├── core.go ├── lstream_client.go ├── lstream_cmd.go ├── lstreams_manager.go ├── lstreams_resolver.go ├── lstreams_resolver_test.go ├── nerdlog_agent.sh ├── nerdlog_agent_test.go ├── nerdlog_agent_testdata │ ├── logfiles │ │ ├── single_file │ │ │ └── syslog │ │ ├── small_dec_jan │ │ │ ├── syslog │ │ │ └── syslog.1 │ │ ├── small_mar │ │ │ ├── syslog │ │ │ └── syslog.1 │ │ └── tiny │ │ │ ├── syslog │ │ │ └── syslog.1 │ └── test_cases │ │ ├── all_existing_logs │ │ ├── 01_from_is_set_to_is_set │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── 01_from_is_set_to_is_unset │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── 01_from_is_unset_to_is_set │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 01_from_is_unset_to_is_unset │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── edge_of_two_fles │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── from_the_beginning_of_prev_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── in_the_middle_latest_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── in_the_middle_of_prev_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── latest_logs_same_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── 02_basic_more_full_amount │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── 04_basic_more_no_more_logs │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 10_to_is_specified_and_is_in_the_future │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── latest_logs_same_file_pattern1 │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── latest_logs_same_file_pattern2 │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── query_range_is_outside │ │ ├── from_is_after_to_is_even_further │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── from_is_after_to_is_unset │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── to_is_before_from_is_even_earlier │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── to_is_before_from_is_unset │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── second_log_file_doesnt_exist │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 02_oldest_logs │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── whole_latest_middle_prev_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ ├── whole_latest_whole_prev_file │ │ ├── 01_basic │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── 03_basic_more_less_than_max │ │ │ ├── test_case.yaml │ │ │ ├── want_stderr │ │ │ └── want_stdout │ │ └── year_infer_edge_of_two_years │ │ ├── 01_logs_in_the_past_cur_apr │ │ ├── test_case.yaml │ │ ├── want_stderr │ │ └── want_stdout │ │ └── 01_logs_in_the_past_cur_jan │ │ ├── test_case.yaml │ │ ├── want_stderr │ │ └── want_stdout ├── parsing_time.go ├── parsing_time_test.go └── resolver_testdata │ └── ssh_config_1 ├── go.mod ├── go.sum ├── images ├── nerdlog_demo.gif ├── nerdlog_intro.png └── nerdlog_query_edit_form.png ├── log └── log.go ├── shellescape ├── shell_escape.go └── shell_escape_test.go └── util ├── copy_test_results.sh └── sysloggen └── generate_syslog.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | cmd/nerdlog-tui/nerdlog-tui 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2025 Dmitry Frank 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: clean nerdlog 2 | 3 | .PHONY: nerdlog 4 | nerdlog: 5 | go build -o bin/nerdlog ./cmd/nerdlog-tui 6 | 7 | .PHONY: clean 8 | clean: 9 | rm -rf bin 10 | 11 | PREFIX ?= /usr/local 12 | DESTDIR ?= 13 | BINDIR := $(DESTDIR)$(PREFIX)/bin 14 | INSTALL := install 15 | INSTALL_FLAGS := -m 755 16 | 17 | .PHONY: install 18 | install: 19 | $(INSTALL) $(INSTALL_FLAGS) -D bin/nerdlog $(BINDIR)/nerdlog 20 | 21 | test: 22 | # The tests run rather slow so we use "-v -p 1" so that we get the unbuffered 23 | # output. 24 | go test ./... -count 1 -v -p 1 25 | 26 | bench: 27 | # The -run=^$ is needed to avoid running the regular tests as well. 28 | go test ./core -bench=BenchmarkNerdlogAgent -benchtime=3s -run=^$ 29 | -------------------------------------------------------------------------------- /benchmark: -------------------------------------------------------------------------------- 1 | go test ./core -bench=BenchmarkNerdlogAgent -benchtime=3s -run=^$ 2 | goos: linux 3 | goarch: amd64 4 | pkg: github.com/dimonomid/nerdlog/core 5 | cpu: 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz 6 | BenchmarkNerdlogAgentSmallLogNoIndex-3 74 54123443 ns/op 7 | BenchmarkNerdlogAgentSmallLogCompleteIndex-3 151 22797961 ns/op 8 | BenchmarkNerdlogAgentLargeLogSmallPortionNoIndex-3 2 1750335450 ns/op 9 | BenchmarkNerdlogAgentLargeLogSmallPortionCompleteIndex-3 20 178455823 ns/op 10 | BenchmarkNerdlogAgentLargeLogTinyPortionCompleteIndex-3 49 80060471 ns/op 11 | PASS 12 | -------------------------------------------------------------------------------- /blhistory/browser_like_history.go: -------------------------------------------------------------------------------- 1 | package blhistory 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // BLHistory is a browser-like history: we can add more items to the history; 8 | // we can go back and forth; when we're a few items back and we add a new item, 9 | // a new item is added at this place in the history and all the previously existing 10 | // newer items are dropped; there is no persistence. 11 | // 12 | // A cache can be added to every history item too, but that's a TODO. 13 | type BLHistory struct { 14 | items []Item 15 | 16 | curIdx int 17 | } 18 | 19 | type Item struct { 20 | Time time.Time 21 | 22 | Str string 23 | } 24 | 25 | func New() *BLHistory { 26 | h := &BLHistory{} 27 | 28 | return h 29 | } 30 | 31 | func (h *BLHistory) Add(s string) { 32 | item := Item{ 33 | Time: time.Now(), 34 | Str: s, 35 | } 36 | 37 | if len(h.items) > 0 && h.curIdx < len(h.items)-1 { 38 | h.items = h.items[:h.curIdx+1] 39 | } 40 | 41 | h.items = append(h.items, item) 42 | h.curIdx = len(h.items) - 1 43 | } 44 | 45 | func (h *BLHistory) Prev() *Item { 46 | if h.curIdx == 0 { 47 | return nil 48 | } 49 | 50 | h.curIdx-- 51 | 52 | item := h.items[h.curIdx] 53 | return &item 54 | } 55 | 56 | func (h *BLHistory) Next() *Item { 57 | if h.curIdx >= len(h.items)-1 { 58 | return nil 59 | } 60 | 61 | h.curIdx++ 62 | 63 | item := h.items[h.curIdx] 64 | return &item 65 | } 66 | -------------------------------------------------------------------------------- /blhistory/browser_like_history_test.go: -------------------------------------------------------------------------------- 1 | package blhistory 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type testCase struct { 10 | add string 11 | prev bool 12 | next bool 13 | 14 | want string 15 | } 16 | 17 | func TestBLHistory(t *testing.T) { 18 | testCases := []testCase{ 19 | testCase{prev: true, want: ""}, 20 | testCase{prev: true, want: ""}, 21 | testCase{next: true, want: ""}, 22 | testCase{next: true, want: ""}, 23 | testCase{add: "item 1"}, 24 | testCase{prev: true, want: ""}, 25 | testCase{next: true, want: ""}, 26 | testCase{add: "item 2"}, 27 | testCase{prev: true, want: "item 1"}, 28 | testCase{prev: true, want: ""}, 29 | testCase{next: true, want: "item 2"}, 30 | testCase{next: true, want: ""}, 31 | testCase{add: "item 3"}, 32 | testCase{prev: true, want: "item 2"}, 33 | testCase{prev: true, want: "item 1"}, 34 | testCase{prev: true, want: ""}, 35 | testCase{prev: true, want: ""}, 36 | testCase{next: true, want: "item 2"}, 37 | testCase{next: true, want: "item 3"}, 38 | testCase{next: true, want: ""}, 39 | testCase{next: true, want: ""}, 40 | testCase{next: true, want: ""}, 41 | testCase{prev: true, want: "item 2"}, 42 | testCase{add: "item 10"}, 43 | testCase{next: true, want: ""}, 44 | testCase{prev: true, want: "item 2"}, 45 | testCase{prev: true, want: "item 1"}, 46 | testCase{prev: true, want: ""}, 47 | testCase{next: true, want: "item 2"}, 48 | testCase{next: true, want: "item 10"}, 49 | testCase{next: true, want: ""}, 50 | } 51 | 52 | h := New() 53 | 54 | for i, tc := range testCases { 55 | if tc.add != "" { 56 | h.Add(tc.add) 57 | } else if tc.prev { 58 | var gotStr string 59 | got := h.Prev() 60 | if got != nil { 61 | gotStr = got.Str 62 | } 63 | 64 | assert.Equal(t, tc.want, gotStr, "testCase #%d (%+v)", i, tc) 65 | } else if tc.next { 66 | var gotStr string 67 | got := h.Next() 68 | if got != nil { 69 | gotStr = got.Str 70 | } 71 | 72 | assert.Equal(t, tc.want, gotStr, "testCase #%d (%+v)", i, tc) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /clhistory/clhistory.go: -------------------------------------------------------------------------------- 1 | package clhistory 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/juju/errors" 13 | ) 14 | 15 | type CLHistory struct { 16 | params CLHistoryParams 17 | 18 | items []Item 19 | 20 | // curHistIdx is used when navigating the history using Prev / Next. 21 | // When navigating isn't in progress (after a new item was added using Add), 22 | // it's reset to -1. 23 | curHistIdx int 24 | lastEphemeralItem Item 25 | } 26 | 27 | type CLHistoryParams struct { 28 | // Filename is where to load the history from and write it to. If it's 29 | // empty, the history is only kept in RAM and not persisted anywhere. 30 | Filename string 31 | } 32 | 33 | type Item struct { 34 | Time time.Time 35 | 36 | Str string 37 | } 38 | 39 | func New(params CLHistoryParams) (*CLHistory, error) { 40 | h := &CLHistory{ 41 | params: params, 42 | 43 | curHistIdx: -1, 44 | } 45 | 46 | if err := h.Load(); err != nil && !os.IsNotExist(errors.Cause(err)) { 47 | return nil, errors.Trace(err) 48 | } 49 | 50 | return h, nil 51 | } 52 | 53 | // Load loads all history from the file. If Filename in params is empty, 54 | // Load is a no-op. 55 | func (h *CLHistory) Load() error { 56 | if h.params.Filename == "" { 57 | return nil 58 | } 59 | 60 | f, err := os.Open(h.params.Filename) 61 | if err != nil { 62 | return errors.Trace(err) 63 | } 64 | 65 | decoder := NewHistoryDecoder(f) 66 | loadedItems, err := decoder.Decode() 67 | if err != nil { 68 | return errors.Trace(err) 69 | } 70 | 71 | h.items = loadedItems 72 | h.resetHistoryNavigation() 73 | 74 | return nil 75 | } 76 | 77 | // Add adds the given string as a new history item to the in-RAM history and, 78 | // if Filename in params was not empty, then also to this file. It also resets 79 | // the history navigation, if any. 80 | func (h *CLHistory) Add(s string) error { 81 | h.resetHistoryNavigation() 82 | 83 | item := Item{ 84 | Time: time.Now(), 85 | Str: s, 86 | } 87 | 88 | h.items = append(h.items, item) 89 | 90 | if h.params.Filename != "" { 91 | f, err := os.OpenFile(h.params.Filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 92 | if err != nil { 93 | return errors.Trace(err) 94 | } 95 | 96 | defer f.Close() 97 | 98 | fmt.Fprint(f, string(marshalItem(item))) 99 | } 100 | 101 | return nil 102 | } 103 | 104 | // Reset resets the history navigation. Typically client code should call it 105 | // when a user edits or aborts/accepts the command line. 106 | func (h *CLHistory) Reset() { 107 | h.resetHistoryNavigation() 108 | } 109 | 110 | // Prev returns what it considers the previous item. 111 | func (h *CLHistory) Prev(s string) (item Item, hasMore bool) { 112 | if h.curHistIdx == -1 { 113 | h.startHistoryNavigation(s) 114 | } 115 | 116 | for { 117 | h.curHistIdx-- 118 | if h.curHistIdx < 0 { 119 | h.curHistIdx = 0 120 | } 121 | 122 | hasMore := h.curHistIdx > 0 123 | 124 | item := h.getItem(h.curHistIdx) 125 | if item.Str != s || !hasMore { 126 | return item, hasMore 127 | } 128 | } 129 | } 130 | 131 | // Next returns what it considers the next item. 132 | func (h *CLHistory) Next(s string) (item Item, hasMore bool) { 133 | if h.curHistIdx == -1 { 134 | h.startHistoryNavigation(s) 135 | } 136 | 137 | for { 138 | h.curHistIdx++ 139 | if h.curHistIdx > len(h.items) { 140 | // We do allow it to exceed the data by 1 item, which means just returning 141 | // lastEphemeralItem; thus we use len(h.items) and not len(h.items)-1. 142 | h.curHistIdx = len(h.items) 143 | } 144 | 145 | hasMore := h.curHistIdx < len(h.items) 146 | 147 | item := h.getItem(h.curHistIdx) 148 | if item.Str != s || !hasMore { 149 | return item, hasMore 150 | } 151 | } 152 | } 153 | 154 | func (h *CLHistory) startHistoryNavigation(s string) { 155 | h.curHistIdx = len(h.items) 156 | h.lastEphemeralItem = Item{Str: s} 157 | } 158 | 159 | func (h *CLHistory) resetHistoryNavigation() { 160 | h.curHistIdx = -1 161 | h.lastEphemeralItem = Item{} 162 | } 163 | 164 | func (h *CLHistory) getItem(idx int) Item { 165 | if idx < len(h.items) { 166 | return h.items[idx] 167 | } 168 | 169 | if idx == len(h.items) { 170 | return h.lastEphemeralItem 171 | } 172 | 173 | panic(fmt.Sprintf("idx=%d, len(items)=%d", idx, len(h.items))) 174 | } 175 | 176 | // :1650712458000000000:12:0:foo bar baz 177 | func marshalItem(item Item) []byte { 178 | b := bytes.Buffer{} 179 | b.WriteRune(':') 180 | b.WriteString(strconv.FormatInt(item.Time.UnixNano(), 10)) 181 | b.WriteRune(':') 182 | b.WriteString(strconv.Itoa(len(item.Str))) 183 | b.WriteString(":0:") // For now, no extra info 184 | b.WriteString(item.Str) 185 | b.WriteRune('\n') 186 | 187 | return b.Bytes() 188 | } 189 | 190 | type HistoryDecoder struct { 191 | r io.Reader 192 | br *bufio.Reader 193 | } 194 | 195 | func NewHistoryDecoder(r io.Reader) *HistoryDecoder { 196 | return &HistoryDecoder{ 197 | r: r, 198 | br: bufio.NewReader(r), 199 | } 200 | } 201 | 202 | func (hd *HistoryDecoder) Decode() ([]Item, error) { 203 | var items []Item 204 | 205 | for i := 0; ; i++ { 206 | item, err := hd.readNextItem() 207 | if err != nil { 208 | if errors.Cause(err) == io.EOF { 209 | break 210 | } 211 | 212 | return nil, errors.Annotatef(err, "%dth item", i) 213 | } 214 | 215 | items = append(items, item) 216 | } 217 | 218 | return items, nil 219 | } 220 | 221 | func (hd *HistoryDecoder) readNextItem() (Item, error) { 222 | //br := bufio.NewReader(r) 223 | //br.ReadSlice(':') 224 | 225 | if err := hd.consumeByte(':'); err != nil { 226 | if errors.Cause(err) == io.EOF { 227 | return Item{}, io.EOF 228 | } 229 | 230 | return Item{}, errors.Annotatef(err, "reading initial colon") 231 | } 232 | 233 | chunk, err := hd.br.ReadBytes(':') 234 | if err != nil { 235 | if errors.Cause(err) == io.EOF { 236 | err = io.ErrUnexpectedEOF 237 | } 238 | 239 | return Item{}, errors.Annotatef(err, "reading timestamp") 240 | } 241 | 242 | nanos, err := strconv.ParseInt(string(chunk[:len(chunk)-1]), 10, 64) 243 | if err != nil { 244 | return Item{}, errors.Annotatef(err, "parsing timestamp") 245 | } 246 | 247 | chunk, err = hd.br.ReadBytes(':') 248 | if err != nil { 249 | if errors.Cause(err) == io.EOF { 250 | err = io.ErrUnexpectedEOF 251 | } 252 | 253 | return Item{}, errors.Annotatef(err, "reading timestamp") 254 | } 255 | 256 | lenData, err := strconv.Atoi(string(chunk[:len(chunk)-1])) 257 | if err != nil { 258 | return Item{}, errors.Annotatef(err, "parsing data length") 259 | } 260 | 261 | chunk, err = hd.br.ReadBytes(':') 262 | if err != nil { 263 | if errors.Cause(err) == io.EOF { 264 | err = io.ErrUnexpectedEOF 265 | } 266 | 267 | return Item{}, errors.Annotatef(err, "reading timestamp") 268 | } 269 | 270 | lenExtra, err := strconv.Atoi(string(chunk[:len(chunk)-1])) 271 | if err != nil { 272 | return Item{}, errors.Annotatef(err, "parsing extra length") 273 | } 274 | 275 | if lenExtra > 0 { 276 | dataIgnored := make([]byte, lenExtra) 277 | _, err := io.ReadFull(hd.br, dataIgnored) 278 | if err != nil { 279 | if errors.Cause(err) == io.EOF { 280 | err = io.ErrUnexpectedEOF 281 | } 282 | 283 | return Item{}, errors.Annotatef(err, "reading extra data") 284 | } 285 | } 286 | 287 | var data []byte 288 | 289 | if lenData > 0 { 290 | data = make([]byte, lenData) 291 | _, err := io.ReadFull(hd.br, data) 292 | if err != nil { 293 | if errors.Cause(err) == io.EOF { 294 | err = io.ErrUnexpectedEOF 295 | } 296 | 297 | return Item{}, errors.Annotatef(err, "reading data") 298 | } 299 | } 300 | 301 | if err := hd.consumeByte('\n'); err != nil { 302 | if errors.Cause(err) == io.EOF { 303 | err = io.ErrUnexpectedEOF 304 | } 305 | 306 | return Item{}, errors.Annotatef(err, "reading final newline") 307 | } 308 | 309 | return Item{ 310 | Time: time.Unix(0, nanos), 311 | Str: string(data), 312 | }, nil 313 | } 314 | 315 | func (hd *HistoryDecoder) consumeByte(want byte) error { 316 | b := make([]byte, 1) 317 | _, err := io.ReadFull(hd.br, b) 318 | if err != nil { 319 | return errors.Trace(err) 320 | } 321 | 322 | if b[0] != want { 323 | return errors.Errorf("expected to read %v, but read %v", want, b[0]) 324 | } 325 | 326 | return nil 327 | } 328 | -------------------------------------------------------------------------------- /cmd/generate_syslog/README.md: -------------------------------------------------------------------------------- 1 | Just run it as: 2 | 3 | ``` 4 | $ go run ./main.go 5 | ``` 6 | 7 | And it'll generate the `randomlog.1` and `randomlog` for you in the current directory. 8 | -------------------------------------------------------------------------------- /cmd/generate_syslog/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/dimonomid/nerdlog/util/sysloggen" 9 | "github.com/juju/errors" 10 | ) 11 | 12 | func main() { 13 | if err := main2(); err != nil { 14 | fmt.Println("error:", err.Error()) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func main2() error { 20 | t, err := time.Parse(time.RFC3339, "2025-03-09T06:00:00Z") 21 | if err != nil { 22 | return errors.Trace(err) 23 | } 24 | 25 | t2, err := time.Parse(time.RFC3339, "2025-03-10T06:00:00Z") 26 | if err != nil { 27 | return errors.Trace(err) 28 | } 29 | 30 | err = sysloggen.GenerateSyslog(sysloggen.Params{ 31 | TimeLayout: "Jan _2 15:04:05", 32 | //TimeLayout: "2006-01-02T15:04:05.000000-07:00", 33 | 34 | StartTime: t, 35 | SecondLogTime: t2, 36 | 37 | LogBasename: "randomlog", 38 | 39 | NumLogs: 4000000, 40 | MinDelayMS: 0, 41 | MaxDelayMS: 80, 42 | 43 | RandomSeed: 123, 44 | }) 45 | if err != nil { 46 | return errors.Trace(err) 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "time" 9 | 10 | "github.com/dimonomid/nerdlog/blhistory" 11 | "github.com/dimonomid/nerdlog/clhistory" 12 | "github.com/dimonomid/nerdlog/core" 13 | "github.com/dimonomid/nerdlog/log" 14 | "github.com/juju/errors" 15 | "github.com/kevinburke/ssh_config" 16 | "github.com/rivo/tview" 17 | ) 18 | 19 | type nerdlogApp struct { 20 | params nerdlogAppParams 21 | 22 | options *OptionsShared 23 | 24 | // tviewApp is the TUI application. NOTE: once TUI exits, tviewApp is reset 25 | // to nil. 26 | tviewApp *tview.Application 27 | 28 | lsman *core.LStreamsManager 29 | mainView *MainView 30 | 31 | // cmdLineHistory is the command line history 32 | cmdLineHistory *clhistory.CLHistory 33 | 34 | // queryBLHistory is the history of queries, as shell strings like this: 35 | // - nerdlog --lstreams 'localhost' --time -10h --pattern '/something/' 36 | // - nerdlog --lstreams 'localhost' --time -2h --pattern '/something/' 37 | queryBLHistory *blhistory.BLHistory 38 | // queryCLHistory is tracking the same data as queryBLHistory (queries like 39 | // nerdlog --lstreams .....), but it's command-line-like, and it can be 40 | // navigated on the query edit form. 41 | queryCLHistory *clhistory.CLHistory 42 | 43 | lastQueryFull QueryFull 44 | 45 | // lastLogResp contains the last response from LStreamsManager. 46 | lastLogResp *core.LogRespTotal 47 | } 48 | 49 | type nerdlogAppParams struct { 50 | initialQueryData QueryFull 51 | connectRightAway bool 52 | enableClipboard bool 53 | logLevel log.LogLevel 54 | } 55 | 56 | type cmdWithOpts struct { 57 | cmd string 58 | opts CmdOpts 59 | } 60 | 61 | func newNerdlogApp( 62 | params nerdlogAppParams, queryCLHistory *clhistory.CLHistory, 63 | ) (*nerdlogApp, error) { 64 | logger := log.NewLogger(params.logLevel) 65 | 66 | homeDir, err := os.UserHomeDir() 67 | if err != nil { 68 | return nil, errors.Annotatef(err, "getting home dir") 69 | } 70 | 71 | cmdLineHistory, err := clhistory.New(clhistory.CLHistoryParams{ 72 | Filename: filepath.Join(homeDir, ".nerdlog_history"), 73 | }) 74 | if err != nil { 75 | return nil, errors.Annotatef(err, "initializing cmdline history") 76 | } 77 | 78 | app := &nerdlogApp{ 79 | params: params, 80 | 81 | options: NewOptionsShared(Options{ 82 | Timezone: time.Local, 83 | MaxNumLines: 250, 84 | }), 85 | 86 | tviewApp: tview.NewApplication(), 87 | 88 | cmdLineHistory: cmdLineHistory, 89 | queryBLHistory: blhistory.New(), 90 | queryCLHistory: queryCLHistory, 91 | } 92 | 93 | cmdCh := make(chan cmdWithOpts, 8) 94 | 95 | app.mainView = NewMainView(&MainViewParams{ 96 | App: app.tviewApp, 97 | Options: app.options, 98 | OnLogQuery: func(params core.QueryLogsParams) { 99 | params.MaxNumLines = app.options.GetMaxNumLines() 100 | 101 | // Get the current QueryFull and marshal it to a shell command. 102 | qf := app.mainView.getQueryFull() 103 | qfStr := qf.MarshalShellCmd() 104 | 105 | // Add this query shell command to the commandline-like history. 106 | app.queryCLHistory.Add(qfStr) 107 | 108 | // If needed, also add it to the browser-like history. 109 | if qf != app.lastQueryFull { 110 | app.lastQueryFull = qf 111 | if !params.DontAddHistoryItem { 112 | app.queryBLHistory.Add(qfStr) 113 | } 114 | } 115 | 116 | app.lsman.QueryLogs(params) 117 | }, 118 | OnLStreamsChange: func(lstreamsSpec string) error { 119 | err := app.lsman.SetLStreams(lstreamsSpec) 120 | if err != nil { 121 | return errors.Trace(err) 122 | } 123 | 124 | return nil 125 | }, 126 | OnDisconnectRequest: func() { 127 | app.lsman.Disconnect() 128 | }, 129 | OnReconnectRequest: func() { 130 | app.lsman.Reconnect() 131 | }, 132 | OnCmd: func(cmd string, opts CmdOpts) { 133 | cmdCh <- cmdWithOpts{ 134 | cmd: cmd, 135 | opts: opts, 136 | } 137 | }, 138 | 139 | CmdHistory: app.cmdLineHistory, 140 | QueryHistory: app.queryCLHistory, 141 | 142 | Logger: logger, 143 | }) 144 | 145 | // NOTE: initLStreamsManager has to be called _after_ app.mainView is initialized. 146 | if err := app.initLStreamsManager("", homeDir, logger); err != nil { 147 | return nil, errors.Trace(err) 148 | } 149 | 150 | if !params.connectRightAway { 151 | app.mainView.params.App.SetFocus(app.mainView.logsTable) 152 | app.mainView.queryEditView.Show(params.initialQueryData) 153 | } else { 154 | if err := app.mainView.applyQueryEditData(params.initialQueryData, doQueryParams{}); err != nil { 155 | panic(err.Error()) 156 | } 157 | } 158 | 159 | go app.handleCmdLine(cmdCh) 160 | 161 | return app, nil 162 | } 163 | 164 | func (app *nerdlogApp) runTViewApp() error { 165 | err := app.tviewApp.SetRoot(app.mainView.GetUIPrimitive(), true).Run() 166 | 167 | // Now that TUI app has finished, remember that by resetting it to nil. 168 | app.tviewApp = nil 169 | 170 | return err 171 | } 172 | 173 | // NOTE: initLStreamsManager has to be called _after_ app.mainView is initialized. 174 | func (app *nerdlogApp) initLStreamsManager( 175 | initialLStreams string, 176 | homeDir string, 177 | logger *log.Logger, 178 | ) error { 179 | updatesCh := make(chan core.LStreamsManagerUpdate, 128) 180 | go func() { 181 | // We don't want to necessarily update UI on _every_ state update, since 182 | // they might be getting a lot of those messages due to those progress 183 | // percentage updates; so we just remember the last state, and only update 184 | // the UI once we don't have more messages yet. 185 | var lastState *core.LStreamsManagerState 186 | var logResps []*core.LogRespTotal // TODO: perhaps we should also only keep the last one? 187 | var bootstrapErrors []error 188 | 189 | handleUpdate := func(upd core.LStreamsManagerUpdate) { 190 | switch { 191 | case upd.State != nil: 192 | lastState = upd.State 193 | case upd.LogResp != nil: 194 | logResps = append(logResps, upd.LogResp) 195 | case upd.BootstrapIssue != nil: 196 | bootstrapErrors = append( 197 | bootstrapErrors, 198 | errors.Errorf("%s: %s", upd.BootstrapIssue.LStreamName, upd.BootstrapIssue.Err), 199 | ) 200 | 201 | default: 202 | panic("empty lstreams manager update") 203 | } 204 | } 205 | 206 | for { 207 | select { 208 | case upd := <-updatesCh: 209 | handleUpdate(upd) 210 | 211 | default: 212 | // If anything has changed, update the UI. 213 | // 214 | // The tviewApp might be nil here if the TUI app has finished, but we're 215 | // still receiving updates during the teardown; so if that's the case, 216 | // just don't update the TUI. 217 | if app.tviewApp != nil && 218 | (lastState != nil || len(logResps) > 0 || len(bootstrapErrors) > 0) { 219 | 220 | app.tviewApp.QueueUpdateDraw(func() { 221 | if lastState != nil { 222 | app.mainView.applyHMState(lastState) 223 | } 224 | 225 | for _, logResp := range logResps { 226 | if len(logResp.Errs) > 0 { 227 | app.mainView.handleQueryError(combineErrors(logResp.Errs)) 228 | return 229 | } 230 | 231 | app.mainView.applyLogs(logResp) 232 | app.lastLogResp = logResp 233 | } 234 | 235 | if len(bootstrapErrors) > 0 { 236 | app.mainView.handleBootstrapError(combineErrors(bootstrapErrors)) 237 | } 238 | }) 239 | 240 | lastState = nil 241 | logResps = nil 242 | bootstrapErrors = nil 243 | } 244 | 245 | // The same select again, but without the default case. 246 | select { 247 | case upd := <-updatesCh: 248 | handleUpdate(upd) 249 | } 250 | } 251 | } 252 | }() 253 | 254 | envUser := os.Getenv("USER") 255 | 256 | var logstreamsCfg core.ConfigLogStreams 257 | logstreamsCfgPath := filepath.Join(homeDir, ".config", "nerdlog", "logstreams.yaml") 258 | _, statErr := os.Stat(logstreamsCfgPath) 259 | if statErr == nil { 260 | appLogstreamsCfg, err := LoadLogstreamsConfigFromFile(logstreamsCfgPath) 261 | if err != nil { 262 | return errors.Trace(err) 263 | } 264 | 265 | logstreamsCfg = appLogstreamsCfg.LogStreams 266 | } 267 | 268 | var sshConfig *ssh_config.Config 269 | sshConfigFile, err := os.Open(filepath.Join(homeDir, ".ssh", "config")) 270 | if err != nil { 271 | if !os.IsNotExist(err) { 272 | // TODO: would perhaps be more useful if we warn the user about it, 273 | // but still proceed. 274 | return errors.Annotatef(err, "reading ssh config") 275 | } 276 | } else { 277 | var err error 278 | sshConfig, err = ssh_config.Decode(sshConfigFile) 279 | if err != nil { 280 | // TODO: would perhaps be more useful if we warn the user about it, 281 | // but still proceed. 282 | return errors.Annotatef(err, "parsing ssh config") 283 | } 284 | } 285 | 286 | app.lsman = core.NewLStreamsManager(core.LStreamsManagerParams{ 287 | Logger: logger, 288 | 289 | ConfigLogStreams: logstreamsCfg, 290 | SSHConfig: sshConfig, 291 | 292 | InitialLStreams: initialLStreams, 293 | 294 | ClientID: envUser, 295 | 296 | UpdatesCh: updatesCh, 297 | }) 298 | 299 | return nil 300 | } 301 | 302 | func (app *nerdlogApp) handleCmdLine(cmdCh <-chan cmdWithOpts) { 303 | for { 304 | cwo := <-cmdCh 305 | app.tviewApp.QueueUpdateDraw(func() { 306 | if !cwo.opts.Internal { 307 | app.cmdLineHistory.Add(cwo.cmd) 308 | } 309 | app.handleCmd(cwo.cmd) 310 | app.mainView.formatTimeRange() 311 | app.mainView.formatLogs() 312 | }) 313 | } 314 | } 315 | 316 | // printError lets user know that there is an error by printing a simple error 317 | // message over the command line, sort of like in Vim. 318 | // Note that if command line is focused atm, the message will not be printed 319 | // and it's a no-op. 320 | func (app *nerdlogApp) printError(msg string) { 321 | app.mainView.printMsg(msg, nlMsgLevelErr) 322 | } 323 | 324 | // printMsg prints a FYI kind of message. Also see notes for printError. 325 | func (app *nerdlogApp) printMsg(msg string) { 326 | app.mainView.printMsg(msg, nlMsgLevelInfo) 327 | } 328 | 329 | func (app *nerdlogApp) Close() { 330 | app.lsman.Close() 331 | } 332 | 333 | func (app *nerdlogApp) Wait() { 334 | app.lsman.Wait() 335 | } 336 | 337 | func combineErrors(errs []error) error { 338 | var totalErr error 339 | if len(errs) == 1 { 340 | totalErr = errs[0] 341 | } else { 342 | var sb strings.Builder 343 | sb.WriteString(fmt.Sprintf("%d errors:", len(errs))) 344 | for i, curErr := range errs { 345 | sb.WriteString("\n") 346 | sb.WriteString(fmt.Sprintf("%d: %s", i+1, curErr.Error())) 347 | } 348 | totalErr = errors.New(sb.String()) 349 | } 350 | 351 | return totalErr 352 | } 353 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/cmdline.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/juju/errors" 9 | "golang.design/x/clipboard" 10 | ) 11 | 12 | // NOTE: handleCmd is always called from the tview's event loop, so it's safe 13 | // to use all UI primitives and nerdlogApp etc. 14 | func (app *nerdlogApp) handleCmd(cmd string) { 15 | parts := strings.Fields(cmd) 16 | if len(parts) == 0 { 17 | return 18 | } 19 | 20 | switch parts[0] { 21 | case "h", "help": 22 | app.mainView.showMessagebox("err", "Fyi", "The only help for now is the README.md in the repo, so check it out", &MessageboxParams{ 23 | Width: 49, 24 | }) 25 | 26 | case "time": 27 | ftr, err := ParseFromToRange(app.options.GetTimezone(), strings.Join(parts[1:], " ")) 28 | if err != nil { 29 | app.printError(err.Error()) 30 | return 31 | } 32 | 33 | app.mainView.setTimeRange(ftr.From, ftr.To) 34 | app.mainView.doQuery(doQueryParams{}) 35 | 36 | case "w", "write": 37 | //if len(parts) < 2 { 38 | //app.printError(":write requires an argument: the filename to write") 39 | //return 40 | //} 41 | 42 | //fname := parts[1] 43 | 44 | fname := "/tmp/last_nerdlog" 45 | if len(parts) >= 2 { 46 | fname = parts[1] 47 | } 48 | 49 | if app.lastLogResp == nil { 50 | app.printError("No logs yet") 51 | return 52 | } 53 | 54 | lfile, err := os.Create(fname) 55 | if err != nil { 56 | app.printError(fmt.Sprintf("Failed to open %s for writing: %s", fname, err)) 57 | return 58 | } 59 | 60 | for _, logMsg := range app.lastLogResp.Logs { 61 | fmt.Fprintf(lfile, "%s \n", 62 | logMsg.OrigLine, 63 | logMsg.Context["lstream"], logMsg.LogLinenumber, logMsg.LogFilename, 64 | ) 65 | } 66 | 67 | lfile.Close() 68 | 69 | app.printMsg(fmt.Sprintf("Saved to %s", fname)) 70 | 71 | case "set": 72 | if len(parts) < 2 || len(parts[1]) == 0 { 73 | app.printError("set requires an argument") 74 | return 75 | } 76 | 77 | // TODO: implement in a generic way 78 | 79 | setParts := strings.SplitN(parts[1], "=", 2) 80 | if len(setParts) == 2 { 81 | optName := setParts[0] 82 | optValue := setParts[1] 83 | 84 | if opt := OptionMetaByName(optName); opt != nil { 85 | var setErr error 86 | app.options.Call(func(o *Options) { 87 | setErr = opt.Set(o, optValue) 88 | }) 89 | 90 | if setErr != nil { 91 | app.printError(setErr.Error()) 92 | return 93 | } 94 | 95 | return 96 | } 97 | 98 | app.printError("Unknown variable " + optName) 99 | return 100 | } 101 | 102 | if parts[1][len(parts[1])-1] == '?' { 103 | optName := parts[1][:len(parts[1])-1] 104 | 105 | if opt := OptionMetaByName(optName); opt != nil { 106 | var optValue string 107 | app.options.Call(func(o *Options) { 108 | optValue = opt.Get(o) 109 | }) 110 | 111 | app.printMsg(fmt.Sprintf("%s is %s", optName, optValue)) 112 | return 113 | } 114 | 115 | app.printError("Unknown variable " + optName) 116 | return 117 | } 118 | 119 | app.printError("Invalid set command") 120 | 121 | case "xc", "xclip": 122 | qf := app.mainView.getQueryFull() 123 | shellCmd := qf.MarshalShellCmd() 124 | if app.params.enableClipboard { 125 | clipboard.Write(clipboard.FmtText, []byte(shellCmd)) 126 | app.printMsg("Copied to clipboard") 127 | } else { 128 | app.printMsg("Clipboard is not available, command: " + shellCmd) 129 | } 130 | 131 | case "nerdlog": 132 | // Mimic as if it was called from a shell 133 | 134 | if err := app.unmarshalAndApplyQuery(cmd, doQueryParams{}); err != nil { 135 | app.printError(err.Error()) 136 | return 137 | } 138 | 139 | case "prev", "bac", "bck", "back": 140 | item := app.queryBLHistory.Prev() 141 | if item == nil { 142 | app.printError("No more history items") 143 | return 144 | } 145 | 146 | if err := app.unmarshalAndApplyQuery(item.Str, doQueryParams{ 147 | dontAddHistoryItem: true, 148 | }); err != nil { 149 | // This shouldn't happen really provided a sane history. 150 | app.printError(err.Error()) 151 | return 152 | } 153 | 154 | // TODO: print history item stats 155 | 156 | case "next", "fwd", "forward": 157 | item := app.queryBLHistory.Next() 158 | if item == nil { 159 | app.printError("No more history items") 160 | return 161 | } 162 | 163 | if err := app.unmarshalAndApplyQuery(item.Str, doQueryParams{ 164 | dontAddHistoryItem: true, 165 | }); err != nil { 166 | // This shouldn't happen really provided a sane history. 167 | app.printError(err.Error()) 168 | return 169 | } 170 | 171 | // TODO: print history item stats 172 | 173 | case "e", "edit": 174 | app.mainView.openQueryEditView() 175 | 176 | case "q", "quit": 177 | app.tviewApp.Stop() 178 | 179 | case "reconnect": 180 | app.mainView.reconnect(true) 181 | 182 | case "disconnect": 183 | app.mainView.disconnect() 184 | 185 | default: 186 | app.printError(fmt.Sprintf("unknown command %q", parts[0])) 187 | } 188 | } 189 | 190 | func (app *nerdlogApp) unmarshalAndApplyQuery(cmd string, dqp doQueryParams) error { 191 | var qf QueryFull 192 | if err := qf.UnmarshalShellCmd(cmd); err != nil { 193 | return errors.Annotatef(err, "parsing") 194 | } 195 | 196 | if err := app.mainView.applyQueryEditData(qf, dqp); err != nil { 197 | return errors.Annotatef(err, "applying") 198 | } 199 | 200 | return nil 201 | } 202 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/column_edit_view.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gdamore/tcell/v2" 5 | "github.com/juju/errors" 6 | "github.com/rivo/tview" 7 | ) 8 | 9 | type ColumnEditViewParams struct { 10 | // DoneFunc is called when the user submits the form. If it returns a non-nil 11 | // error, the form will show that error and will not be submitted. 12 | DoneFunc func(field SelectQueryField) error 13 | } 14 | 15 | type ColumnEditView struct { 16 | params ColumnEditViewParams 17 | mainView *MainView 18 | 19 | field SelectQueryField 20 | 21 | flex *tview.Flex 22 | 23 | nameInput *tview.InputField 24 | displayNameInput *tview.InputField 25 | stickyCheckbox *tview.Checkbox 26 | frame *tview.Frame 27 | } 28 | 29 | func NewColumnEditView( 30 | mainView *MainView, params *ColumnEditViewParams, 31 | ) *ColumnEditView { 32 | cev := &ColumnEditView{ 33 | params: *params, 34 | mainView: mainView, 35 | } 36 | 37 | var focusers []tview.Primitive 38 | getGenericTabHandler := func(curPrimitive tview.Primitive) func(event *tcell.EventKey) *tcell.EventKey { 39 | return func(event *tcell.EventKey) *tcell.EventKey { 40 | key := event.Key() 41 | 42 | nextIdx := 0 43 | prevIdx := 0 44 | 45 | for i, p := range focusers { 46 | if p != curPrimitive { 47 | continue 48 | } 49 | 50 | prevIdx = i - 1 51 | if prevIdx < 0 { 52 | prevIdx = len(focusers) - 1 53 | } 54 | 55 | nextIdx = i + 1 56 | if nextIdx >= len(focusers) { 57 | nextIdx = 0 58 | } 59 | } 60 | 61 | switch key { 62 | case tcell.KeyTab: 63 | cev.mainView.params.App.SetFocus(focusers[nextIdx]) 64 | return nil 65 | 66 | case tcell.KeyBacktab: 67 | cev.mainView.params.App.SetFocus(focusers[prevIdx]) 68 | return nil 69 | } 70 | 71 | return event 72 | } 73 | } 74 | 75 | cev.flex = tview.NewFlex().SetDirection(tview.FlexRow) 76 | 77 | nameLabel := tview.NewTextView() 78 | nameLabel.SetText("Name:") 79 | nameLabel.SetDynamicColors(true) 80 | cev.flex.AddItem(nameLabel, 1, 0, false) 81 | 82 | cev.nameInput = tview.NewInputField() 83 | cev.nameInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 84 | event = cev.genericInputHandler(event, getGenericTabHandler(cev.nameInput), nil, nil) 85 | if event == nil { 86 | return nil 87 | } 88 | 89 | return event 90 | }) 91 | cev.flex.AddItem(cev.nameInput, 1, 0, true) 92 | focusers = append(focusers, cev.nameInput) 93 | 94 | cev.flex.AddItem(nil, 1, 0, false) 95 | 96 | displayNameLabel := tview.NewTextView() 97 | displayNameLabel.SetText("Display name (empty means the same as Name):") 98 | displayNameLabel.SetDynamicColors(true) 99 | cev.flex.AddItem(displayNameLabel, 1, 0, false) 100 | 101 | cev.displayNameInput = tview.NewInputField() 102 | cev.displayNameInput.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 103 | event = cev.genericInputHandler(event, getGenericTabHandler(cev.displayNameInput), nil, nil) 104 | if event == nil { 105 | return nil 106 | } 107 | 108 | return event 109 | }) 110 | cev.flex.AddItem(cev.displayNameInput, 1, 0, true) 111 | focusers = append(focusers, cev.displayNameInput) 112 | 113 | cev.flex.AddItem(nil, 1, 0, false) 114 | 115 | stickyLabel := tview.NewTextView() 116 | stickyLabel.SetText("Sticky") 117 | stickyLabel.SetDynamicColors(true) 118 | 119 | cev.stickyCheckbox = tview.NewCheckbox() 120 | cev.stickyCheckbox.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 121 | event = cev.genericInputHandler(event, getGenericTabHandler(cev.stickyCheckbox), nil, nil) 122 | if event == nil { 123 | return nil 124 | } 125 | 126 | return event 127 | }) 128 | focusers = append(focusers, cev.stickyCheckbox) 129 | 130 | stickyFlex := tview.NewFlex().SetDirection(tview.FlexColumn) 131 | stickyFlex. 132 | AddItem(tview.NewTextView().SetText("["), 1, 0, false). 133 | AddItem(cev.stickyCheckbox, 1, 0, false). 134 | AddItem(tview.NewTextView().SetText("]"), 1, 0, false). 135 | AddItem(nil, 1, 0, false). 136 | AddItem(stickyLabel, 0, 1, false). 137 | AddItem(nil, 0, 1, false) 138 | cev.flex.AddItem(stickyFlex, 1, 0, true) 139 | 140 | cev.flex.AddItem(nil, 0, 1, false) 141 | 142 | cev.frame = tview.NewFrame(cev.flex).SetBorders(0, 0, 0, 0, 0, 0) 143 | cev.frame.SetBorder(true).SetBorderPadding(0, 0, 1, 1) 144 | cev.frame.SetTitle("Edit column") 145 | 146 | return cev 147 | } 148 | 149 | func (cev *ColumnEditView) Show(field SelectQueryField) { 150 | cev.nameInput.SetText(field.Name) 151 | if field.DisplayName != field.Name { 152 | cev.displayNameInput.SetText(field.DisplayName) 153 | } else { 154 | cev.displayNameInput.SetText("") 155 | } 156 | cev.stickyCheckbox.SetChecked(field.Sticky) 157 | 158 | cev.mainView.showModal( 159 | pageNameColumnDetails, cev.frame, 160 | 60, 161 | 9, 162 | true, 163 | ) 164 | } 165 | 166 | func (cev *ColumnEditView) Hide() { 167 | cev.mainView.hideModal(pageNameColumnDetails, true) 168 | } 169 | 170 | func (cev *ColumnEditView) GetSelectQueryField() SelectQueryField { 171 | field := SelectQueryField{ 172 | Name: cev.nameInput.GetText(), 173 | DisplayName: cev.displayNameInput.GetText(), 174 | Sticky: cev.stickyCheckbox.IsChecked(), 175 | } 176 | 177 | if field.DisplayName == "" { 178 | field.DisplayName = field.Name 179 | } 180 | 181 | return field 182 | } 183 | 184 | func (cev *ColumnEditView) genericInputHandler( 185 | event *tcell.EventKey, 186 | genericTabHandler func(event *tcell.EventKey) *tcell.EventKey, 187 | getQFPart func(qf QueryFull) string, 188 | setQFPart func(qf *QueryFull, part string), 189 | ) *tcell.EventKey { 190 | event = genericTabHandler(event) 191 | if event == nil { 192 | return nil 193 | } 194 | 195 | switch event.Key() { 196 | case tcell.KeyEnter: 197 | if err := cev.apply(); err != nil { 198 | cev.mainView.showMessagebox("err", "Error", err.Error(), nil) 199 | } 200 | return nil 201 | 202 | case tcell.KeyEsc: 203 | cev.Hide() 204 | return nil 205 | } 206 | 207 | return event 208 | } 209 | 210 | func (cev *ColumnEditView) apply() error { 211 | err := cev.params.DoneFunc(cev.GetSelectQueryField()) 212 | if err != nil { 213 | return errors.Trace(err) 214 | } 215 | 216 | cev.Hide() 217 | 218 | return nil 219 | } 220 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/dimonomid/nerdlog/core" 8 | "github.com/juju/errors" 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | type ConfigLogStreams struct { 13 | LogStreams core.ConfigLogStreams `yaml:"log_streams"` 14 | } 15 | 16 | func LoadLogstreamsConfigFromFile(path string) (*ConfigLogStreams, error) { 17 | file, err := os.Open(path) 18 | if err != nil { 19 | return nil, errors.Annotatef(err, "opening config file: %s", path) 20 | } 21 | defer file.Close() 22 | 23 | data, err := ioutil.ReadAll(file) 24 | if err != nil { 25 | return nil, errors.Annotatef(err, "reading config file %s", path) 26 | } 27 | 28 | var cfg ConfigLogStreams 29 | if err := yaml.Unmarshal(data, &cfg); err != nil { 30 | return nil, errors.Annotatef(err, "unmarshaling yaml from %s", path) 31 | } 32 | 33 | return &cfg, nil 34 | } 35 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/from_to_range.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/dimonomid/nerdlog/core" 8 | "github.com/juju/errors" 9 | ) 10 | 11 | type FromToRange struct { 12 | From TimeOrDur 13 | To TimeOrDur 14 | } 15 | 16 | func ParseFromToRange(timezone *time.Location, s string) (FromToRange, error) { 17 | flds := strings.Split(s, " to ") 18 | if len(flds) == 0 { 19 | return FromToRange{}, errors.New("time can't be empty. try -5h") 20 | } 21 | 22 | var from, to TimeOrDur 23 | var err error 24 | 25 | fromStr := flds[0] 26 | 27 | from, err = parseAndInferTimeOrDur(timezone, inputTimeLayout, fromStr) 28 | if err != nil { 29 | return FromToRange{}, errors.Annotatef(err, "invalid 'from' duration") 30 | } 31 | 32 | to = TimeOrDur{} 33 | 34 | if len(flds) > 1 { 35 | toStr := flds[1] 36 | 37 | // If there's no date, prepend date 38 | if len(toStr) <= 5 && len(fromStr) > 5 { 39 | toStr = fromStr[:5] + " " + toStr 40 | } 41 | 42 | var err error 43 | to, err = parseAndInferTimeOrDur(timezone, inputTimeLayout, toStr) 44 | if err != nil { 45 | return FromToRange{}, errors.Annotatef(err, "invalid 'to' duration") 46 | } 47 | } 48 | 49 | return FromToRange{ 50 | From: from, 51 | To: to, 52 | }, nil 53 | } 54 | 55 | func (ftr *FromToRange) String() string { 56 | fromStr := ftr.From.Format(inputTimeLayout) 57 | 58 | if ftr.To.IsZero() { 59 | return fromStr 60 | } 61 | 62 | // If both From and To are absolute and have the same day, then omit day for 63 | // the To. 64 | format := inputTimeLayout 65 | _, fm, fd := ftr.From.Time.Date() 66 | _, tm, td := ftr.To.Time.Date() 67 | if fm == tm && fd == td { 68 | format = inputTimeLayoutMMHH 69 | } 70 | 71 | return fromStr + " to " + ftr.To.Format(format) 72 | } 73 | 74 | func parseAndInferTimeOrDur(timezone *time.Location, layout, s string) (TimeOrDur, error) { 75 | t, err := ParseTimeOrDur(timezone, layout, s) 76 | if err != nil { 77 | return TimeOrDur{}, err 78 | } 79 | 80 | if t.IsAbsolute() { 81 | t.Time = core.InferYear(t.Time) 82 | } 83 | 84 | return t, nil 85 | } 86 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/histogram_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestClearTviewFormatting(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | input string 16 | expected string 17 | }{ 18 | {"Simple color", "[yellow]Yellow text", "Yellow text"}, 19 | {"Background color", "[yellow:red]Yellow text on red background", "Yellow text on red background"}, 20 | {"Only background", "[:red]Red background", "Red background"}, 21 | {"Underline", "[yellow::u]Underlined text", "Underlined text"}, 22 | {"Bold and blinking", "[::bl]Bold blinking", "Bold blinking"}, 23 | {"Reset styles", "[::-]Text after reset", "Text after reset"}, 24 | {"Reset foreground", "[-]No color", "No color"}, 25 | {"Italic on", "[::i]Italic text", "Italic text"}, 26 | {"Italic off", "[::I]Normal text", "Normal text"}, 27 | {"Link formatting", "Click [:::https://example.com]here[:::-] for more", "Click here for more"}, 28 | {"Multiple mailto links", "Email [:::mailto:a@x]a/[:::mail:b@x]b/[:::mail:c@x]c[:::-]", "Email a/b/c"}, 29 | {"Reset everything", "[-:-:-:-]Clean", "Clean"}, 30 | {"No-op tag", "[:]Still here", "Still here"}, 31 | {"Invalid tag shows brackets", "[]Bracket tag", "Bracket tag"}, 32 | {"Escaped tag", "[red[]Text", "[red]Text"}, 33 | {"Escaped quoted tag", "[\"123\"[]hello", "[\"123\"]hello"}, 34 | {"Escaped color tag", "[#6aff00[[]Greenish", "[#6aff00[]Greenish"}, 35 | {"Escaped nonsense", "[a#\"[[[]stuff", "[a#\"[[]stuff"}, 36 | {"Mixed content", "Start [yellow]middle[::u]under[:::-]end", "Start middleunderend"}, 37 | } 38 | 39 | for _, tt := range tests { 40 | t.Run(tt.name, func(t *testing.T) { 41 | result := clearTviewFormatting(tt.input) 42 | if result != tt.expected { 43 | t.Errorf("Expected %q, got %q", tt.expected, result) 44 | } 45 | }) 46 | } 47 | } 48 | 49 | func TestGetOptimalScale_Custom(t *testing.T) { 50 | const binSize = 60 // 1 minute 51 | 52 | tests := []struct { 53 | name string 54 | from int 55 | to int 56 | width int 57 | expected *histogramScale 58 | }{ 59 | { 60 | name: "BasicCase", 61 | from: 0, 62 | to: 3600, 63 | width: 100, 64 | expected: &histogramScale{ 65 | from: 0, 66 | to: 3600, 67 | numDataBins: 60, 68 | dataBinsInChartBar: 1, 69 | chartBarWidth: 1, 70 | }, 71 | }, 72 | { 73 | name: "SnappingUp", 74 | from: 0, 75 | to: 60*18*100 - 2, 76 | width: 100, 77 | expected: &histogramScale{ 78 | from: 0, 79 | to: 60*18*100 - 0, 80 | numDataBins: 1800, 81 | dataBinsInChartBar: 20, 82 | chartBarWidth: 1, 83 | }, 84 | }, 85 | { 86 | name: "ExactSnapTo2", 87 | from: 0, 88 | to: 3600, 89 | width: 30, 90 | expected: &histogramScale{ 91 | from: 0, 92 | to: 3600, 93 | numDataBins: 60, 94 | dataBinsInChartBar: 2, 95 | chartBarWidth: 1, 96 | }, 97 | }, 98 | { 99 | name: "NonAlignedInputWithSnapTo5", 100 | from: 1000065, 101 | to: 1001130, 102 | width: 100, 103 | expected: &histogramScale{ 104 | from: 1000020, 105 | to: 1001160, 106 | numDataBins: 19, 107 | dataBinsInChartBar: 1, 108 | chartBarWidth: 5, 109 | }, 110 | }, 111 | { 112 | name: "WideRangeMinimalWidth", 113 | from: 0, 114 | to: 60 * 10000, 115 | width: 1, 116 | expected: &histogramScale{ 117 | from: 0, 118 | to: 60 * 10080, 119 | numDataBins: 10080, 120 | dataBinsInChartBar: 10080, 121 | chartBarWidth: 1, 122 | }, 123 | }, 124 | { 125 | name: "TooSmallToHaveABin", 126 | from: 0, 127 | to: 59, 128 | width: 10, 129 | expected: nil, 130 | }, 131 | { 132 | name: "chart bar width increases to 2", 133 | from: 0, 134 | to: 660 * 60, 135 | width: 320, 136 | expected: &histogramScale{ 137 | from: 0, 138 | to: 660 * 60, 139 | numDataBins: 660, 140 | dataBinsInChartBar: 5, 141 | chartBarWidth: 2, 142 | }, 143 | }, 144 | } 145 | 146 | for _, tt := range tests { 147 | t.Run(tt.name, func(t *testing.T) { 148 | actual := getOptimalScale(tt.from, tt.to, binSize, tt.width, snapDataBinsInChartDot) 149 | assert.Equal(t, tt.expected, actual) 150 | }) 151 | } 152 | } 153 | 154 | // TestGetOptimalScale_Random uses random inputs, and makes sure that the 155 | // output always satisifies the following conditions: 156 | // 157 | // - The resulting chart width never exceeds the given width 158 | // - If we increase chartBarWidth by just 1, it would already exceed the chart 159 | // width 160 | // - The dataBinsInChartBar is snapped (snapDataBinsInChartDot returns the same 161 | // value) 162 | // - If we try to increase resolution by reducing dataBinsInChartBar to the next 163 | // smallest snap, and set chartBarWidth to 1, we would exceed the chart width 164 | func TestGetOptimalScale_Random(t *testing.T) { 165 | rand.Seed(time.Now().UnixNano()) 166 | 167 | const binSize = 100 168 | const iterations = 200000 169 | 170 | for i := 0; i < iterations; i++ { 171 | from := rand.Intn(1000) // 0–999 172 | duration := (rand.Intn(1440) + 1) * binSize // 1 min to 24 hrs 173 | to := from + duration 174 | width := rand.Intn(2000) + 1 175 | 176 | scale := getOptimalScale(from, to, binSize, width, snapDataBinsInChartDot) 177 | if scale == nil { 178 | continue 179 | } 180 | 181 | numBars := scale.numDataBins / scale.dataBinsInChartBar 182 | totalChartWidth := numBars * scale.chartBarWidth 183 | 184 | detailsStr := fmt.Sprintf( 185 | "inputs: from=%d, to=%d, width=%d; outputs=%+v (numBars=%d, totalWidth=%d)", 186 | from, to, width, 187 | scale, 188 | numBars, totalChartWidth, 189 | ) 190 | 191 | // 1. Total chart width must not exceed the available width 192 | assert.LessOrEqual(t, totalChartWidth, width, 193 | fmt.Sprintf("total chart width should not exceed the provided width. %s", detailsStr)) 194 | 195 | // 2. If we increase chartBarWidth by 1, total width would exceed provided width 196 | extraWidth := numBars * (scale.chartBarWidth + 1) 197 | assert.Greater(t, extraWidth, width, 198 | fmt.Sprintf("increasing chartBarWidth by 1 should exceed the available width. %s", detailsStr), 199 | ) 200 | 201 | // 3. The dataBinsInChartBar is correctly snapped 202 | snapped := snapDataBinsInChartDot(scale.dataBinsInChartBar) 203 | assert.Equal(t, scale.dataBinsInChartBar, snapped, 204 | fmt.Sprintf("dataBinsInChartBar should match the snapped value. %s", detailsStr), 205 | ) 206 | 207 | // 4. If we reduce dataBinsInChartBar to the next smallest snap and set chartBarWidth = 1, 208 | // total chart width should exceed available width 209 | var prevSnap int 210 | for _, snap := range snaps { 211 | snapMinutes := int(snap / time.Minute) 212 | if snapMinutes >= scale.dataBinsInChartBar { 213 | break 214 | } 215 | prevSnap = snapMinutes 216 | } 217 | 218 | if prevSnap > 0 { 219 | numBarsAtHigherRes := scale.numDataBins / prevSnap 220 | if scale.numDataBins%prevSnap != 0 { 221 | numBarsAtHigherRes++ 222 | } 223 | higherResTotalWidth := numBarsAtHigherRes * 1 224 | assert.Greater(t, higherResTotalWidth, width, 225 | "trying smaller dataBinsInChartBar at chartBarWidth=1 should exceed width") 226 | } 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/dimonomid/nerdlog/clhistory" 9 | "github.com/dimonomid/nerdlog/log" 10 | "github.com/spf13/pflag" 11 | "golang.design/x/clipboard" 12 | ) 13 | 14 | // TODO: make multiple of them 15 | const inputTimeLayout = "Jan2 15:04" 16 | const inputTimeLayoutMMHH = "15:04" 17 | 18 | var ( 19 | flagTime = pflag.StringP("time", "t", "", "Time range in the same format as accepted by the UI. Examples: '1h', 'Mar27 12:00'") 20 | flagLStreams = pflag.StringP("lstreams", "h", "", "Logstreams to connect to, as comma-separated glob patterns, e.g. 'foo-*,bar-*'") 21 | flagQuery = pflag.StringP("pattern", "p", "", "Initial awk pattern to use") 22 | flagSelectQuery = pflag.StringP("selquery", "s", "", "SELECT-like query to specify which fields to show, like 'time STICKY, message, lstream, level_name AS level, *'") 23 | flagLogLevel = pflag.String("loglevel", "error", "This is NOT about the logs that nerdlog fetches from the remote servers, it's rather about nerdlog's own log. Valid values are: error, warning, info, verbose1, verbose2 or verbose3") 24 | ) 25 | 26 | func main() { 27 | pflag.Parse() 28 | 29 | // As of today, the only way to connect to a logstream is to use ssh via agent, 30 | // so check if the agent env var is present, and fail quickly if it's not. 31 | if os.Getenv("SSH_AUTH_SOCK") == "" { 32 | fmt.Fprintf(os.Stderr, "SSH_AUTH_SOCK env var is not present, which means ssh agent is not running, or at least is not accessible to Nerdlog. As of today, ssh agent is the only way for Nerdlog to connect to logstreams, so please start one, make sure that all the necessary keys are added to it, and retry.\n") 33 | os.Exit(1) 34 | } 35 | 36 | homeDir, err := os.UserHomeDir() 37 | if err != nil { 38 | fmt.Fprintf(os.Stderr, "Error getting home dir: %s\n", err) 39 | os.Exit(1) 40 | } 41 | 42 | queryCLHistory, err := clhistory.New(clhistory.CLHistoryParams{ 43 | Filename: filepath.Join(homeDir, ".nerdlog_query_history"), 44 | }) 45 | if err != nil { 46 | fmt.Fprintf(os.Stderr, "Error initializing query history: %s\n", err) 47 | os.Exit(1) 48 | } 49 | 50 | initialTime := "-1h" 51 | initialLStreams := "localhost" 52 | initialQuery := "" 53 | initialSelectQuery := DefaultSelectQuery 54 | connectRightAway := false 55 | 56 | if *flagTime != "" { 57 | initialTime = *flagTime 58 | connectRightAway = true 59 | } 60 | 61 | if *flagLStreams != "" { 62 | initialLStreams = *flagLStreams 63 | connectRightAway = true 64 | } 65 | 66 | if *flagQuery != "" { 67 | initialQuery = *flagQuery 68 | connectRightAway = true 69 | } 70 | 71 | if *flagSelectQuery != "" { 72 | initialSelectQuery = SelectQuery(*flagSelectQuery) 73 | connectRightAway = true 74 | } 75 | 76 | initialQueryData := QueryFull{ 77 | Time: initialTime, 78 | Query: initialQuery, 79 | LStreams: initialLStreams, 80 | SelectQuery: initialSelectQuery, 81 | } 82 | 83 | if !connectRightAway { 84 | // No query params were given, try to get the last one from the history. 85 | item, _ := queryCLHistory.Prev("") 86 | if item.Str != "" { 87 | var qf QueryFull 88 | if err := qf.UnmarshalShellCmd(item.Str); err != nil { 89 | // Ignore the error, just use the defaults 90 | } else { 91 | // Successfully parsed the last item from query history, use that. 92 | initialQueryData = qf 93 | } 94 | } 95 | } 96 | 97 | enableClipboard := true 98 | if err := clipboard.Init(); err != nil { 99 | enableClipboard = false 100 | fmt.Println("NOTE: X Clipboard is not available") 101 | } 102 | 103 | logLevel := log.Info 104 | if *flagLogLevel == "error" { 105 | logLevel = log.Error 106 | } else if *flagLogLevel == "warning" { 107 | logLevel = log.Warning 108 | } else if *flagLogLevel == "info" { 109 | logLevel = log.Info 110 | } else if *flagLogLevel == "verbose1" { 111 | logLevel = log.Verbose1 112 | } else if *flagLogLevel == "verbose2" { 113 | logLevel = log.Verbose2 114 | } else if *flagLogLevel == "verbose3" { 115 | logLevel = log.Verbose3 116 | } else { 117 | fmt.Fprintf(os.Stderr, "Invalid --loglevel, try error, warning, info, verbose1, verbose2 or verbose3") 118 | os.Exit(1) 119 | } 120 | 121 | app, err := newNerdlogApp( 122 | nerdlogAppParams{ 123 | initialQueryData: initialQueryData, 124 | connectRightAway: connectRightAway, 125 | enableClipboard: enableClipboard, 126 | logLevel: logLevel, 127 | }, 128 | queryCLHistory, 129 | ) 130 | if err != nil { 131 | fmt.Fprintf(os.Stderr, "Error: %s\n", err) 132 | os.Exit(1) 133 | } 134 | 135 | fmt.Println("Starting UI ...") 136 | if err := app.runTViewApp(); err != nil { 137 | fmt.Fprintf(os.Stderr, "Error: %s\n", err) 138 | os.Exit(1) 139 | } 140 | 141 | // We end up here when the user quits the UI 142 | 143 | fmt.Println("") 144 | fmt.Println("Closing connections...") 145 | 146 | app.Close() 147 | app.Wait() 148 | 149 | fmt.Println("Have a nice day.") 150 | } 151 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/menu.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type menuItem struct { 4 | Title string 5 | Handler func(mv *MainView) 6 | } 7 | 8 | var mainMenu = []menuItem{ 9 | { 10 | Title: "Back :back ", 11 | Handler: func(mv *MainView) { 12 | mv.params.OnCmd("back", CmdOpts{Internal: true}) 13 | }, 14 | }, 15 | { 16 | Title: "Forward :fwd ", 17 | Handler: func(mv *MainView) { 18 | mv.params.OnCmd("fwd", CmdOpts{Internal: true}) 19 | }, 20 | }, 21 | { 22 | Title: "Copy query command :xclip", 23 | Handler: func(mv *MainView) { 24 | mv.params.OnCmd("xclip", CmdOpts{Internal: true}) 25 | }, 26 | }, 27 | } 28 | 29 | func getMainMenuTitles() []string { 30 | ret := make([]string, 0, len(mainMenu)) 31 | for _, item := range mainMenu { 32 | ret = append(ret, item.Title) 33 | } 34 | 35 | return ret 36 | } 37 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/message_view.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gdamore/tcell/v2" 7 | "github.com/rivo/tview" 8 | ) 9 | 10 | type MessageViewParams struct { 11 | App *tview.Application 12 | 13 | MessageID string 14 | Title string 15 | Message string 16 | Buttons []string 17 | OnButtonPressed func(label string, idx int) 18 | OnEsc func() 19 | 20 | // Width and Height are 40 and 10 by default 21 | Width, Height int 22 | 23 | // By default, tview.AlignLeft (because it happens to be 0) 24 | Align int 25 | 26 | NoFocus bool 27 | 28 | BackgroundColor tcell.Color 29 | } 30 | 31 | type MessageView struct { 32 | params MessageViewParams 33 | mainView *MainView 34 | 35 | msgboxFlex *tview.Flex 36 | buttonsFlex *tview.Flex 37 | frame *tview.Frame 38 | 39 | textView *tview.TextView 40 | focusers []tview.Primitive 41 | } 42 | 43 | // getMaxLineLength returns the length of the longest line in the given string. 44 | func getMaxLineLength(s string) int { 45 | maxLen := 0 46 | start := 0 47 | 48 | for i, c := range s { 49 | if c == '\n' { 50 | lineLen := i - start 51 | if lineLen > maxLen { 52 | maxLen = lineLen 53 | } 54 | start = i + 1 55 | } 56 | } 57 | 58 | // Handle the last line if it doesn't end with a newline 59 | if len(s)-start > maxLen { 60 | maxLen = len(s) - start 61 | } 62 | 63 | return maxLen 64 | } 65 | 66 | // getNumLines returns the number of lines that are needed to draw the given 67 | // text. 68 | func getNumLines(s string, screenWidth int) int { 69 | if screenWidth <= 0 { 70 | return 0 71 | } 72 | 73 | lines := strings.Split(s, "\n") 74 | numLines := 0 75 | for _, line := range lines { 76 | // Divide line length by screen width and round up 77 | lineLen := len(line) 78 | numLines += (lineLen + screenWidth - 1) / screenWidth 79 | } 80 | return numLines 81 | } 82 | 83 | func NewMessageView( 84 | mainView *MainView, params *MessageViewParams, 85 | ) *MessageView { 86 | msgv := &MessageView{ 87 | params: *params, 88 | mainView: mainView, 89 | } 90 | 91 | // extraWidth covers padding and border 92 | extraWidth := 4 93 | // extraHeight covers padding, border and buttons 94 | extraHeight := 6 95 | 96 | if msgv.params.Width == 0 { 97 | // Set it to fit the longest line (if the terminal window allows), 98 | // plus the padding and the border. 99 | msgv.params.Width = getMaxLineLength(params.Message) + extraWidth 100 | } 101 | 102 | if msgv.params.Height == 0 { 103 | msgv.params.Height = extraHeight + getNumLines(params.Message, mainView.screenWidth-extraWidth) 104 | } 105 | 106 | msgv.msgboxFlex = tview.NewFlex().SetDirection(tview.FlexRow) 107 | 108 | msgv.textView = tview.NewTextView() 109 | msgv.textView.SetText(params.Message) 110 | msgv.textView.SetTextAlign(msgv.params.Align) 111 | msgv.textView.SetDynamicColors(true) 112 | 113 | if msgv.params.BackgroundColor != tcell.ColorDefault { 114 | msgv.textView.SetBackgroundColor(msgv.params.BackgroundColor) 115 | } 116 | 117 | msgv.msgboxFlex.AddItem(msgv.textView, 0, 1, len(params.Buttons) == 0) 118 | 119 | msgv.buttonsFlex = tview.NewFlex().SetDirection(tview.FlexColumn) 120 | msgv.msgboxFlex.AddItem(msgv.buttonsFlex, 1, 1, len(params.Buttons) != 0) 121 | 122 | // Add a spacer at the left of the buttons, to make them centered 123 | // (there's also a spacer at the right, added later) 124 | msgv.buttonsFlex.AddItem(nil, 0, 1, false) 125 | 126 | for i := 0; i < len(params.Buttons); i++ { 127 | btnLabel := params.Buttons[i] 128 | btnIdx := i 129 | btn := tview.NewButton(btnLabel).SetSelectedFunc(func() { 130 | params.OnButtonPressed(btnLabel, btnIdx) 131 | }) 132 | msgv.focusers = append(msgv.focusers, btn) 133 | tabHandler := msgv.getGenericTabHandler(btn) 134 | btn.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 135 | // Handle Esc key 136 | switch event.Key() { 137 | case tcell.KeyEsc: 138 | if params.OnEsc != nil { 139 | params.OnEsc() 140 | } 141 | } 142 | 143 | event = tabHandler(event) 144 | if event == nil { 145 | return nil 146 | } 147 | 148 | return event 149 | }) 150 | 151 | // Unless it's the first button, add a 1-char spacing. 152 | if i > 0 { 153 | msgv.buttonsFlex.AddItem(nil, 1, 0, false) 154 | } 155 | 156 | // Add the button itself: spacing of 2 chars at each side, and min 10 chars total. 157 | // Focus the first one. 158 | buttonSize := len(btnLabel) + 2*2 159 | if buttonSize < 10 { 160 | buttonSize = 10 161 | } 162 | msgv.buttonsFlex.AddItem(btn, buttonSize, 0, i == 0) 163 | } 164 | 165 | // Add a spacer at the right of the buttons, to make them centered 166 | // (there's also a spacer at the left, added before) 167 | msgv.buttonsFlex.AddItem(nil, 0, 1, false) 168 | 169 | msgv.frame = tview.NewFrame(msgv.msgboxFlex).SetBorders(0, 0, 0, 0, 0, 0) 170 | msgv.frame.SetBorder(true).SetBorderPadding(1, 1, 1, 1) 171 | msgv.frame.SetTitle(params.Title) 172 | if msgv.params.BackgroundColor != tcell.ColorDefault { 173 | msgv.frame.SetBackgroundColor(msgv.params.BackgroundColor) 174 | } 175 | 176 | return msgv 177 | } 178 | 179 | func (msgv *MessageView) Show() { 180 | msgv.mainView.showModal( 181 | pageNameMessage+msgv.params.MessageID, msgv.frame, 182 | msgv.params.Width, 183 | msgv.params.Height, 184 | !msgv.params.NoFocus, 185 | ) 186 | } 187 | 188 | func (msgv *MessageView) Hide() { 189 | msgv.mainView.hideModal(pageNameMessage+msgv.params.MessageID, !msgv.params.NoFocus) 190 | } 191 | 192 | func (msgv *MessageView) getGenericTabHandler(curPrimitive tview.Primitive) func(event *tcell.EventKey) *tcell.EventKey { 193 | return func(event *tcell.EventKey) *tcell.EventKey { 194 | key := event.Key() 195 | 196 | nextIdx := 0 197 | prevIdx := 0 198 | 199 | for i, p := range msgv.focusers { 200 | if p != curPrimitive { 201 | continue 202 | } 203 | 204 | prevIdx = i - 1 205 | if prevIdx < 0 { 206 | prevIdx = len(msgv.focusers) - 1 207 | } 208 | 209 | nextIdx = i + 1 210 | if nextIdx >= len(msgv.focusers) { 211 | nextIdx = 0 212 | } 213 | } 214 | 215 | switch key { 216 | case tcell.KeyTab: 217 | msgv.params.App.SetFocus(msgv.focusers[nextIdx]) 218 | return nil 219 | 220 | case tcell.KeyBacktab: 221 | msgv.params.App.SetFocus(msgv.focusers[prevIdx]) 222 | return nil 223 | } 224 | 225 | return event 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/my_text_view.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gdamore/tcell/v2" 5 | "github.com/rivo/tview" 6 | "golang.design/x/clipboard" 7 | ) 8 | 9 | type MyTextViewParams struct { 10 | Title string 11 | Text string 12 | } 13 | 14 | type MyTextView struct { 15 | params MyTextViewParams 16 | mainView *MainView 17 | 18 | flex *tview.Flex 19 | tv *tview.TextView 20 | okBtn *tview.Button 21 | copyToClipboardBtn *tview.Button 22 | frame *tview.Frame 23 | } 24 | 25 | func NewMyTextView( 26 | mainView *MainView, params *MyTextViewParams, 27 | ) *MyTextView { 28 | rdv := &MyTextView{ 29 | params: *params, 30 | mainView: mainView, 31 | } 32 | 33 | var focusers []tview.Primitive 34 | getGenericTabHandler := func(curPrimitive tview.Primitive) func(event *tcell.EventKey) *tcell.EventKey { 35 | return func(event *tcell.EventKey) *tcell.EventKey { 36 | key := event.Key() 37 | 38 | nextIdx := 0 39 | prevIdx := 0 40 | 41 | for i, p := range focusers { 42 | if p != curPrimitive { 43 | continue 44 | } 45 | 46 | prevIdx = i - 1 47 | if prevIdx < 0 { 48 | prevIdx = len(focusers) - 1 49 | } 50 | 51 | nextIdx = i + 1 52 | if nextIdx >= len(focusers) { 53 | nextIdx = 0 54 | } 55 | } 56 | 57 | switch key { 58 | case tcell.KeyTab: 59 | rdv.mainView.params.App.SetFocus(focusers[nextIdx]) 60 | return nil 61 | 62 | case tcell.KeyBacktab: 63 | rdv.mainView.params.App.SetFocus(focusers[prevIdx]) 64 | return nil 65 | } 66 | 67 | return event 68 | } 69 | } 70 | 71 | rdv.flex = tview.NewFlex().SetDirection(tview.FlexRow) 72 | 73 | rdv.tv = tview.NewTextView() 74 | rdv.tv.SetText(params.Text) 75 | rdv.tv.SetDynamicColors(true) 76 | 77 | rdv.tv.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 78 | switch event.Key() { 79 | case tcell.KeyEnter: 80 | rdv.Hide() 81 | return nil 82 | 83 | case tcell.KeyCtrlD: 84 | // TODO: ideally we'd want to only go half a page down, but for now just 85 | // return Ctrl+F which will go the full page down 86 | return tcell.NewEventKey(tcell.KeyCtrlF, 0, tcell.ModNone) 87 | case tcell.KeyCtrlU: 88 | // TODO: ideally we'd want to only go half a page up, but for now just 89 | // return Ctrl+B which will go the full page up 90 | return tcell.NewEventKey(tcell.KeyCtrlB, 0, tcell.ModNone) 91 | } 92 | 93 | event = rdv.genericInputHandler(event, getGenericTabHandler(rdv.tv), nil, nil) 94 | if event == nil { 95 | return nil 96 | } 97 | 98 | return event 99 | }) 100 | rdv.flex.AddItem(rdv.tv, 0, 1, true) 101 | focusers = append(focusers, rdv.tv) 102 | 103 | rdv.flex.AddItem(nil, 1, 0, false) 104 | 105 | rdv.okBtn = tview.NewButton("OK") 106 | rdv.okBtn.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 107 | switch event.Key() { 108 | case tcell.KeyEnter: 109 | rdv.Hide() 110 | return nil 111 | } 112 | 113 | event = rdv.genericInputHandler(event, getGenericTabHandler(rdv.okBtn), nil, nil) 114 | if event == nil { 115 | return nil 116 | } 117 | 118 | return event 119 | }) 120 | focusers = append(focusers, rdv.okBtn) 121 | 122 | rdv.copyToClipboardBtn = tview.NewButton("Copy to clipboard") 123 | rdv.copyToClipboardBtn.SetInputCapture(func(event *tcell.EventKey) *tcell.EventKey { 124 | switch event.Key() { 125 | case tcell.KeyEnter: 126 | // TODO: check if clipboard is actually available 127 | clipboard.Write(clipboard.FmtText, []byte(rdv.params.Text)) 128 | rdv.copyToClipboardBtn.SetLabel("Copied!") 129 | return nil 130 | } 131 | 132 | rdv.copyToClipboardBtn.SetLabel("Copy to clipboard") 133 | 134 | event = rdv.genericInputHandler(event, getGenericTabHandler(rdv.copyToClipboardBtn), nil, nil) 135 | if event == nil { 136 | return nil 137 | } 138 | 139 | return event 140 | }) 141 | focusers = append(focusers, rdv.copyToClipboardBtn) 142 | 143 | bottomFlex := tview.NewFlex().SetDirection(tview.FlexColumn) 144 | bottomFlex. 145 | AddItem(rdv.okBtn, 10, 0, false). 146 | AddItem(nil, 1, 0, false). 147 | AddItem(rdv.copyToClipboardBtn, 25, 0, false). 148 | AddItem(nil, 1, 0, false) 149 | 150 | rdv.flex.AddItem(bottomFlex, 1, 0, false) 151 | 152 | rdv.frame = tview.NewFrame(rdv.flex).SetBorders(0, 0, 0, 0, 0, 0) 153 | rdv.frame.SetBorder(true).SetBorderPadding(0, 0, 1, 1) 154 | rdv.frame.SetTitle(rdv.params.Title) 155 | 156 | return rdv 157 | } 158 | 159 | func (rdv *MyTextView) Show() { 160 | rdv.mainView.showModal( 161 | pageNameTextView, rdv.frame, 162 | 121, 163 | 35, 164 | true, 165 | ) 166 | } 167 | 168 | func (rdv *MyTextView) Hide() { 169 | rdv.mainView.hideModal(pageNameTextView, true) 170 | } 171 | 172 | func (rdv *MyTextView) genericInputHandler( 173 | event *tcell.EventKey, 174 | genericTabHandler func(event *tcell.EventKey) *tcell.EventKey, 175 | getQFPart func(qf QueryFull) string, 176 | setQFPart func(qf *QueryFull, part string), 177 | ) *tcell.EventKey { 178 | event = genericTabHandler(event) 179 | if event == nil { 180 | return nil 181 | } 182 | 183 | switch event.Key() { 184 | case tcell.KeyEsc, tcell.KeyEnter: 185 | rdv.Hide() 186 | return nil 187 | 188 | case tcell.KeyRune: 189 | if event.Rune() == 'q' { 190 | rdv.Hide() 191 | return nil 192 | } 193 | } 194 | 195 | return event 196 | } 197 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/options.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "sync" 7 | "time" 8 | 9 | "github.com/juju/errors" 10 | ) 11 | 12 | type Options struct { 13 | Timezone *time.Location 14 | 15 | // MaxNumLines is how many log lines the nerdlog_agent.sh will return at 16 | // most. Initially it's set to 250. 17 | MaxNumLines int 18 | } 19 | 20 | type OptionsShared struct { 21 | mtx *sync.Mutex 22 | options Options 23 | } 24 | 25 | func NewOptionsShared(options Options) *OptionsShared { 26 | return &OptionsShared{ 27 | mtx: &sync.Mutex{}, 28 | options: options, 29 | } 30 | } 31 | 32 | func (o *OptionsShared) GetTimezone() *time.Location { 33 | o.mtx.Lock() 34 | defer o.mtx.Unlock() 35 | return o.options.Timezone 36 | } 37 | 38 | func (o *OptionsShared) GetMaxNumLines() int { 39 | o.mtx.Lock() 40 | defer o.mtx.Unlock() 41 | return o.options.MaxNumLines 42 | } 43 | 44 | func (o *OptionsShared) GetAll() Options { 45 | o.mtx.Lock() 46 | defer o.mtx.Unlock() 47 | return o.options 48 | } 49 | 50 | func (o *OptionsShared) Call(f func(o *Options)) { 51 | o.mtx.Lock() 52 | defer o.mtx.Unlock() 53 | f(&o.options) 54 | } 55 | 56 | type OptionMeta struct { 57 | // If AliasOf is non-empty, all the other fields are ignored. 58 | AliasOf string 59 | 60 | Get func(o *Options) string 61 | Set func(o *Options, value string) error 62 | Help string 63 | } 64 | 65 | var AllOptions = map[string]*OptionMeta{ 66 | "timezone": { // {{{ 67 | Get: func(o *Options) string { 68 | return o.Timezone.String() 69 | }, 70 | Set: func(o *Options, value string) error { 71 | loc, err := time.LoadLocation(value) 72 | if err != nil { 73 | return errors.Trace(err) 74 | } 75 | 76 | o.Timezone = loc 77 | return nil 78 | }, 79 | Help: "Timezone to use in the UI.", 80 | }, // }}} 81 | "maxnumlines": { // {{{ 82 | Get: func(o *Options) string { 83 | return fmt.Sprint(o.MaxNumLines) 84 | }, 85 | Set: func(o *Options, value string) error { 86 | maxNumLines, err := strconv.Atoi(value) 87 | if err != nil { 88 | return errors.Trace(err) 89 | } 90 | 91 | if maxNumLines < 2 { 92 | return errors.Errorf("numlines must be at least 2") 93 | } 94 | 95 | o.MaxNumLines = maxNumLines 96 | return nil 97 | }, 98 | Help: "How many log lines to fetch from each logstream in one query", 99 | }, 100 | "numlines": { 101 | AliasOf: "maxnumlines", 102 | }, // }}} 103 | } 104 | 105 | func OptionMetaByName(name string) *OptionMeta { 106 | meta, ok := AllOptions[name] 107 | if !ok { 108 | return nil 109 | } 110 | 111 | if meta.AliasOf != "" { 112 | var ok bool 113 | meta, ok = AllOptions[meta.AliasOf] 114 | if !ok { 115 | // This one would mean a programmer error, so we panic here. 116 | panic(fmt.Sprintf("option %s is defined as an alias of non-existing option %s", name, meta.AliasOf)) 117 | } 118 | } 119 | 120 | if meta.AliasOf != "" { 121 | panic(fmt.Sprintf("option %s is defined as an alias of another alias %s", name, meta.AliasOf)) 122 | } 123 | 124 | return meta 125 | } 126 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/query.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/dimonomid/nerdlog/shellescape" 5 | "github.com/juju/errors" 6 | ) 7 | 8 | // QueryFull contains everything that defines a query: the logstreams filter, time range, 9 | // and the query to filter logs. 10 | type QueryFull struct { 11 | LStreams string 12 | Time string 13 | Query string 14 | 15 | SelectQuery SelectQuery 16 | } 17 | 18 | var execName = "nerdlog" 19 | 20 | // numShellParts defines how many shell parts should be in the 21 | // shell-command-marshalled form. It looks like this: 22 | // 23 | // nerdlog --lstreams --time --pattern 24 | // 25 | // Therefore, there are 7 parts. 26 | var numShellParts = 1 + 3*2 27 | 28 | func (qf *QueryFull) MarshalShellCmd() string { 29 | parts := qf.MarshalShellCmdParts() 30 | return shellescape.Escape(parts) 31 | } 32 | 33 | func (qf *QueryFull) UnmarshalShellCmd(cmd string) error { 34 | parts, err := shellescape.Parse(cmd) 35 | if err != nil { 36 | return errors.Trace(err) 37 | } 38 | 39 | if err := qf.UnmarshalShellCmdParts(parts); err != nil { 40 | return errors.Trace(err) 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func (qf *QueryFull) MarshalShellCmdParts() []string { 47 | parts := make([]string, 0, numShellParts) 48 | 49 | parts = append(parts, execName) 50 | parts = append(parts, "--lstreams", qf.LStreams) 51 | parts = append(parts, "--time", qf.Time) 52 | parts = append(parts, "--pattern", qf.Query) 53 | parts = append(parts, "--selquery", string(qf.SelectQuery)) 54 | 55 | return parts 56 | } 57 | 58 | // UnmarshalShellCmdParts unmarshals shell command parts to the receiver 59 | // QueryFull. Note that no checks are performed as to whether LStreams, 60 | // Time or Query are actually valid strings. 61 | func (qf *QueryFull) UnmarshalShellCmdParts(parts []string) error { 62 | if len(parts) < numShellParts { 63 | return errors.Errorf( 64 | "not enough parts; should be at least %d, got %d", numShellParts, len(parts), 65 | ) 66 | } 67 | 68 | if parts[0] != execName { 69 | return errors.Errorf("command should begin from %q, but it's %q", execName, parts[0]) 70 | } 71 | 72 | parts = parts[1:] 73 | 74 | var lstreamsSet, timeSet, querySet, selectQuerySet bool 75 | 76 | for ; len(parts) >= 2; parts = parts[2:] { 77 | switch parts[0] { 78 | case "--lstreams": 79 | qf.LStreams = parts[1] 80 | lstreamsSet = true 81 | case "--time": 82 | qf.Time = parts[1] 83 | timeSet = true 84 | case "--pattern": 85 | qf.Query = parts[1] 86 | querySet = true 87 | case "--selquery": 88 | qf.SelectQuery = SelectQuery(parts[1]) 89 | selectQuerySet = true 90 | } 91 | } 92 | 93 | if !lstreamsSet { 94 | return errors.Errorf("--lstreams is missing") 95 | } 96 | 97 | if !timeSet { 98 | return errors.Errorf("--time is missing") 99 | } 100 | 101 | if !querySet { 102 | return errors.Errorf("--pattern is missing") 103 | } 104 | 105 | if !selectQuerySet { 106 | // NOTE: we can't return an error here since selquery was not there from the beginning 107 | qf.SelectQuery = DefaultSelectQuery 108 | } 109 | 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/rune_buffer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // RuneBuffer allows writing strings at arbitrary positions, expanding as needed 4 | type RuneBuffer struct { 5 | data []rune 6 | } 7 | 8 | // WriteAt writes a string at the given rune index, expanding the buffer if needed 9 | func (b *RuneBuffer) WriteAt(index int, s string) { 10 | runes := []rune(s) 11 | end := index + len(runes) 12 | 13 | // Expand the buffer if necessary 14 | if end > len(b.data) { 15 | newData := make([]rune, end) 16 | copy(newData, b.data) 17 | b.data = newData 18 | } 19 | 20 | copy(b.data[index:end], runes) 21 | } 22 | 23 | func (b *RuneBuffer) Rune(index int) (rune, bool) { 24 | if index >= 0 && index < len(b.data) { 25 | return b.data[index], true 26 | } 27 | return ' ', false 28 | } 29 | 30 | // String returns the current buffer as a string 31 | func (b *RuneBuffer) String() string { 32 | return string(b.data) 33 | } 34 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/select_query.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/juju/errors" 7 | ) 8 | 9 | var DefaultSelectQuery SelectQuery = FieldNameTime + " STICKY, " + FieldNameMessage + ", lstream, *" 10 | 11 | const ( 12 | FieldNameTime = "time" 13 | FieldNameMessage = "message" 14 | ) 15 | 16 | var FieldNamesSpecial = map[string]struct{}{ 17 | FieldNameTime: {}, 18 | FieldNameMessage: {}, 19 | } 20 | 21 | // TODO explain 22 | type SelectQuery string 23 | 24 | type SelectQueryParsed struct { 25 | Fields []SelectQueryField 26 | 27 | // If IncludeAll is true, then after all the fields in the Fields slice, all other 28 | // fields will be included in lexicographical order. 29 | IncludeAll bool 30 | } 31 | 32 | type SelectQueryField struct { 33 | // Name is the actual field name as is in logs. 34 | Name string 35 | 36 | // DisplayName is how it's displayed in the table header. Often it's the same 37 | // as Name. 38 | DisplayName string 39 | 40 | // If Sticky is true, the column will always be visible. 41 | Sticky bool 42 | } 43 | 44 | func ParseSelectQuery(sq SelectQuery) (*SelectQueryParsed, error) { 45 | ret := &SelectQueryParsed{} 46 | 47 | if sq == "" { 48 | return nil, errors.Errorf("no fields selected") 49 | } 50 | 51 | fields := strings.Split(string(sq), ",") 52 | 53 | for i, fldStr := range fields { 54 | parts := strings.Fields(fldStr) 55 | 56 | if len(parts) == 0 { 57 | return nil, errors.Errorf("empty field #%d", i) 58 | } 59 | 60 | if parts[0] == "*" { 61 | if len(parts) != 1 { 62 | return nil, errors.Errorf("invalid wildcard specifier") 63 | } 64 | 65 | ret.IncludeAll = true 66 | continue 67 | } 68 | 69 | if ret.IncludeAll { 70 | return nil, errors.Errorf("wildcard can only be the last item") 71 | } 72 | 73 | field := SelectQueryField{ 74 | Name: parts[0], 75 | DisplayName: parts[0], 76 | } 77 | 78 | const ( 79 | stateRoot = iota 80 | stateWaitDisplayName 81 | ) 82 | 83 | state := stateRoot 84 | for _, token := range parts[1:] { 85 | switch state { 86 | case stateRoot: 87 | switch strings.ToLower(token) { 88 | case "as": 89 | if field.Name != field.DisplayName { 90 | return nil, errors.Errorf("syntax error for field %s: more than a single 'AS'", field.Name) 91 | } 92 | 93 | state = stateWaitDisplayName 94 | continue 95 | case "sticky": 96 | if field.Sticky { 97 | return nil, errors.Errorf("syntax error for field %s: more than a single 'STICKY'", field.Name) 98 | } 99 | 100 | field.Sticky = true 101 | continue 102 | default: 103 | return nil, errors.Errorf("syntax error for field %s", field.Name) 104 | } 105 | 106 | case stateWaitDisplayName: 107 | field.DisplayName = token 108 | state = stateRoot 109 | } 110 | } 111 | 112 | if state != stateRoot { 113 | return nil, errors.Errorf("incomplete field %s", field.Name) 114 | } 115 | 116 | ret.Fields = append(ret.Fields, field) 117 | } 118 | 119 | return ret, nil 120 | } 121 | 122 | func (sqp *SelectQueryParsed) Marshal() SelectQuery { 123 | var sb strings.Builder 124 | 125 | add := func(i int, s string) { 126 | if i != 0 { 127 | sb.WriteString(", ") 128 | } 129 | 130 | sb.WriteString(s) 131 | } 132 | 133 | var n int 134 | for i, fld := range sqp.Fields { 135 | v := fld.Name 136 | 137 | if fld.DisplayName != fld.Name { 138 | v += " AS " + fld.DisplayName 139 | } 140 | 141 | if fld.Sticky { 142 | v += " STICKY" 143 | } 144 | 145 | add(i, v) 146 | n = i 147 | } 148 | 149 | if sqp.IncludeAll { 150 | add(n, "*") 151 | n++ 152 | } 153 | 154 | return SelectQuery(sb.String()) 155 | } 156 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/select_query_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSelectQuery(t *testing.T) { 10 | type testCase struct { 11 | descr string 12 | 13 | // str is the string select query that we try to parse. 14 | str SelectQuery 15 | 16 | // If strRemarshaled is not empty, it means that after we remarshal the query, 17 | // the result is expected to be different from str. 18 | strRemarshaled SelectQuery 19 | 20 | wantParsed *SelectQueryParsed 21 | wantErr string 22 | } 23 | 24 | testCases := []testCase{ 25 | testCase{descr: "close to default one", // {{{ 26 | str: "time STICKY, message, lstream, level_name AS lvl, redacted_id_int AS ds", 27 | wantParsed: &SelectQueryParsed{ 28 | Fields: []SelectQueryField{ 29 | SelectQueryField{ 30 | Name: "time", 31 | DisplayName: "time", 32 | Sticky: true, 33 | }, 34 | SelectQueryField{ 35 | Name: "message", 36 | DisplayName: "message", 37 | }, 38 | SelectQueryField{ 39 | Name: "lstream", 40 | DisplayName: "lstream", 41 | }, 42 | SelectQueryField{ 43 | Name: "level_name", 44 | DisplayName: "lvl", 45 | }, 46 | SelectQueryField{ 47 | Name: "redacted_id_int", 48 | DisplayName: "ds", 49 | }, 50 | }, 51 | }, 52 | }, // }}} 53 | testCase{descr: "two sticky fields", // {{{ 54 | str: "time STICKY, message, lstream, level_name AS lvl STICKY, redacted_id_int AS ds", 55 | wantParsed: &SelectQueryParsed{ 56 | Fields: []SelectQueryField{ 57 | SelectQueryField{ 58 | Name: "time", 59 | DisplayName: "time", 60 | Sticky: true, 61 | }, 62 | SelectQueryField{ 63 | Name: "message", 64 | DisplayName: "message", 65 | }, 66 | SelectQueryField{ 67 | Name: "lstream", 68 | DisplayName: "lstream", 69 | }, 70 | // NOTE that in the actual app it will be moved to the front (right after time), 71 | // but it's not done on the marshaling level, to avoid the remarshaled string 72 | // look differently. 73 | SelectQueryField{ 74 | Name: "level_name", 75 | DisplayName: "lvl", 76 | Sticky: true, 77 | }, 78 | SelectQueryField{ 79 | Name: "redacted_id_int", 80 | DisplayName: "ds", 81 | }, 82 | }, 83 | }, 84 | }, // }}} 85 | testCase{descr: "close to default one, with a wildcard", // {{{ 86 | str: "time STICKY, message, lstream, level_name AS lvl, redacted_id_int AS ds, *", 87 | wantParsed: &SelectQueryParsed{ 88 | Fields: []SelectQueryField{ 89 | SelectQueryField{ 90 | Name: "time", 91 | DisplayName: "time", 92 | Sticky: true, 93 | }, 94 | SelectQueryField{ 95 | Name: "message", 96 | DisplayName: "message", 97 | }, 98 | SelectQueryField{ 99 | Name: "lstream", 100 | DisplayName: "lstream", 101 | }, 102 | SelectQueryField{ 103 | Name: "level_name", 104 | DisplayName: "lvl", 105 | }, 106 | SelectQueryField{ 107 | Name: "redacted_id_int", 108 | DisplayName: "ds", 109 | }, 110 | }, 111 | 112 | IncludeAll: true, 113 | }, 114 | }, // }}} 115 | testCase{descr: "only wildcard", // {{{ 116 | str: "*", 117 | wantParsed: &SelectQueryParsed{ 118 | IncludeAll: true, 119 | }, 120 | }, // }}} 121 | testCase{descr: "no uppercase", // {{{ 122 | str: "time sticky, message, lstream, level_name as lvl, redacted_id_int as ds", 123 | strRemarshaled: "time STICKY, message, lstream, level_name AS lvl, redacted_id_int AS ds", 124 | wantParsed: &SelectQueryParsed{ 125 | Fields: []SelectQueryField{ 126 | SelectQueryField{ 127 | Name: "time", 128 | DisplayName: "time", 129 | Sticky: true, 130 | }, 131 | SelectQueryField{ 132 | Name: "message", 133 | DisplayName: "message", 134 | }, 135 | SelectQueryField{ 136 | Name: "lstream", 137 | DisplayName: "lstream", 138 | }, 139 | SelectQueryField{ 140 | Name: "level_name", 141 | DisplayName: "lvl", 142 | }, 143 | SelectQueryField{ 144 | Name: "redacted_id_int", 145 | DisplayName: "ds", 146 | }, 147 | }, 148 | }, 149 | }, // }}} 150 | 151 | testCase{descr: "wildcard as a non-last item: error", // {{{ 152 | str: "time STICKY, message, lstream, level_name AS lvl, *, redacted_id_int AS ds", 153 | wantErr: "wildcard can only be the last item", 154 | }, // }}} 155 | testCase{descr: "more than one AS: error", // {{{ 156 | str: "time STICKY, message, lstream, level_name AS foo AS bar, redacted_id_int as ds", 157 | wantErr: "syntax error for field level_name: more than a single 'AS'", 158 | }, // }}} 159 | testCase{descr: "empty: error", // {{{ 160 | str: "", 161 | wantErr: "no fields selected", 162 | }, // }}} 163 | } 164 | 165 | for i, tc := range testCases { 166 | assertArgs := []interface{}{"test case #%d (%s)", i, tc.descr} 167 | 168 | gotParsed, gotErr := ParseSelectQuery(tc.str) 169 | 170 | assert.Equal(t, tc.wantParsed, gotParsed, assertArgs...) 171 | 172 | if tc.wantErr == "" { 173 | assert.Nil(t, gotErr, assertArgs...) 174 | } else { 175 | assert.Equal(t, tc.wantErr, gotErr.Error(), assertArgs...) 176 | } 177 | 178 | if gotParsed != nil { 179 | gotRemarshaled := gotParsed.Marshal() 180 | wantRemarshaled := tc.strRemarshaled 181 | if wantRemarshaled == "" { 182 | wantRemarshaled = tc.str 183 | } 184 | 185 | assert.Equal(t, wantRemarshaled, gotRemarshaled, assertArgs...) 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/time_or_dur.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/juju/errors" 7 | ) 8 | 9 | type TimeOrDur struct { 10 | Time time.Time 11 | Dur time.Duration 12 | } 13 | 14 | func (t TimeOrDur) IsZero() bool { 15 | return t.Time.IsZero() && t.Dur == 0 16 | } 17 | 18 | func (t TimeOrDur) In(loc *time.Location) TimeOrDur { 19 | if t.Time.IsZero() { 20 | return t 21 | } 22 | 23 | t.Time = t.Time.In(loc) 24 | return t 25 | } 26 | 27 | func (t TimeOrDur) IsAbsolute() bool { 28 | return !t.Time.IsZero() 29 | } 30 | 31 | // AbsoluteTime returns the exact point in time, either relative to the 32 | // provided relativeTo, or if it represents an absolute point in time already, 33 | // then just returns it (and then relativeTo is ignored). 34 | // 35 | // If relativeTo is zero, AbsoluteTime panics. 36 | // 37 | // AbsoluteTime can never return a zero time. 38 | func (t TimeOrDur) AbsoluteTime(relativeTo time.Time) time.Time { 39 | if relativeTo.IsZero() { 40 | panic("relativeTo can't be zero") 41 | } 42 | 43 | if !t.Time.IsZero() { 44 | return t.Time 45 | } 46 | 47 | return relativeTo.Add(t.Dur) 48 | } 49 | 50 | func (t TimeOrDur) String() string { 51 | if !t.Time.IsZero() { 52 | return t.Time.String() 53 | } 54 | 55 | return formatDuration(t.Dur) 56 | } 57 | 58 | func (t TimeOrDur) Format(layout string) string { 59 | if !t.Time.IsZero() { 60 | return t.Time.Format(layout) 61 | } 62 | 63 | return formatDuration(t.Dur) 64 | } 65 | 66 | // ParseTimeOrDur tries to parse a string as either time or duration. 67 | // If parsing as a duration succeeds, then layout is ignored; otherwise it's 68 | // used to parse it as time. 69 | func ParseTimeOrDur(timezone *time.Location, layout, s string) (TimeOrDur, error) { 70 | // Try to parse as a duration first 71 | dur, err := time.ParseDuration(s) 72 | if err == nil { 73 | return TimeOrDur{ 74 | Dur: dur, 75 | }, nil 76 | } 77 | 78 | // Now try to parse as a time 79 | t, err := time.ParseInLocation(layout, s, timezone) 80 | if err != nil { 81 | return TimeOrDur{}, errors.Trace(err) 82 | } 83 | 84 | return TimeOrDur{ 85 | Time: t, 86 | }, nil 87 | } 88 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/ui/table_with_dropdown.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "github.com/gdamore/tcell/v2" 5 | "github.com/rivo/tview" 6 | ) 7 | 8 | type TableWithDropdown struct { 9 | *tview.Table 10 | 11 | // The options from which the user can choose. 12 | options []*dropDownOption 13 | 14 | // Strings to be placed before and after each drop-down option. 15 | optionPrefix, optionSuffix string 16 | 17 | // The index of the currently selected option. Negative if no option is 18 | // currently selected. 19 | currentOption int 20 | 21 | list *tview.List 22 | 23 | dropdownRow int 24 | dropdownCol int 25 | } 26 | 27 | func NewTableWithDropdown() *TableWithDropdown { 28 | t := &TableWithDropdown{ 29 | Table: tview.NewTable(), 30 | 31 | optionPrefix: " ", 32 | optionSuffix: " ", 33 | 34 | currentOption: -1, 35 | 36 | list: tview.NewList(), 37 | 38 | dropdownRow: -1, 39 | dropdownCol: -1, 40 | } 41 | 42 | t.list.ShowSecondaryText(false). 43 | SetMainTextColor(tview.Styles.PrimitiveBackgroundColor). 44 | SetSelectedTextColor(tview.Styles.PrimitiveBackgroundColor). 45 | SetSelectedBackgroundColor(tview.Styles.PrimaryTextColor). 46 | SetHighlightFullLine(true). 47 | SetBackgroundColor(tview.Styles.MoreContrastBackgroundColor) 48 | 49 | t.list.SetFocusFunc(func() { 50 | // We want selection to still show when the dropdown is focused 51 | t.SetSelectable(true, false) 52 | }) 53 | 54 | return t 55 | } 56 | 57 | func (t *TableWithDropdown) IsDropdownOpen() bool { 58 | return t.dropdownRow >= 0 && t.dropdownCol >= 0 59 | } 60 | 61 | func (t *TableWithDropdown) GetDropdownList() *tview.List { 62 | return t.list 63 | } 64 | 65 | // Focus is called by the application when the primitive receives focus. 66 | func (t *TableWithDropdown) Focus(delegate func(p tview.Primitive)) { 67 | if t.IsDropdownOpen() { 68 | delegate(t.list) 69 | } else { 70 | t.Box.Focus(delegate) 71 | } 72 | } 73 | 74 | // HasFocus returns whether or not this primitive has focus. 75 | func (t *TableWithDropdown) HasFocus() bool { 76 | if t.IsDropdownOpen() { 77 | return t.list.HasFocus() 78 | } 79 | return t.Box.HasFocus() 80 | } 81 | 82 | func (t *TableWithDropdown) InputHandler() func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { 83 | return t.WrapInputHandler(func(event *tcell.EventKey, setFocus func(p tview.Primitive)) { 84 | // If the list has focus, let it process its own key events. 85 | if t.list.HasFocus() { 86 | if handler := t.list.InputHandler(); handler != nil { 87 | handler(event, setFocus) 88 | } 89 | return 90 | } 91 | 92 | t.Table.InputHandler()(event, setFocus) 93 | }) 94 | } 95 | 96 | func (t *TableWithDropdown) ClearOptions() *TableWithDropdown { 97 | t.options = nil 98 | t.list.Clear() 99 | return t 100 | } 101 | 102 | func (t *TableWithDropdown) AddOption(text string, selected func()) *TableWithDropdown { 103 | t.options = append(t.options, &dropDownOption{Text: text, Selected: selected}) 104 | t.list.AddItem(t.optionPrefix+text+t.optionSuffix, "", 0, nil) 105 | return t 106 | } 107 | 108 | func (t *TableWithDropdown) SetListStyles(unselected, selected tcell.Style) *TableWithDropdown { 109 | fg, bg, _ := unselected.Decompose() 110 | t.list.SetMainTextColor(fg).SetBackgroundColor(bg) 111 | fg, bg, _ = selected.Decompose() 112 | t.list.SetSelectedTextColor(fg).SetSelectedBackgroundColor(bg) 113 | return t 114 | } 115 | 116 | func (t *TableWithDropdown) CloseDropdownList(setFocus func(p tview.Primitive)) { 117 | t.closeList(setFocus) 118 | } 119 | 120 | func (t *TableWithDropdown) OpenDropdownList(row, col int, setFocus func(p tview.Primitive)) { 121 | t.dropdownRow = row 122 | t.dropdownCol = col 123 | 124 | t.list.SetSelectedFunc(func(index int, mainText, secondaryText string, shortcut rune) { 125 | // An option was selected. Close the list again. 126 | t.currentOption = index 127 | t.closeList(setFocus) 128 | 129 | if t.options[t.currentOption].Selected != nil { 130 | t.options[t.currentOption].Selected() 131 | } 132 | }) 133 | 134 | setFocus(t.list) 135 | } 136 | 137 | func (t *TableWithDropdown) closeList(setFocus func(tview.Primitive)) { 138 | t.dropdownRow = -1 139 | t.dropdownCol = -1 140 | if t.list.HasFocus() { 141 | setFocus(t) 142 | } 143 | } 144 | 145 | func (t *TableWithDropdown) Draw(screen tcell.Screen) { 146 | t.Table.Draw(screen) 147 | 148 | if t.IsDropdownOpen() { 149 | x, y, _ := t.GetCell(t.dropdownRow, t.dropdownCol).GetLastPosition() 150 | 151 | // What's the longest option text? 152 | maxWidth := 0 153 | optionWrapWidth := tview.TaggedStringWidth(t.optionPrefix + t.optionSuffix) 154 | for _, option := range t.options { 155 | strWidth := tview.TaggedStringWidth(option.Text) + optionWrapWidth 156 | if strWidth > maxWidth { 157 | maxWidth = strWidth 158 | } 159 | } 160 | 161 | lx := x 162 | ly := y + 1 163 | lwidth := maxWidth 164 | lheight := len(t.options) 165 | swidth, sheight := screen.Size() 166 | // We prefer to align the list left side with the main widget left size, 167 | // but if there is no space, then shift the list to the left. 168 | if lx+lwidth >= swidth { 169 | lx = swidth - lwidth 170 | if lx < 0 { 171 | lx = 0 172 | } 173 | } 174 | // We prefer to drop down but if there is no space, maybe drop up? 175 | if ly+lheight >= sheight && ly-2 > lheight-ly { 176 | ly = y - lheight 177 | if ly < 0 { 178 | ly = 0 179 | } 180 | } 181 | if ly+lheight >= sheight { 182 | lheight = sheight - ly 183 | } 184 | t.list.SetRect(lx, ly, lwidth, lheight) 185 | t.list.Draw(screen) 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/ui/util.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "github.com/mattn/go-runewidth" 5 | "github.com/rivo/uniseg" 6 | ) 7 | 8 | // stringWidth returns the number of horizontal cells needed to print the given 9 | // text. It splits the text into its grapheme clusters, calculates each 10 | // cluster's width, and adds them up to a total. 11 | func stringWidth(text string) (width int) { 12 | g := uniseg.NewGraphemes(text) 13 | for g.Next() { 14 | var chWidth int 15 | for _, r := range g.Runes() { 16 | chWidth = runewidth.RuneWidth(r) 17 | if chWidth > 0 { 18 | break // Our best guess at this point is to use the width of the first non-zero-width rune. 19 | } 20 | } 21 | width += chWidth 22 | } 23 | return 24 | } 25 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/xmarks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // getXMarksForTimeRange takes a time range which represent a timescale, and 8 | // returns timestamps of a few marks that make sense to put on the timescale 9 | // for the human to see; the number of marks is as close to maxNumMarks, but not 10 | // larger than this. 11 | // 12 | // The returned marks are on the most round places: e.g. if there are multiple 13 | // days, then at least some marks must be on the day boundary; the marks are 14 | // usually divisible by 5, 10, 30, or 60 mins, etc. 15 | func getXMarksForTimeRange(timezone *time.Location, from, to time.Time, maxNumMarks int) []time.Time { 16 | if !from.Before(to) || maxNumMarks <= 0 { 17 | return nil 18 | } 19 | 20 | duration := to.Sub(from) 21 | step := chooseStep(duration, maxNumMarks) 22 | if step == 0 { 23 | return nil 24 | } 25 | 26 | // Align the first mark to the step boundary. 27 | start := truncateAlignedToMidnight(from, step, timezone) 28 | if start.Before(from) { 29 | start = start.Add(step) 30 | } 31 | 32 | var marks []time.Time 33 | for t := start; !t.After(to); t = t.Add(step) { 34 | marks = append(marks, t) 35 | if len(marks) >= maxNumMarks { 36 | break 37 | } 38 | } 39 | return marks 40 | } 41 | 42 | // Like time.Truncate, but instead of aligning with zero time, aligns with the 43 | // closest midnight earlier than the given t in the given location. 44 | func truncateAlignedToMidnight(t time.Time, d time.Duration, loc *time.Location) time.Time { 45 | t = t.In(loc) 46 | 47 | // If the location is UTC (or otherwise with zero offset), just use 48 | // standard Truncate. 49 | _, offset := t.Zone() 50 | if offset == 0 { 51 | return t.Truncate(d) 52 | } 53 | 54 | // Looks like the offset is non-zero, so align with the midnight in that timezone. 55 | midnight := time.Date( 56 | t.Year(), 57 | t.Month(), 58 | t.Day(), 59 | 0, 0, 0, 0, 60 | loc, 61 | ) 62 | 63 | sinceMidnight := t.Sub(midnight) 64 | truncatedSinceMidnight := sinceMidnight.Truncate(d) 65 | 66 | return midnight.Add(truncatedSinceMidnight) 67 | } 68 | 69 | var snaps = []time.Duration{ 70 | time.Minute * 1, 71 | time.Minute * 2, 72 | time.Minute * 5, 73 | time.Minute * 10, 74 | time.Minute * 15, 75 | time.Minute * 20, 76 | time.Minute * 30, 77 | time.Hour * 1, 78 | time.Hour * 2, 79 | time.Hour * 3, 80 | time.Hour * 6, 81 | time.Hour * 12, 82 | time.Hour * 24, 83 | time.Hour * 24 * 2, 84 | time.Hour * 24 * 7, 85 | time.Hour * 24 * 30, 86 | time.Hour * 24 * 365, 87 | } 88 | 89 | // chooseStep picks a "round" duration step that will produce close to maxNumMarks marks. 90 | func chooseStep(duration time.Duration, maxNumMarks int) time.Duration { 91 | for _, step := range snaps { 92 | if int(duration/step) <= maxNumMarks { 93 | return step 94 | } 95 | } 96 | 97 | return snaps[len(snaps)-1] 98 | } 99 | 100 | func getXMarksForHistogram(timezone *time.Location, from, to int, numChars int) []int { 101 | const minCharsDistanceBetweenMarks = 15 102 | numMarks := numChars / minCharsDistanceBetweenMarks 103 | 104 | fromTime := time.Unix(int64(from), 0).In(timezone) 105 | toTime := time.Unix(int64(to), 0).In(timezone) 106 | 107 | marksTime := getXMarksForTimeRange(timezone, fromTime, toTime, numMarks) 108 | ret := make([]int, 0, len(marksTime)) 109 | for _, v := range marksTime { 110 | ret = append(ret, int(v.Unix())) 111 | } 112 | 113 | return ret 114 | } 115 | 116 | func snapDataBinsInChartDot(dataBinsInChartDot int) int { 117 | for _, snap := range snaps { 118 | snapMinutes := int(snap / time.Minute) 119 | 120 | if dataBinsInChartDot <= snapMinutes { 121 | return snapMinutes 122 | } 123 | } 124 | 125 | return int(snaps[len(snaps)-1] / time.Minute) 126 | } 127 | -------------------------------------------------------------------------------- /cmd/nerdlog-tui/xmarks_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func formatRFC3339Slice(vals []time.Time) []string { 11 | var result []string 12 | for _, v := range vals { 13 | s := v.Format(time.RFC3339) 14 | result = append(result, s) 15 | } 16 | return result 17 | } 18 | 19 | func TestGetXMarksForTimeRange(t *testing.T) { 20 | tests := []struct { 21 | name string 22 | from, to string 23 | maxMarks int 24 | expected []string 25 | }{ 26 | { 27 | name: "1-hour range, 10 marks", 28 | from: "2023-01-01T12:00:00Z", 29 | to: "2023-01-01T13:00:00Z", 30 | maxMarks: 10, 31 | expected: []string{ 32 | "2023-01-01T12:00:00Z", 33 | "2023-01-01T12:10:00Z", 34 | "2023-01-01T12:20:00Z", 35 | "2023-01-01T12:30:00Z", 36 | "2023-01-01T12:40:00Z", 37 | "2023-01-01T12:50:00Z", 38 | "2023-01-01T13:00:00Z", 39 | }, 40 | }, 41 | { 42 | name: "24-hour range, 5 marks", 43 | from: "2023-01-01T00:00:00Z", 44 | to: "2023-01-02T00:00:00Z", 45 | maxMarks: 5, 46 | expected: []string{ 47 | "2023-01-01T00:00:00Z", 48 | "2023-01-01T06:00:00Z", 49 | "2023-01-01T12:00:00Z", 50 | "2023-01-01T18:00:00Z", 51 | "2023-01-02T00:00:00Z", 52 | }, 53 | }, 54 | { 55 | name: "10-day range, 7 marks", 56 | from: "2023-01-01T00:00:00Z", 57 | to: "2023-01-11T00:00:00Z", 58 | maxMarks: 7, 59 | expected: []string{ 60 | "2023-01-01T00:00:00Z", 61 | "2023-01-03T00:00:00Z", 62 | "2023-01-05T00:00:00Z", 63 | "2023-01-07T00:00:00Z", 64 | "2023-01-09T00:00:00Z", 65 | "2023-01-11T00:00:00Z", 66 | }, 67 | }, 68 | { 69 | name: "30-minute range, 4 marks", 70 | from: "2023-01-01T10:00:00Z", 71 | to: "2023-01-01T10:30:00Z", 72 | maxMarks: 4, 73 | expected: []string{ 74 | "2023-01-01T10:00:00Z", 75 | "2023-01-01T10:10:00Z", 76 | "2023-01-01T10:20:00Z", 77 | "2023-01-01T10:30:00Z", 78 | }, 79 | }, 80 | { 81 | name: "Invalid range", 82 | from: "2023-01-02T00:00:00Z", 83 | to: "2023-01-01T00:00:00Z", 84 | maxMarks: 5, 85 | expected: nil, 86 | }, 87 | 88 | { 89 | name: "Odd 1h17m range, 5 marks", 90 | from: "2023-01-01T09:13:00Z", 91 | to: "2023-01-01T10:30:00Z", 92 | maxMarks: 5, 93 | expected: []string{ 94 | "2023-01-01T09:15:00Z", 95 | "2023-01-01T09:30:00Z", 96 | "2023-01-01T09:45:00Z", 97 | "2023-01-01T10:00:00Z", 98 | "2023-01-01T10:15:00Z", 99 | }, 100 | }, 101 | { 102 | name: "Odd 2h37m range, 6 marks", 103 | from: "2023-01-01T03:27:00Z", 104 | to: "2023-01-01T06:04:00Z", 105 | maxMarks: 6, 106 | expected: []string{ 107 | "2023-01-01T03:30:00Z", 108 | "2023-01-01T04:00:00Z", 109 | "2023-01-01T04:30:00Z", 110 | "2023-01-01T05:00:00Z", 111 | "2023-01-01T05:30:00Z", 112 | "2023-01-01T06:00:00Z", 113 | }, 114 | }, 115 | { 116 | name: "3-day non-midnight range", 117 | from: "2023-01-01T08:20:00Z", 118 | to: "2023-01-04T17:40:00Z", 119 | maxMarks: 7, 120 | expected: []string{ 121 | "2023-01-01T12:00:00Z", 122 | "2023-01-02T00:00:00Z", 123 | "2023-01-02T12:00:00Z", 124 | "2023-01-03T00:00:00Z", 125 | "2023-01-03T12:00:00Z", 126 | "2023-01-04T00:00:00Z", 127 | "2023-01-04T12:00:00Z", 128 | }, 129 | }, 130 | { 131 | name: "2-day non-midnight range", 132 | from: "2023-01-01T08:20:00Z", 133 | to: "2023-01-03T17:40:00Z", 134 | maxMarks: 7, 135 | expected: []string{ 136 | "2023-01-01T12:00:00Z", 137 | "2023-01-02T00:00:00Z", 138 | "2023-01-02T12:00:00Z", 139 | "2023-01-03T00:00:00Z", 140 | "2023-01-03T12:00:00Z", 141 | }, 142 | }, 143 | } 144 | 145 | for _, tt := range tests { 146 | t.Run(tt.name, func(t *testing.T) { 147 | from, _ := time.Parse(time.RFC3339, tt.from) 148 | to, _ := time.Parse(time.RFC3339, tt.to) 149 | 150 | actual := getXMarksForTimeRange(time.UTC, from, to, tt.maxMarks) 151 | actualStrs := formatRFC3339Slice(actual) 152 | 153 | assert.Equal(t, tt.expected, actualStrs) 154 | }) 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /core/config.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "sort" 4 | 5 | type ConfigLogStreams map[string]ConfigLogStream 6 | 7 | type ConfigLogStream struct { 8 | // HostAddr is the actual host to connect to. 9 | // 10 | // If empty, we'll resort to the ssh config, and if there's no info for that 11 | // host either, we'll try to get host addr from the key in 12 | // ConfigLogStreams.LogStreams. 13 | Hostname string `yaml:"hostname"` 14 | 15 | // Port is the actual port to connect to. If empty, the same overriding rules 16 | // apply. 17 | Port string `yaml:"port"` 18 | 19 | // User is the user to authenticate as. If empty, same overriding rules 20 | // apply. 21 | User string `yaml:"user"` 22 | 23 | // TODO: optional Jumphost configuration, also with addr and user. 24 | 25 | // LogFiles contains a list of files which are part of the logstream, like 26 | // ["/var/log/syslog", "/var/log/syslog.1"]. The [0]th item is the latest log 27 | // file [1]st is the previous one, etc. 28 | // 29 | // During the final usage (after resolving everything), it must contain at 30 | // least a single item, otherwise LogStream is invalid. However in the configs, 31 | // it's optional (and eventually, if empty, will be set to default values by 32 | // the LStreamsResolver). 33 | LogFiles []string `yaml:"log_files"` 34 | } 35 | 36 | func (lss ConfigLogStreams) Keys() []string { 37 | keys := make([]string, 0, len(lss)) 38 | for k := range lss { 39 | keys = append(keys, k) 40 | } 41 | 42 | sort.Strings(keys) 43 | 44 | return keys 45 | } 46 | -------------------------------------------------------------------------------- /core/core.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "time" 4 | 5 | const ( 6 | // MaxNumLinesDefault is a default for QueryLogsParams.MaxNumLines below. 7 | MaxNumLinesDefault = 250 8 | ) 9 | 10 | type QueryLogsParams struct { 11 | // maxNumLines is how many log lines the nerdlog_agent.sh will return at 12 | // most. 13 | MaxNumLines int 14 | 15 | From time.Time 16 | To time.Time 17 | 18 | Query string 19 | 20 | // If LoadEarlier is true, it means we're only loading the logs _before_ the ones 21 | // we already had. 22 | LoadEarlier bool 23 | 24 | // If DontAddHistoryItem is true, the browser-like history will not be 25 | // populated with a new item (it should be used exactly when we're navigating 26 | // this browser-like history back and forth) 27 | DontAddHistoryItem bool 28 | } 29 | 30 | // LogResp is a log response from a single logstream 31 | type LogResp struct { 32 | // MinuteStats is a map from the unix timestamp (in seconds) to the stats for 33 | // the minute starting at this timestamp. 34 | MinuteStats map[int64]MinuteStatsItem 35 | 36 | Logs []LogMsg 37 | 38 | // NumMsgsTotal is the total number of messages in the time range (and 39 | // included in MinuteStats). This number is usually larger than len(Logs). 40 | NumMsgsTotal int 41 | } 42 | 43 | // LogRespTotal is a log response from a LStreamsManager. It's merged from 44 | // multiple LogResp's and it also contains some extra field(s), e.g. LoadedEarlier. 45 | type LogRespTotal struct { 46 | // If LoadedEarlier is true, it means we've just loaded more logs instead of replacing 47 | // the logs (the Logs slice still contains everything though). 48 | LoadedEarlier bool 49 | 50 | // MinuteStats is a map from the unix timestamp (in seconds) to the stats for 51 | // the minute starting at this timestamp. 52 | MinuteStats map[int64]MinuteStatsItem 53 | 54 | Logs []LogMsg 55 | 56 | // NumMsgsTotal is the total number of messages in the time range (and 57 | // included in MinuteStats). This number is usually larger than len(Logs). 58 | NumMsgsTotal int 59 | 60 | Errs []error 61 | 62 | // QueryDur shows how long the query took. 63 | QueryDur time.Duration 64 | } 65 | 66 | type MinuteStatsItem struct { 67 | NumMsgs int 68 | } 69 | 70 | type LogMsg struct { 71 | Time time.Time 72 | DecreasedTimestamp bool 73 | 74 | // LogFilename and LogLinenumber are file ane line number in that file 75 | LogFilename string 76 | LogLinenumber int 77 | 78 | // CombinedLinenumber is the line number in pseudo-file: all (actually just 79 | // two) log files concatenated. This is the linenumbers output by the 80 | // nerdlog_agent.sh for every "msg:" line, and this is the linenumber 81 | // which should be used for --lines-until param. 82 | CombinedLinenumber int 83 | 84 | Msg string 85 | Context map[string]string 86 | Level LogLevel 87 | 88 | OrigLine string 89 | } 90 | 91 | type LogLevel string 92 | 93 | const LogLevelUnknown LogLevel = "" 94 | const LogLevelDebug LogLevel = "debug" 95 | const LogLevelInfo LogLevel = "info" 96 | const LogLevelWarn LogLevel = "warn" 97 | const LogLevelError LogLevel = "error" 98 | -------------------------------------------------------------------------------- /core/lstream_cmd.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "time" 4 | 5 | type lstreamCmd struct { 6 | // respCh must be either nil, or 1-buffered and it'll receive exactly one 7 | // message. 8 | respCh chan lstreamCmdRes 9 | 10 | // Exactly one of the fields below must be non-nil. 11 | 12 | bootstrap *lstreamCmdBootstrap 13 | ping *lstreamCmdPing 14 | queryLogs *lstreamCmdQueryLogs 15 | } 16 | 17 | type lstreamCmdCtx struct { 18 | cmd lstreamCmd 19 | 20 | idx int 21 | 22 | bootstrapCtx *lstreamCmdCtxBootstrap 23 | pingCtx *lstreamCmdCtxPing 24 | queryLogsCtx *lstreamCmdCtxQueryLogs 25 | 26 | // Initially, stdoutDoneIdx and stderrDoneIdx are set to false. Once we 27 | // receive the "command_done" marker from either stdout or stderr, we set the 28 | // corresponding bool here to true. Once both are set, we consider the 29 | // command execution done, and parse the results. 30 | stdoutDone bool 31 | stderrDone bool 32 | 33 | // errs contains all errors accumulated during command execution. This 34 | // includes errors printed by the nerdlog_agent.sh (lines starting from 35 | // "error:", on either stderr or stdout), as well as any errors generated 36 | // on the Go side, e.g. failure to parse some other output. 37 | errs []error 38 | 39 | exitCode string 40 | 41 | // unhandledStdout and unhandledStderr contain the lines which the Go app did 42 | // not make sense of. These are usually ignored, but if the the 43 | // nerdlog_agent.sh returns an error code, and there are no specific errors 44 | // printed (lines with the "error:" prefix), then we'll print all these 45 | // as an error message. 46 | unhandledStdout []string 47 | unhandledStderr []string 48 | } 49 | 50 | type lstreamCmdRes struct { 51 | hostname string 52 | 53 | err error 54 | resp interface{} 55 | } 56 | 57 | type lstreamCmdBootstrap struct{} 58 | 59 | type lstreamCmdCtxBootstrap struct { 60 | receivedSuccess bool 61 | receivedFailure bool 62 | } 63 | 64 | type lstreamCmdPing struct{} 65 | 66 | type lstreamCmdCtxPing struct { 67 | } 68 | 69 | type lstreamCmdQueryLogs struct { 70 | maxNumLines int 71 | 72 | from time.Time 73 | to time.Time 74 | 75 | query string 76 | 77 | // If linesUntil is not zero, it'll be passed to nerdlog_agent.sh as --lines-until. 78 | // Effectively, only logs BEFORE this log line (not including it) will be output. 79 | linesUntil int 80 | } 81 | 82 | type lstreamCmdCtxQueryLogs struct { 83 | Resp *LogResp 84 | 85 | logfiles []logfileWithStartingLinenumber 86 | lastTime time.Time 87 | } 88 | 89 | type logfileWithStartingLinenumber struct { 90 | filename string 91 | fromLinenumber int 92 | } 93 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/logfiles/tiny/syslog: -------------------------------------------------------------------------------- 1 | Mar 10 10:00:01 myhost kern[5159]: Disk space reclaimed 2 | Mar 10 10:14:05 myhost auth[8368]: Database schema updated 3 | Mar 10 10:20:17 myhost syslog[4163]: System health check failed 4 | Mar 10 10:20:46 myhost lpr[891]: User session timed out 5 | Mar 10 10:24:32 myhost user[8515]: Cache cleared 6 | Mar 10 10:27:26 myhost kern[2205]: Session token expired 7 | Mar 10 10:27:26 myhost cron[9005]: File transfer completed 8 | Mar 10 10:32:21 myhost daemon[8000]: Failed login attempt 9 | Mar 10 10:32:21 myhost mail[7726]: Error reading file 10 | Mar 10 10:33:00 myhost kern[4506]: Service request queued 11 | Mar 10 10:34:31 myhost cron[935]: Database connection error 12 | Mar 10 10:36:14 myhost user[2831]: File system full 13 | Mar 10 10:38:25 myhost mail[8342]: User account disabled 14 | Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 15 | Mar 10 10:51:01 myhost user[3758]: System running low on resources 16 | Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 17 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/logfiles/tiny/syslog.1: -------------------------------------------------------------------------------- 1 | Mar 10 09:00:36 myhost ftp[3406]: Timeout occurred 2 | Mar 10 09:02:02 myhost authpriv[1893]: CPU temperature critical 3 | Mar 10 09:02:02 myhost cron[424]: System running low on resources 4 | Mar 10 09:02:02 myhost authpriv[1827]: Cache cleared 5 | Mar 10 09:05:07 myhost cron[5530]: Firewall rule deleted 6 | Mar 10 09:05:07 myhost daemon[5617]: File upload completed 7 | Mar 10 09:05:44 myhost auth[6052]: Certificate expiration warning 8 | Mar 10 09:05:46 myhost auth[4149]: Memory leak detected 9 | Mar 10 09:14:40 myhost authpriv[3851]: Log file archived 10 | Mar 10 09:22:23 myhost auth[3925]: Server started successfully 11 | Mar 10 09:28:01 myhost news[9026]: Error reading file 12 | Mar 10 09:31:23 myhost authpriv[5771]: User session ended 13 | Mar 10 09:31:23 myhost authpriv[2976]: Cache cleared 14 | Mar 10 09:35:23 myhost kern[3027]: SMTP server connection error 15 | Mar 10 09:35:23 myhost syslog[3626]: Application crash reported 16 | Mar 10 09:39:31 myhost auth[8464]: User session started 17 | Mar 10 09:44:56 myhost news[3840]: System health check completed 18 | Mar 10 09:53:11 myhost news[816]: System configuration restored 19 | Mar 10 09:59:58 myhost ftp[3724]: Out of memory error 20 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_set/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-10-00:00", 10 | "--to", "2025-03-11-00:00" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_set/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-10-00:00 isn't found, will use the beginning 21 | debug:the to 2025-03-11-00:00 isn't found, will use the end 22 | p:stage:3:querying logs 23 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_set/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_set/logfile 24 | p:stage:4:done 25 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_set/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_set/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_set/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 29 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 30 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 31 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 32 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 33 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 34 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 35 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 36 | exit_code:0 37 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_unset/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-10-00:00" 10 | ] 11 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_unset/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-10-00:00 isn't found, will use the beginning 21 | p:stage:3:querying logs 22 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_unset/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_unset/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_set_to_is_unset/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_unset/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_set_to_is_unset/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 29 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 30 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 31 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 32 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 33 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 34 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 35 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 36 | exit_code:0 37 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_set/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--to", "2025-03-11-00:00" 10 | ] 11 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_set/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the to 2025-03-11-00:00 isn't found, will use the end 21 | p:stage:3:querying logs 22 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_set/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_set/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_set/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_set/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_set/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 29 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 30 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 31 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 32 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 33 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 34 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 35 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 36 | exit_code:0 37 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_unset/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8" 9 | ] 10 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_unset/want_stderr: -------------------------------------------------------------------------------- 1 | debug:neither --from or --to are given, but index doesn't exist at all, gonna rebuild 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | p:stage:3:querying logs 21 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_unset/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_unset/logfile 22 | p:stage:4:done 23 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/all_existing_logs/01_from_is_unset_to_is_unset/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_unset/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/all_existing_logs/01_from_is_unset_to_is_unset/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 29 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 30 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 31 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 32 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 33 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 34 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 35 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 36 | exit_code:0 37 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-10-09:30", 10 | "--to", "2025-03-10-10:30" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-09:30 is found: 280 (18618) 24 | debug:the to 2025-03-10-10:30 is found: 295 (19615) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 18618 in prev /tmp/nerdlog_agent_test_output/edge_of_two_fles/01_basic/logfile.1 to offset 458 in latest /tmp/nerdlog_agent_test_output/edge_of_two_fles/01_basic/logfile 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/edge_of_two_fles/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/edge_of_two_fles/01_basic/logfile:287 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:59,1 6 | s:Mar 10 10:14,1 7 | s:Mar 10 10:24,1 8 | s:Mar 10 09:31,2 9 | s:Mar 10 10:27,2 10 | s:Mar 10 09:53,1 11 | s:Mar 10 09:44,1 12 | s:Mar 10 09:35,2 13 | s:Mar 10 10:00,1 14 | m:287:Mar 10 09:59:58 myhost ftp[3724]: Out of memory error 15 | m:288:Mar 10 10:00:01 myhost kern[5159]: Disk space reclaimed 16 | m:289:Mar 10 10:14:05 myhost auth[8368]: Database schema updated 17 | m:290:Mar 10 10:20:17 myhost syslog[4163]: System health check failed 18 | m:291:Mar 10 10:20:46 myhost lpr[891]: User session timed out 19 | m:292:Mar 10 10:24:32 myhost user[8515]: Cache cleared 20 | m:293:Mar 10 10:27:26 myhost kern[2205]: Session token expired 21 | m:294:Mar 10 10:27:26 myhost cron[9005]: File transfer completed 22 | exit_code:0 23 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-10-09:30", 10 | "--to", "2025-03-10-10:30", 11 | "--lines-until", "287" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-09:30 is found: 280 (18618) 24 | debug:the to 2025-03-10-10:30 is found: 295 (19615) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 18618 in prev /tmp/nerdlog_agent_test_output/edge_of_two_fles/03_basic_more_less_than_max/logfile.1 to offset 458 in latest /tmp/nerdlog_agent_test_output/edge_of_two_fles/03_basic_more_less_than_max/logfile 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/edge_of_two_fles/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/edge_of_two_fles/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/edge_of_two_fles/03_basic_more_less_than_max/logfile:287 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:59,1 6 | s:Mar 10 10:14,1 7 | s:Mar 10 10:24,1 8 | s:Mar 10 09:31,2 9 | s:Mar 10 10:27,2 10 | s:Mar 10 09:53,1 11 | s:Mar 10 09:44,1 12 | s:Mar 10 09:35,2 13 | s:Mar 10 10:00,1 14 | m:280:Mar 10 09:31:23 myhost authpriv[5771]: User session ended 15 | m:281:Mar 10 09:31:23 myhost authpriv[2976]: Cache cleared 16 | m:282:Mar 10 09:35:23 myhost kern[3027]: SMTP server connection error 17 | m:283:Mar 10 09:35:23 myhost syslog[3626]: Application crash reported 18 | m:284:Mar 10 09:39:31 myhost auth[8464]: User session started 19 | m:285:Mar 10 09:44:56 myhost news[3840]: System health check completed 20 | m:286:Mar 10 09:53:11 myhost news[816]: System configuration restored 21 | exit_code:0 22 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-08-16:00", 10 | "--to", "2025-03-09-16:00" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-08-16:00 isn't found, will use the beginning 24 | debug:the to 2025-03-09-16:00 is found: 12 (741) 25 | p:stage:3:querying logs 26 | debug:Getting logs from the very beginning to offset 740, all in the prev /tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/01_basic/logfile.1. 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/01_basic/logfile:287 3 | s:Mar 9 15:52,1 4 | s:Mar 9 15:16,1 5 | s:Mar 9 15:07,1 6 | s:Mar 9 15:44,1 7 | s:Mar 9 15:35,1 8 | s:Mar 9 15:36,1 9 | s:Mar 9 15:04,1 10 | s:Mar 9 15:32,1 11 | s:Mar 9 15:23,3 12 | m:4:Mar 9 15:23:17 myhost syslog[4229]: Security patch applied 13 | m:5:Mar 9 15:23:17 myhost lpr[8539]: Cache update completed 14 | m:6:Mar 9 15:23:17 myhost kern[3862]: Permission denied 15 | m:7:Mar 9 15:32:07 myhost news[596]: User permissions updated 16 | m:8:Mar 9 15:35:19 myhost authpriv[7019]: Disk usage critical 17 | m:9:Mar 9 15:36:33 myhost authpriv[7830]: Certificate expiration warning 18 | m:10:Mar 9 15:44:23 myhost lpr[3187]: Service initialization failed 19 | m:11:Mar 9 15:52:34 myhost lpr[3574]: Disk space low 20 | exit_code:0 21 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-08-16:00", 10 | "--to", "2025-03-09-16:00", 11 | "--lines-until", "4" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-08-16:00 isn't found, will use the beginning 24 | debug:the to 2025-03-09-16:00 is found: 12 (741) 25 | p:stage:3:querying logs 26 | debug:Getting logs from the very beginning to offset 740, all in the prev /tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/03_basic_more_less_than_max/logfile.1. 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/from_the_beginning_of_prev_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/from_the_beginning_of_prev_file/03_basic_more_less_than_max/logfile:287 3 | s:Mar 9 15:52,1 4 | s:Mar 9 15:16,1 5 | s:Mar 9 15:07,1 6 | s:Mar 9 15:44,1 7 | s:Mar 9 15:35,1 8 | s:Mar 9 15:36,1 9 | s:Mar 9 15:04,1 10 | s:Mar 9 15:32,1 11 | s:Mar 9 15:23,3 12 | m:1:Mar 9 15:04:05 myhost mail[8554]: High CPU usage detected 13 | m:2:Mar 9 15:07:54 myhost auth[3421]: Security breach detected 14 | m:3:Mar 9 15:16:07 myhost ftp[1118]: File copied successfully 15 | exit_code:0 16 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-12-09:00", 10 | "--to", "2025-03-12-10:00" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-09:00 is found: 1022 (67792) 24 | debug:the to 2025-03-12-10:00 is found: 1033 (68556) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 48636, only 764 bytes, all in the latest /tmp/nerdlog_agent_test_output/in_the_middle_latest_file/01_basic/logfile 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_latest_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_latest_file/01_basic/logfile:287 3 | s:Mar 12 09:09,1 4 | s:Mar 12 09:31,1 5 | s:Mar 12 09:22,1 6 | s:Mar 12 09:05,1 7 | s:Mar 12 09:42,3 8 | s:Mar 12 09:33,1 9 | s:Mar 12 09:15,2 10 | s:Mar 12 09:52,1 11 | m:1025:Mar 12 09:15:54 myhost lpr[8694]: File copied successfully 12 | m:1026:Mar 12 09:22:38 myhost auth[7805]: Service dependency failure 13 | m:1027:Mar 12 09:31:50 myhost news[1141]: User session ended 14 | m:1028:Mar 12 09:33:12 myhost daemon[8974]: Cache update completed 15 | m:1029:Mar 12 09:42:44 myhost news[1075]: System configuration restored 16 | m:1030:Mar 12 09:42:44 myhost user[3514]: Service initialization failed 17 | m:1031:Mar 12 09:42:46 myhost syslog[2812]: Database query failed 18 | m:1032:Mar 12 09:52:46 myhost user[7102]: Insufficient privileges 19 | exit_code:0 20 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-12-09:00", 10 | "--to", "2025-03-12-10:00", 11 | "--lines-until", "1025" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-09:00 is found: 1022 (67792) 24 | debug:the to 2025-03-12-10:00 is found: 1033 (68556) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 48636, only 764 bytes, all in the latest /tmp/nerdlog_agent_test_output/in_the_middle_latest_file/03_basic_more_less_than_max/logfile 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_latest_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_latest_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_latest_file/03_basic_more_less_than_max/logfile:287 3 | s:Mar 12 09:09,1 4 | s:Mar 12 09:31,1 5 | s:Mar 12 09:22,1 6 | s:Mar 12 09:05,1 7 | s:Mar 12 09:42,3 8 | s:Mar 12 09:33,1 9 | s:Mar 12 09:15,2 10 | s:Mar 12 09:52,1 11 | m:1022:Mar 12 09:05:46 myhost daemon[7290]: SMTP server connection error 12 | m:1023:Mar 12 09:09:30 myhost cron[3864]: Software version updated 13 | m:1024:Mar 12 09:15:54 myhost ftp[6693]: Database migration completed 14 | exit_code:0 15 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "10", 9 | "--from", "2025-03-09-23:30", 10 | "--to", "2025-03-10-00:30" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-09-23:30 is found: 132 (8680) 24 | debug:the to 2025-03-10-00:30 is found: 148 (9734) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 8680, only 1054 bytes, all in the prev /tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/01_basic/logfile.1 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/01_basic/logfile:287 3 | s:Mar 9 23:31,1 4 | s:Mar 9 23:50,1 5 | s:Mar 9 23:41,1 6 | s:Mar 10 00:01,2 7 | s:Mar 9 23:42,1 8 | s:Mar 9 23:33,1 9 | s:Mar 9 23:43,1 10 | s:Mar 10 00:22,1 11 | s:Mar 9 23:54,1 12 | s:Mar 9 23:45,1 13 | s:Mar 10 00:17,2 14 | s:Mar 10 00:08,1 15 | s:Mar 9 23:49,1 16 | s:Mar 10 00:29,1 17 | m:138:Mar 9 23:49:53 myhost lpr[7525]: Service started 18 | m:139:Mar 9 23:50:16 myhost news[1351]: Disk space reclaimed 19 | m:140:Mar 9 23:54:28 myhost kern[108]: Database connection error 20 | m:141:Mar 10 00:01:58 myhost cron[3725]: API request failed 21 | m:142:Mar 10 00:01:58 myhost uucp[2334]: Database migration completed 22 | m:143:Mar 10 00:08:34 myhost lpr[3966]: CPU temperature critical 23 | m:144:Mar 10 00:17:17 myhost user[3135]: Application crash reported 24 | m:145:Mar 10 00:17:17 myhost ftp[8324]: Error handling request 25 | m:146:Mar 10 00:22:38 myhost ftp[864]: Server shutting down 26 | m:147:Mar 10 00:29:08 myhost lpr[3704]: Configuration applied successfully 27 | exit_code:0 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-09-23:30", 10 | "--to", "2025-03-10-00:30", 11 | "--lines-until", "138" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-09-23:30 is found: 132 (8680) 24 | debug:the to 2025-03-10-00:30 is found: 148 (9734) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 8680, only 1054 bytes, all in the prev /tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/03_basic_more_less_than_max/logfile.1 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/in_the_middle_of_prev_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/in_the_middle_of_prev_file/03_basic_more_less_than_max/logfile:287 3 | s:Mar 9 23:31,1 4 | s:Mar 9 23:50,1 5 | s:Mar 9 23:41,1 6 | s:Mar 10 00:01,2 7 | s:Mar 9 23:42,1 8 | s:Mar 9 23:33,1 9 | s:Mar 9 23:43,1 10 | s:Mar 10 00:22,1 11 | s:Mar 9 23:54,1 12 | s:Mar 9 23:45,1 13 | s:Mar 10 00:17,2 14 | s:Mar 10 00:08,1 15 | s:Mar 9 23:49,1 16 | s:Mar 10 00:29,1 17 | m:132:Mar 9 23:31:13 myhost news[1390]: Scheduled task executed 18 | m:133:Mar 9 23:33:06 myhost uucp[3943]: Process crashed 19 | m:134:Mar 9 23:41:35 myhost cron[313]: Process started 20 | m:135:Mar 9 23:42:07 myhost uucp[3229]: Disk format completed 21 | m:136:Mar 9 23:43:58 myhost lpr[4421]: Insufficient privileges 22 | m:137:Mar 9 23:45:15 myhost news[7029]: System time drift detected 23 | exit_code:0 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "8", "--from", "2025-03-12-10:00"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-10:00 is found: 1033 (68556) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file/01_basic/logfile. 26 | p:stage:4:done 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/01_basic/logfile:287 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | m:1046:Mar 12 10:16:59 myhost cron[3281]: Timeout occurred 16 | m:1047:Mar 12 10:19:44 myhost user[3462]: User session timed out 17 | m:1048:Mar 12 10:27:16 myhost mail[8396]: New update available 18 | m:1049:Mar 12 10:32:05 myhost syslog[6387]: System clock synchronized 19 | m:1050:Mar 12 10:38:23 myhost auth[1783]: User login successful 20 | m:1051:Mar 12 10:45:36 myhost lpr[6125]: Service request queued 21 | m:1052:Mar 12 10:53:36 myhost ftp[4422]: Configuration reload successful 22 | m:1053:Mar 12 10:56:46 myhost cron[3690]: Memory leak detected 23 | exit_code:0 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/02_basic_more_full_amount/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "8", "--from", "2025-03-12-10:00", "--lines-until", "1046"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/02_basic_more_full_amount/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-10:00 is found: 1033 (68556) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file/02_basic_more_full_amount/logfile. 26 | p:stage:4:done 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/02_basic_more_full_amount/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/02_basic_more_full_amount/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/02_basic_more_full_amount/logfile:287 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | m:1038:Mar 12 10:10:05 myhost authpriv[3500]: System clock synchronized 16 | m:1039:Mar 12 10:10:10 myhost authpriv[3500]: Database query failed 17 | m:1040:Mar 12 10:10:12 myhost authpriv[3500]: System clock synchronized 18 | m:1041:Mar 12 10:10:15 myhost authpriv[3500]: System clock synchronized 19 | m:1042:Mar 12 10:10:15 myhost authpriv[3500]: System clock synchronized 20 | m:1043:Mar 12 10:10:15 myhost authpriv[3500]: System clock synchronized 21 | m:1044:Mar 12 10:14:06 myhost mail[173]: User session ended 22 | m:1045:Mar 12 10:16:00 myhost ftp[8866]: User session started 23 | exit_code:0 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "8", "--from", "2025-03-12-10:00", "--lines-until", "1038"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-10:00 is found: 1033 (68556) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file/03_basic_more_less_than_max/logfile. 26 | p:stage:4:done 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/03_basic_more_less_than_max/logfile:287 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | m:1033:Mar 12 10:01:02 myhost lpr[6903]: User account enabled 16 | m:1034:Mar 12 10:03:46 myhost syslog[2812]: Database query failed 17 | m:1035:Mar 12 10:10:05 myhost authpriv[3500]: System clock synchronized 18 | m:1036:Mar 12 10:10:05 myhost authpriv[3500]: System clock synchronized 19 | m:1037:Mar 12 10:10:05 myhost authpriv[3500]: System clock synchronized 20 | exit_code:0 21 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/04_basic_more_no_more_logs/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "8", "--from", "2025-03-12-10:00", "--lines-until", "1033"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/04_basic_more_no_more_logs/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-10:00 is found: 1033 (68556) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file/04_basic_more_no_more_logs/logfile. 26 | p:stage:4:done 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/04_basic_more_no_more_logs/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/04_basic_more_no_more_logs/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/04_basic_more_no_more_logs/logfile:287 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | exit_code:0 16 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-12-10:00", 10 | "--to", "2025-05-01-00:00", 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-10:00 is found: 1033 (68556) 24 | debug:the to 2025-05-01-00:00 isn't found, will use the end 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/logfile. 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file/10_to_is_specified_and_is_in_the_future/logfile:287 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | m:1046:Mar 12 10:16:59 myhost cron[3281]: Timeout occurred 16 | m:1047:Mar 12 10:19:44 myhost user[3462]: User session timed out 17 | m:1048:Mar 12 10:27:16 myhost mail[8396]: New update available 18 | m:1049:Mar 12 10:32:05 myhost syslog[6387]: System clock synchronized 19 | m:1050:Mar 12 10:38:23 myhost auth[1783]: User login successful 20 | m:1051:Mar 12 10:45:36 myhost lpr[6125]: Service request queued 21 | m:1052:Mar 12 10:53:36 myhost ftp[4422]: Configuration reload successful 22 | m:1053:Mar 12 10:56:46 myhost cron[3690]: Memory leak detected 23 | exit_code:0 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "5", "--from", "2025-03-10-15:00", "/Backup completed/"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-15:00 is found: 411 (27328) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 8172 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/01_basic/logfile. 26 | p:p:15 27 | p:p:30 28 | p:p:45 29 | p:p:60 30 | p:p:75 31 | p:p:90 32 | p:stage:4:done 33 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/01_basic/logfile:287 3 | s:Mar 10 17:37,1 4 | s:Mar 12 03:10,1 5 | s:Mar 11 13:56,1 6 | s:Mar 11 08:21,1 7 | s:Mar 10 18:01,1 8 | s:Mar 10 16:35,1 9 | s:Mar 11 21:12,1 10 | m:450:Mar 10 18:01:32 myhost uucp[136]: Backup completed 11 | m:663:Mar 11 08:21:42 myhost user[4017]: Backup completed 12 | m:751:Mar 11 13:56:18 myhost uucp[8088]: Backup completed 13 | m:846:Mar 11 21:12:15 myhost auth[1817]: Backup completed 14 | m:939:Mar 12 03:10:17 myhost lpr[4051]: Backup completed 15 | exit_code:0 16 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "5", 9 | "--from", "2025-03-10-15:00", 10 | "--lines-until", "450", 11 | "/Backup completed/" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-15:00 is found: 411 (27328) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 8172 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/03_basic_more_less_than_max/logfile. 26 | p:p:15 27 | p:p:30 28 | p:p:45 29 | p:p:60 30 | p:p:75 31 | p:p:90 32 | p:stage:4:done 33 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern1/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern1/03_basic_more_less_than_max/logfile:287 3 | s:Mar 10 17:37,1 4 | s:Mar 12 03:10,1 5 | s:Mar 11 13:56,1 6 | s:Mar 11 08:21,1 7 | s:Mar 10 18:01,1 8 | s:Mar 10 16:35,1 9 | s:Mar 11 21:12,1 10 | m:432:Mar 10 16:35:56 myhost daemon[7460]: Backup completed 11 | m:447:Mar 10 17:37:49 myhost news[3166]: Backup completed 12 | exit_code:0 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "4", "--from", "2025-03-10-15:00", "/Backup completed/ && !/warning/"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-15:00 is found: 411 (27328) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 8172 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/01_basic/logfile. 26 | p:p:15 27 | p:p:30 28 | p:p:45 29 | p:p:60 30 | p:p:75 31 | p:p:90 32 | p:stage:4:done 33 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/01_basic/logfile:287 3 | s:Mar 10 17:37,1 4 | s:Mar 12 03:10,1 5 | s:Mar 11 13:56,1 6 | s:Mar 10 18:01,1 7 | s:Mar 10 16:35,1 8 | m:447:Mar 10 17:37:49 myhost news[3166]: Backup completed 9 | m:450:Mar 10 18:01:32 myhost uucp[136]: Backup completed 10 | m:751:Mar 11 13:56:18 myhost uucp[8088]: Backup completed 11 | m:939:Mar 12 03:10:17 myhost lpr[4051]: Backup completed 12 | exit_code:0 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "4", 9 | "--from", "2025-03-10-15:00", 10 | "--lines-until", "447", 11 | "/Backup completed/ && !/warning/" 12 | ] 13 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-10-15:00 is found: 411 (27328) 24 | p:stage:3:querying logs 25 | debug:Getting logs from offset 8172 until the end of latest /tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/03_basic_more_less_than_max/logfile. 26 | p:p:15 27 | p:p:30 28 | p:p:45 29 | p:p:60 30 | p:p:75 31 | p:p:90 32 | p:stage:4:done 33 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/latest_logs_same_file_pattern2/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/latest_logs_same_file_pattern2/03_basic_more_less_than_max/logfile:287 3 | s:Mar 10 17:37,1 4 | s:Mar 12 03:10,1 5 | s:Mar 11 13:56,1 6 | s:Mar 10 18:01,1 7 | s:Mar 10 16:35,1 8 | m:432:Mar 10 16:35:56 myhost daemon[7460]: Backup completed 9 | exit_code:0 10 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_even_further/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-12-11:00", 10 | "--to", "2025-03-12-12:00", 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_even_further/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-11:00 is after the latest log we have, will return nothing 24 | debug:the to 2025-03-12-12:00 isn't found, will use the end 25 | p:stage:4:done 26 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_even_further/want_stdout: -------------------------------------------------------------------------------- 1 | exit_code:0 2 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_unset/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-12-11:00", 10 | ] 11 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_unset/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-12-11:00 is after the latest log we have, will return nothing 24 | p:stage:4:done 25 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/from_is_after_to_is_unset/want_stdout: -------------------------------------------------------------------------------- 1 | exit_code:0 2 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_even_earlier/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--from", "2025-03-09-14:00", 10 | "--to", "2025-03-09-15:00", 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_even_earlier/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2025-03-09-14:00 isn't found, will use the beginning 24 | debug:the to 2025-03-09-15:00 is before the first log we have, will return nothing 25 | p:stage:4:done 26 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_even_earlier/want_stdout: -------------------------------------------------------------------------------- 1 | exit_code:0 2 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_unset/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_mar 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "8", 9 | "--to", "2025-03-09-15:00", 10 | ] 11 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_unset/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the to 2025-03-09-15:00 is before the first log we have, will return nothing 24 | p:stage:4:done 25 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/query_range_is_outside/to_is_before_from_is_unset/want_stdout: -------------------------------------------------------------------------------- 1 | exit_code:0 2 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/single_file 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "8", "--from", "2025-03-12-10:00"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:prev logfile /tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/01_basic/logfile.1 doesn't exist, using a dummy empty file /tmp/nerdlog-empty-file 2 | debug:index file doesn't exist or is empty, gonna refresh it 3 | p:stage:1:indexing from scratch 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:30 9 | p:p:35 10 | p:p:40 11 | p:p:45 12 | p:p:50 13 | p:p:55 14 | p:p:60 15 | p:p:65 16 | p:p:70 17 | p:p:75 18 | p:p:80 19 | p:p:85 20 | p:p:90 21 | p:p:95 22 | debug:the from 2025-03-12-10:00 is found: 746 (49400) 23 | p:stage:3:querying logs 24 | debug:Getting logs from offset 49400 until the end of latest /tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/01_basic/logfile. 25 | p:stage:4:done 26 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog-empty-file:0 2 | logfile:/tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/01_basic/logfile:0 3 | s:Mar 12 10:03,1 4 | s:Mar 12 10:32,1 5 | s:Mar 12 10:14,1 6 | s:Mar 12 10:16,2 7 | s:Mar 12 10:53,1 8 | s:Mar 12 10:45,1 9 | s:Mar 12 10:27,1 10 | s:Mar 12 10:19,1 11 | s:Mar 12 10:56,1 12 | s:Mar 12 10:38,1 13 | s:Mar 12 10:10,9 14 | s:Mar 12 10:01,1 15 | m:759:Mar 12 10:16:59 myhost cron[3281]: Timeout occurred 16 | m:760:Mar 12 10:19:44 myhost user[3462]: User session timed out 17 | m:761:Mar 12 10:27:16 myhost mail[8396]: New update available 18 | m:762:Mar 12 10:32:05 myhost syslog[6387]: System clock synchronized 19 | m:763:Mar 12 10:38:23 myhost auth[1783]: User login successful 20 | m:764:Mar 12 10:45:36 myhost lpr[6125]: Service request queued 21 | m:765:Mar 12 10:53:36 myhost ftp[4422]: Configuration reload successful 22 | m:766:Mar 12 10:56:46 myhost cron[3690]: Memory leak detected 23 | exit_code:0 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/02_oldest_logs/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/single_file 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "10", 9 | "--from", "2025-03-09-10:30", 10 | "--to", "2025-03-10-10:30", 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/02_oldest_logs/want_stderr: -------------------------------------------------------------------------------- 1 | debug:prev logfile /tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/02_oldest_logs/logfile.1 doesn't exist, using a dummy empty file /tmp/nerdlog-empty-file 2 | debug:index file doesn't exist or is empty, gonna refresh it 3 | p:stage:1:indexing from scratch 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:30 9 | p:p:35 10 | p:p:40 11 | p:p:45 12 | p:p:50 13 | p:p:55 14 | p:p:60 15 | p:p:65 16 | p:p:70 17 | p:p:75 18 | p:p:80 19 | p:p:85 20 | p:p:90 21 | p:p:95 22 | debug:the from 2025-03-09-10:30 isn't found, will use the beginning 23 | debug:the to 2025-03-10-10:30 is found: 8 (459) 24 | p:stage:3:querying logs 25 | debug:Getting logs from the very beginning in prev /tmp/nerdlog-empty-file to offset 458 in latest /tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/02_oldest_logs/logfile 26 | p:stage:4:done 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/second_log_file_doesnt_exist/02_oldest_logs/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog-empty-file:0 2 | logfile:/tmp/nerdlog_agent_test_output/second_log_file_doesnt_exist/02_oldest_logs/logfile:0 3 | s:Mar 10 10:20,2 4 | s:Mar 10 10:14,1 5 | s:Mar 10 10:24,1 6 | s:Mar 10 10:27,2 7 | s:Mar 10 10:00,1 8 | m:1:Mar 10 10:00:01 myhost kern[5159]: Disk space reclaimed 9 | m:2:Mar 10 10:14:05 myhost auth[8368]: Database schema updated 10 | m:3:Mar 10 10:20:17 myhost syslog[4163]: System health check failed 11 | m:4:Mar 10 10:20:46 myhost lpr[891]: User session timed out 12 | m:5:Mar 10 10:24:32 myhost user[8515]: Cache cleared 13 | m:6:Mar 10 10:27:26 myhost kern[2205]: Session token expired 14 | m:7:Mar 10 10:27:26 myhost cron[9005]: File transfer completed 15 | exit_code:0 16 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "20", "--from", "2025-03-10-09:30"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-10-09:30 is found: 12 (733) 21 | p:stage:3:querying logs 22 | debug:Getting logs from offset 733 in prev /tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/01_basic/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/01_basic/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/01_basic/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:59,1 6 | s:Mar 10 10:32,2 7 | s:Mar 10 10:14,1 8 | s:Mar 10 10:51,1 9 | s:Mar 10 10:33,1 10 | s:Mar 10 10:24,1 11 | s:Mar 10 10:34,1 12 | s:Mar 10 09:31,2 13 | s:Mar 10 10:45,1 14 | s:Mar 10 10:36,1 15 | s:Mar 10 10:27,2 16 | s:Mar 10 10:38,1 17 | s:Mar 10 09:53,1 18 | s:Mar 10 09:44,1 19 | s:Mar 10 09:35,2 20 | s:Mar 10 10:57,1 21 | s:Mar 10 10:00,1 22 | m:16:Mar 10 09:39:31 myhost auth[8464]: User session started 23 | m:17:Mar 10 09:44:56 myhost news[3840]: System health check completed 24 | m:18:Mar 10 09:53:11 myhost news[816]: System configuration restored 25 | m:19:Mar 10 09:59:58 myhost ftp[3724]: Out of memory error 26 | m:20:Mar 10 10:00:01 myhost kern[5159]: Disk space reclaimed 27 | m:21:Mar 10 10:14:05 myhost auth[8368]: Database schema updated 28 | m:22:Mar 10 10:20:17 myhost syslog[4163]: System health check failed 29 | m:23:Mar 10 10:20:46 myhost lpr[891]: User session timed out 30 | m:24:Mar 10 10:24:32 myhost user[8515]: Cache cleared 31 | m:25:Mar 10 10:27:26 myhost kern[2205]: Session token expired 32 | m:26:Mar 10 10:27:26 myhost cron[9005]: File transfer completed 33 | m:27:Mar 10 10:32:21 myhost daemon[8000]: Failed login attempt 34 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 35 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 36 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 37 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 38 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 39 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 40 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 41 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 42 | exit_code:0 43 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "20", 9 | "--from", "2025-03-10-09:30", 10 | "--lines-until", "16" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-10-09:30 is found: 12 (733) 21 | p:stage:3:querying logs 22 | debug:Getting logs from offset 733 in prev /tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/03_basic_more_less_than_max/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/03_basic_more_less_than_max/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_middle_prev_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_middle_prev_file/03_basic_more_less_than_max/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:59,1 6 | s:Mar 10 10:32,2 7 | s:Mar 10 10:14,1 8 | s:Mar 10 10:51,1 9 | s:Mar 10 10:33,1 10 | s:Mar 10 10:24,1 11 | s:Mar 10 10:34,1 12 | s:Mar 10 09:31,2 13 | s:Mar 10 10:45,1 14 | s:Mar 10 10:36,1 15 | s:Mar 10 10:27,2 16 | s:Mar 10 10:38,1 17 | s:Mar 10 09:53,1 18 | s:Mar 10 09:44,1 19 | s:Mar 10 09:35,2 20 | s:Mar 10 10:57,1 21 | s:Mar 10 10:00,1 22 | m:12:Mar 10 09:31:23 myhost authpriv[5771]: User session ended 23 | m:13:Mar 10 09:31:23 myhost authpriv[2976]: Cache cleared 24 | m:14:Mar 10 09:35:23 myhost kern[3027]: SMTP server connection error 25 | m:15:Mar 10 09:35:23 myhost syslog[3626]: Application crash reported 26 | exit_code:0 27 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/01_basic/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: ["--max-num-lines", "20", "--from", "2025-03-01-00:00"] 8 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/01_basic/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-01-00:00 isn't found, will use the beginning 21 | p:stage:3:querying logs 22 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/01_basic/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/01_basic/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/01_basic/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/01_basic/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/01_basic/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:16:Mar 10 09:39:31 myhost auth[8464]: User session started 29 | m:17:Mar 10 09:44:56 myhost news[3840]: System health check completed 30 | m:18:Mar 10 09:53:11 myhost news[816]: System configuration restored 31 | m:19:Mar 10 09:59:58 myhost ftp[3724]: Out of memory error 32 | m:20:Mar 10 10:00:01 myhost kern[5159]: Disk space reclaimed 33 | m:21:Mar 10 10:14:05 myhost auth[8368]: Database schema updated 34 | m:22:Mar 10 10:20:17 myhost syslog[4163]: System health check failed 35 | m:23:Mar 10 10:20:46 myhost lpr[891]: User session timed out 36 | m:24:Mar 10 10:24:32 myhost user[8515]: Cache cleared 37 | m:25:Mar 10 10:27:26 myhost kern[2205]: Session token expired 38 | m:26:Mar 10 10:27:26 myhost cron[9005]: File transfer completed 39 | m:27:Mar 10 10:32:21 myhost daemon[8000]: Failed login attempt 40 | m:28:Mar 10 10:32:21 myhost mail[7726]: Error reading file 41 | m:29:Mar 10 10:33:00 myhost kern[4506]: Service request queued 42 | m:30:Mar 10 10:34:31 myhost cron[935]: Database connection error 43 | m:31:Mar 10 10:36:14 myhost user[2831]: File system full 44 | m:32:Mar 10 10:38:25 myhost mail[8342]: User account disabled 45 | m:33:Mar 10 10:45:04 myhost authpriv[7892]: Memory usage high 46 | m:34:Mar 10 10:51:01 myhost user[3758]: System running low on resources 47 | m:35:Mar 10 10:57:37 myhost news[5185]: Insufficient privileges 48 | exit_code:0 49 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/03_basic_more_less_than_max/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "Initial basic test case" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/tiny 5 | cur_year: 2025 6 | cur_month: 3 7 | args: [ 8 | "--max-num-lines", "20", 9 | "--from", "2025-03-01-00:00", 10 | "--lines-until", "16" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/03_basic_more_less_than_max/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:10 4 | p:p:20 5 | p:p:25 6 | p:p:30 7 | p:p:35 8 | p:p:40 9 | p:p:45 10 | p:p:50 11 | p:p:50 12 | p:p:55 13 | p:p:60 14 | p:p:65 15 | p:p:70 16 | p:p:80 17 | p:p:85 18 | p:p:90 19 | p:p:95 20 | debug:the from 2025-03-01-00:00 isn't found, will use the beginning 21 | p:stage:3:querying logs 22 | debug:Getting logs from the very beginning in prev /tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/03_basic_more_less_than_max/logfile.1 until the end of latest /tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/03_basic_more_less_than_max/logfile 23 | p:stage:4:done 24 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/whole_latest_whole_prev_file/03_basic_more_less_than_max/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/03_basic_more_less_than_max/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/whole_latest_whole_prev_file/03_basic_more_less_than_max/logfile:19 3 | s:Mar 10 10:20,2 4 | s:Mar 10 09:39,1 5 | s:Mar 10 09:00,1 6 | s:Mar 10 09:59,1 7 | s:Mar 10 10:32,2 8 | s:Mar 10 10:14,1 9 | s:Mar 10 09:02,3 10 | s:Mar 10 10:51,1 11 | s:Mar 10 10:33,1 12 | s:Mar 10 10:24,1 13 | s:Mar 10 10:34,1 14 | s:Mar 10 09:31,2 15 | s:Mar 10 09:22,1 16 | s:Mar 10 09:14,1 17 | s:Mar 10 09:05,4 18 | s:Mar 10 10:45,1 19 | s:Mar 10 10:36,1 20 | s:Mar 10 10:27,2 21 | s:Mar 10 10:38,1 22 | s:Mar 10 09:53,1 23 | s:Mar 10 09:44,1 24 | s:Mar 10 09:35,2 25 | s:Mar 10 10:57,1 26 | s:Mar 10 10:00,1 27 | s:Mar 10 09:28,1 28 | m:1:Mar 10 09:00:36 myhost ftp[3406]: Timeout occurred 29 | m:2:Mar 10 09:02:02 myhost authpriv[1893]: CPU temperature critical 30 | m:3:Mar 10 09:02:02 myhost cron[424]: System running low on resources 31 | m:4:Mar 10 09:02:02 myhost authpriv[1827]: Cache cleared 32 | m:5:Mar 10 09:05:07 myhost cron[5530]: Firewall rule deleted 33 | m:6:Mar 10 09:05:07 myhost daemon[5617]: File upload completed 34 | m:7:Mar 10 09:05:44 myhost auth[6052]: Certificate expiration warning 35 | m:8:Mar 10 09:05:46 myhost auth[4149]: Memory leak detected 36 | m:9:Mar 10 09:14:40 myhost authpriv[3851]: Log file archived 37 | m:10:Mar 10 09:22:23 myhost auth[3925]: Server started successfully 38 | m:11:Mar 10 09:28:01 myhost news[9026]: Error reading file 39 | m:12:Mar 10 09:31:23 myhost authpriv[5771]: User session ended 40 | m:13:Mar 10 09:31:23 myhost authpriv[2976]: Cache cleared 41 | m:14:Mar 10 09:35:23 myhost kern[3027]: SMTP server connection error 42 | m:15:Mar 10 09:35:23 myhost syslog[3626]: Application crash reported 43 | exit_code:0 44 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_dec_jan 5 | cur_year: 2021 6 | cur_month: 4 7 | args: [ 8 | "--max-num-lines", "50", 9 | "--from", "2020-12-31-23:30", 10 | "--to", "2021-01-01-00:30" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2020-12-31-23:30 is found: 132 (8680) 24 | debug:the to 2021-01-01-00:30 is found: 148 (9734) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 8680, only 1054 bytes, all in the prev /tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/logfile.1 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_apr/logfile:287 3 | s:Jan 1 00:22,1 4 | s:Dec 31 23:31,1 5 | s:Dec 31 23:50,1 6 | s:Dec 31 23:41,1 7 | s:Dec 31 23:42,1 8 | s:Dec 31 23:33,1 9 | s:Dec 31 23:43,1 10 | s:Jan 1 00:17,2 11 | s:Jan 1 00:08,1 12 | s:Dec 31 23:54,1 13 | s:Dec 31 23:45,1 14 | s:Jan 1 00:29,1 15 | s:Dec 31 23:49,1 16 | s:Jan 1 00:01,2 17 | m:132:Dec 31 23:31:13 myhost news[1390]: Scheduled task executed 18 | m:133:Dec 31 23:33:06 myhost uucp[3943]: Process crashed 19 | m:134:Dec 31 23:41:35 myhost cron[313]: Process started 20 | m:135:Dec 31 23:42:07 myhost uucp[3229]: Disk format completed 21 | m:136:Dec 31 23:43:58 myhost lpr[4421]: Insufficient privileges 22 | m:137:Dec 31 23:45:15 myhost news[7029]: System time drift detected 23 | m:138:Dec 31 23:49:53 myhost lpr[7525]: Service started 24 | m:139:Dec 31 23:50:16 myhost news[1351]: Disk space reclaimed 25 | m:140:Dec 31 23:54:28 myhost kern[108]: Database connection error 26 | m:141:Jan 1 00:01:58 myhost cron[3725]: API request failed 27 | m:142:Jan 1 00:01:58 myhost uucp[2334]: Database migration completed 28 | m:143:Jan 1 00:08:34 myhost lpr[3966]: CPU temperature critical 29 | m:144:Jan 1 00:17:17 myhost user[3135]: Application crash reported 30 | m:145:Jan 1 00:17:17 myhost ftp[8324]: Error handling request 31 | m:146:Jan 1 00:22:38 myhost ftp[864]: Server shutting down 32 | m:147:Jan 1 00:29:08 myhost lpr[3704]: Configuration applied successfully 33 | exit_code:0 34 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/test_case.yaml: -------------------------------------------------------------------------------- 1 | descr: "" 2 | logfiles: 3 | kind: all_from_dir 4 | dir: ../../../logfiles/small_dec_jan 5 | cur_year: 2021 6 | cur_month: 1 7 | args: [ 8 | "--max-num-lines", "50", 9 | "--from", "2020-12-31-23:30", 10 | "--to", "2021-01-01-00:30" 11 | ] 12 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/want_stderr: -------------------------------------------------------------------------------- 1 | debug:index file doesn't exist or is empty, gonna refresh it 2 | p:stage:1:indexing from scratch 3 | p:p:5 4 | p:p:10 5 | p:p:15 6 | p:p:20 7 | p:p:25 8 | p:p:25 9 | p:p:30 10 | p:p:35 11 | p:p:40 12 | p:p:45 13 | p:p:50 14 | p:p:55 15 | p:p:60 16 | p:p:65 17 | p:p:70 18 | p:p:75 19 | p:p:80 20 | p:p:85 21 | p:p:90 22 | p:p:95 23 | debug:the from 2020-12-31-23:30 is found: 132 (8680) 24 | debug:the to 2021-01-01-00:30 is found: 148 (9734) 25 | p:stage:3:querying logs 26 | debug:Getting logs from offset 8680, only 1054 bytes, all in the prev /tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/logfile.1 27 | p:stage:4:done 28 | -------------------------------------------------------------------------------- /core/nerdlog_agent_testdata/test_cases/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/want_stdout: -------------------------------------------------------------------------------- 1 | logfile:/tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/logfile.1:0 2 | logfile:/tmp/nerdlog_agent_test_output/year_infer_edge_of_two_years/01_logs_in_the_past_cur_jan/logfile:287 3 | s:Jan 1 00:22,1 4 | s:Dec 31 23:31,1 5 | s:Dec 31 23:50,1 6 | s:Dec 31 23:41,1 7 | s:Dec 31 23:42,1 8 | s:Dec 31 23:33,1 9 | s:Dec 31 23:43,1 10 | s:Jan 1 00:17,2 11 | s:Jan 1 00:08,1 12 | s:Dec 31 23:54,1 13 | s:Dec 31 23:45,1 14 | s:Jan 1 00:29,1 15 | s:Dec 31 23:49,1 16 | s:Jan 1 00:01,2 17 | m:132:Dec 31 23:31:13 myhost news[1390]: Scheduled task executed 18 | m:133:Dec 31 23:33:06 myhost uucp[3943]: Process crashed 19 | m:134:Dec 31 23:41:35 myhost cron[313]: Process started 20 | m:135:Dec 31 23:42:07 myhost uucp[3229]: Disk format completed 21 | m:136:Dec 31 23:43:58 myhost lpr[4421]: Insufficient privileges 22 | m:137:Dec 31 23:45:15 myhost news[7029]: System time drift detected 23 | m:138:Dec 31 23:49:53 myhost lpr[7525]: Service started 24 | m:139:Dec 31 23:50:16 myhost news[1351]: Disk space reclaimed 25 | m:140:Dec 31 23:54:28 myhost kern[108]: Database connection error 26 | m:141:Jan 1 00:01:58 myhost cron[3725]: API request failed 27 | m:142:Jan 1 00:01:58 myhost uucp[2334]: Database migration completed 28 | m:143:Jan 1 00:08:34 myhost lpr[3966]: CPU temperature critical 29 | m:144:Jan 1 00:17:17 myhost user[3135]: Application crash reported 30 | m:145:Jan 1 00:17:17 myhost ftp[8324]: Error handling request 31 | m:146:Jan 1 00:22:38 myhost ftp[864]: Server shutting down 32 | m:147:Jan 1 00:29:08 myhost lpr[3704]: Configuration applied successfully 33 | exit_code:0 34 | -------------------------------------------------------------------------------- /core/parsing_time_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type detectTimeTestCase struct { 10 | name string 11 | logLine string 12 | wantLayout string 13 | } 14 | 15 | func TestDetectTimeFormat(t *testing.T) { 16 | testCases := []detectTimeTestCase{ 17 | { 18 | name: "rsyslog no year, 1-digit", 19 | logLine: "Apr 8 01:02:03 somehost systemd[1]: Started something.", 20 | wantLayout: "Jan _2 15:04:05", 21 | }, 22 | { 23 | name: "rsyslog no year, 2-digit", 24 | logLine: "Apr 18 01:02:03 somehost systemd[1]: Started something.", 25 | wantLayout: "Jan _2 15:04:05", 26 | }, 27 | { 28 | name: "ISO8601 non-UTC full with microseconds", 29 | logLine: "2024-04-19T14:23:45.123456+02:00 INFO something happened", 30 | wantLayout: "2006-01-02T15:04:05.000000Z07:00", 31 | }, 32 | { 33 | name: "ISO8601 UTC full with microseconds", 34 | logLine: "2024-04-19T14:23:45.123456Z INFO something happened", 35 | wantLayout: "2006-01-02T15:04:05.000000Z07:00", 36 | }, 37 | { 38 | name: "RFC3339", 39 | logLine: "2024-04-19T14:23:45+02:00 Starting server", 40 | wantLayout: "2006-01-02T15:04:05Z07:00", 41 | }, 42 | { 43 | name: "No timestamp in line", 44 | logLine: "This is a log line without a timestamp.", 45 | wantLayout: "", 46 | }, 47 | } 48 | 49 | for _, tc := range testCases { 50 | t.Run(tc.name, func(t *testing.T) { 51 | layout := DetectTimeLayout(tc.logLine) 52 | assert.Equal(t, tc.wantLayout, layout) 53 | }) 54 | } 55 | } 56 | 57 | type timeDescrTestCase struct { 58 | name string 59 | layout string 60 | expected *TimeFormatDescr 61 | expectErr string 62 | } 63 | 64 | func TestGenerateTimeDescr(t *testing.T) { 65 | tests := []timeDescrTestCase{ 66 | { 67 | name: "Traditional syslog", 68 | layout: "Jan _2 15:04:05", 69 | expected: &TimeFormatDescr{ 70 | TimestampLayout: "Jan _2 15:04:05", 71 | MinuteKeyLayout: "Jan _2 15:04", 72 | AWKExpr: TimeFormatAWKExpr{ 73 | Month: "monthByName[substr($0, 1, 3)]", 74 | Year: "yearByMonth[month]", 75 | Day: `(substr($0, 5, 1) == " ") ? "0" substr($0, 6, 1) : substr($0, 5, 2)`, 76 | HHMM: "substr($0, 8, 5)", 77 | MinuteKey: "substr($0, 1, 12)", 78 | }, 79 | }, 80 | }, 81 | { 82 | name: "ISO8601 with microseconds and timezone", 83 | layout: "2006-01-02T15:04:05.000000Z07:00", 84 | expected: &TimeFormatDescr{ 85 | TimestampLayout: "2006-01-02T15:04:05.000000Z07:00", 86 | MinuteKeyLayout: "01-02T15:04", 87 | AWKExpr: TimeFormatAWKExpr{ 88 | Month: "substr($0, 6, 2)", 89 | Year: "substr($0, 1, 4)", 90 | Day: "substr($0, 9, 2)", 91 | HHMM: "substr($0, 12, 5)", 92 | MinuteKey: "substr($0, 6, 11)", 93 | }, 94 | }, 95 | }, 96 | { 97 | name: "Custom 24-hour format", 98 | layout: "2006-01-02 15:04:05", 99 | expected: &TimeFormatDescr{ 100 | TimestampLayout: "2006-01-02 15:04:05", 101 | MinuteKeyLayout: "01-02 15:04", 102 | AWKExpr: TimeFormatAWKExpr{ 103 | Month: "substr($0, 6, 2)", 104 | Year: "substr($0, 1, 4)", 105 | Day: "substr($0, 9, 2)", 106 | HHMM: "substr($0, 12, 5)", 107 | MinuteKey: "substr($0, 6, 11)", 108 | }, 109 | }, 110 | }, 111 | { 112 | name: "ISO8601 without timezone", 113 | layout: "2006-01-02T15:04:05", 114 | expected: &TimeFormatDescr{ 115 | TimestampLayout: "2006-01-02T15:04:05", 116 | MinuteKeyLayout: "01-02T15:04", 117 | AWKExpr: TimeFormatAWKExpr{ 118 | Month: "substr($0, 6, 2)", 119 | Year: "substr($0, 1, 4)", 120 | Day: "substr($0, 9, 2)", 121 | HHMM: "substr($0, 12, 5)", 122 | MinuteKey: "substr($0, 6, 11)", 123 | }, 124 | }, 125 | }, 126 | { 127 | name: "Seconds are in between, unsupported", 128 | layout: "15:04:05 Jan _2 2006", 129 | expectErr: "seconds are in between of month, day, hour and min; can't extract MinuteKey", 130 | }, 131 | { 132 | name: "Non-fixed length (the date Jan 2 can also be Jan 12), unsupported", 133 | layout: "Jan 2 15:04:05", 134 | expectErr: "unsupported layout: required components not found", 135 | }, 136 | } 137 | 138 | for _, tc := range tests { 139 | t.Run(tc.name, func(t *testing.T) { 140 | result, err := GenerateTimeDescr(tc.layout) 141 | 142 | if tc.expectErr != "" { 143 | assert.EqualError(t, err, tc.expectErr) 144 | assert.Nil(t, result) 145 | } else { 146 | assert.NoError(t, err) 147 | assert.Equal(t, tc.expected, result) 148 | } 149 | }) 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /core/resolver_testdata/ssh_config_1: -------------------------------------------------------------------------------- 1 | Host * 2 | AddKeysToAgent yes 3 | IdentityFile ~/.ssh/id_rsa 4 | 5 | Host sshfoo-01 6 | User user-foo-from-ssh-config-01 7 | HostName host-foo-from-ssh-config-01.com 8 | Port 3001 9 | 10 | Host sshfoo-02 11 | User user-foo-from-ssh-config-02 12 | HostName host-foo-from-ssh-config-02.com 13 | Port 3002 14 | 15 | # This one must be ignored by our code 16 | Host sshfoo-* 17 | User someuser 18 | HostName somehost 19 | Port 3000 20 | 21 | Host sshbar-01 22 | User user-bar-from-ssh-config-01 23 | HostName host-bar-from-ssh-config-01.com 24 | Port 3001 25 | 26 | Host sshbar-02 27 | User user-bar-from-ssh-config-02 28 | HostName host-bar-from-ssh-config-02.com 29 | Port 3002 30 | 31 | Host sshrealhost.com 32 | User user-from-ssh-config 33 | Port 4001 34 | 35 | Host sshnoport-01 36 | User user-noport-from-ssh-config-01 37 | HostName host-noport-from-ssh-config-01.com 38 | 39 | Host foo-01 40 | User user-foo-from-ssh-config-01 41 | HostName host-foo-from-ssh-config-01.com 42 | Port 5001 43 | 44 | Host foo-02 45 | User user-foo-from-ssh-config-02 46 | HostName host-foo-from-ssh-config-02.com 47 | Port 5002 48 | 49 | Host host-bar-from-nerdlog-config-01.com 50 | User user-bar-from-ssh-config-01 51 | Port 6001 52 | 53 | Host host-bar-from-nerdlog-config-02.com 54 | User user-bar-from-ssh-config-02 55 | Port 6002 56 | 57 | Host baz-01 58 | User user-baz-from-ssh-config-01 59 | HostName host-baz-from-ssh-config-01.com 60 | Port 7001 61 | 62 | Host baz-02 63 | User user-baz-from-ssh-config-02 64 | HostName host-baz-from-ssh-config-02.com 65 | Port 7002 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/dimonomid/nerdlog 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 7 | github.com/gobwas/glob v0.2.3 8 | github.com/juju/errors v0.0.0-20220324005906-d8c5072c94ab 9 | github.com/kevinburke/ssh_config v1.2.0 10 | github.com/mattn/go-runewidth v0.0.13 11 | github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8 12 | github.com/rivo/uniseg v0.2.0 13 | github.com/spf13/pflag v1.0.5 14 | github.com/stretchr/testify v1.7.1 15 | golang.design/x/clipboard v0.6.2 16 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 17 | gopkg.in/yaml.v2 v2.4.0 18 | ) 19 | 20 | require ( 21 | github.com/davecgh/go-spew v1.1.0 // indirect 22 | github.com/gdamore/encoding v1.0.0 // indirect 23 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 24 | github.com/pmezard/go-difflib v1.0.0 // indirect 25 | golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 // indirect 26 | golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect 27 | golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 // indirect 28 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect 29 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect 30 | golang.org/x/text v0.3.6 // indirect 31 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= 2 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= 6 | github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= 7 | github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 h1:QqwPZCwh/k1uYqq6uXSb9TRDhTkfQbO80v8zhnIe5zM= 8 | github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1/go.mod h1:Az6Jt+M5idSED2YPGtwnfJV0kXohgdCBPmHGSYc1r04= 9 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 10 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 11 | github.com/juju/errors v0.0.0-20220324005906-d8c5072c94ab h1:tDk/iwzLX311+QPYXSjilLHWSdwRAfQWbDiAD0nrWT4= 12 | github.com/juju/errors v0.0.0-20220324005906-d8c5072c94ab/go.mod h1:jMGj9DWF/qbo91ODcfJq6z/RYc3FX3taCBZMCcpI4Ls= 13 | github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= 14 | github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= 15 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 16 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 17 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 18 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 19 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 20 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 21 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 22 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 23 | github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= 24 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8 h1:xe+mmCnDN82KhC010l3NfYlA8ZbOuzbXAzSYBa6wbMc= 28 | github.com/rivo/tview v0.0.0-20220307222120-9994674d60a8/go.mod h1:WIfMkQNY+oq/mWwtsjOYHIZBuwthioY2srOmljJkTnk= 29 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 30 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 31 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 32 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 33 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 34 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 35 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 36 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 37 | golang.design/x/clipboard v0.6.2 h1:a3Np4qfKnLWwfFJQhUWU3IDeRfmVuqWl+QPtP4CSYGw= 38 | golang.design/x/clipboard v0.6.2/go.mod h1:kqBSweBP0/im4SZGGjLrppH0D400Hnfo5WbFKSNK8N4= 39 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 40 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 41 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 42 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s= 43 | golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 44 | golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 h1:estk1glOnSVeJ9tdEZZc5mAMDZk5lNJNyJ6DvrBkTEU= 45 | golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= 46 | golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= 47 | golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 48 | golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ= 49 | golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= 50 | golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= 51 | golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554 h1:3In5TnfvnuXTF/uflgpYxSCEGP2NdYT37KsPh3VjZYU= 52 | golang.org/x/mobile v0.0.0-20210716004757-34ab1303b554/go.mod h1:jFTmtFYCV0MFtXBU+J5V/+5AUeVS0ON/0WkE/KSrl6E= 53 | golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= 54 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 55 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 56 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 57 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 58 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 59 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 60 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 61 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 64 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 66 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 67 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 68 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= 70 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 71 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 72 | golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 73 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= 74 | golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 75 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 76 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 77 | golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= 78 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 79 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 80 | golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 81 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 82 | golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 83 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 84 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 85 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 86 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 87 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 88 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 89 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 90 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 91 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 92 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 93 | -------------------------------------------------------------------------------- /images/nerdlog_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimonomid/nerdlog/bd920d17e19e7d9e522d929bbf9cc047e648f25a/images/nerdlog_demo.gif -------------------------------------------------------------------------------- /images/nerdlog_intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimonomid/nerdlog/bd920d17e19e7d9e522d929bbf9cc047e648f25a/images/nerdlog_intro.png -------------------------------------------------------------------------------- /images/nerdlog_query_edit_form.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimonomid/nerdlog/bd920d17e19e7d9e522d929bbf9cc047e648f25a/images/nerdlog_query_edit_form.png -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type LogLevel int 14 | 15 | const ( 16 | Verbose3 LogLevel = iota 17 | Verbose2 18 | Verbose1 19 | Info 20 | Warning 21 | Error 22 | ) 23 | 24 | var logFile *os.File 25 | var logFileMtx sync.Mutex 26 | 27 | // printf prints a formatted message to the log file ~/.nerdlog.log 28 | func printf(format string, a ...interface{}) { 29 | w := writer() 30 | 31 | fmt.Fprintf(w, "%s: ", time.Now().Format("2006-01-02T15:04:05.999")) 32 | 33 | if !strings.HasSuffix(format, "\n") { 34 | format += "\n" 35 | } 36 | 37 | fmt.Fprintf(w, format, a...) 38 | } 39 | 40 | func writer() io.Writer { 41 | logFileMtx.Lock() 42 | defer logFileMtx.Unlock() 43 | 44 | if logFile == nil { 45 | homeDir, err := os.UserHomeDir() 46 | if err != nil { 47 | panic(err.Error()) 48 | } 49 | 50 | fname := filepath.Join(homeDir, ".nerdlog.log") 51 | 52 | logFile, err = os.OpenFile(fname, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 53 | if err != nil { 54 | panic(err.Error()) 55 | } 56 | } 57 | 58 | return logFile 59 | } 60 | 61 | type Logger struct { 62 | minLevel LogLevel 63 | 64 | namespace string 65 | context map[string]string 66 | } 67 | 68 | func NewLogger(minLevel LogLevel) *Logger { 69 | return &Logger{ 70 | minLevel: minLevel, 71 | } 72 | } 73 | 74 | func (l *Logger) thisOrDefault() *Logger { 75 | if l != nil { 76 | return l 77 | } 78 | 79 | return &Logger{ 80 | minLevel: Info, 81 | } 82 | } 83 | 84 | func (l *Logger) WithNamespaceAppended(n string) *Logger { 85 | l = l.thisOrDefault() 86 | 87 | ns := l.namespace 88 | if ns != "" { 89 | ns += "/" 90 | } 91 | ns += n 92 | 93 | newLogger := *l 94 | newLogger.namespace = ns 95 | return &newLogger 96 | } 97 | 98 | func (l *Logger) Verbose3f(format string, a ...interface{}) { 99 | l.Printf(Verbose3, format, a...) 100 | } 101 | 102 | func (l *Logger) Verbose2f(format string, a ...interface{}) { 103 | l.Printf(Verbose2, format, a...) 104 | } 105 | 106 | func (l *Logger) Verbose1f(format string, a ...interface{}) { 107 | l.Printf(Verbose1, format, a...) 108 | } 109 | 110 | func (l *Logger) Infof(format string, a ...interface{}) { 111 | l.Printf(Info, format, a...) 112 | } 113 | 114 | func (l *Logger) Warnf(format string, a ...interface{}) { 115 | l.Printf(Warning, format, a...) 116 | } 117 | 118 | func (l *Logger) Errorf(format string, a ...interface{}) { 119 | l.Printf(Error, format, a...) 120 | } 121 | 122 | func (l *Logger) Printf(level LogLevel, format string, a ...interface{}) { 123 | l = l.thisOrDefault() 124 | 125 | if level < l.minLevel { 126 | return 127 | } 128 | 129 | if l.namespace != "" { 130 | printf("[%s] %s", l.namespace, fmt.Sprintf(format, a...)) 131 | } else { 132 | printf("%s", l.namespace, fmt.Sprintf(format, a...)) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /shellescape/shell_escape.go: -------------------------------------------------------------------------------- 1 | package shellescape 2 | 3 | import ( 4 | "strings" 5 | "unicode" 6 | 7 | "github.com/juju/errors" 8 | ) 9 | 10 | func Escape(parts []string) string { 11 | eParts := make([]string, 0, len(parts)) 12 | 13 | for _, part := range parts { 14 | needEscape := false 15 | for _, r := range part { 16 | if !unicode.IsLetter(r) && !unicode.IsNumber(r) && r != '-' && r != '_' && r != '.' && r != '/' { 17 | needEscape = true 18 | break 19 | } 20 | } 21 | 22 | if len(part) == 0 { 23 | needEscape = true 24 | } 25 | 26 | if needEscape { 27 | part = "'" + strings.Replace(part, "'", "'\"'\"'", -1) + "'" 28 | } 29 | eParts = append(eParts, part) 30 | } 31 | 32 | return strings.Join(eParts, " ") 33 | } 34 | 35 | type parserQuoteState int 36 | 37 | const ( 38 | parserQuoteStateNone parserQuoteState = iota 39 | parserQuoteStateSingle 40 | parserQuoteStateDouble 41 | parserQuoteStateDoubleEscaped 42 | ) 43 | 44 | func Parse(shellCmd string) ([]string, error) { 45 | var parts []string 46 | 47 | partBuilder := strings.Builder{} 48 | 49 | inPart := false 50 | quoteState := parserQuoteStateNone 51 | 52 | finalizePart := func() { 53 | parts = append(parts, partBuilder.String()) 54 | partBuilder.Reset() 55 | } 56 | 57 | for _, r := range shellCmd { 58 | // Sanity check, TODO: perhaps remove it 59 | if !inPart && quoteState != parserQuoteStateNone { 60 | panic("should never be here") 61 | } 62 | 63 | isSpace := unicode.IsSpace(r) 64 | 65 | if !inPart { 66 | if !isSpace { 67 | inPart = true 68 | } else { 69 | // We're in the whitespace, nothing else to do here. 70 | continue 71 | } 72 | } 73 | 74 | switch quoteState { 75 | case parserQuoteStateNone: 76 | switch r { 77 | case '\'': 78 | quoteState = parserQuoteStateSingle 79 | case '"': 80 | quoteState = parserQuoteStateDouble 81 | default: 82 | if !isSpace { 83 | partBuilder.WriteRune(r) 84 | } else { 85 | finalizePart() 86 | inPart = false 87 | } 88 | } 89 | 90 | case parserQuoteStateSingle: 91 | switch r { 92 | case '\'': 93 | quoteState = parserQuoteStateNone 94 | default: 95 | partBuilder.WriteRune(r) 96 | } 97 | 98 | case parserQuoteStateDouble: 99 | switch r { 100 | case '"': 101 | quoteState = parserQuoteStateNone 102 | case '\\': 103 | quoteState = parserQuoteStateDoubleEscaped 104 | default: 105 | partBuilder.WriteRune(r) 106 | } 107 | 108 | case parserQuoteStateDoubleEscaped: 109 | switch r { 110 | case '\\': 111 | partBuilder.WriteRune(r) 112 | case '"': 113 | partBuilder.WriteRune(r) 114 | default: 115 | partBuilder.WriteRune('\\') 116 | partBuilder.WriteRune(r) 117 | } 118 | 119 | quoteState = parserQuoteStateDouble 120 | } 121 | } 122 | 123 | if inPart { 124 | if quoteState != parserQuoteStateNone { 125 | return nil, errors.Errorf("unfinished quote") 126 | } 127 | 128 | finalizePart() 129 | } 130 | 131 | return parts, nil 132 | } 133 | -------------------------------------------------------------------------------- /shellescape/shell_escape_test.go: -------------------------------------------------------------------------------- 1 | package shellescape 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type parseTC struct { 10 | shellCmd string 11 | want []string 12 | wantErr string 13 | } 14 | 15 | func TestParse(t *testing.T) { 16 | testCases := []parseTC{ 17 | parseTC{shellCmd: ``, want: nil}, 18 | parseTC{shellCmd: `foo bar bazzzz`, want: []string{`foo`, `bar`, `bazzzz`}}, 19 | parseTC{shellCmd: `foo bar bazzzz`, want: []string{`foo`, `bar`, `bazzzz`}}, 20 | parseTC{shellCmd: ` foo bar bazzzz `, want: []string{`foo`, `bar`, `bazzzz`}}, 21 | parseTC{shellCmd: `foo 'bar' bazzzz`, want: []string{`foo`, `bar`, `bazzzz`}}, 22 | parseTC{shellCmd: `'foo' 'bar' 'bazzzz'`, want: []string{`foo`, `bar`, `bazzzz`}}, 23 | parseTC{shellCmd: `' foo' 'bar ' 'bazzzz'`, want: []string{` foo`, `bar `, `bazzzz`}}, 24 | parseTC{shellCmd: `foo 'bar bazz'zz`, want: []string{`foo`, `bar bazzzz`}}, 25 | parseTC{shellCmd: `foo 'bar ba'"zz z"z`, want: []string{`foo`, `bar bazz zz`}}, 26 | parseTC{shellCmd: `'foo bar bazzzz'`, want: []string{`foo bar bazzzz`}}, 27 | parseTC{shellCmd: `"foo bar bazzzz"`, want: []string{`foo bar bazzzz`}}, 28 | parseTC{shellCmd: `"foo bar" bazzzz`, want: []string{`foo bar`, `bazzzz`}}, 29 | parseTC{shellCmd: `"foo bar" bazzzz`, want: []string{`foo bar`, `bazzzz`}}, 30 | parseTC{shellCmd: `"foo \"bar" bazzzz`, want: []string{`foo "bar`, `bazzzz`}}, 31 | parseTC{shellCmd: `"foo \" bar" bazzzz`, want: []string{`foo " bar`, `bazzzz`}}, 32 | parseTC{shellCmd: `"foo \\ bar" bazzzz`, want: []string{`foo \ bar`, `bazzzz`}}, 33 | parseTC{shellCmd: `'foo \" bar' bazzzz`, want: []string{`foo \" bar`, `bazzzz`}}, 34 | parseTC{shellCmd: `'foo '"'"'bar'"'"' baz'`, want: []string{`foo 'bar' baz`}}, 35 | 36 | parseTC{shellCmd: `"foo \" bar bazzzz`, wantErr: "unfinished quote"}, 37 | parseTC{shellCmd: `'foo \" bar bazzzz`, wantErr: "unfinished quote"}, 38 | } 39 | 40 | for i, tc := range testCases { 41 | assertArgs := []interface{}{"testCase %d %q", i, tc.shellCmd} 42 | 43 | got, gotErr := Parse(tc.shellCmd) 44 | 45 | if tc.wantErr != "" { 46 | assert.Nil(t, got) 47 | assert.Equal(t, tc.wantErr, gotErr.Error(), assertArgs...) 48 | 49 | if gotErr != nil { 50 | assert.Equal(t, tc.wantErr, gotErr.Error(), assertArgs...) 51 | } 52 | } else { 53 | assert.Equal(t, tc.want, got, assertArgs...) 54 | assert.Nil(t, gotErr, assertArgs...) 55 | } 56 | } 57 | } 58 | 59 | type escapeTC struct { 60 | parts []string 61 | want string 62 | } 63 | 64 | func TestEscape(t *testing.T) { 65 | testCases := []escapeTC{ 66 | escapeTC{parts: nil, want: ""}, 67 | escapeTC{parts: []string{`foo`, `bar`, `baz`}, want: `foo bar baz`}, 68 | escapeTC{parts: []string{`foo/bar-baz/hey_foo`, `bar`, `baz`}, want: `foo/bar-baz/hey_foo bar baz`}, 69 | escapeTC{parts: []string{`foo/bar-\baz/hey_foo`, `bar`, `baz`}, want: `'foo/bar-\baz/hey_foo' bar baz`}, 70 | escapeTC{parts: []string{`foo13`, `bar`, `baz`}, want: `foo13 bar baz`}, 71 | escapeTC{parts: []string{`foo bar`, `baz`}, want: `'foo bar' baz`}, 72 | escapeTC{parts: []string{`foo "bar`, `baz`}, want: `'foo "bar' baz`}, 73 | escapeTC{parts: []string{`'foo bar`, `baz`}, want: `''"'"'foo bar' baz`}, 74 | escapeTC{parts: []string{`foo`, `bar, baz`}, want: `foo 'bar, baz'`}, 75 | } 76 | 77 | for i, tc := range testCases { 78 | assertArgs := []interface{}{"testCase %d %q", i, tc.parts} 79 | 80 | got := Escape(tc.parts) 81 | assert.Equal(t, tc.want, got, assertArgs...) 82 | 83 | // Also make sure we can parse it back and get the same result 84 | gotParsed, gotParsedErr := Parse(got) 85 | assert.Equal(t, tc.parts, gotParsed, assertArgs...) 86 | assert.Nil(t, gotParsedErr, assertArgs...) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /util/copy_test_results.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" 4 | 5 | # Define the paths for both directory trees 6 | tests_output_root_dir="/tmp/nerdlog_agent_test_output" 7 | tests_input_root_dir="${SCRIPT_DIR}/../core/nerdlog_agent_testdata/test_cases" 8 | 9 | # Convert tests_output_root_dir to absolute path for consistency 10 | tests_output_root_dir=$(realpath "$tests_output_root_dir") 11 | 12 | # Function to copy stderr and stdout files recursively 13 | copy_logs() { 14 | local current_dir="$1" 15 | 16 | # Get the relative path of current_dir from tests_output_root_dir 17 | relative_path="${current_dir#$tests_output_root_dir}" 18 | 19 | second_dir="$tests_input_root_dir/$relative_path" 20 | 21 | # Check if the corresponding directory exists in the second tree 22 | if [ ! -d "$second_dir" ]; then 23 | echo "Warning: Directory $second_dir does not exist. Skipping..." 24 | return 25 | fi 26 | 27 | # Check if the current directory has the expected files 28 | if [ -f "$current_dir/nerdlog_agent_stderr" ]; then 29 | # Instead of copying, echo the command 30 | echo -n . 31 | cp "$current_dir/nerdlog_agent_stderr" "$second_dir/want_stderr" || exit 1 32 | fi 33 | 34 | if [ -f "$current_dir/nerdlog_agent_stdout" ]; then 35 | # Instead of copying, echo the command 36 | echo -n . 37 | cp "$current_dir/nerdlog_agent_stdout" "$second_dir/want_stdout" || exit 1 38 | fi 39 | 40 | # Recurse into subdirectories 41 | for subdir in "$current_dir"/*; do 42 | if [ -d "$subdir" ]; then 43 | copy_logs "$subdir" || exit 1 44 | fi 45 | done 46 | } 47 | 48 | # Start recursion from the top of the first tree 49 | copy_logs "$tests_output_root_dir" 50 | echo "" 51 | echo "All done" 52 | --------------------------------------------------------------------------------