├── .github └── workflows │ └── go.yml ├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── assets ├── ayano-logrotate ├── ayano.service ├── fail2ban │ ├── filter.d │ │ └── ayano.conf │ └── jail.d │ │ └── ayano.conf └── goaccess.conf ├── cmd ├── list.go ├── root.go └── run.go ├── go.mod ├── go.sum ├── main.go └── pkg ├── analyze ├── analyze.go ├── analyze_test.go ├── file.go ├── log.go ├── sortfunc.go ├── util.go └── util_test.go ├── fileiter └── iterator.go ├── info └── info.go ├── parser ├── caddy.go ├── caddy_test.go ├── common.go ├── common_test.go ├── goaccess.go ├── goaccess_test.go ├── nginx-combined.go ├── nginx-combined_test.go ├── nginx-json.go ├── nginx-json_test.go ├── parser.go ├── rsync-proxy.go ├── rsync-proxy_test.go ├── tencent-cdn.go └── tencent-cdn_test.go ├── systemd └── notify.go └── tui ├── interface.go ├── routines.go └── term.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | 9 | build: 10 | name: Build 11 | runs-on: ubuntu-latest 12 | steps: 13 | 14 | - uses: actions/checkout@v4 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: Setup Go 19 | id: go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: stable 23 | check-latest: true 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | - name: Test 33 | run: | 34 | go test -race ./... && go vet ./... 35 | 36 | - name: Build 37 | uses: goreleaser/goreleaser-action@v6 38 | with: 39 | args: build --snapshot --clean 40 | 41 | - name: Release 42 | uses: goreleaser/goreleaser-action@v6 43 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') 44 | with: 45 | version: latest 46 | args: release --clean 47 | env: 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ayano 2 | ayano-* 3 | ayano.test 4 | *.pprof 5 | __debug_bin* 6 | /dist/ 7 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - go mod tidy 6 | 7 | builds: 8 | - id: ayano 9 | binary: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}{{ if .Amd64 }}-{{ .Amd64 }}{{ end }}" 10 | env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | goarch: 15 | - amd64 16 | - arm64 17 | goarm: 18 | - 7 19 | goamd64: 20 | - v2 21 | - v3 22 | flags: 23 | - -trimpath 24 | ldflags: 25 | - -s -w -X github.com/taoky/ayano/pkg/info.Version={{.Version}} -X github.com/taoky/ayano/pkg/info.BuildDate={{.Date}} -X github.com/taoky/ayano/pkg/info.GitCommit={{.Commit}} 26 | no_unique_dist_dir: true 27 | 28 | archives: 29 | - format: binary 30 | name_template: "{{ .Binary }}" 31 | 32 | checksum: 33 | name_template: 'checksums.txt' 34 | 35 | snapshot: 36 | name_template: "{{ incpatch .Version }}-next" 37 | 38 | changelog: 39 | use: github-native 40 | 41 | # modelines, feel free to remove those if you don't want/use them: 42 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 43 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 taoky 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ayano 2 | 3 | Follow nginx log, and find out bad guys! Ayano parses web server log and shows clients eating most bandwidth every few seconds. 4 | 5 | ## Build 6 | 7 | ```shell 8 | CGO_ENABLED=0 go build 9 | ``` 10 | 11 | ## Usage 12 | 13 | ```console 14 | $ ./ayano 15 | A simple log analysis tool for Nginx, Apache, or other web server logs 16 | 17 | Usage: 18 | ayano [flags] 19 | ayano [command] 20 | 21 | Available Commands: 22 | analyze Log analyse mode (no tail following, only show top N at the end, and implies --whole) 23 | completion Generate the autocompletion script for the specified shell 24 | daemon Daemon mode, prints out IP CIDR and total size every 1 GiB 25 | help Help about any command 26 | list List various items 27 | run Run and follow the log file(s) 28 | 29 | Flags: 30 | -h, --help help for ayano 31 | 32 | Use "ayano [command] --help" for more information about a command. 33 | $ ./ayano run --help 34 | Run and follow the log file(s) 35 | 36 | Usage: 37 | ayano run [filename...] [flags] 38 | 39 | Flags: 40 | -a, --absolute Show absolute time for each item 41 | -g, --group Try to group CIDRs 42 | -h, --help help for run 43 | --no-netstat Do not detect active connections 44 | -o, --outlog string Change log output file 45 | -p, --parser string Log parser (see "ayano list parsers") (default "nginx-json") 46 | --prefixv4 int Group IPv4 by prefix (default 24) 47 | --prefixv6 int Group IPv6 by prefix (default 48) 48 | -r, --refresh int Refresh interval in seconds (default 5) 49 | -s, --server string Server IP to filter (nginx-json only) 50 | -S, --sort-by string Sort result by (size|requests) (default "size") 51 | -t, --threshold size Threshold size for request (only requests at least this large will be counted) (default 10 MB) 52 | -n, --top int Number of top items to show (default 10) 53 | --truncate Truncate long URLs from output 54 | --truncate-to int Truncate URLs to given length, overrides --truncate 55 | -w, --whole Analyze whole log file and then tail it 56 | 57 | # Example 1 58 | $ ./ayano run -n 20 --threshold 50M /var/log/nginx/access_json.log 59 | # Example 2 60 | $ ./ayano run -n 50 --whole --parser nginx-combined /var/log/nginx/access.log 61 | # Example 3. This will use fast path to analyse log, and just print result and quit. 62 | $ ./ayano analyze -n 100 /var/log/nginx/access_json.log 63 | ``` 64 | 65 | Ayano would output a table which is easy for humans to read. 66 | 67 | ### Daemon mode (experimental) 68 | 69 | Daemon mode is a simple log output mode that intended to work with fail2ban. 70 | 71 | Current log format looks like this (`log_time client_cidr total_gib GiB first_time path`): 72 | 73 | ```log 74 | 2024/06/25 01:03:17 172.26.3.0/24 1.0 GiB 2024-06-25 01:03:17 /big 75 | 2024/06/25 01:03:29 172.26.3.0/24 2.0 GiB 2024-06-25 01:03:17 /big 76 | 2024/06/25 01:03:42 172.26.3.0/24 3.0 GiB 2024-06-25 01:03:17 /big 77 | 2024/06/25 01:03:56 172.26.3.0/24 4.0 GiB 2024-06-25 01:03:17 /big 78 | 2024/06/25 01:04:09 172.26.3.0/24 5.0 GiB 2024-06-25 01:03:17 /big 79 | ``` 80 | 81 | A reference systemd service file, logrotate file and fail2ban configs are provided in [assets/](assets/). 82 | 83 | Please note that the stats output would NOT be rotated (unless you restart ayano). 84 | 85 | If you don't like to use fail2ban, you could also use this simple one-liner to check stats. Here is an example: 86 | 87 | ```console 88 | $ awk '{print $3}' record.log | sort | uniq -c | sort -nr 89 | 36 114.5.14.0/24 90 | 3 191.9.81.0/24 91 | ``` 92 | 93 | which means that "114.5.14.0/24" takes at least 36GiB bandwidth, and "191.9.81.0/24" takes at least 3GiB bandwidth, for the time period this log file covers. 94 | 95 | ## Format support 96 | 97 | Ayano supports following types of log format. You could also use `ayano list parsers` to check. 98 | 99 | 1. Standard "combined" format access log. 100 | 2. JSON format access log configured as: 101 | 102 | ```nginx 103 | log_format ngx_json escape=json '{' 104 | '"timestamp":$msec,' 105 | '"clientip":"$remote_addr",' 106 | '"serverip":"$server_addr",' 107 | '"method":"$request_method",' 108 | '"url":"$request_uri",' 109 | '"status":$status,' 110 | '"size":$body_bytes_sent,' 111 | '"resp_time":$request_time,' 112 | '"http_host":"$host",' 113 | '"referer":"$http_referer",' 114 | '"user_agent":"$http_user_agent"' 115 | '}'; 116 | ``` 117 | 118 | 3. Caddy default JSON format like [this](https://caddyserver.com/docs/logging#structured-logs): 119 | 120 | ```json 121 | {"level":"info","ts":1646861401.5241024,"logger":"http.log.access","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"41342","client_ip":"127.0.0.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/7.82.0"],"Accept":["*/*"],"Accept-Encoding":["gzip, deflate, br"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"example.com"}},"bytes_read": 0,"user_id":"","duration":0.000929675,"size":10900,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Encoding":["gzip"],"Content-Type":["text/html; charset=utf-8"],"Vary":["Accept-Encoding"]}} 122 | ``` 123 | 124 | **Note**: If you are using Caddy behind a reverse proxy, please upgrade Caddy to 2.7.0+ and set `trusted_proxies` (and `client_ip_headers`) in configuration file to let log have `client_ip` field outputted. 125 | 126 | 4. GoAccess format string. You shall set `GOACCESS_CONFIG` env to a goaccess config file beforehand ([format recognized](https://github.com/taoky/goaccessfmt?tab=readme-ov-file#config-file-format), [example](assets/goaccess.conf)). 127 | 5. Tencent CDN log format. 128 | 129 | ## Note 130 | 131 | ### Memory footprint 132 | 133 | If you have literally A LOT OF logs to analyze, and you're running ayano on a server with very low RAM, you could use `systemd-run` to restrict its memory footprint like this: 134 | 135 | ```shell 136 | GOMEMLIMIT=270MiB systemd-run --user --scope -p MemoryMax=300M ayano analyze ... 137 | ``` 138 | 139 | `GOMEMLIMIT` is a soft limit -- it helps go runtime GC to do its job more aggressively when it would reach the limit (at the cost of more CPU time). Please read [A Guide to the Go Garbage Collector](https://tip.golang.org/doc/gc-guide#Memory_limit) for more information. 140 | 141 | Also, when in interactive mode (`ayano run`), `ayano` might take double memory if log format has server IP set, to support filtering by server IP without restarting. 142 | 143 | ## Naming 144 | 145 | Ayano is named after *Sugiura Ayano*, the Student Council vice-president in [*Yuru Yuri*](https://en.wikipedia.org/wiki/YuruYuri#Student_Council). 146 | 147 | Also, if you want something easier to use than iftop... Please try my new little project [chitose](https://github.com/taoky/chitose)! 148 | -------------------------------------------------------------------------------- /assets/ayano-logrotate: -------------------------------------------------------------------------------- 1 | /var/log/ayano/*.log 2 | { 3 | create 0644 nobody nogroup 4 | daily 5 | size 100M 6 | rotate 200 7 | dateext 8 | compress 9 | notifempty 10 | missingok 11 | 12 | postrotate 13 | systemctl reload ayano.service 14 | endscript 15 | } 16 | -------------------------------------------------------------------------------- /assets/ayano.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Nginx log analyzer for OSS mirrors, working with fail2ban 3 | 4 | [Service] 5 | Restart=on-failure 6 | StartLimitInterval=10s 7 | MemoryMax=5G 8 | ExecStart=/usr/local/bin/ayano daemon --outlog /var/log/ayano/record.log --parser nginx-combined /var/log/nginx/access.log 9 | 10 | # Systemd in Debian Bookworm does not support notify-reload 11 | # Type=notify-reload 12 | Type=notify 13 | ExecReload=/bin/kill -HUP $MAINPID 14 | 15 | LogsDirectory=ayano 16 | # Run `adduser --system ayano` first! 17 | User=ayano 18 | Group=nogroup 19 | # The group of /var/log/nginx 20 | SupplementaryGroups=adm 21 | 22 | [Install] 23 | WantedBy=multi-user.target 24 | -------------------------------------------------------------------------------- /assets/fail2ban/filter.d/ayano.conf: -------------------------------------------------------------------------------- 1 | [Definition] 2 | failregex = \d+\.?\d+? GiB \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d .+ 3 | -------------------------------------------------------------------------------- /assets/fail2ban/jail.d/ayano.conf: -------------------------------------------------------------------------------- 1 | [ayano] 2 | enabled = true 3 | filter = ayano 4 | banaction = iptables-multiport-log 5 | logpath = /var/log/ayano/record.log 6 | # example: ban 2 days if downloads large files more than 1TB for 12 hours 7 | maxretry = 1024 8 | findtime = 43200 9 | bantime = 172800 10 | usedns = no 11 | -------------------------------------------------------------------------------- /assets/goaccess.conf: -------------------------------------------------------------------------------- 1 | log-format { "server": "%S", "ts": "%x.%^", "request": { "client_ip": "%h", "proto":"%H", "method": "%m", "host": "%v", "uri": "%U", "headers": {"User-Agent": ["%u"], "Referer": ["%R"] }, "tls": { "cipher_suite":"%k", "proto": "%K" } }, "duration": "%T", "size": "%b","status": "%s", "resp_headers": { "Content-Type": ["%M"] } } 2 | date-format %s 3 | time-format %s 4 | -------------------------------------------------------------------------------- /cmd/list.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "slices" 5 | "strings" 6 | 7 | "github.com/olekukonko/tablewriter" 8 | "github.com/spf13/cobra" 9 | "github.com/taoky/ayano/pkg/parser" 10 | ) 11 | 12 | func listCmd() *cobra.Command { 13 | cmd := &cobra.Command{ 14 | Use: "list ", 15 | Short: "List various items", 16 | Args: cobra.NoArgs, 17 | RunE: showHelp, 18 | } 19 | cmd.AddCommand(listParsersCmd()) 20 | return cmd 21 | } 22 | 23 | func listParsersCmd() *cobra.Command { 24 | cmd := &cobra.Command{ 25 | Use: "parsers", 26 | Short: "List available log parsers", 27 | Args: cobra.NoArgs, 28 | } 29 | var all bool 30 | cmd.Flags().BoolVarP(&all, "all", "a", false, "Show all parsers") 31 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 32 | table := tablewriter.NewWriter(cmd.OutOrStdout()) 33 | table.SetAutoWrapText(false) 34 | table.SetAutoFormatHeaders(true) 35 | table.SetCenterSeparator("") 36 | table.SetColumnSeparator("") 37 | table.SetRowLine(false) 38 | table.SetRowSeparator("") 39 | table.SetTablePadding(" ") 40 | table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) 41 | table.SetAlignment(tablewriter.ALIGN_LEFT) 42 | table.SetHeaderLine(false) 43 | table.SetBorder(false) 44 | table.SetNoWhiteSpace(true) 45 | 46 | table.SetHeader([]string{"Name", "Description"}) 47 | 48 | parsers := parser.All() 49 | slices.SortFunc(parsers, func(a, b parser.ParserMeta) int { 50 | return strings.Compare(a.Name, b.Name) 51 | }) 52 | for _, p := range parsers { 53 | if all || !p.Hidden { 54 | table.Append([]string{p.Name, p.Description}) 55 | } 56 | } 57 | table.Render() 58 | return nil 59 | } 60 | return cmd 61 | } 62 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | func showHelp(cmd *cobra.Command, args []string) error { 8 | return cmd.Help() 9 | } 10 | 11 | func RootCmd() *cobra.Command { 12 | rootCmd := &cobra.Command{ 13 | Use: "ayano", 14 | Short: "A simple log analysis tool for Nginx, Apache, or other web server logs", 15 | Args: cobra.NoArgs, 16 | RunE: showHelp, 17 | } 18 | rootCmd.AddCommand( 19 | runCmd(), 20 | analyzeCmd(), 21 | daemonCmd(), 22 | listCmd(), 23 | ) 24 | return rootCmd 25 | } 26 | -------------------------------------------------------------------------------- /cmd/run.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "runtime" 9 | "runtime/pprof" 10 | "syscall" 11 | 12 | "github.com/spf13/cobra" 13 | "github.com/taoky/ayano/pkg/analyze" 14 | "github.com/taoky/ayano/pkg/fileiter" 15 | "github.com/taoky/ayano/pkg/systemd" 16 | "github.com/taoky/ayano/pkg/tui" 17 | ) 18 | 19 | const defaultFilename = "/var/log/nginx/mirrors/access_json.log" 20 | 21 | func filenamesFromArgs(args []string) []string { 22 | if len(args) == 0 { 23 | return []string{defaultFilename} 24 | } 25 | return args 26 | } 27 | 28 | func runWithConfig(cmd *cobra.Command, args []string, config analyze.AnalyzerConfig) error { 29 | // Sanily check 30 | if config.Analyze && config.Daemon { 31 | return errors.New("analyze mode and daemonizing are mutually exclusive") 32 | } 33 | 34 | filenames := filenamesFromArgs(args) 35 | fmt.Fprintln(cmd.ErrOrStderr(), "Using log files:", filenames) 36 | cmd.SilenceUsage = true 37 | 38 | analyzer, err := analyze.NewAnalyzer(config) 39 | if err != nil { 40 | return fmt.Errorf("failed to create analyzer: %w", err) 41 | } 42 | 43 | // setup SIGHUP to reopen log file 44 | c := make(chan os.Signal, 1) 45 | signal.Notify(c, syscall.SIGHUP) 46 | go func() { 47 | for range c { 48 | systemd.MustNotifyReloading() 49 | analyzer.OpenLogFile() 50 | // Let GC close the old file 51 | runtime.GC() 52 | systemd.MustNotifyReady() 53 | } 54 | }() 55 | 56 | if config.Analyze { 57 | for _, filename := range filenames { 58 | err = analyzer.AnalyzeFile(filename) 59 | if err != nil { 60 | break 61 | } 62 | } 63 | analyzer.PrintTopValues(nil, config.SortBy, "") 64 | return err 65 | } else { 66 | // Tail mode 67 | var iters []fileiter.Iterator 68 | for _, filename := range filenames { 69 | iter, err := analyzer.OpenTailIterator(filename) 70 | if err != nil { 71 | return err 72 | } 73 | iters = append(iters, iter) 74 | } 75 | 76 | if config.Daemon { 77 | if err := systemd.NotifyReady(); err != nil { 78 | return fmt.Errorf("failed to notify systemd: %w", err) 79 | } 80 | } else { 81 | go tui.New(analyzer).Run() 82 | } 83 | 84 | if len(iters) == 1 { 85 | return analyzer.RunLoop(iters[0]) 86 | } else { 87 | return analyzer.RunLoopWithMultipleIterators(iters) 88 | } 89 | } 90 | } 91 | 92 | func runCmd() *cobra.Command { 93 | cmd := &cobra.Command{ 94 | Use: "run [filename...]", 95 | Short: "Run and follow the log file(s)", 96 | } 97 | config := analyze.DefaultConfig() 98 | config.InstallFlags(cmd.Flags(), cmd.Name()) 99 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 100 | return runWithConfig(cmd, args, config) 101 | } 102 | return cmd 103 | } 104 | 105 | func analyzeCmd() *cobra.Command { 106 | cmd := &cobra.Command{ 107 | Use: "analyze [filename...]", 108 | Aliases: []string{"analyse"}, 109 | Short: "Log analyse mode (no tail following, only show top N at the end, and implies --whole)", 110 | } 111 | config := analyze.DefaultConfig() 112 | config.InstallFlags(cmd.Flags(), cmd.Name()) 113 | 114 | var cpuProf string 115 | var memProf string 116 | cmd.Flags().StringVar(&cpuProf, "cpuprof", "", "write CPU pprof data to file") 117 | cmd.Flags().StringVar(&memProf, "memprof", "", "write memory pprof data to file") 118 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 119 | config.Analyze = true 120 | if cpuProf != "" { 121 | f, err := os.Create(cpuProf) 122 | if err != nil { 123 | return fmt.Errorf("failed to create CPU pprof file: %w", err) 124 | } 125 | defer f.Close() 126 | if err := pprof.StartCPUProfile(f); err != nil { 127 | return fmt.Errorf("failed to start CPU pprof: %w", err) 128 | } 129 | defer pprof.StopCPUProfile() 130 | } 131 | err := runWithConfig(cmd, args, config) 132 | if memProf != "" { 133 | f, err := os.Create(memProf) 134 | if err != nil { 135 | return fmt.Errorf("failed to create memory pprof file: %w", err) 136 | } 137 | defer f.Close() 138 | if err := pprof.WriteHeapProfile(f); err != nil { 139 | return fmt.Errorf("failed to write memory pprof: %w", err) 140 | } 141 | } 142 | return err 143 | } 144 | return cmd 145 | } 146 | 147 | func daemonCmd() *cobra.Command { 148 | cmd := &cobra.Command{ 149 | Use: "daemon [filename]", 150 | Short: "Daemon mode, prints out IP CIDR and total size every 1 GiB", 151 | Args: cobra.MaximumNArgs(1), 152 | } 153 | config := analyze.DefaultConfig() 154 | config.InstallFlags(cmd.Flags(), cmd.Name()) 155 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 156 | config.Daemon = true 157 | return runWithConfig(cmd, args, config) 158 | } 159 | return cmd 160 | } 161 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/taoky/ayano 2 | 3 | go 1.23.0 4 | 5 | require ( 6 | github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 7 | github.com/dustin/go-humanize v1.0.1 8 | github.com/goccy/go-json v0.10.5 9 | github.com/nxadm/tail v1.4.11 10 | github.com/olekukonko/tablewriter v0.0.5 11 | github.com/schollz/progressbar/v3 v3.18.0 12 | github.com/spf13/cobra v1.9.1 13 | github.com/spf13/pflag v1.0.6 14 | github.com/stretchr/testify v1.9.0 15 | github.com/taoky/goaccessfmt v0.0.0-20240824074420-af31a41470aa 16 | golang.org/x/sys v0.31.0 17 | ) 18 | 19 | require ( 20 | github.com/davecgh/go-spew v1.1.1 // indirect 21 | github.com/fsnotify/fsnotify v1.8.0 // indirect 22 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 23 | github.com/itchyny/timefmt-go v0.1.6 // indirect 24 | github.com/mattn/go-runewidth v0.0.16 // indirect 25 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect 26 | github.com/pmezard/go-difflib v1.0.0 // indirect 27 | github.com/rivo/uniseg v0.4.7 // indirect 28 | golang.org/x/term v0.30.0 // indirect 29 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 30 | gopkg.in/yaml.v3 v3.0.1 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= 2 | github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= 3 | github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= 4 | github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 9 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 10 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 11 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 12 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 13 | github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 14 | github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 15 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 16 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 17 | github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q= 18 | github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg= 19 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 20 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 21 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 22 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= 23 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= 24 | github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= 25 | github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= 26 | github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= 27 | github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= 28 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 29 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 30 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 31 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 32 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 33 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 34 | github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= 35 | github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= 36 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 37 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 38 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 39 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 40 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 41 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 42 | github.com/taoky/goaccessfmt v0.0.0-20240824074420-af31a41470aa h1:+yzNM1meB5B2Yw8FiaP0IeDZf0aSy8fldLxEq5OnW60= 43 | github.com/taoky/goaccessfmt v0.0.0-20240824074420-af31a41470aa/go.mod h1:TOGR4KBT75UkXmBzXh2rvkmb1dYSKWtGbV5o8MTJ4dM= 44 | golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 45 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 46 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 47 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 48 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 49 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 52 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 53 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 54 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/taoky/ayano/cmd" 4 | 5 | func main() { 6 | cmd.RootCmd().Execute() 7 | } 8 | -------------------------------------------------------------------------------- /pkg/analyze/analyze.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/netip" 10 | "os" 11 | "slices" 12 | "strconv" 13 | "sync" 14 | "time" 15 | "unique" 16 | 17 | "github.com/cakturk/go-netstat/netstat" 18 | "github.com/dustin/go-humanize" 19 | "github.com/olekukonko/tablewriter" 20 | "github.com/schollz/progressbar/v3" 21 | "github.com/spf13/pflag" 22 | "github.com/taoky/ayano/pkg/fileiter" 23 | "github.com/taoky/ayano/pkg/parser" 24 | ) 25 | 26 | const TimeFormat = time.DateTime 27 | 28 | var ( 29 | tableColorNone = tablewriter.Colors{tablewriter.Normal} 30 | tableColorBold = tablewriter.Colors{tablewriter.Bold} 31 | ) 32 | 33 | type UAKeyType = unique.Handle[string] 34 | 35 | type IPStats struct { 36 | Size uint64 37 | Requests uint64 38 | LastURL string 39 | 40 | // Used with daemon mode only 41 | LastSize uint64 42 | FirstSeen time.Time 43 | 44 | // Record time of last URL change 45 | LastURLUpdate time.Time 46 | 47 | // Record time of last URL access 48 | LastURLAccess time.Time 49 | 50 | // User-agent 51 | UAStore map[UAKeyType]struct{} 52 | } 53 | 54 | func (i IPStats) UpdateWith(item parser.LogItem) IPStats { 55 | i.Size += item.Size 56 | i.Requests += 1 57 | if item.URL != i.LastURL { 58 | if i.LastURLUpdate.Before(item.Time) { 59 | i.LastURL = item.URL 60 | i.LastURLUpdate = item.Time 61 | i.LastURLAccess = item.Time 62 | } 63 | } else { 64 | if i.LastURLAccess.Before(item.Time) { 65 | i.LastURLAccess = item.Time 66 | } 67 | } 68 | if i.UAStore == nil { 69 | i.UAStore = make(map[UAKeyType]struct{}) 70 | } 71 | i.UAStore[unique.Make(item.Useragent)] = struct{}{} 72 | return i 73 | } 74 | 75 | func (i IPStats) MergeWith(other IPStats) IPStats { 76 | i.Size += other.Size 77 | i.Requests += other.Requests 78 | if i.LastURL == other.LastURL { 79 | if other.LastURLAccess.After(i.LastURLAccess) { 80 | i.LastURLAccess = other.LastURLAccess 81 | } 82 | if other.LastURLUpdate.Before(i.LastURLUpdate) { 83 | i.LastURLUpdate = other.LastURLUpdate 84 | } 85 | } else if other.LastURLUpdate.After(i.LastURLUpdate) { 86 | i.LastURL = other.LastURL 87 | i.LastURLUpdate = other.LastURLUpdate 88 | i.LastURLAccess = other.LastURLAccess 89 | } 90 | for k := range other.UAStore { 91 | i.UAStore[k] = struct{}{} 92 | } 93 | return i 94 | } 95 | 96 | type StatKey struct { 97 | Server string 98 | Prefix netip.Prefix 99 | } 100 | 101 | type Analyzer struct { 102 | Config AnalyzerConfig 103 | 104 | // [server, ip prefix] -> IPStats 105 | stats map[StatKey]IPStats 106 | mu sync.Mutex 107 | 108 | logParser parser.Parser 109 | logger *log.Logger 110 | bar *progressbar.ProgressBar 111 | } 112 | 113 | type AnalyzerConfig struct { 114 | Absolute bool 115 | Group bool 116 | LogOutput string 117 | NoNetstat bool 118 | Parser string 119 | PrefixV4 int 120 | PrefixV6 int 121 | PrintDelta SizeFlag 122 | RefreshSec int 123 | Server string 124 | SortBy SortByFlag 125 | Threshold SizeFlag 126 | TopN int 127 | Truncate bool 128 | Truncate2 int 129 | Whole bool 130 | 131 | Analyze bool 132 | Daemon bool 133 | } 134 | 135 | func (c *AnalyzerConfig) InstallFlags(flags *pflag.FlagSet, cmdname string) { 136 | flags.BoolVarP(&c.Absolute, "absolute", "a", c.Absolute, "Show absolute time for each item") 137 | flags.StringVarP(&c.LogOutput, "outlog", "o", c.LogOutput, "Change log output file") 138 | flags.BoolVarP(&c.NoNetstat, "no-netstat", "", c.NoNetstat, "Do not detect active connections") 139 | flags.StringVarP(&c.Parser, "parser", "p", c.Parser, "Log parser (see \"ayano list parsers\")") 140 | flags.IntVar(&c.PrefixV4, "prefixv4", c.PrefixV4, "Group IPv4 by prefix") 141 | flags.IntVar(&c.PrefixV6, "prefixv6", c.PrefixV6, "Group IPv6 by prefix") 142 | flags.IntVarP(&c.RefreshSec, "refresh", "r", c.RefreshSec, "Refresh interval in seconds") 143 | flags.StringVarP(&c.Server, "server", "s", c.Server, "Server IP to filter (nginx-json only)") 144 | flags.VarP(&c.SortBy, "sort-by", "S", "Sort result by (size|requests)") 145 | flags.VarP(&c.Threshold, "threshold", "t", "Threshold size for request (only requests at least this large will be counted)") 146 | flags.IntVarP(&c.TopN, "top", "n", c.TopN, "Number of top items to show") 147 | flags.BoolVar(&c.Truncate, "truncate", c.Truncate, "Truncate long URLs from output") 148 | flags.IntVar(&c.Truncate2, "truncate-to", c.Truncate2, "Truncate URLs to given length, overrides --truncate") 149 | 150 | if cmdname == "analyze" { 151 | c.Whole = true 152 | flags.BoolVarP(&c.Group, "group", "g", c.Group, "Try to group CIDRs") 153 | flags.BoolVarP(new(bool), "whole", "w", false, "(This flag is implied in analyze mode)") 154 | } else { 155 | flags.BoolVarP(&c.Whole, "whole", "w", c.Whole, "Analyze whole log file and then tail it") 156 | } 157 | 158 | if cmdname == "daemon" { 159 | flags.Var(&c.PrintDelta, "print-delta", "Size interval for printing lines") 160 | } 161 | } 162 | 163 | func (c *AnalyzerConfig) UseLock() bool { 164 | return !c.Analyze && !c.Daemon 165 | } 166 | 167 | func DefaultConfig() AnalyzerConfig { 168 | return AnalyzerConfig{ 169 | Parser: "nginx-json", 170 | PrefixV4: 24, 171 | PrefixV6: 48, 172 | PrintDelta: SizeFlag(1e9), 173 | RefreshSec: 5, 174 | SortBy: SortBySize, 175 | Threshold: SizeFlag(10e6), 176 | TopN: 10, 177 | } 178 | } 179 | 180 | func NewAnalyzer(c AnalyzerConfig) (*Analyzer, error) { 181 | logParser, err := parser.GetParser(c.Parser) 182 | if err != nil { 183 | return nil, fmt.Errorf("invalid parser: %w", err) 184 | } 185 | 186 | if c.Analyze { 187 | c.Whole = true 188 | } 189 | 190 | logger := log.New(os.Stdout, "", log.LstdFlags) 191 | if c.LogOutput != "" { 192 | f, err := os.OpenFile(c.LogOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 193 | if err != nil { 194 | return nil, fmt.Errorf("open log file error: %w", err) 195 | } 196 | c.LogOutput = f.Name() 197 | logger.SetOutput(f) 198 | } 199 | if c.Analyze { 200 | logger.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime)) 201 | } 202 | 203 | return &Analyzer{ 204 | Config: c, 205 | stats: make(map[StatKey]IPStats), 206 | logParser: logParser, 207 | logger: logger, 208 | bar: progressbar.Default(-1, "analyzing"), 209 | }, nil 210 | } 211 | 212 | func (a *Analyzer) RunLoop(iter fileiter.Iterator) error { 213 | a.bar.Reset() 214 | defer a.bar.Finish() 215 | for { 216 | line, err := iter.Next() 217 | if err != nil { 218 | return err 219 | } 220 | if line == nil { 221 | break 222 | } 223 | if err := a.handleLine(line); err != nil { 224 | // TODO: replace with logger 225 | a.logger.Printf("analyze error: %v", err) 226 | } 227 | } 228 | return nil 229 | } 230 | 231 | func (a *Analyzer) RunLoopWithMultipleIterators(iters []fileiter.Iterator) error { 232 | a.bar.Reset() 233 | defer a.bar.Finish() 234 | 235 | var wg sync.WaitGroup 236 | linesChan := make(chan []byte, 2*len(iters)) 237 | 238 | var errorMu sync.Mutex 239 | var collectedErrors []error 240 | 241 | for _, iter := range iters { 242 | wg.Add(1) 243 | go func() { 244 | defer wg.Done() 245 | for { 246 | line, err := iter.Next() 247 | if err != nil { 248 | errorMu.Lock() 249 | collectedErrors = append(collectedErrors, err) 250 | errorMu.Unlock() 251 | return 252 | } 253 | if line == nil { 254 | return 255 | } 256 | linesChan <- line 257 | } 258 | }() 259 | } 260 | 261 | go func() { 262 | wg.Wait() 263 | close(linesChan) 264 | }() 265 | 266 | for result := range linesChan { 267 | if err := a.handleLine(result); err != nil { 268 | a.logger.Printf("analyze error: %v", err) 269 | } 270 | } 271 | 272 | if len(collectedErrors) > 0 { 273 | return errors.Join(collectedErrors...) 274 | } 275 | 276 | return nil 277 | } 278 | 279 | func (a *Analyzer) AnalyzeFile(filename string) error { 280 | f, err := OpenFile(filename) 281 | if err != nil { 282 | return err 283 | } 284 | if closer, ok := f.(io.Closer); ok { 285 | defer closer.Close() 286 | } 287 | return a.RunLoop(fileiter.NewWithScanner(f)) 288 | } 289 | 290 | func (a *Analyzer) TailFile(filename string) error { 291 | iter, err := a.OpenTailIterator(filename) 292 | if err != nil { 293 | return err 294 | } 295 | return a.RunLoop(iter) 296 | } 297 | 298 | func (a *Analyzer) handleLine(line []byte) error { 299 | a.bar.Add64(1) 300 | logItem, err := a.logParser.Parse(line) 301 | if err != nil { 302 | return fmt.Errorf("parse error: %w\ngot line: %q", err, line) 303 | } 304 | return a.handleLogItem(logItem) 305 | } 306 | 307 | func (a *Analyzer) handleLogItem(logItem parser.LogItem) error { 308 | if logItem.Discard { 309 | return nil 310 | } 311 | 312 | // Filter by server 313 | if a.Config.Server != "" && logItem.Server != a.Config.Server { 314 | return nil 315 | } 316 | 317 | // Filter by sent size 318 | size := logItem.Size 319 | if size < uint64(a.Config.Threshold) { 320 | return nil 321 | } 322 | 323 | clientip, err := netip.ParseAddr(logItem.Client) 324 | if err != nil { 325 | return fmt.Errorf("parse ip error: %w", err) 326 | } 327 | clientPrefix := a.IPPrefix(clientip) 328 | 329 | if a.Config.UseLock() { 330 | a.mu.Lock() 331 | defer a.mu.Unlock() 332 | } 333 | 334 | updateStats := func(key StatKey) { 335 | a.stats[key] = a.stats[key].UpdateWith(logItem) 336 | } 337 | 338 | if a.Config.Analyze || a.Config.Daemon { 339 | // Avoid using double memory when not in interactive mode 340 | updateStats(StatKey{a.Config.Server, clientPrefix}) 341 | } else { 342 | updateStats(StatKey{logItem.Server, clientPrefix}) 343 | 344 | // Write it twice (to total here) when we have multiple servers 345 | if logItem.Server != "" { 346 | updateStats(StatKey{"", clientPrefix}) 347 | } 348 | } 349 | 350 | if a.Config.Daemon { 351 | ipStats := a.stats[StatKey{a.Config.Server, clientPrefix}] 352 | delta := ipStats.Size - ipStats.LastSize 353 | if ipStats.LastSize == 0 { 354 | ipStats.FirstSeen = logItem.Time 355 | } 356 | printTimes := delta / uint64(a.Config.PrintDelta) 357 | for range printTimes { 358 | a.logger.Printf("%s %s %s %s", 359 | clientPrefix.String(), 360 | humanize.IBytes(ipStats.Size), 361 | ipStats.FirstSeen.Format(TimeFormat), 362 | logItem.URL) 363 | } 364 | ipStats.LastSize += printTimes * uint64(a.Config.PrintDelta) 365 | // Just update [StatKey{a.Config.Server, clientPrefix}] here, as the config would not be updated runtime now 366 | a.stats[StatKey{a.Config.Server, clientPrefix}] = ipStats 367 | } 368 | 369 | return nil 370 | } 371 | 372 | func filterSockTabEntry(s *netstat.SockTabEntry) bool { 373 | switch s.LocalAddr.Port { 374 | case 80, 443, 873: 375 | default: 376 | return false 377 | } 378 | return s.State == netstat.Established 379 | } 380 | 381 | func (a *Analyzer) GetActiveConns(activeConn map[netip.Prefix]int) { 382 | // Get active connections 383 | tabs, err := netstat.TCPSocks(filterSockTabEntry) 384 | if err != nil { 385 | a.logger.Printf("netstat error: %v", err) 386 | } else { 387 | for _, tab := range tabs { 388 | ip, ok := netip.AddrFromSlice(tab.RemoteAddr.IP) 389 | if !ok { 390 | continue 391 | } 392 | activeConn[a.IPPrefix(ip)] += 1 393 | } 394 | } 395 | tabs, err = netstat.TCP6Socks(filterSockTabEntry) 396 | if err != nil { 397 | a.logger.Printf("netstat error: %v", err) 398 | } else { 399 | for _, tab := range tabs { 400 | ip, ok := netip.AddrFromSlice(tab.RemoteAddr.IP) 401 | if !ok { 402 | continue 403 | } 404 | activeConn[a.IPPrefix(ip)] += 1 405 | } 406 | } 407 | } 408 | 409 | // SortedKeys returns stat keys sorted by value 410 | func (a *Analyzer) SortedKeys(sortBy SortByFlag, serverFilter string) []StatKey { 411 | keys := make([]StatKey, 0) 412 | for s := range a.stats { 413 | if s.Server != serverFilter { 414 | continue 415 | } 416 | keys = append(keys, s) 417 | } 418 | sortFunc := GetSortFunc(sortBy, a.stats) 419 | if sortFunc != nil { 420 | slices.SortFunc(keys, sortFunc) 421 | } 422 | return keys 423 | } 424 | 425 | func (a *Analyzer) PrintTopValues(displayRecord map[netip.Prefix]time.Time, sortBy SortByFlag, serverFilter string) { 426 | activeConn := make(map[netip.Prefix]int) 427 | if !a.Config.NoNetstat { 428 | a.GetActiveConns(activeConn) 429 | } 430 | 431 | if a.Config.UseLock() { 432 | a.mu.Lock() 433 | defer a.mu.Unlock() 434 | } 435 | 436 | keys := a.SortedKeys(sortBy, serverFilter) 437 | 438 | // print top N 439 | top := a.Config.TopN 440 | if len(keys) < a.Config.TopN { 441 | top = len(keys) 442 | } else if a.Config.TopN == 0 { 443 | // no limit 444 | top = len(keys) 445 | } 446 | 447 | if a.Config.Group { 448 | groupedKeys := make(map[StatKey]struct{}) 449 | for _, key := range keys { 450 | if key.Prefix.Bits() == 0 { 451 | continue 452 | } 453 | adjacentKey := StatKey{key.Server, AdjacentPrefix(key.Prefix)} 454 | _, ok := groupedKeys[adjacentKey] 455 | if !ok { 456 | // would insert into groupedKeys 457 | if len(groupedKeys) >= top { 458 | break 459 | } 460 | groupedKeys[key] = struct{}{} 461 | } 462 | for ok { 463 | newStat := a.stats[key].MergeWith(a.stats[adjacentKey]) 464 | mergedPrefix := netip.PrefixFrom(key.Prefix.Addr(), key.Prefix.Bits()-1).Masked() 465 | newKey := StatKey{key.Server, mergedPrefix} 466 | 467 | a.stats[newKey] = newStat 468 | delete(a.stats, key) 469 | delete(a.stats, adjacentKey) 470 | 471 | groupedKeys[newKey] = struct{}{} 472 | delete(groupedKeys, key) 473 | delete(groupedKeys, adjacentKey) 474 | 475 | if newKey.Prefix.Bits() == 0 { 476 | break 477 | } 478 | key = newKey 479 | adjacentKey = StatKey{key.Server, AdjacentPrefix(key.Prefix)} 480 | _, ok = groupedKeys[adjacentKey] 481 | } 482 | } 483 | keys = a.SortedKeys(sortBy, serverFilter) 484 | if len(keys) < top { 485 | top = len(keys) 486 | } 487 | } 488 | 489 | tableBuf := new(bytes.Buffer) 490 | table := tablewriter.NewWriter(tableBuf) 491 | table.SetCenterSeparator(" ") 492 | table.SetColumnSeparator("") 493 | table.SetRowSeparator("") 494 | table.SetTablePadding(" ") 495 | table.SetAutoFormatHeaders(false) 496 | table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) 497 | table.SetAlignment(tablewriter.ALIGN_LEFT) 498 | table.SetHeaderLine(false) 499 | table.SetBorder(false) 500 | table.SetNoWhiteSpace(true) 501 | tAlignment := []int{ 502 | tablewriter.ALIGN_RIGHT, 503 | tablewriter.ALIGN_RIGHT, 504 | tablewriter.ALIGN_RIGHT, 505 | tablewriter.ALIGN_RIGHT, 506 | tablewriter.ALIGN_RIGHT, 507 | tablewriter.ALIGN_DEFAULT, 508 | tablewriter.ALIGN_RIGHT, 509 | tablewriter.ALIGN_RIGHT, 510 | tablewriter.ALIGN_RIGHT, 511 | } 512 | tHeaders := []string{"CIDR", "Conn", "Bytes", "Reqs", "Avg", "URL", "URL Since", "URL Last", "UA"} 513 | if a.Config.NoNetstat { 514 | tAlignment = append(tAlignment[:1], tAlignment[2:]...) 515 | tHeaders = append(tHeaders[:1], tHeaders[2:]...) 516 | } 517 | table.SetColumnAlignment(tAlignment) 518 | table.SetHeader(tHeaders) 519 | 520 | for i := range top { 521 | key := keys[i] 522 | ipStats := a.stats[key] 523 | total := ipStats.Size 524 | reqTotal := ipStats.Requests 525 | last := ipStats.LastURL 526 | agents := len(ipStats.UAStore) 527 | if a.Config.Truncate2 > 0 { 528 | last = TruncateURLPathLen(last, a.Config.Truncate2) 529 | } else if a.Config.Truncate { 530 | last = TruncateURLPath(last) 531 | } 532 | 533 | var lastUpdateTime, lastAccessTime string 534 | if a.Config.Absolute { 535 | lastUpdateTime = ipStats.LastURLUpdate.Format(TimeFormat) 536 | lastAccessTime = ipStats.LastURLAccess.Format(TimeFormat) 537 | } else { 538 | lastUpdateTime = humanize.Time(ipStats.LastURLUpdate) 539 | lastAccessTime = humanize.Time(ipStats.LastURLAccess) 540 | } 541 | 542 | average := total / uint64(reqTotal) 543 | boldLine := false 544 | if displayRecord != nil && displayRecord[key.Prefix] != ipStats.LastURLAccess { 545 | // display this line in bold 546 | boldLine = true 547 | displayRecord[key.Prefix] = ipStats.LastURLAccess 548 | } 549 | 550 | row := []string{ 551 | key.Prefix.String(), "", humanize.IBytes(total), strconv.FormatUint(reqTotal, 10), 552 | humanize.IBytes(average), last, lastUpdateTime, lastAccessTime, strconv.Itoa(agents), 553 | } 554 | rowColors := slices.Repeat([]tablewriter.Colors{tableColorNone}, len(row)) 555 | if boldLine { 556 | rowColors = slices.Repeat([]tablewriter.Colors{tableColorBold}, len(row)) 557 | } else { 558 | // Bold color for 2nd column (connections) 559 | rowColors[1] = tableColorBold 560 | } 561 | 562 | if !a.Config.NoNetstat { 563 | if _, ok := activeConn[key.Prefix]; ok { 564 | row[1] = strconv.Itoa(activeConn[key.Prefix]) 565 | } 566 | } else { 567 | // Remove connections column 568 | row = append(row[:1], row[2:]...) 569 | rowColors = append(rowColors[:1], rowColors[2:]...) 570 | } 571 | 572 | table.Rich(row, rowColors) 573 | } 574 | table.Render() 575 | a.logger.Writer().Write(tableBuf.Bytes()) 576 | } 577 | 578 | func (a *Analyzer) GetCurrentServers() []string { 579 | if a.Config.UseLock() { 580 | a.mu.Lock() 581 | defer a.mu.Unlock() 582 | } 583 | servers := make(map[string]struct{}) 584 | for sp := range a.stats { 585 | if sp.Server != "" { 586 | servers[sp.Server] = struct{}{} 587 | } 588 | } 589 | keys := make([]string, 0, len(servers)) 590 | for key := range servers { 591 | keys = append(keys, key) 592 | } 593 | return keys 594 | } 595 | 596 | func (a *Analyzer) PrintTotal() { 597 | type kv struct { 598 | server string 599 | value uint64 600 | } 601 | if a.Config.UseLock() { 602 | a.mu.Lock() 603 | defer a.mu.Unlock() 604 | } 605 | 606 | totals := make(map[string]uint64) 607 | for sp, value := range a.stats { 608 | totals[sp.Server] += value.Size 609 | } 610 | 611 | var totalSlice []kv 612 | for k, v := range totals { 613 | totalSlice = append(totalSlice, kv{k, v}) 614 | } 615 | slices.SortFunc(totalSlice, func(i, j kv) int { 616 | return int(j.value - i.value) 617 | }) 618 | 619 | for _, kv := range totalSlice { 620 | a.logger.Printf("%s: %s\n", kv.server, humanize.IBytes(kv.value)) 621 | } 622 | } 623 | -------------------------------------------------------------------------------- /pkg/analyze/analyze_test.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func benchmarkAnalyzeLoop(b *testing.B, parserStr string) { 9 | // get logPath from env 10 | logPath := os.Getenv("LOG_PATH") 11 | if logPath == "" { 12 | b.Fatal("LOG_PATH is not set") 13 | } 14 | c := AnalyzerConfig{ 15 | NoNetstat: true, 16 | Parser: parserStr, 17 | Server: "", 18 | RefreshSec: 5, 19 | Threshold: 100 * (1 << 20), 20 | TopN: 20, 21 | Whole: true, 22 | 23 | Analyze: true, 24 | } 25 | 26 | a, err := NewAnalyzer(c) 27 | if err != nil { 28 | b.Fatal(err) 29 | } 30 | 31 | err = a.AnalyzeFile(logPath) 32 | if err != nil { 33 | b.Fatal(err) 34 | } 35 | a.PrintTopValues(nil, SortBySize, "") 36 | } 37 | 38 | func BenchmarkAnalyzeLoopNgxJSON(b *testing.B) { 39 | benchmarkAnalyzeLoop(b, "nginx-json") 40 | } 41 | 42 | func BenchmarkAnalyzeLoopCombined(b *testing.B) { 43 | benchmarkAnalyzeLoop(b, "nginx-combined") 44 | } 45 | -------------------------------------------------------------------------------- /pkg/analyze/file.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | 10 | "github.com/nxadm/tail" 11 | "github.com/taoky/ayano/pkg/fileiter" 12 | ) 13 | 14 | const oneMiB = 1024 * 1024 15 | 16 | type filteredReader struct { 17 | cmd *exec.Cmd 18 | r io.ReadCloser 19 | } 20 | 21 | func (fr *filteredReader) Read(p []byte) (n int, err error) { 22 | return fr.r.Read(p) 23 | } 24 | 25 | func (fr *filteredReader) Close() error { 26 | return errors.Join(fr.cmd.Wait(), fr.r.Close()) 27 | } 28 | 29 | func filterByCommand(r io.Reader, args []string) (io.ReadCloser, error) { 30 | cmd := exec.Command(args[0], args[1:]...) 31 | cmd.Stdin = r 32 | stdout, err := cmd.StdoutPipe() 33 | if err != nil { 34 | return nil, err 35 | } 36 | if err := cmd.Start(); err != nil { 37 | return nil, err 38 | } 39 | return &filteredReader{cmd: cmd, r: stdout}, nil 40 | } 41 | 42 | type filterFunc func(r io.ReadCloser) (io.ReadCloser, error) 43 | 44 | var fileTypes = map[string]filterFunc{ 45 | ".gz": func(r io.ReadCloser) (io.ReadCloser, error) { 46 | return filterByCommand(r, []string{"gzip", "-cd"}) 47 | }, 48 | ".xz": func(r io.ReadCloser) (io.ReadCloser, error) { 49 | return filterByCommand(r, []string{"xz", "-cd", "-T", "0"}) 50 | }, 51 | ".zst": func(r io.ReadCloser) (io.ReadCloser, error) { 52 | return filterByCommand(r, []string{"zstd", "-cd", "-T0"}) 53 | }, 54 | } 55 | 56 | func OpenFile(filename string) (io.ReadCloser, error) { 57 | f, err := os.Open(filename) 58 | if err != nil { 59 | return nil, err 60 | } 61 | if filter, ok := fileTypes[filepath.Ext(filename)]; ok { 62 | return filter(f) 63 | } 64 | return f, nil 65 | } 66 | 67 | func (a *Analyzer) OpenTailIterator(filename string) (fileiter.Iterator, error) { 68 | var seekInfo *tail.SeekInfo 69 | if a.Config.Whole { 70 | seekInfo = &tail.SeekInfo{ 71 | Offset: 0, 72 | Whence: io.SeekStart, 73 | } 74 | } else { 75 | // Workaround: In this case seek does not support to keep seek at start when file < 1MiB 76 | // So here we check file size first, though it could have race condition, 77 | // at least it's better than crashing later 78 | fileInfo, err := os.Stat(filename) 79 | if err != nil { 80 | return nil, err 81 | } 82 | fileSize := fileInfo.Size() 83 | if fileSize < oneMiB { 84 | // The log file is too small so let's just start from the beginning 85 | seekInfo = &tail.SeekInfo{ 86 | Offset: 0, 87 | Whence: io.SeekStart, 88 | } 89 | } else { 90 | seekInfo = &tail.SeekInfo{ 91 | Offset: -oneMiB, 92 | Whence: io.SeekEnd, 93 | } 94 | } 95 | } 96 | t, err := tail.TailFile(filename, tail.Config{ 97 | Follow: true, 98 | ReOpen: true, 99 | Location: seekInfo, 100 | CompleteLines: true, 101 | MustExist: true, 102 | }) 103 | if err != nil { 104 | return nil, err 105 | } 106 | if !a.Config.Whole { 107 | // Eat a line from t.Lines, as first line may be incomplete 108 | <-t.Lines 109 | } 110 | return fileiter.NewWithTail(t), nil 111 | } 112 | -------------------------------------------------------------------------------- /pkg/analyze/log.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | func (a *Analyzer) OpenLogFile() error { 9 | if a.Config.LogOutput == "" { 10 | return nil 11 | } 12 | 13 | logFile, err := os.OpenFile(a.Config.LogOutput, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 14 | if err != nil { 15 | return err 16 | } 17 | log.SetOutput(logFile) 18 | return nil 19 | } 20 | -------------------------------------------------------------------------------- /pkg/analyze/sortfunc.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "slices" 5 | ) 6 | 7 | type SortFunc func(l, r StatKey) int 8 | 9 | var sortFuncs = map[SortByFlag]func(a map[StatKey]IPStats) SortFunc{ 10 | SortBySize: func(i map[StatKey]IPStats) SortFunc { 11 | return func(l, r StatKey) int { 12 | return int(i[r].Size - i[l].Size) 13 | } 14 | }, 15 | SortByRequests: func(i map[StatKey]IPStats) SortFunc { 16 | return func(l, r StatKey) int { 17 | return int(i[r].Requests - i[l].Requests) 18 | } 19 | }, 20 | } 21 | 22 | func GetSortFunc(name SortByFlag, i map[StatKey]IPStats) SortFunc { 23 | fn, ok := sortFuncs[name] 24 | if !ok { 25 | return nil 26 | } 27 | return fn(i) 28 | } 29 | 30 | func ListSortFuncs() []SortByFlag { 31 | ret := make([]SortByFlag, 0, len(sortFuncs)) 32 | for key := range sortFuncs { 33 | ret = append(ret, key) 34 | } 35 | slices.Sort(ret) 36 | return ret 37 | } 38 | -------------------------------------------------------------------------------- /pkg/analyze/util.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/netip" 7 | "path/filepath" 8 | "slices" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/dustin/go-humanize" 13 | ) 14 | 15 | type SizeFlag uint64 16 | 17 | func (s SizeFlag) String() string { 18 | return humanize.Bytes(uint64(s)) 19 | } 20 | 21 | func (s *SizeFlag) Set(value string) error { 22 | // First try parsing as a plain number 23 | size, err := strconv.ParseUint(value, 10, 64) 24 | if err == nil { 25 | *s = SizeFlag(size) 26 | return nil 27 | } 28 | 29 | size, err = humanize.ParseBytes(value) 30 | if err != nil { 31 | return err 32 | } 33 | *s = SizeFlag(size) 34 | return nil 35 | } 36 | 37 | func (s SizeFlag) Type() string { 38 | return "size" 39 | } 40 | 41 | type SortByFlag string 42 | 43 | const ( 44 | SortBySize SortByFlag = "size" 45 | SortByRequests SortByFlag = "requests" 46 | ) 47 | 48 | func (s SortByFlag) String() string { 49 | return string(s) 50 | } 51 | 52 | func (s *SortByFlag) Set(value string) error { 53 | switch value { 54 | case "size": 55 | *s = SortBySize 56 | case "requests", "reqs": 57 | *s = SortByRequests 58 | default: 59 | return errors.New(`must be one of "size" or "requests"`) 60 | } 61 | return nil 62 | } 63 | 64 | func (s SortByFlag) Type() string { 65 | return "string" 66 | } 67 | 68 | func (a *Analyzer) IPPrefix(ip netip.Addr) netip.Prefix { 69 | var clientPrefix netip.Prefix 70 | if ip.Is4() { 71 | clientPrefix = netip.PrefixFrom(ip, a.Config.PrefixV4) 72 | } else { 73 | clientPrefix = netip.PrefixFrom(ip, a.Config.PrefixV6) 74 | } 75 | return clientPrefix.Masked() 76 | } 77 | 78 | func AdjacentPrefix(p netip.Prefix) netip.Prefix { 79 | bits := p.Bits() 80 | if bits == 0 { 81 | return p 82 | } 83 | a := p.Addr() 84 | if a.Is4() { 85 | addr := a.As4() 86 | addr[(bits-1)/8] ^= uint8(1 << (7 - (bits-1)%8)) 87 | return netip.PrefixFrom(netip.AddrFrom4(addr), bits) 88 | } else { 89 | addr := a.As16() 90 | addr[(bits-1)/8] ^= uint8(1 << (7 - (bits-1)%8)) 91 | return netip.PrefixFrom(netip.AddrFrom16(addr), bits) 92 | } 93 | } 94 | 95 | func TruncateURLPath(input string) string { 96 | parts := strings.SplitN(input, "?", 2) 97 | path := filepath.Clean(parts[0]) 98 | if strings.HasSuffix(parts[0], "/") { 99 | // filepath.Clean removes trailing slash 100 | // Add it back to preserve directory notation 101 | path += "/" 102 | } 103 | args := "" 104 | if len(parts) == 2 { 105 | args = "?..." 106 | } 107 | count := strings.Count(path, "/") 108 | if count <= 2 { 109 | return path + args 110 | } 111 | parts = strings.Split(path, "/") 112 | if parts[len(parts)-1] == "" { 113 | if count == 3 { 114 | return path + args 115 | } 116 | count-- 117 | parts[count] += "/" 118 | } 119 | return fmt.Sprintf("/%s/.../%s%s", parts[1], parts[count], args) 120 | } 121 | 122 | func TruncateURLPathLen(input string, target int) string { 123 | stub := TruncateURLPath(input) 124 | if len(stub) <= target { 125 | return stub 126 | } 127 | 128 | // if removing query string suffices, do it 129 | parts := strings.SplitN(stub, "?", 2) 130 | stub = parts[0] 131 | if len(stub) < target { 132 | return stub + "?" 133 | } else if len(stub) == target { 134 | return stub 135 | } 136 | 137 | // stub contains at most 3 slashes 138 | parts = strings.SplitN(stub, "/", 4) 139 | filename := parts[len(parts)-1] 140 | isDirectory := false 141 | if filename == "" { 142 | isDirectory = true 143 | filename = parts[len(parts)-2] 144 | } 145 | filenameTarget := len(filename) - (len(stub) - target) 146 | if filenameTarget > 0 { 147 | filename = TruncateFilenameLen(filename, filenameTarget) 148 | if isDirectory { 149 | parts[len(parts)-2] = filename 150 | } else { 151 | parts[len(parts)-1] = filename 152 | } 153 | return strings.Join(parts, "/") 154 | } 155 | 156 | // give up and truncate directly 157 | return stub[:target] 158 | } 159 | 160 | var compressionSuffixes = []string{".gz", ".bz2", ".xz", ".zst"} 161 | 162 | func TruncateFilenameLen(input string, target int) string { 163 | if len(input) <= target { 164 | return input 165 | } 166 | ext := filepath.Ext(input) 167 | if slices.Contains(compressionSuffixes, ext) { 168 | ext = filepath.Ext(strings.TrimSuffix(input, ext)) + ext 169 | } 170 | basename := strings.TrimSuffix(input, ext) 171 | if len(basename) > len(input)-target { 172 | toTruncate := len(basename) - (len(input) - target) 173 | if ext != "" && toTruncate > 2 { 174 | // ext will begin with a dot already 175 | return basename[:toTruncate-2] + ".." + ext 176 | } else if toTruncate > 3 { 177 | return basename[:toTruncate-3] + "..." 178 | } else { 179 | // basename too short, keep just an asterisk 180 | return "*" + ext 181 | } 182 | } 183 | // truncating basename alone would not suffice, keep characters from end 184 | return input[len(input)-target:] 185 | } 186 | -------------------------------------------------------------------------------- /pkg/analyze/util_test.go: -------------------------------------------------------------------------------- 1 | package analyze 2 | 3 | import ( 4 | "net/netip" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestAdjacentPrefix(t *testing.T) { 11 | testCases := [][2]string{ 12 | {"1.2.3.0/24", "1.2.2.0/24"}, 13 | {"1.2.2.0/23", "1.2.0.0/23"}, 14 | {"2001:db8:114:514::/64", "2001:db8:114:515::/64"}, 15 | {"2001:db8:114:514::/63", "2001:db8:114:516::/63"}, 16 | } 17 | for _, c := range testCases { 18 | assert.Equal(t, netip.MustParsePrefix(c[1]), AdjacentPrefix(netip.MustParsePrefix(c[0]))) 19 | assert.Equal(t, netip.MustParsePrefix(c[0]), AdjacentPrefix(netip.MustParsePrefix(c[1]))) 20 | } 21 | } 22 | 23 | func TestTruncateURLPath(t *testing.T) { 24 | testCases := [][2]string{ 25 | {"/example/a/b/c/d/e/file.ext", "/example/.../file.ext"}, 26 | {"///example//merge/slashes///file.ext", "/example/.../file.ext"}, 27 | {"/example/a/b/c/d/e/dir/", "/example/.../dir/"}, 28 | {"/short/", "/short/"}, 29 | {"/short/file.ext", "/short/file.ext"}, 30 | {"/short/dir/", "/short/dir/"}, 31 | {"/with/args/?a=1&b=2", "/with/args/?..."}, 32 | } 33 | for _, c := range testCases { 34 | assert.Equal(t, c[1], TruncateURLPath(c[0])) 35 | } 36 | } 37 | 38 | func TestTruncateURLPathLen(t *testing.T) { 39 | type testCase struct { 40 | input string 41 | target int 42 | expected string 43 | } 44 | testCases := []testCase{ 45 | {"/example/a/b/c/d/e/file.ext", 30, "/example/.../file.ext"}, 46 | {"/example/a/b/c/d/e/dir/", 30, "/example/.../dir/"}, 47 | {"///example//merge/slashes///dir/", 30, "/example/.../dir/"}, 48 | {"/with/args/?a=1&b=2", 30, "/with/args/?..."}, 49 | {"/with/args/?a=1&b=2", 13, "/with/args/?"}, 50 | {"/with/args/?a=1&b=2", 12, "/with/args/?"}, 51 | {"/with/args/?a=1&b=2", 11, "/with/args/"}, 52 | {"/with/args/?a=1&b=2", 8, "/with/*/"}, 53 | {"/with/args/?a=1&b=2", 4, "/wit"}, 54 | {"///with//args/?a=1&b=2", 4, "/wit"}, 55 | 56 | {"/example/a/b/c/d/e/file.with.long.name.ext", 30, "/example/.../file.with.l...ext"}, 57 | {"/example/file.with.very.long.name.ext", 30, "/example/file.with.very....ext"}, 58 | } 59 | for _, c := range testCases { 60 | assert.Equal(t, c.expected, TruncateURLPathLen(c.input, c.target)) 61 | } 62 | } 63 | 64 | func TestTruncateFilenameLen(t *testing.T) { 65 | type testCase struct { 66 | input string 67 | target int 68 | expected string 69 | } 70 | testCases := []testCase{ 71 | {"file.ext", 8, "file.ext"}, 72 | {"file.ext", 10, "file.ext"}, 73 | {"file.ext", 80, "file.ext"}, 74 | {"file.long.long.ext", 8, "fi...ext"}, 75 | {"file.long.long.ext", 10, "file...ext"}, 76 | {"file.long.long.ext.gz", 10, "f...ext.gz"}, 77 | // short basename = asterisk 78 | {"file.long.long.ext.bz2", 10, "*.ext.bz2"}, 79 | // very short basename = keep ext only 80 | {"file.long.long.ext.bz2", 6, "xt.bz2"}, 81 | } 82 | for _, c := range testCases { 83 | assert.Equal(t, c.expected, TruncateFilenameLen(c.input, c.target)) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/fileiter/iterator.go: -------------------------------------------------------------------------------- 1 | package fileiter 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | 7 | "github.com/nxadm/tail" 8 | ) 9 | 10 | type Iterator interface { 11 | Next() ([]byte, error) 12 | } 13 | 14 | type scannerIterator struct { 15 | scanner *bufio.Scanner 16 | } 17 | 18 | func NewWithScanner(r io.Reader) Iterator { 19 | // Prepare a large buffer 20 | const bufSz = 1024 * 1024 21 | scanner := bufio.NewScanner(r) 22 | scanner.Buffer(make([]byte, bufSz), bufSz) 23 | return &scannerIterator{scanner: scanner} 24 | } 25 | 26 | func (s *scannerIterator) Next() ([]byte, error) { 27 | if s.scanner.Scan() { 28 | return s.scanner.Bytes(), nil 29 | } else { 30 | return nil, s.scanner.Err() 31 | } 32 | } 33 | 34 | type tailIterator struct { 35 | tail *tail.Tail 36 | } 37 | 38 | func (t tailIterator) Next() ([]byte, error) { 39 | return []byte((<-t.tail.Lines).Text), nil 40 | } 41 | 42 | func NewWithTail(tail *tail.Tail) Iterator { 43 | return &tailIterator{tail: tail} 44 | } 45 | -------------------------------------------------------------------------------- /pkg/info/info.go: -------------------------------------------------------------------------------- 1 | package info 2 | 3 | var ( 4 | Version string 5 | BuildDate string 6 | GitCommit string 7 | ) 8 | -------------------------------------------------------------------------------- /pkg/parser/caddy.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "math" 5 | "strings" 6 | "time" 7 | 8 | "github.com/goccy/go-json" 9 | ) 10 | 11 | func init() { 12 | newFunc := func() Parser { 13 | return ParserFunc(ParseCaddyJSON) 14 | } 15 | 16 | RegisterParser(ParserMeta{ 17 | Name: "caddy-json", 18 | Description: "Caddy's default JSON format", 19 | F: newFunc, 20 | }) 21 | RegisterParser(ParserMeta{ 22 | Name: "caddy", 23 | Description: "An alias for `caddy-json`", 24 | Hidden: true, 25 | F: newFunc, 26 | }) 27 | } 28 | 29 | type CaddyJsonLogHeader struct { 30 | Useragent []string `json:"User-Agent"` 31 | } 32 | 33 | type CaddyJsonLogRequest struct { 34 | RemoteIP string `json:"remote_ip"` 35 | ClientIP string `json:"client_ip"` 36 | Uri string `json:"uri"` 37 | Headers CaddyJsonLogHeader `json:"headers"` 38 | } 39 | 40 | type CaddyJsonLog struct { 41 | Msg string `json:"msg"` 42 | Timestamp float64 `json:"ts"` // (unix_seconds_float) 43 | Request CaddyJsonLogRequest `json:"request"` 44 | Size uint64 `json:"size"` 45 | } 46 | 47 | func ParseCaddyJSON(line []byte) (LogItem, error) { 48 | var logItem CaddyJsonLog 49 | err := json.Unmarshal(line, &logItem) 50 | if err != nil { 51 | return LogItem{}, err 52 | } 53 | if logItem.Msg != "handled request" { 54 | return LogItem{Discard: true}, nil 55 | } 56 | sec, dec := math.Modf(logItem.Timestamp) 57 | t := time.Unix(int64(sec), int64(dec*1e9)) 58 | client := logItem.Request.ClientIP 59 | if client == "" { 60 | client = logItem.Request.RemoteIP 61 | } 62 | return LogItem{ 63 | Size: logItem.Size, 64 | Client: client, 65 | Time: t, 66 | URL: logItem.Request.Uri, 67 | Useragent: strings.Join(logItem.Request.Headers.Useragent, ", "), 68 | }, nil 69 | } 70 | -------------------------------------------------------------------------------- /pkg/parser/caddy_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestCaddyJSONParser(t *testing.T) { 11 | as := assert.New(t) 12 | p := ParserFunc(ParseCaddyJSON) 13 | line := `{"level":"info","ts":1646861401.5241024,"logger":"http.log.access","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"41342","client_ip":"127.0.0.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/7.82.0"],"Accept":["*/*"],"Accept-Encoding":["gzip, deflate, br"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"example.com"}},"bytes_read": 0,"user_id":"","duration":0.000929675,"size":10900,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Encoding":["gzip"],"Content-Type":["text/html; charset=utf-8"],"Vary":["Accept-Encoding"]}}` 14 | log, err := p.Parse([]byte(line)) 15 | as.NoError(err) 16 | as.Equal("/", log.URL) 17 | as.EqualValues(10900, log.Size) 18 | as.Equal("127.0.0.1", log.Client) 19 | expectedTime := time.Unix(1646861401, 524102400) 20 | as.WithinDuration(expectedTime, log.Time, time.Microsecond) 21 | as.Equal("curl/7.82.0", log.Useragent) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/parser/common.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | const CommonLogFormat = "02/Jan/2006:15:04:05 -0700" 10 | 11 | func clfDateParse(s []byte) time.Time { 12 | return clfDateParseString(string(s)) 13 | } 14 | 15 | func clfDateParseString(s string) time.Time { 16 | t, _ := time.Parse(CommonLogFormat, s) 17 | return t 18 | } 19 | 20 | // Nginx escapes `"`, `\` to `\xXX` 21 | // Apache esacpes `"`, `\` to `\"` `\\` 22 | func findEndingDoubleQuote(data []byte) int { 23 | inEscape := false 24 | for i := 0; i < len(data); i++ { 25 | if inEscape { 26 | inEscape = false 27 | } else { 28 | if data[i] == '\\' { 29 | inEscape = true 30 | } else if data[i] == '"' { 31 | return i 32 | } 33 | } 34 | } 35 | return -1 36 | } 37 | 38 | func splitFields(line []byte) ([][]byte, error) { 39 | res := make([][]byte, 0, 16) 40 | loop: 41 | for baseIdx := 0; baseIdx < len(line); { 42 | switch line[baseIdx] { 43 | case '"': 44 | quoteIdx := findEndingDoubleQuote(line[baseIdx+1:]) 45 | if quoteIdx == -1 { 46 | return res, fmt.Errorf("unexpected format: unbalanced quotes [ at %d", baseIdx) 47 | } 48 | res = append(res, line[baseIdx+1:baseIdx+quoteIdx+1]) 49 | baseIdx += quoteIdx + 2 50 | case '[': 51 | closingIdx := bytes.IndexByte(line[baseIdx+1:], ']') 52 | if closingIdx == -1 { 53 | return res, fmt.Errorf("unexpected format: unmatched [ at %d", baseIdx) 54 | } 55 | res = append(res, line[baseIdx+1:baseIdx+closingIdx+1]) 56 | baseIdx += closingIdx + 2 57 | default: 58 | spaceIdx := bytes.IndexByte(line[baseIdx:], ' ') 59 | if spaceIdx == -1 { 60 | res = append(res, line[baseIdx:]) 61 | break loop 62 | } 63 | res = append(res, line[baseIdx:baseIdx+spaceIdx]) 64 | baseIdx += spaceIdx 65 | } 66 | if baseIdx < len(line) && line[baseIdx] == ' ' { 67 | baseIdx++ 68 | } 69 | } 70 | return res, nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/parser/common_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestClfDateParse(t *testing.T) { 11 | expected := time.Date(2006, time.January, 2, 15, 4, 5, 0, time.FixedZone("", -7*60*60)) 12 | assert.Equal(t, expected, clfDateParse([]byte(CommonLogFormat))) 13 | assert.Equal(t, expected, clfDateParseString(CommonLogFormat)) 14 | } 15 | 16 | func TestFindEndingDoubleQuote(t *testing.T) { 17 | type testCase struct { 18 | input []byte 19 | expected int 20 | } 21 | testCases := []testCase{ 22 | {[]byte(`abc"`), 3}, 23 | {[]byte(`ab\"c"`), 5}, 24 | {[]byte(`ab\\c"`), 5}, 25 | {[]byte(`ab`), -1}, 26 | } 27 | for _, c := range testCases { 28 | assert.Equal(t, c.expected, findEndingDoubleQuote(c.input)) 29 | } 30 | } 31 | 32 | func TestSplitFields(t *testing.T) { 33 | type testCase struct { 34 | line []byte 35 | expected [][]byte 36 | } 37 | testCases := []testCase{ 38 | { 39 | []byte(`127.0.0.1 - - [2/Jan/2006:15:04:05 -0700] "GET /blog/2021/01/hello-world HTTP/1.1" 200 512`), 40 | [][]byte{ 41 | []byte(`127.0.0.1`), 42 | []byte(`-`), 43 | []byte(`-`), 44 | []byte(`2/Jan/2006:15:04:05 -0700`), 45 | []byte(`GET /blog/2021/01/hello-world HTTP/1.1`), 46 | []byte(`200`), 47 | []byte(`512`), 48 | }, 49 | }, 50 | } 51 | for _, c := range testCases { 52 | res, err := splitFields(c.line) 53 | if assert.NoError(t, err) { 54 | assert.Equal(t, c.expected, res) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/parser/goaccess.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/taoky/goaccessfmt/pkg/goaccessfmt" 8 | ) 9 | 10 | func init() { 11 | RegisterParser(ParserMeta{ 12 | Name: "goaccess", 13 | Description: "GoAccess output format", 14 | F: func() Parser { 15 | parser, err := GoAccessFormatParser{}.new() 16 | if err != nil { 17 | log.Fatalln("goaccess init failed (You might need to set GOACCESS_CONFIG env)\n", err) 18 | } 19 | return parser 20 | }, 21 | }) 22 | } 23 | 24 | type GoAccessFormatParser struct { 25 | conf goaccessfmt.Config 26 | } 27 | 28 | func (p GoAccessFormatParser) new() (GoAccessFormatParser, error) { 29 | confFile := os.Getenv("GOACCESS_CONFIG") 30 | file, err := os.Open(confFile) 31 | if err != nil { 32 | return p, err 33 | } 34 | conf, err := goaccessfmt.ParseConfigReader(file) 35 | if err != nil { 36 | return p, err 37 | } 38 | p.conf = conf 39 | return p, nil 40 | } 41 | 42 | func (p GoAccessFormatParser) Parse(line []byte) (LogItem, error) { 43 | glogitem, err := goaccessfmt.ParseLine(p.conf, string(line)) 44 | if err != nil { 45 | return LogItem{}, err 46 | } 47 | 48 | return LogItem{ 49 | Size: glogitem.RespSize, 50 | Client: glogitem.Host, 51 | Time: glogitem.Dt, 52 | URL: glogitem.Req, 53 | Server: glogitem.Server, 54 | Useragent: glogitem.Agent, 55 | }, nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/parser/goaccess_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestGoAccessParser(t *testing.T) { 12 | configEnvName := "GOACCESS_CONFIG" 13 | originalValue, hadValue := os.LookupEnv(configEnvName) 14 | 15 | err := os.Setenv(configEnvName, "../../assets/goaccess.conf") 16 | if err != nil { 17 | t.Fatalf("Error setting environment variable: %v", err) 18 | } 19 | 20 | t.Cleanup(func() { 21 | if hadValue { 22 | os.Setenv(configEnvName, originalValue) 23 | } else { 24 | os.Unsetenv(configEnvName) 25 | } 26 | }) 27 | 28 | as := assert.New(t) 29 | p, err := GetParser("goaccess") 30 | as.NoError(err) 31 | line := `{"level":"info","ts":1646861401.5241024,"logger":"http.log.access","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"41342","client_ip":"127.0.0.1","proto":"HTTP/2.0","method":"GET","host":"localhost","uri":"/","headers":{"User-Agent":["curl/7.82.0"],"Accept":["*/*"],"Accept-Encoding":["gzip, deflate, br"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h2","server_name":"example.com"}},"bytes_read": 0,"user_id":"","duration":0.000929675,"size":1090000000,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Encoding":["gzip"],"Content-Type":["text/html; charset=utf-8"],"Vary":["Accept-Encoding"]},"server":"1.3.5.7"}` 32 | log, err := p.Parse([]byte(line)) 33 | as.NoError(err) 34 | as.Equal("/", log.URL) 35 | as.EqualValues(1090000000, log.Size) 36 | as.Equal("127.0.0.1", log.Client) 37 | // Currently goaccess would ignore nsec part. 38 | expectedTime := time.Unix(1646861401, 0) 39 | as.WithinDuration(expectedTime, log.Time, 0) 40 | as.Equal("1.3.5.7", log.Server) 41 | as.Equal("curl/7.82.0", log.Useragent) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/parser/nginx-combined.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "regexp" 8 | "strconv" 9 | ) 10 | 11 | func init() { 12 | newFunc := func() Parser { return ParserFunc(ParseNginxCombined) } 13 | RegisterParser(ParserMeta{ 14 | Name: "nginx-combined", 15 | Description: "For nginx's default `combined` format", 16 | F: newFunc, 17 | }) 18 | RegisterParser(ParserMeta{ 19 | Name: "combined", 20 | Description: "An alias for `nginx-combined`", 21 | Hidden: true, 22 | F: newFunc, 23 | }) 24 | 25 | newFuncRegex := func() Parser { return ParserFunc(ParseNginxCombinedRegex) } 26 | RegisterParser(ParserMeta{ 27 | Name: "nginx-combined-regex", 28 | Description: "For nginx's default `combined` format, using regular expressions", 29 | F: newFuncRegex, 30 | }) 31 | RegisterParser(ParserMeta{ 32 | Name: "combined-regex", 33 | Description: "An alias for `nginx-combined-regex`", 34 | Hidden: true, 35 | F: newFuncRegex, 36 | }) 37 | } 38 | 39 | func ParseNginxCombined(line []byte) (logItem LogItem, err error) { 40 | fields, err := splitFields(line) 41 | if err != nil { 42 | return logItem, err 43 | } 44 | if len(fields) != 9 { 45 | return logItem, fmt.Errorf("invalid format: expected 9 fields, got %d", len(fields)) 46 | } 47 | 48 | if string(fields[1]) != "-" { 49 | return logItem, errors.New("unexpected format: no - (empty identity)") 50 | } 51 | 52 | logItem.Client = string(fields[0]) 53 | logItem.Time = clfDateParse(fields[3]) 54 | 55 | requestLine := fields[4] 56 | url := requestLine 57 | // strip HTTP method in url 58 | spaceIndex := bytes.IndexByte(url, ' ') 59 | if spaceIndex == -1 { 60 | // Some abnormal requests do not have a HTTP method 61 | // Sliently ignore this case 62 | } else { 63 | url = url[spaceIndex+1:] 64 | } 65 | spaceIndex = bytes.IndexByte(url, ' ') 66 | if spaceIndex == -1 { 67 | // Some abnormal requests do not have a HTTP version 68 | // Sliently ignore this case 69 | } else { 70 | url = url[:spaceIndex] 71 | } 72 | logItem.URL = string(url) 73 | 74 | sizeBytes := fields[6] 75 | logItem.Size, err = strconv.ParseUint(string(sizeBytes), 10, 64) 76 | if err != nil { 77 | return logItem, err 78 | } 79 | 80 | logItem.Useragent = string(fields[8]) 81 | return 82 | } 83 | 84 | var nginxCombinedRe = regexp.MustCompile( 85 | //1 2 3 4 5 6 7 8 9 10 86 | `^(\S+) - ([^[]+) \[([^]]+)\] "([^ ]+ )?([^ ]+)( HTTP/[\d.]+)?" (\d+) (\d+) "((?:[^\"]|\\.)*)" "((?:[^\"]|\\.)*)"\s*$`) 87 | 88 | func ParseNginxCombinedRegex(line []byte) (LogItem, error) { 89 | m := nginxCombinedRe.FindStringSubmatch(string(line)) 90 | if m == nil { 91 | return LogItem{}, errors.New("unexpected format") 92 | } 93 | size, err := strconv.ParseUint(m[8], 10, 64) 94 | if err != nil { 95 | return LogItem{}, fmt.Errorf("invalid size %s: %w", m[8], err) 96 | } 97 | return LogItem{ 98 | Client: m[1], 99 | Time: clfDateParseString(m[3]), 100 | URL: m[5], 101 | Size: size, 102 | Useragent: m[10], 103 | }, nil 104 | } 105 | -------------------------------------------------------------------------------- /pkg/parser/nginx-combined_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func testNginxCombinedParser(t *testing.T, p Parser) { 11 | as := assert.New(t) 12 | line := `123.45.67.8 - - [12/Mar/2023:00:15:32 +0800] "GET /path/to/a/file HTTP/1.1" 200 3009 "-" ""` 13 | log, err := p.Parse([]byte(line)) 14 | if as.NoError(err) { 15 | as.EqualValues(3009, log.Size) 16 | as.Equal("123.45.67.8", log.Client) 17 | as.Equal("/path/to/a/file", log.URL) 18 | expectedTime := time.Date(2023, 3, 12, 0, 15, 32, 0, time.FixedZone("CST", 8*60*60)) 19 | as.WithinDuration(expectedTime, log.Time, 0) 20 | as.Equal("", log.Useragent) 21 | } 22 | } 23 | 24 | func TestNginxCombinedParser(t *testing.T) { 25 | testNginxCombinedParser(t, ParserFunc(ParseNginxCombined)) 26 | testNginxCombinedParser(t, ParserFunc(ParseNginxCombinedRegex)) 27 | } 28 | 29 | func testNginxCombinedParserWithUnusualInputs(t *testing.T, p Parser) { 30 | as := assert.New(t) 31 | line := `114.5.1.4 - - [04/Apr/2024:08:01:12 +0800] "\x16\x03\x01\x00\xCA\x01\x00\x00\xC6\x03\x03\x94b\x22\x06u\xBEi\xF6\xC5cA\x97eq\xF0\xD5\xD3\xE6\x08I" 400 163 "-" "-"` 32 | log, err := p.Parse([]byte(line)) 33 | if as.NoError(err) { 34 | as.Equal(`\x16\x03\x01\x00\xCA\x01\x00\x00\xC6\x03\x03\x94b\x22\x06u\xBEi\xF6\xC5cA\x97eq\xF0\xD5\xD3\xE6\x08I`, log.URL) 35 | as.Equal(uint64(163), log.Size) 36 | } 37 | 38 | line = `114.5.1.5 - - [04/Apr/2024:09:02:13 +0800] "\x16\x03\x01\x00\xEE\x01\x00\x00\xEA\x03\x03\x9C\xB4\x92\xC5{\xE9\xEC\x18\xB1\x17\x04f\xCA\x0F\xF3\xFD\xAA\x98H\xA5N\xBC\xC9\xD7\xF8\x95.H\x15\x13\xF2\xF9 ~W\xB9\x94Qs\x01\x02\xE3c'\xA8pB\xC5\xCC\x10c\xC9\xF4\x99{\x0E1\x90\x81\xBD4J\x10y\x17\x00&\xC0+\xC0/\xC0,\xC00\xCC\xA9\xCC\xA8\xC0\x09\xC0\x13\xC0" 400 163 "-" "-"` 39 | log, err = p.Parse([]byte(line)) 40 | if as.NoError(err) { 41 | // When the abnormal request have a space in the URL, we ignore things before the space (shall be "method") 42 | as.Equal(`~W\xB9\x94Qs\x01\x02\xE3c'\xA8pB\xC5\xCC\x10c\xC9\xF4\x99{\x0E1\x90\x81\xBD4J\x10y\x17\x00&\xC0+\xC0/\xC0,\xC00\xCC\xA9\xCC\xA8\xC0\x09\xC0\x13\xC0`, log.URL) 43 | as.Equal(uint64(163), log.Size) 44 | } 45 | 46 | // In nginx double quote is escaped as \x22, but in apache2 it's escaped as \" 47 | line = `172.17.0.1 - - [10/Sep/2024:21:18:44 +0000] "GET /aaaaa\"\"\" HTTP/1.1" 404 196 "http://referer.example.com/\"example/" "Useragent\"\"test"` 48 | log, err = p.Parse([]byte(line)) 49 | if as.NoError(err) { 50 | as.Equal(`/aaaaa\"\"\"`, log.URL) 51 | as.Equal(`Useragent\"\"test`, log.Useragent) 52 | } 53 | } 54 | 55 | func TestNginxCombinedParserWithUnusualInputs(t *testing.T) { 56 | testNginxCombinedParserWithUnusualInputs(t, ParserFunc(ParseNginxCombined)) 57 | testNginxCombinedParserWithUnusualInputs(t, ParserFunc(ParseNginxCombinedRegex)) 58 | } 59 | 60 | func testNginxNotProperlyEscaped(t *testing.T, p Parser) { 61 | as := assert.New(t) 62 | line := `114.5.1.4 - - [01/May/2024:01:02:03 +0800] "GET /echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23'%20&echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23|"%20&echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23/fonts/ HTTP/1.1" 404 178 "" "Example UA"` 63 | log, err := p.Parse([]byte(line)) 64 | as.NoError(err) 65 | as.Equal(`/echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23'%20&echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23|"%20&echo%20ebisyz$()\%20ziyihu\nz^xyu||a%20%23/fonts/`, log.URL) 66 | as.Equal(uint64(178), log.Size) 67 | as.Equal("Example UA", log.Useragent) 68 | } 69 | 70 | func TestNginxNotProperlyEscaped(t *testing.T) { 71 | // Cases `"` is not escaped at all. This case would NOT be supported by ParseNginxCombined. 72 | testNginxNotProperlyEscaped(t, ParserFunc(ParseNginxCombinedRegex)) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/parser/nginx-json.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "math" 5 | "time" 6 | 7 | "github.com/goccy/go-json" 8 | ) 9 | 10 | func init() { 11 | newFunc := func() Parser { 12 | return ParserFunc(ParseNginxJSON) 13 | } 14 | RegisterParser(ParserMeta{ 15 | Name: "nginx-json", 16 | Description: "`nginx-json` format, see README.md for details", 17 | F: newFunc, 18 | }) 19 | RegisterParser(ParserMeta{ 20 | Name: "ngx_json", 21 | Description: "An alias for `nginx-json`", 22 | Hidden: true, 23 | F: newFunc, 24 | }) 25 | } 26 | 27 | type NginxJSONLog struct { 28 | Size uint64 `json:"size"` 29 | Client string `json:"clientip"` 30 | Url string `json:"url"` 31 | Timestamp float64 `json:"timestamp"` 32 | ServerIP string `json:"serverip"` 33 | Useragent string `json:"user_agent"` 34 | } 35 | 36 | func ParseNginxJSON(line []byte) (LogItem, error) { 37 | var logItem NginxJSONLog 38 | err := json.Unmarshal(line, &logItem) 39 | if err != nil { 40 | return LogItem{}, err 41 | } 42 | sec, dec := math.Modf(logItem.Timestamp) 43 | t := time.Unix(int64(sec), int64(dec*1e9)) 44 | return LogItem{ 45 | Size: logItem.Size, 46 | Client: logItem.Client, 47 | Time: t, 48 | URL: logItem.Url, 49 | Server: logItem.ServerIP, 50 | Useragent: logItem.Useragent, 51 | }, nil 52 | } 53 | -------------------------------------------------------------------------------- /pkg/parser/nginx-json_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNginxJsonParser(t *testing.T) { 11 | as := assert.New(t) 12 | p := ParserFunc(ParseNginxJSON) 13 | line := `{"timestamp":1678551332.293,"clientip":"123.45.67.8","serverip":"87.65.4.32","method":"GET","url":"/path/to/a/file","status":200,"size":3009,"resp_time":0.000,"http_host":"example.com","referer":"","user_agent":""}` 14 | log, err := p.Parse([]byte(line)) 15 | as.NoError(err) 16 | as.EqualValues(3009, log.Size) 17 | as.Equal("123.45.67.8", log.Client) 18 | as.Equal("/path/to/a/file", log.URL) 19 | expectedTime := time.Unix(1678551332, 293000000) 20 | as.WithinDuration(expectedTime, log.Time, time.Microsecond) 21 | as.Equal("", log.Useragent) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | ) 7 | 8 | type LogItem struct { 9 | Size uint64 10 | Client string 11 | Time time.Time 12 | URL string 13 | Server string 14 | Useragent string 15 | 16 | // Parsers wishing to discard this log item can set Discard to true. 17 | Discard bool 18 | } 19 | 20 | type Parser interface { 21 | Parse(line []byte) (LogItem, error) 22 | } 23 | 24 | type ParserFunc func(line []byte) (LogItem, error) 25 | 26 | func (p ParserFunc) Parse(line []byte) (LogItem, error) { 27 | return p(line) 28 | } 29 | 30 | type NewFunc func() Parser 31 | 32 | type ParserMeta struct { 33 | Name string 34 | Description string 35 | Hidden bool 36 | F NewFunc 37 | } 38 | 39 | var ( 40 | registry = make(map[string]ParserMeta) 41 | ) 42 | 43 | func RegisterParser(m ParserMeta) { 44 | registry[m.Name] = m 45 | } 46 | 47 | func GetParser(name string) (Parser, error) { 48 | m, ok := registry[name] 49 | if !ok { 50 | return nil, errors.New(name) 51 | } 52 | return m.F(), nil 53 | } 54 | 55 | func All() []ParserMeta { 56 | result := make([]ParserMeta, 0, len(registry)) 57 | for _, m := range registry { 58 | result = append(result, m) 59 | } 60 | return result 61 | } 62 | -------------------------------------------------------------------------------- /pkg/parser/rsync-proxy.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | const ( 11 | goLogTime = "2006/01/02 15:04:05" 12 | listModules = "/" 13 | ) 14 | 15 | func init() { 16 | newFunc := func() Parser { return ParserFunc(ParseRsyncProxy) } 17 | RegisterParser(ParserMeta{ 18 | Name: "rsync-proxy", 19 | Description: "rsync-proxy's access.log", 20 | F: newFunc, 21 | }) 22 | } 23 | 24 | func ParseRsyncProxy(line []byte) (LogItem, error) { 25 | fields := strings.Fields(string(line)) 26 | if len(fields) != 9 && len(fields) != 12 { 27 | return LogItem{}, fmt.Errorf("invalid format: expected 9 or 12 fields, got %d", len(fields)) 28 | } 29 | 30 | logTime, err := time.ParseInLocation(goLogTime, fields[0]+" "+fields[1], time.Local) 31 | if err != nil { 32 | return LogItem{}, fmt.Errorf("invalid log time: %w", err) 33 | } 34 | 35 | logItem := LogItem{ 36 | Client: fields[4], 37 | Time: logTime, 38 | } 39 | 40 | switch fields[5] { 41 | case "starts": 42 | return LogItem{Discard: true}, nil 43 | case "finishes": 44 | logItem.URL = fields[7] 45 | size, err := strconv.ParseUint(strings.TrimSuffix(fields[9], ","), 10, 64) 46 | if err != nil { 47 | return logItem, fmt.Errorf("invalid size: %w", err) 48 | } 49 | logItem.Size = size 50 | case "requests": 51 | if fields[6] == "listing" { 52 | logItem.URL = listModules 53 | // chop off port 54 | if strings.HasPrefix(logItem.Client, "[") { 55 | logItem.Client = strings.TrimPrefix(logItem.Client, "[") 56 | logItem.Client = strings.Split(logItem.Client, "]")[0] 57 | } else { 58 | logItem.Client = strings.Split(logItem.Client, ":")[0] 59 | } 60 | } else { 61 | // requests non-existing module 62 | logItem.URL = fields[8] 63 | } 64 | } 65 | return logItem, nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/parser/rsync-proxy_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestRsyncProxyParser(t *testing.T) { 11 | as := assert.New(t) 12 | parser, err := GetParser("rsync-proxy") 13 | if !as.NoError(err) { 14 | return 15 | } 16 | 17 | line := `2024/10/01 00:00:00 server.go:279: client 123.45.67.89 starts requesting module ubuntu` 18 | log, err := parser.Parse([]byte(line)) 19 | if as.NoError(err) { 20 | as.True(log.Discard) 21 | } 22 | 23 | line = `2024/10/01 00:00:00 server.go:279: client 123.45.67.89 finishes module ubuntu (sent: 1841, received: 208)` 24 | log, err = parser.Parse([]byte(line)) 25 | if as.NoError(err) { 26 | as.False(log.Discard) 27 | as.Equal("123.45.67.89", log.Client) 28 | as.Equal("ubuntu", log.URL) 29 | as.EqualValues(1841, log.Size) 30 | expectedTime := time.Date(2024, 10, 1, 0, 0, 0, 0, time.Local) 31 | as.WithinDuration(expectedTime, log.Time, 0) 32 | } 33 | 34 | line = `2024/10/01 00:00:00 server.go:279: client 123.45.67.89 requests non-existing module centos` 35 | log, err = parser.Parse([]byte(line)) 36 | if as.NoError(err) { 37 | as.False(log.Discard) 38 | as.Equal("123.45.67.89", log.Client) 39 | as.Equal("centos", log.URL) 40 | as.EqualValues(0, log.Size) 41 | expectedTime := time.Date(2024, 10, 1, 0, 0, 0, 0, time.Local) 42 | as.WithinDuration(expectedTime, log.Time, 0) 43 | } 44 | 45 | line = `2024/10/01 00:00:00 server.go:279: client 123.45.67.89:2333 requests listing all modules` 46 | log, err = parser.Parse([]byte(line)) 47 | if as.NoError(err) { 48 | as.False(log.Discard) 49 | as.Equal("123.45.67.89", log.Client) 50 | as.Equal(listModules, log.URL) 51 | as.EqualValues(0, log.Size) 52 | expectedTime := time.Date(2024, 10, 1, 0, 0, 0, 0, time.Local) 53 | as.WithinDuration(expectedTime, log.Time, 0) 54 | } 55 | 56 | line = `2024/10/01 00:00:00 server.go:279: client [2001:db8:666:6969::1]:12345 requests listing all modules` 57 | log, err = parser.Parse([]byte(line)) 58 | if as.NoError(err) { 59 | as.False(log.Discard) 60 | as.Equal("2001:db8:666:6969::1", log.Client) 61 | as.Equal(listModules, log.URL) 62 | as.EqualValues(0, log.Size) 63 | expectedTime := time.Date(2024, 10, 1, 0, 0, 0, 0, time.Local) 64 | as.WithinDuration(expectedTime, log.Time, 0) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/parser/tencent-cdn.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | func init() { 10 | newFunc := func() Parser { return ParserFunc(ParseTencentCDN) } 11 | RegisterParser(ParserMeta{ 12 | Name: "tencent-cdn", 13 | Description: "Tencent CDN log format", 14 | F: newFunc, 15 | }) 16 | RegisterParser(ParserMeta{ 17 | Name: "tcdn", 18 | Description: "An alias for `tencent-cdn`", 19 | Hidden: true, 20 | F: newFunc, 21 | }) 22 | } 23 | 24 | const compactDateTime = "20060102150405" 25 | 26 | func compactDateTimeParse(s []byte) time.Time { 27 | return compactDateTimeParseString(string(s)) 28 | } 29 | 30 | func compactDateTimeParseString(s string) time.Time { 31 | t, _ := time.ParseInLocation(compactDateTime, s, time.Local) 32 | return t 33 | } 34 | 35 | func ParseTencentCDN(line []byte) (logItem LogItem, err error) { 36 | fields, err := splitFields(line) 37 | if err != nil { 38 | return logItem, err 39 | } 40 | if len(fields) != 16 { 41 | return logItem, fmt.Errorf("invalid format: expected 16 fields, got %d", len(fields)) 42 | } 43 | size, err := strconv.ParseUint(string(fields[4]), 10, 64) 44 | if err != nil { 45 | return logItem, fmt.Errorf("invalid size %s: %w", fields[4], err) 46 | } 47 | return LogItem{ 48 | Size: size, 49 | Client: string(fields[1]), 50 | Time: compactDateTimeParse(fields[0]), 51 | URL: string(fields[3]), 52 | Server: string(fields[2]), 53 | Useragent: string(fields[10]), 54 | }, nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/parser/tencent-cdn_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTencentCDNParser(t *testing.T) { 11 | as := assert.New(t) 12 | p, err := GetParser("tencent-cdn") 13 | if !(as.NoError(err) && as.NotNil(p)) { 14 | return 15 | } 16 | line := []byte(`20240930180135 123.45.67.8 www.example.com /wp-content/favicon.ico 6969 120 2 200 https://www.example.com/ 3 "Mozilla/5.0 () Chrome/96.0.4664.104 Mobile Safari/537.36" "(null)" GET HTTPS hit 32768`) 17 | log, err := p.Parse(line) 18 | if !as.NoError(err) { 19 | return 20 | } 21 | as.EqualValues(6969, log.Size) 22 | as.Equal("123.45.67.8", log.Client) 23 | as.Equal("/wp-content/favicon.ico", log.URL) 24 | expectedTime := time.Date(2024, 9, 30, 18, 1, 35, 0, time.Local) 25 | as.WithinDuration(expectedTime, log.Time, time.Microsecond) 26 | as.Equal("Mozilla/5.0 () Chrome/96.0.4664.104 Mobile Safari/537.36", log.Useragent) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/systemd/notify.go: -------------------------------------------------------------------------------- 1 | package systemd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "os" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | func NotifyReady() error { 13 | return Notify("READY=1") 14 | } 15 | 16 | func MustNotifyReady() { 17 | if err := NotifyReady(); err != nil { 18 | panic(err) 19 | } 20 | } 21 | 22 | func getMonoTime() (uint64, error) { 23 | var ts unix.Timespec 24 | err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts) 25 | if err != nil { 26 | return 0, err 27 | } 28 | return uint64(ts.Sec)*1e6 + uint64(ts.Nsec)/1e3, nil 29 | } 30 | 31 | func NotifyReloading() error { 32 | microsecs, err := getMonoTime() 33 | if err != nil { 34 | return err 35 | } 36 | msg := fmt.Sprintf("RELOADING=1\nRELOAD_TIMESTAMP=%d", microsecs) 37 | return Notify(msg) 38 | } 39 | 40 | func MustNotifyReloading() { 41 | if err := NotifyReloading(); err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | func Notify(message string) error { 47 | if len(message) == 0 { 48 | return errors.New("requires a message") 49 | } 50 | name := os.Getenv("NOTIFY_SOCKET") 51 | if name == "" { 52 | // If not set, nothing to do 53 | return nil 54 | } 55 | if name[0] != '@' && name[0] != '/' { 56 | return errors.New("unsupported socket type") 57 | } 58 | 59 | if name[0] == '@' { 60 | name = "\x00" + name[1:] 61 | } 62 | 63 | conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: name, Net: "unixgram"}) 64 | if err != nil { 65 | return err 66 | } 67 | defer conn.Close() 68 | 69 | _, err = conn.Write([]byte(message)) 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /pkg/tui/interface.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "fmt" 5 | "net/netip" 6 | "sync/atomic" 7 | "time" 8 | 9 | "github.com/taoky/ayano/pkg/analyze" 10 | ) 11 | 12 | type ShowMode int 13 | 14 | const ( 15 | TopValues ShowMode = iota 16 | Total 17 | ) 18 | 19 | const helpMsg = `Available shortcuts: 20 | t/T: print total size aggregated by server 21 | s: set server filtering 22 | S: change sort by 23 | ?: help` 24 | 25 | type Tui struct { 26 | analyzer *analyze.Analyzer 27 | // displayRecord is used to remember latest accesses time by specific IP 28 | displayRecord map[netip.Prefix]time.Time 29 | serverFilter string 30 | mode ShowMode 31 | sortBy analyze.SortByFlag 32 | ticker *time.Ticker 33 | refreshChan chan struct{} 34 | inputChan chan byte 35 | noPrint atomic.Bool 36 | } 37 | 38 | func New(analyzer *analyze.Analyzer) *Tui { 39 | return &Tui{ 40 | analyzer: analyzer, 41 | displayRecord: make(map[netip.Prefix]time.Time), 42 | mode: TopValues, 43 | sortBy: analyzer.Config.SortBy, 44 | refreshChan: make(chan struct{}), 45 | inputChan: make(chan byte), 46 | } 47 | } 48 | 49 | func (t *Tui) Run() { 50 | a := t.analyzer 51 | go t.timerRoutine() 52 | go t.waitForOneByte() 53 | 54 | for { 55 | select { 56 | case k := <-t.inputChan: 57 | t.handleInput(k) 58 | case <-t.refreshChan: 59 | if t.mode == TopValues { 60 | a.PrintTopValues(t.displayRecord, t.sortBy, t.serverFilter) 61 | } else { 62 | a.PrintTotal() 63 | } 64 | fmt.Println() 65 | } 66 | } 67 | } 68 | 69 | func (t *Tui) handleInput(key byte) { 70 | switch key { 71 | case 's': 72 | t.handles() 73 | case 'S': 74 | if t.sortBy == analyze.SortBySize { 75 | t.sortBy = analyze.SortByRequests 76 | fmt.Println("Switched to sort by requests") 77 | } else { 78 | t.sortBy = analyze.SortBySize 79 | fmt.Println("Switched to sort by size") 80 | } 81 | case 'T', 't': 82 | if t.mode == TopValues { 83 | t.mode = Total 84 | fmt.Println("Switched to showing total") 85 | } else { 86 | t.mode = TopValues 87 | fmt.Println("Switched to showing top values") 88 | } 89 | case '?': 90 | fmt.Println(helpMsg) 91 | fmt.Println() 92 | } 93 | go t.waitForOneByte() 94 | } 95 | 96 | func (t *Tui) handles() { 97 | t.noPrint.Store(true) 98 | defer t.noPrint.Store(false) 99 | 100 | servers := t.analyzer.GetCurrentServers() 101 | if len(servers) == 1 { 102 | serverFmt := "" 103 | if len(servers[0]) > 0 { 104 | serverFmt = " (" + servers[0] + ")" 105 | } 106 | fmt.Printf("Only one server%s is available.\n", serverFmt) 107 | } else if len(servers) != 0 { 108 | fmt.Println("Please give the server name you want to view. Enter to remove filtering.") 109 | if t.serverFilter != "" { 110 | fmt.Println("Current:", t.serverFilter) 111 | } 112 | // Get all servers available 113 | fmt.Println("Available servers:") 114 | for _, s := range servers { 115 | fmt.Println(s) 116 | } 117 | var input string 118 | n, err := fmt.Scanln(&input) 119 | if err != nil { 120 | if n != 0 { 121 | fmt.Println("Failed to get input:", err) 122 | } else { 123 | if t.serverFilter != "" { 124 | fmt.Println("(filter removed)") 125 | } 126 | t.serverFilter = "" 127 | } 128 | } else { 129 | found := false 130 | for _, str := range servers { 131 | if str == input { 132 | found = true 133 | t.serverFilter = input 134 | break 135 | } 136 | } 137 | if !found { 138 | fmt.Println("Input does not match existing server.") 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /pkg/tui/routines.go: -------------------------------------------------------------------------------- 1 | package tui 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | ) 8 | 9 | func (t *Tui) timerRoutine() { 10 | t.ticker = time.NewTicker(time.Duration(t.analyzer.Config.RefreshSec) * time.Second) 11 | for range t.ticker.C { 12 | if !t.noPrint.Load() { 13 | t.refreshChan <- struct{}{} 14 | } 15 | } 16 | } 17 | 18 | func (t *Tui) waitForOneByte() { 19 | oldState, err := makeRaw(int(os.Stdin.Fd())) 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | if oldState == nil { 24 | return 25 | } 26 | defer restore(int(os.Stdin.Fd()), oldState) 27 | 28 | b := make([]byte, 1) 29 | n, err := os.Stdin.Read(b) 30 | if err != nil { 31 | log.Println(err) 32 | return 33 | } 34 | if n == 0 { 35 | return 36 | } 37 | t.inputChan <- b[0] 38 | } 39 | -------------------------------------------------------------------------------- /pkg/tui/term.go: -------------------------------------------------------------------------------- 1 | // Modified from https://github.com/golang/term/blob/5b15d269ba1f54e8da86c8aa5574253aea0c2198/term_unix.go#L22 2 | // It only changes input flags to disable echo and canonical mode. 3 | // BSD-3-Clause License 4 | package tui 5 | 6 | import ( 7 | "log" 8 | 9 | "golang.org/x/sys/unix" 10 | ) 11 | 12 | const ioctlReadTermios = unix.TCGETS 13 | 14 | type state struct { 15 | termios unix.Termios 16 | } 17 | 18 | func makeRaw(fd int) (*state, error) { 19 | termios, err := unix.IoctlGetTermios(fd, ioctlReadTermios) 20 | if err != nil { 21 | log.Println("Not a terminal. Shortcuts will be disabled.") 22 | return nil, nil 23 | } 24 | oldState := state{termios: *termios} 25 | 26 | termios.Lflag &^= unix.ECHO | unix.ICANON 27 | if err := unix.IoctlSetTermios(fd, unix.TCSETS, termios); err != nil { 28 | return nil, err 29 | } 30 | 31 | return &oldState, nil 32 | } 33 | 34 | func restore(fd int, oldState *state) error { 35 | if oldState == nil { 36 | return nil 37 | } 38 | return unix.IoctlSetTermios(fd, unix.TCSETS, &oldState.termios) 39 | } 40 | --------------------------------------------------------------------------------