├── .github └── FUNDING.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── bacon.toml ├── benches ├── filter_resources.rs └── load_base.rs ├── compile-all-targets.sh ├── doc ├── download-filter.png ├── hits-or-bytes.png ├── intro.png ├── logo-rhit.png ├── mixed-filter.png └── tables-choice.png ├── release.sh ├── src ├── cli │ ├── args.rs │ ├── help.rs │ └── mod.rs ├── csv.rs ├── date.rs ├── date_histogram.rs ├── date_idx.rs ├── date_time.rs ├── error.rs ├── fields.rs ├── filters │ ├── date_time_filter.rs │ ├── method_filter.rs │ ├── mod.rs │ ├── status_filter.rs │ ├── str_filter.rs │ └── time_filter.rs ├── histo_line.rs ├── json.rs ├── key.rs ├── leak.rs ├── lib.rs ├── line_group.rs ├── main.rs ├── md │ ├── addr.rs │ ├── methods.rs │ ├── mod.rs │ ├── paths.rs │ ├── printer.rs │ ├── referers.rs │ ├── section.rs │ ├── skin.rs │ ├── status.rs │ └── summary.rs ├── method.rs ├── nginx_log │ ├── file_finder.rs │ ├── file_reader.rs │ ├── line_consumer.rs │ ├── log_base.rs │ ├── log_line.rs │ ├── mod.rs │ └── ranger.rs ├── output.rs ├── raw.rs ├── time.rs ├── time_histogram.rs ├── trend.rs └── trend_computer.rs ├── test-data ├── README.md └── access.log ├── version.sh └── website ├── .gitignore ├── README.md ├── custom_theme ├── main.html └── toc.html ├── docs ├── README.md ├── community.md ├── css │ ├── extra.css │ └── link-to-dystroy.css ├── export.md ├── img │ ├── changes-path.png │ ├── changes-ref.png │ ├── dystroy-rust-white.svg │ ├── export-jq.png │ ├── export-raw.png │ ├── export-tables.png │ ├── favicon.ico │ ├── favicon.png │ ├── fields-all-paths.png │ ├── fields-date.png │ ├── fields-ip.png │ ├── fields-method.png │ ├── fields-path.png │ ├── fields-referer.png │ ├── fields-status.png │ ├── filter-date-no-year.png │ ├── filter-date.png │ ├── filter-path-ogh.png │ ├── filter-referer.png │ ├── intro.png │ ├── lines.png │ ├── logo-rhit.png │ ├── logo-rhit.svg │ ├── referer-changes.png │ ├── status-413.png │ ├── two-keys.png │ └── whiskey.jpg ├── index.md ├── install.md ├── js │ └── link-to-dystroy.js ├── usage-changes.md ├── usage-fields.md ├── usage-filters.md ├── usage-key.md └── usage-overview.md └── mkdocs.yml /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [Canop] 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /releases 3 | deploy.sh 4 | rhit_*.zip 5 | /build 6 | /trav 7 | !*.log 8 | rhit.log 9 | glassbench_v1.db 10 | .bacon-locations 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | ### v2.0.3 - 2024-10-01 3 | - right align the 'hits' column of histograms 4 | 5 | 6 | ### v2.0.2 - 2024-09-07 7 | - allow some additional spaces in filters of several kinds, eg `-d '> 12/25'` 8 | - fix some problems in filters using several operators without parenthesis 9 | 10 | 11 | ### v2.0.1 - 2023-09-03 12 | - fix app name in `--version` 13 | - better error message on path not found - thanks @orhun 14 | 15 | 16 | ### v2.0.0 - 2023-08-30 17 | - `--output` parameter lets you choose between summary tables (default) or the log lines, either raw, as CSV, or as JSON 18 | - `--lines` parameter removed (use `--output raw` or `-o r` instead) 19 | - `--date` precision now the second 20 | - `--time` filter 21 | - new time histogram (time of the day, in the server's timezone) 22 | - more helpful `--help` 23 | - more targets for binaries in the official archives, especially ARM 32/64 both gnu and musl 24 | 25 | 26 | ### v1.7.2 - 2023-04-23 27 | - dependency managment - Fix #22 28 | 29 | 30 | ### v1.7.1 - 2022-06-05 31 | - mostly dependency updates and compilation fixes 32 | 33 | 34 | ### v1.7.0 - 2022-01-16 35 | - allow passing several paths as arguments - Fix #14 36 | 37 | 38 | ### v1.6.0 - 2021-12-25 39 | - better table fitting algorithm, less frequently breaking the histogram columns 40 | 41 | 42 | ### v1.5.5 - 2021-12-21 43 | - don't write an error when no log line matches the query 44 | 45 | 46 | ### v1.5.4 - 2021-11-27 47 | - fix compilation broken by patch release 1.0.49 of anyhow 48 | 49 | 50 | ### v1.5.3 - 2021-07-13 51 | - nothing new visible, small internal upgrades 52 | 53 | 54 | ### v1.5.2 - 2021-06-29 55 | - fix inability to render on narrow terminals 56 | 57 | 58 | ### v1.5.1 - 2021-05-01 59 | - look up to 3 lines of a file for a log line when checking whether it's a log file - Fix #8 60 | - faster log parsing (about 7%) 61 | - IP filtering allow regexes or any string based filtering 62 | 63 | 64 | ### v1.5.0 - 2021-03-19 65 | - new syntax to specify fields, allow adding from default, removing from all, etc. (the old syntax still works) 66 | - compiles on windows (but doesn't know where the log files are) - I need testers to confirm it works 67 | - change error message "no log found" into a more appropriate one when there was an error reading (usually lack of permission) 68 | 69 | 70 | ### v1.4.1 - 2021-03-07 71 | - small details, like the order of arguments in help 72 | 73 | 74 | ### v1.4.0 - 2021-03-03 75 | - `--lines` option to output log lines to stdout 76 | - accept date in ISO 8601 format (previously, only the "common log format" was accepted) - Fix #3 77 | 78 | 79 | ### v1.3.2 - 2021-02-23 80 | - fix wrong version number in rhit.log file 81 | - any file whose name contains "access.log" is considered a probable log file 82 | - when a single file is given to rhit, its name isn't checked 83 | - no file name is checked with `--no-name-check` 84 | 85 | 86 | ### v1.3.1 - 2021-02-19 87 | - `--all` argument to remove the filter excluding "resources" from the paths tables 88 | 89 | 90 | ### v1.3.0 - 2021-02-18 91 | Many changes in the arguments you give to rhit: 92 | - `tables` have been renamed `fields` 93 | - `addr` (remote IP addresses) has been changed to `ip` both in fields list and as filter 94 | - instead of a `trends` table, there's a `--changes` argument (short: `-c`) 95 | - with `--changes`, you see more popular and less popular referers 96 | - with `--changes`, you see more popular and less popular remote ip adresses if the ip field is shown (eg with `rhit -f date,ip -c`) 97 | - date filters can be negative or inequalities (eg: `-d '>2021/02/10'`) 98 | 99 | 100 | ### v1.2.0 - 2021-02-12 101 | - the `--key` argument defines the key measure, either 'hits' (default) or 'bytes' (of the response) used for sorting and filtering, and highlighted in pink 102 | - you can filter on year or month (eg `rhit -d 2021/02`) 103 | - trends in all tables 104 | 105 | 106 | ### v1.1.1 - 2021-02-10 107 | - when you pipe the output of rhit to a file, there's no style information. You can choose explicitly to have or not the styles and colors with the `--color` argument - Fix #1 108 | 109 | 110 | ### v1.1.0 - 2021-02-09 111 | - trends 112 | 113 | 114 | ### v1.0.0 - 2021-01-29 115 | - first public release 116 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rhit" 3 | version = "2.0.3" 4 | authors = ["dystroy "] 5 | repository = "https://github.com/Canop/rhit" 6 | description = "nginx log analyzer" 7 | edition = "2021" 8 | keywords = ["log", "nginx", "analyzer"] 9 | license = "MIT" 10 | categories = ["command-line-utilities"] 11 | rust-version = "1.60" 12 | 13 | [dependencies] 14 | bet = "1.0.4" 15 | clap = { version = "4.4", features = ["derive", "cargo"] } 16 | clap-help = "1.4.0" 17 | cli-log = "2.0" 18 | file-size = "1.0.3" 19 | flate2 = "1.0.30" 20 | have = "0.1.1" 21 | itertools = "0.13" 22 | lazy-regex = "3.3" 23 | num-format = "0.4" 24 | smallvec = "1.11" 25 | termimad = { version = "0.32", default-features = false, features = ["special-renders"] } 26 | thiserror = "1.0" 27 | 28 | [dev-dependencies] 29 | glassbench = "0.4" 30 | 31 | [profile.release] 32 | debug = false 33 | lto = true 34 | codegen-units = 1 35 | strip = true 36 | 37 | [[bench]] 38 | name = "load_base" 39 | harness = false 40 | [[bench]] 41 | name = "filter_resources" 42 | harness = false 43 | 44 | [patch.crates-io] 45 | # bet = { path = "../bet" } 46 | # cli-log = { path = "../cli-log" } 47 | # have = { path = "../have" } 48 | # minimad = { path = "../minimad" } 49 | # termimad = { path = "../termimad" } 50 | # lazy-regex = { path = "../lazy-regex" } 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Canop 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | [![Latest Version][s1]][l1] [![MIT][s2]][l2] [![Chat on Miaou][s3]][l3] [![Packaging status][srep]][lrep] 3 | 4 | [s1]: https://img.shields.io/crates/v/rhit.svg 5 | [l1]: https://crates.io/crates/rhit 6 | 7 | [s2]: https://img.shields.io/badge/license-MIT-blue.svg 8 | [l2]: LICENSE 9 | 10 | [s3]: https://miaou.dystroy.org/static/shields/room.svg 11 | [l3]: https://miaou.dystroy.org/3768?rust 12 | 13 | [srep]: https://repology.org/badge/tiny-repos/rhit.svg 14 | [lrep]: https://repology.org/project/rhit/versions 15 | 16 | ![logo](doc/logo-rhit.png) 17 | 18 | **[Rhit](https://dystroy.org/rhit)** reads your nginx log files in their standard location(even gzipped), does some analysis and tells you about it in pretty tables in your console, storing and polluting nothing. 19 | 20 | It lets you filter hits by dates, status, referers or paths, and does trend analysis. 21 | 22 | And it's fast enough (about one second per million lines) so you can iteratively try queries to build your insight. 23 | 24 | Here's looking at dates and trends on January hits with status 2xx and 3xx: 25 | 26 | ![intro](doc/intro.png) 27 | 28 | 29 | **[Installation instructions and documentation on Rhit's website](https://dystroy.org/rhit)** 30 | 31 | -------------------------------------------------------------------------------- /bacon.toml: -------------------------------------------------------------------------------- 1 | # This is a configuration file for the bacon tool 2 | # 3 | # Bacon repository: https://github.com/Canop/bacon 4 | # Complete help on configuration: https://dystroy.org/bacon/config/ 5 | # You can also check bacon's own bacon.toml file 6 | # as an example: https://github.com/Canop/bacon/blob/main/bacon.toml 7 | 8 | default_job = "check" 9 | 10 | [jobs.check] 11 | command = ["cargo", "check", "--color", "always"] 12 | need_stdout = false 13 | 14 | [jobs.check-all] 15 | command = ["cargo", "check", "--all-targets", "--color", "always"] 16 | need_stdout = false 17 | 18 | [jobs.clippy] 19 | command = [ 20 | "cargo", "clippy", 21 | "--all-targets", 22 | "--color", "always", 23 | "--", 24 | "-A", "clippy::manual_range_contains", 25 | "-A", "clippy::match_like_matches_macro", 26 | "-A", "clippy::manual_clamp", 27 | "-A", "clippy::if_same_then_else", 28 | "-A", "clippy::manual_range_contains", 29 | ] 30 | need_stdout = false 31 | 32 | [jobs.test] 33 | command = [ 34 | "cargo", "test", "--color", "always", 35 | "--", "--color", "always", # see https://github.com/Canop/bacon/issues/124 36 | ] 37 | need_stdout = true 38 | 39 | [jobs.doc] 40 | command = ["cargo", "doc", "--color", "always", "--no-deps"] 41 | need_stdout = false 42 | 43 | # If the doc compiles, then it opens in your browser and bacon switches 44 | # to the previous job 45 | [jobs.doc-open] 46 | command = ["cargo", "doc", "--color", "always", "--no-deps", "--open"] 47 | need_stdout = false 48 | on_success = "back" # so that we don't open the browser at each change 49 | 50 | # You can run your application and have the result displayed in bacon, 51 | # *if* it makes sense for this crate. You can run an example the same 52 | # way. Don't forget the `--color always` part or the errors won't be 53 | # properly parsed. 54 | [jobs.run] 55 | command = [ 56 | "cargo", "run", 57 | "--color", "always", 58 | # put launch parameters for your program behind a `--` separator 59 | ] 60 | need_stdout = true 61 | allow_warnings = true 62 | 63 | # You may define here keybindings that would be specific to 64 | # a project, for example a shortcut to launch a specific job. 65 | # Shortcuts to internal functions (scrolling, toggling, etc.) 66 | # should go in your personal global prefs.toml file instead. 67 | [keybindings] 68 | # alt-m = "job:my-job" 69 | c = "job:clippy" 70 | -------------------------------------------------------------------------------- /benches/filter_resources.rs: -------------------------------------------------------------------------------- 1 | 2 | use { 3 | glassbench::*, 4 | rhit::*, 5 | rhit::args::Args, 6 | std::path::PathBuf, 7 | }; 8 | 9 | fn filter_resources(bench: &mut Bench) { 10 | bench.task("filter resources", |task| { 11 | let package_dir = std::env::var_os("CARGO_MANIFEST_DIR").expect("manifest dir not set"); 12 | let paths = vec![PathBuf::from(package_dir).join("test-data/")]; 13 | let args = Args { 14 | silent_load: true, 15 | ..Default::default() 16 | }; 17 | let base = LogBase::new(&paths, &args).unwrap(); 18 | assert_eq!(base.lines.len(), 33468); 19 | task.iter(|| { 20 | let count = base.lines.iter() 21 | .filter(|line| line.is_resource()) 22 | .count(); 23 | assert_eq!(count, 5772); 24 | pretend_used(count); 25 | }); 26 | }); 27 | } 28 | 29 | glassbench!( 30 | "Resources filtering", 31 | filter_resources, 32 | ); 33 | -------------------------------------------------------------------------------- /benches/load_base.rs: -------------------------------------------------------------------------------- 1 | 2 | use { 3 | glassbench::*, 4 | rhit::*, 5 | rhit::args::Args, 6 | std::path::PathBuf, 7 | }; 8 | 9 | fn bench_base_loading(bench: &mut Bench) { 10 | bench.task("read access.log", |task| { 11 | let package_dir = std::env::var_os("CARGO_MANIFEST_DIR").expect("manifest dir not set"); 12 | let paths = vec![PathBuf::from(package_dir).join("test-data/")]; 13 | let args = Args { 14 | silent_load: true, 15 | ..Default::default() 16 | }; 17 | task.iter(|| { 18 | let base = LogBase::new(&paths, &args).unwrap(); 19 | assert_eq!(base.lines.len(), 33468); 20 | pretend_used(base); 21 | }); 22 | }); 23 | } 24 | 25 | glassbench!( 26 | "Log Base Loading", 27 | bench_base_loading, 28 | ); 29 | -------------------------------------------------------------------------------- /compile-all-targets.sh: -------------------------------------------------------------------------------- 1 | #WARNING: This script is NOT meant for normal installation, it's dedicated 2 | # to the compilation of all supported targets. 3 | # This is a long process and it involves specialized toolchains. 4 | # For usual compilation do 5 | # cargo build --release 6 | # or read all possible installation solutions on 7 | # https://dystroy.org/rhit/install 8 | 9 | H1="\n\e[30;104;1m\e[2K\n\e[A" # style first header 10 | H2="\n\e[30;104m\e[1K\n\e[A" # style second header 11 | EH="\e[00m\n\e[2K" # end header 12 | NAME=rhit 13 | version=$(./version.sh) 14 | 15 | echo -e "${H1}Compilation of all targets for $NAME $version${EH}" 16 | 17 | # Clean previous build 18 | rm -rf build 19 | mkdir build 20 | echo " build cleaned" 21 | 22 | # Build versions for other platforms using cargo cross 23 | cross_build() { 24 | target_name="$1" 25 | target="$2" 26 | echo -e "${H2}Compiling the $target_name version for target $target ${EH}" 27 | cargo clean 28 | cross build --target "$target" --release 29 | mkdir "build/$target" 30 | if [[ $target_name == 'Windows' ]] 31 | then 32 | exec="$NAME.exe" 33 | else 34 | exec="$NAME" 35 | fi 36 | cp "target/$target/release/$exec" "build/$target/" 37 | } 38 | cross_build "Windows" "x86_64-pc-windows-gnu" 39 | cross_build "MUSL" "x86_64-unknown-linux-musl" 40 | cross_build "Linux GLIBC" "x86_64-unknown-linux-gnu" 41 | cross_build "ARM 32" "armv7-unknown-linux-gnueabihf" 42 | cross_build "ARM 32 MUSL" "armv7-unknown-linux-musleabi" 43 | cross_build "ARM 64" "aarch64-unknown-linux-gnu" 44 | cross_build "ARM 64 MUSL" "aarch64-unknown-linux-musl" 45 | 46 | # Build the default linux version 47 | # recent glibc 48 | echo -e "${H2}Compiling the standard linux version${EH}" 49 | cargo build --release 50 | strip "target/release/$NAME" 51 | mkdir build/x86_64-linux/ 52 | cp "target/release/$NAME" build/x86_64-linux/ 53 | 54 | # add a summary of content 55 | echo ' 56 | This archive contains pre-compiled binaries 57 | 58 | For more information, or if you prefer to compile yourself, see https://dystroy.org/rhit/install 59 | ' > build/install.md 60 | 61 | echo -e "${H1}FINISHED${EH}" 62 | -------------------------------------------------------------------------------- /doc/download-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/download-filter.png -------------------------------------------------------------------------------- /doc/hits-or-bytes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/hits-or-bytes.png -------------------------------------------------------------------------------- /doc/intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/intro.png -------------------------------------------------------------------------------- /doc/logo-rhit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/logo-rhit.png -------------------------------------------------------------------------------- /doc/mixed-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/mixed-filter.png -------------------------------------------------------------------------------- /doc/tables-choice.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Canop/rhit/eb77279531624595fdc56b1eeec7a9b8edab059b/doc/tables-choice.png -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | # build a new release of rhit 2 | # This isn't used for normal compilation but for the building of the official releases 3 | version=$(sed 's/version = "\([0-9.]\{1,\}\)"/\1/;t;d' Cargo.toml | head -1) 4 | 5 | echo "Building release $version" 6 | 7 | # make the build directory and compile for all targets 8 | ./compile-all-targets.sh 9 | 10 | # add the readme, changelog and license in the build directory 11 | echo "This is rhit. More info and installation instructions on https://github.com/Canop/rhit" > build/README.md 12 | cp CHANGELOG.md build 13 | cp LICENSE build 14 | 15 | # publish version number 16 | echo "$version" > build/version 17 | 18 | # prepare the release archive 19 | rm rhit_*.zip 20 | zip -r "rhit_$version.zip" build/* 21 | 22 | # copy it to releases folder 23 | mkdir releases 24 | cp "rhit_$version.zip" releases 25 | -------------------------------------------------------------------------------- /src/cli/args.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | Key, 4 | Fields, 5 | Output, 6 | }, 7 | clap::{Parser, ValueEnum}, 8 | std::path::PathBuf, 9 | termimad::crossterm::tty::IsTty, 10 | }; 11 | 12 | /// Program launch argument 13 | #[derive(Debug, Default, Parser)] 14 | #[command(author, about, name = "rhit", disable_version_flag = true, version, disable_help_flag = true)] 15 | pub struct Args { 16 | 17 | /// Print help information 18 | #[arg(long)] 19 | pub help: bool, 20 | 21 | /// Print the version 22 | #[arg(long)] 23 | pub version: bool, 24 | 25 | /// Whether to have styles and colors 26 | #[arg(long, default_value="auto", value_name = "color")] 27 | pub color: TriBool, 28 | 29 | /// Key used in sorting and histogram, either `hits` or `bytes` 30 | #[arg(short, long, default_value="hits")] 31 | pub key: Key, 32 | 33 | /// Detail level, from `0` to `6`, impacts the lengths of tables 34 | #[arg(short, long, default_value = "1")] 35 | pub length: usize, 36 | 37 | /// Comma separated list of hit fields to display. 38 | /// Use `-f a` to get all fields. 39 | /// Use `-f +i` to add ip. 40 | /// Available fields: `date,time,method,status,ip,ref,path`. 41 | #[arg(short, long, default_value = "date,status,ref,path")] 42 | pub fields: Fields, 43 | 44 | /// Add tables with more popular and less popular entries (ip, referers and paths) 45 | #[arg(short, long)] 46 | pub changes: bool, 47 | 48 | /// Filter the dates on a precise day or in an inclusive range 49 | /// (eg: `-d 12/24` or `-d '2021/12/24-2022/01/21'`) 50 | #[arg(short, long)] 51 | pub date: Option, 52 | 53 | /// Ip address to filter by. May be negated with a `!` 54 | #[arg(short, long)] 55 | pub ip: Option, 56 | 57 | /// HTTP method to filter by. Make it negative with a `!`. 58 | /// (eg: `-m PUT` or `-m !DELETE` or `-m none` or `-m other`) 59 | #[arg(short, long)] 60 | pub method: Option, 61 | 62 | /// Pattern for path filtering 63 | /// (eg: `-p broot` or `-p '^/\d+'` or `-p 'miaou | blog'`) 64 | #[arg(short, long)] 65 | pub path: Option, 66 | 67 | /// Referrer filter 68 | #[arg(short, long)] 69 | pub referer: Option, 70 | 71 | /// Comma separated list of statuses or status ranges to filter by 72 | /// (eg: `-s 514` or `-s 4xx,5xx`, or `-s 310-340,400-450` or `-s 5xx,!502`) 73 | #[arg(short, long)] 74 | pub status: Option, 75 | 76 | /// Filter the time of the day, in the logs' timezone 77 | /// (eg: `-t '>19:30'` to get evening hits) 78 | #[arg(short, long)] 79 | pub time: Option, 80 | 81 | /// Show all paths, including resources 82 | #[arg(short, long)] 83 | pub all: bool, 84 | 85 | /// Try to open all files, whatever their names 86 | #[arg(long)] 87 | pub no_name_check: bool, 88 | 89 | /// Output: by default pretty summary tables but you can also 90 | /// output log lines as `csv`, `json`, or `raw` (as they appear in the log files) 91 | #[arg(short, long, default_value="tables")] 92 | pub output: Output, 93 | 94 | /// Don't print anything during load: no progress bar or file list 95 | #[arg(long)] 96 | pub silent_load: bool, 97 | 98 | /// The log file or folder to analyze. It not provided, logs will be opened 99 | /// at their standard location 100 | pub files: Vec, 101 | } 102 | 103 | #[derive(ValueEnum)] 104 | #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] 105 | pub enum TriBool { 106 | #[default] 107 | Auto, 108 | Yes, 109 | No, 110 | } 111 | impl TriBool { 112 | pub fn unwrap_or_else(self, f: F) -> bool 113 | where 114 | F: FnOnce() -> bool 115 | { 116 | match self { 117 | Self::Auto => f(), 118 | Self::Yes => true, 119 | Self::No => false, 120 | } 121 | } 122 | } 123 | 124 | impl Args { 125 | pub fn color(&self) -> bool { 126 | self.color.unwrap_or_else(|| std::io::stdout().is_tty()) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/cli/help.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | args::*, 4 | }, 5 | clap::CommandFactory, 6 | termimad::{ 7 | ansi, 8 | CompoundStyle, 9 | crossterm::style::{Attribute, Color}, 10 | }, 11 | }; 12 | 13 | static INTRO_TEMPLATE: &str = " 14 | **Rhit** analyzes your nginx logs. 15 | 16 | Complete documentation at *https://dystroy.org/rhit* 17 | 18 | "; 19 | 20 | pub fn print() { 21 | let mut printer = clap_help::Printer::new(Args::command()) 22 | .with("introduction", INTRO_TEMPLATE) 23 | .without("author"); 24 | let skin = printer.skin_mut(); 25 | skin.headers[0].compound_style.set_fg(ansi(204)); 26 | skin.italic = CompoundStyle::with_attr(Attribute::Underlined); 27 | skin.bold.set_fg(Color::AnsiValue(204)); 28 | printer.print_help(); 29 | } 30 | -------------------------------------------------------------------------------- /src/cli/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod args; 2 | mod help; 3 | 4 | use { 5 | crate::*, 6 | args::Args, 7 | clap::Parser, 8 | cli_log::*, 9 | std::path::PathBuf, 10 | }; 11 | 12 | const DEFAULT_NGINX_LOCATION: &str = "/var/log/nginx"; 13 | static MISSING_DEFAULT_MESSAGE: &str = "\ 14 | No nginx log found at default location, do you have nginx set up? 15 | If necessary, provide the path to the log file(s) as argument. 16 | More information with 'rhit --help'."; 17 | 18 | fn print_analysis(paths: &[PathBuf], args: &args::Args) -> Result<(), RhitError> { 19 | let mut log_base = time!("LogBase::new", LogBase::new(paths, args))?; 20 | let printer = md::Printer::new(args, &log_base); 21 | let base = &mut log_base; 22 | let trend_computer = time!("Trend computer initialization", TrendComputer::new(base, args))?; 23 | md::summary::print_summary(base, &printer); 24 | time!("Analysis & Printing", md::print_analysis(&log_base, &printer, trend_computer.as_ref())); 25 | Ok(()) 26 | } 27 | 28 | pub fn run() -> Result<(), RhitError> { 29 | let args = Args::parse(); 30 | debug!("args: {:#?}", &args); 31 | if args.version { 32 | println!("{} {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); 33 | return Ok(()); 34 | } 35 | if args.help { 36 | help::print(); 37 | return Ok(()); 38 | } 39 | let mut paths = args.files.clone(); 40 | if paths.is_empty() { 41 | paths.push(PathBuf::from(DEFAULT_NGINX_LOCATION)); 42 | } 43 | let result = match args.output { 44 | Output::Raw => print_raw_lines(&paths, &args), 45 | Output::Tables => print_analysis(&paths, &args), 46 | Output::Csv => print_csv_lines(&paths, &args), 47 | Output::Json => print_json_lines(&paths, &args), 48 | }; 49 | if let Err(RhitError::PathNotFound(ref path)) = result { 50 | if path == &PathBuf::from(DEFAULT_NGINX_LOCATION) { 51 | eprintln!("{}", MISSING_DEFAULT_MESSAGE); 52 | } 53 | } 54 | log_mem(Level::Info); 55 | result 56 | } 57 | 58 | -------------------------------------------------------------------------------- /src/csv.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::*, 3 | std::{ 4 | path::PathBuf, 5 | }, 6 | }; 7 | 8 | /// A printer writing lines as a CSV file 9 | struct CsvPrinter {} 10 | 11 | impl LineConsumer for CsvPrinter { 12 | fn start_eating( 13 | &mut self, 14 | _first_date: Date, 15 | ) { 16 | println!("date,time,remote address,method,path,status,bytes sent,referer"); 17 | } 18 | fn eat_line( 19 | &mut self, 20 | line: LogLine, 21 | _raw_line: &str, 22 | filtered_out: bool, 23 | ) { 24 | if filtered_out { return; } 25 | println!( 26 | r#"{},{},{},{},"{}",{},{},"{}""#, 27 | line.date(), 28 | line.time(), 29 | line.remote_addr, 30 | line.method, 31 | line.path, 32 | line.status, 33 | line.bytes_sent, 34 | line.referer, 35 | ); 36 | } 37 | } 38 | 39 | pub fn print_csv_lines( 40 | path: &[PathBuf], 41 | args: &args::Args, 42 | ) -> Result<(), RhitError> { 43 | let mut printer = CsvPrinter{}; 44 | let mut file_reader = FileReader::new(path, args, &mut printer)?; 45 | time!("reading files", file_reader.read_all_files())?; 46 | Ok(()) 47 | } 48 | 49 | -------------------------------------------------------------------------------- /src/date.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::ParseDateTimeError, 3 | std::fmt, 4 | }; 5 | 6 | pub static MONTHS_3_LETTERS: &[&str] = &[ 7 | "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" 8 | ]; 9 | 10 | /// a not precise date, only valid in the context 11 | /// of the local set of log files. 12 | /// It's implicitely in the timezone of the log files 13 | /// (assuming all the files have the same one). 14 | /// As nginx didn't exist before JC, a u16 is good enough 15 | /// for the year. 16 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 17 | pub struct Date { 18 | pub year: u16, 19 | pub month: u8, // in [1,12] 20 | pub day: u8, // in [1,31] 21 | } 22 | 23 | impl Date { 24 | pub fn new(year: u16, month: u8, day: u8) -> Result { 25 | if day < 1 || day > 31 { 26 | return Err(ParseDateTimeError::InvalidDay(day)); 27 | } 28 | if month < 1 || month > 12 { 29 | return Err(ParseDateTimeError::InvalidMonth(month)); 30 | } 31 | Ok(Self { year, month, day }) 32 | } 33 | /// parse the date part of a nginx datetime. 34 | /// 35 | /// a datetime in nginx is either in 36 | /// - "common log format", eg `10/Jan/2021:10:27:01 +0000` 37 | /// - ISO 8601, eg `1977-04-22T01:00:00-05:00` 38 | pub fn from_nginx(s: &str) -> Result { 39 | if s.len()<11 { 40 | return Err(ParseDateTimeError::UnexpectedEnd); 41 | } 42 | if let Ok(year) = s[0..4].parse() { 43 | // let's go with ISO 8601 44 | let month = s[5..7].parse()?; 45 | let day = s[8..10].parse()?; 46 | Self::new(year, month, day) 47 | } else { 48 | // maybe common log format ? 49 | let day = s[0..2].parse()?; 50 | let month = &s[3..6]; 51 | let month = MONTHS_3_LETTERS 52 | .iter() 53 | .position(|&m| m == month) 54 | .ok_or_else(|| ParseDateTimeError::UnrecognizedMonth(s.to_owned()))?; 55 | let month = (month + 1) as u8; 56 | let year = s[7..11].parse()?; 57 | Self::new(year, month, day) 58 | } 59 | } 60 | /// parse a numeric date with optionally implicit parts 61 | /// The part separator is the '/' 62 | pub fn with_implicit( 63 | s: &str, 64 | default_year: Option, 65 | default_month: Option, 66 | ) -> Result { 67 | let mut t = s.split('/'); 68 | match (t.next(), t.next(), t.next()) { 69 | (Some(year), Some(month), Some(day)) => { 70 | Date::new(year.parse()?, month.parse()?, day.parse()?) 71 | } 72 | (Some(month), Some(day), None) => { 73 | if let Some(year) = default_year { 74 | Date::new(year, month.parse()?, day.parse()?) 75 | } else { 76 | Err(ParseDateTimeError::AmbiguousDate(s.to_owned())) 77 | } 78 | } 79 | (Some(day), None, None) => { 80 | if let (Some(year), Some(month)) = (default_year, default_month) { 81 | Date::new(year, month, day.parse()?) 82 | } else { 83 | Err(ParseDateTimeError::AmbiguousDate(s.to_owned())) 84 | } 85 | } 86 | _ => unsafe { std::hint::unreachable_unchecked() }, 87 | } 88 | } 89 | } 90 | 91 | impl fmt::Display for Date { 92 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 93 | write!(f, "{}/{:0>2}/{:0>2}", self.year, self.month, self.day) 94 | } 95 | } 96 | 97 | pub fn unique_year_month(start_date: Date, end_date: Date) -> (Option, Option) { 98 | let y1 = start_date.year; 99 | let y2 = end_date.year; 100 | if y1 == y2 { 101 | let m1 = start_date.month; 102 | let m2 = end_date.month; 103 | if m1 == m2 { 104 | (Some(y1), Some(m1)) 105 | } else { 106 | (Some(y1), None) 107 | } 108 | } else { 109 | (None, None) 110 | } 111 | } 112 | 113 | #[cfg(test)] 114 | mod date_parsing_tests { 115 | 116 | use super::*; 117 | 118 | #[test] 119 | fn parse_nginx_date_common_log_format() { 120 | assert_eq!( 121 | Date::from_nginx("10/Jan/2021:10:27:01 +0000").unwrap(), 122 | Date::new(2021, 1, 10).unwrap(), 123 | ); 124 | } 125 | #[test] 126 | fn parse_nginx_date_iso_8601() { 127 | assert_eq!( 128 | Date::from_nginx("1977-04-22T01:00:00-05:00").unwrap(), 129 | Date::new(1977, 4, 22).unwrap(), 130 | ); 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/date_histogram.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::*, 3 | minimad::OwningTemplateExpander, 4 | termimad::*, 5 | }; 6 | 7 | static MD: &str = r#" 8 | |:-:|:-:|:-:|:- 9 | |**date**|**hits**|**bytes**|**${scale}** 10 | |:-|-:|-:|:- 11 | ${bars 12 | |${date}|${hits}|${bytes}|*${bar}* 13 | } 14 | |-: 15 | "#; 16 | 17 | #[derive(Clone)] 18 | pub struct DateBar { 19 | pub date: Date, 20 | pub hits: u64, 21 | pub bytes_sent: u64, 22 | } 23 | 24 | impl DateBar { 25 | pub fn new(date: Date) -> Self { 26 | Self { 27 | date, 28 | hits: 0, 29 | bytes_sent: 0, 30 | } 31 | } 32 | } 33 | 34 | #[derive(Clone, Default)] 35 | pub struct DateHistogram { 36 | pub bars: Vec, 37 | } 38 | 39 | impl DateHistogram { 40 | 41 | pub fn from(base: &LogBase) -> Self { 42 | let mut bars: Vec = base.dates.iter() 43 | .map(|&date| DateBar { date, bytes_sent: 0, hits: 0 }) 44 | .collect(); 45 | for line in &base.lines { 46 | bars[line.date_idx].hits += 1; 47 | bars[line.date_idx].bytes_sent += line.bytes_sent; 48 | } 49 | Self { bars } 50 | } 51 | 52 | pub fn print( 53 | &self, 54 | printer: &md::Printer, 55 | ) { 56 | let mut expander = OwningTemplateExpander::new(); 57 | let max_bar = self.bars 58 | .iter() 59 | .map(|b| if printer.key==Key::Hits { b.hits } else { b.bytes_sent }) 60 | .max().unwrap(); 61 | expander.set( 62 | "scale", 63 | format!("0 {:>4}", file_size::fit_4(max_bar)), 64 | ); 65 | let max_bar = max_bar as f32; 66 | for bar in &self.bars { 67 | if printer.date_filter.map_or(true, |f| f.overlaps(bar.date)) { 68 | let value = if printer.key == Key::Hits { bar.hits } else { bar.bytes_sent }; 69 | let part = (value as f32) / max_bar; 70 | expander.sub("bars") 71 | .set("date", bar.date) 72 | .set_md("hits", printer.md_hits(bar.hits as usize)) 73 | .set_md("bytes", printer.md_bytes(bar.bytes_sent)) 74 | .set("bar", ProgressBar::new(part, 20)); 75 | } 76 | } 77 | printer.print(expander, MD); 78 | } 79 | pub fn total_hits(&self) -> u64 { 80 | self.bars.iter().map(|b| b.hits).sum() 81 | } 82 | pub fn total_bytes_sent(&self) -> u64 { 83 | self.bars.iter().map(|b| b.bytes_sent).sum() 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/date_idx.rs: -------------------------------------------------------------------------------- 1 | 2 | /// A trait for structs which hold the index of a date 3 | pub trait DateIndexed { 4 | fn date_idx(&self) -> usize; 5 | fn bytes(&self) -> u64; 6 | } 7 | -------------------------------------------------------------------------------- /src/date_time.rs: -------------------------------------------------------------------------------- 1 | use { 2 | crate::{ 3 | Date, 4 | MONTHS_3_LETTERS, 5 | Time, 6 | }, 7 | std::{ 8 | fmt, 9 | num::ParseIntError, 10 | }, 11 | thiserror::Error, 12 | }; 13 | 14 | #[derive(Debug, Error)] 15 | pub enum ParseDateTimeError { 16 | 17 | #[error("unexpected end")] 18 | UnexpectedEnd, 19 | 20 | #[error("invalid day {0:?}")] 21 | InvalidDay(u8), 22 | 23 | #[error("date is ambiguous in context {0:?}")] 24 | AmbiguousDate(String), 25 | 26 | #[error("invalid month {0:?}")] 27 | InvalidMonth(u8), 28 | 29 | #[error("unrecognized month {0:?}")] 30 | UnrecognizedMonth(String), 31 | 32 | #[error("invalid hour {0:?}")] 33 | InvalidHour(u8), 34 | 35 | #[error("invalid minute {0:?}")] 36 | InvalidMinute(u8), 37 | 38 | #[error("invalid second {0:?}")] 39 | InvalidSecond(u8), 40 | 41 | #[error("expected int")] 42 | IntExpected(#[from] ParseIntError), 43 | 44 | #[error("expected int")] 45 | IntExpectedInternal, 46 | } 47 | 48 | 49 | /// a date with time. 50 | /// 51 | /// It's implicitely in the timezone of the log files 52 | /// (assuming all the files have the same one). 53 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] 54 | pub struct DateTime { 55 | pub date: Date, 56 | pub time: Time, 57 | } 58 | 59 | impl DateTime { 60 | pub fn new( 61 | year: u16, // in [0-4000] 62 | month: u8, // in [1-12] 63 | day: u8, // in [1-31] 64 | hour: u8, // in [0-23] 65 | minute: u8, 66 | second: u8, 67 | ) -> Result { 68 | Ok(Self { 69 | date: Date::new(year, month, day)?, 70 | time: Time::new(hour, minute, second)?, 71 | }) 72 | } 73 | /// parse the date_time part of a nginx log line 74 | /// 75 | /// a datetime in nginx is either in 76 | /// - "common log format", eg `10/Jan/2021:10:27:01 +0000` 77 | /// - ISO 8601, eg `1977-04-22T01:00:00-05:00` 78 | pub fn from_nginx(s: &str) -> Result { 79 | if s.len()<20 { 80 | return Err(ParseDateTimeError::UnexpectedEnd); 81 | } 82 | if let Ok(year) = s[0..4].parse() { 83 | // let's go with ISO 8601 84 | let month = s[5..7].parse()?; 85 | let day = s[8..10].parse()?; 86 | let hour = s[11..13].parse()?; 87 | let minute = s[14..16].parse()?; 88 | let second = s[17..19].parse()?; 89 | Self::new(year, month, day, hour, minute, second) 90 | } else { 91 | // maybe common log format ? 92 | let day = s[0..2].parse()?; 93 | let month = &s[3..6]; 94 | let month = MONTHS_3_LETTERS 95 | .iter() 96 | .position(|&m| m == month) 97 | .ok_or_else(|| ParseDateTimeError::UnrecognizedMonth(s.to_owned()))?; 98 | let month = (month + 1) as u8; 99 | let year = s[7..11].parse()?; 100 | let hour = s[12..14].parse()?; 101 | let minute = s[15..17].parse()?; 102 | let second = s[18..20].parse()?; 103 | Self::new(year, month, day, hour, minute, second) 104 | } 105 | } 106 | pub fn round_up(date: Date, time: Option