├── .github └── workflows │ ├── ci.yaml │ └── release.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── aliases ├── apache.toml ├── k8s-ingress-nginx.toml ├── multi-operator.toml └── nginx.toml ├── benches ├── 10k.inp ├── 1k.inp ├── 1m.inp ├── e2e.rs └── parse.rs ├── ci ├── build.bash ├── common.bash ├── set_rust_version.bash └── test.bash ├── pkg └── brew │ └── agrind-bin.rb ├── screen_shots ├── count.gif ├── filter.gif ├── json.gif ├── overview.gif └── parse.gif ├── src ├── alias.rs ├── bin │ └── agrind.rs ├── data.rs ├── errors.rs ├── filter.rs ├── funcs.rs ├── lang.rs ├── lib.rs ├── operator.rs ├── operator │ ├── average.rs │ ├── count.rs │ ├── count_distinct.rs │ ├── expr.rs │ ├── fields.rs │ ├── limit.rs │ ├── max.rs │ ├── min.rs │ ├── parse.rs │ ├── percentile.rs │ ├── sort.rs │ ├── split.rs │ ├── sum.rs │ ├── timeslice.rs │ ├── total.rs │ └── where_op.rs ├── printer.rs ├── render.rs └── typecheck.rs ├── test_files ├── angle_grinder_releases.json ├── basic ├── binary_data.bin ├── empty ├── filter_test.log ├── gen_logs.py ├── long_lines.log ├── multiline ├── test_json.log ├── test_logfmt.log ├── test_nested_logfmt.log ├── test_parse.log └── test_partial_json.log └── tests ├── code_blocks.rs ├── integration.rs └── structured_tests ├── agg_of_agg.toml ├── aliases ├── alias_with_failure.toml ├── apache.toml ├── k8s-ingress_nginx.toml ├── multi-operator.toml └── nginx.toml ├── arrays_1.toml ├── count_conditional.toml ├── count_distinct.toml ├── count_distinct_error.toml ├── count_distinct_error_2.toml ├── escaped_ident.toml ├── field_expr-1.toml ├── fields_after_agg.toml ├── fields_except.toml ├── filters.toml ├── func_arg_error_1.toml ├── func_arg_error_2.toml ├── if-1.toml ├── if-2.toml ├── json_from.toml ├── json_output.toml ├── json_output_sorted.toml ├── limit.toml ├── limit_agg.toml ├── limit_agg_tail.toml ├── limit_error.toml ├── limit_error_2.toml ├── limit_tail.toml ├── logfmt.toml ├── logfmt_output.toml ├── logfmt_output_with_sort.toml ├── logical-expr-1.toml ├── logical-expr-2.toml ├── longlines.toml ├── math_funcs_1.toml ├── min_max.toml ├── min_max_none.toml ├── min_max_rounded.toml ├── missing_optional_match.toml ├── nested_values_1.toml ├── nested_values_2.toml ├── nested_values_3.toml ├── not_an_agg.toml ├── not_an_agg_2.toml ├── parseDate_1.toml ├── parseDate_2.toml ├── parse_drop.toml ├── parse_error_double_from.toml ├── parse_error_missing_field_name.toml ├── parse_error_missing_operand.toml ├── parse_error_unterminated.toml ├── parse_error_unterminated_sq.toml ├── parse_literal_tab.toml ├── parse_nodrop.toml ├── parse_plain.toml ├── parse_plain_no_convert.toml ├── parse_regex.toml ├── percentile_1.toml ├── sort_by_expr.toml ├── sort_order.toml ├── split_1.toml ├── split_10.toml ├── split_2.toml ├── split_3.toml ├── split_4.toml ├── split_5.toml ├── split_6.toml ├── split_7.toml ├── split_8.toml ├── split_9.toml ├── string_funcs_1.toml ├── string_funcs_2.toml ├── sum.toml ├── timeslice_1.toml ├── timeslice_2.toml ├── total.toml ├── total_agg.toml ├── where-1.toml ├── where-10.toml ├── where-11.toml ├── where-12.toml ├── where-2.toml ├── where-3.toml ├── where-4.toml ├── where-5.toml ├── where-6.toml ├── where-7.toml ├── where-8.toml └── where-9.toml /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | windows: 7 | runs-on: windows-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v4 11 | - name: Build binary 12 | uses: houseabsolute/actions-rust-cross@v1 13 | with: 14 | command: build 15 | target: x86_64-pc-windows-msvc 16 | args: "--locked" 17 | strip: true 18 | 19 | macos: 20 | runs-on: macos-latest 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | - name: Build and test binary 25 | uses: houseabsolute/actions-rust-cross@v1 26 | with: 27 | command: both # build and test 28 | target: x86_64-apple-darwin 29 | args: "--locked" 30 | strip: true 31 | 32 | linux: 33 | runs-on: ubuntu-24.04 34 | strategy: 35 | fail-fast: true 36 | matrix: 37 | toolchain: [stable, beta] 38 | target: 39 | - x86_64-unknown-linux-gnu 40 | - x86_64-unknown-linux-musl 41 | # Uncomment if you need these targets 42 | # - aarch64-unknown-linux-gnu 43 | # - aarch64-unknown-linux-musl 44 | # - x86_64-unknown-netbsd 45 | steps: 46 | - name: Checkout 47 | uses: actions/checkout@v4 48 | - name: Build binary 49 | uses: houseabsolute/actions-rust-cross@v1 50 | with: 51 | command: build 52 | target: ${{ matrix.target }} 53 | toolchain: ${{ matrix.toolchain }} 54 | args: "--locked" 55 | strip: true 56 | - name: Test binary 57 | uses: houseabsolute/actions-rust-cross@v1 58 | with: 59 | command: test 60 | target: ${{ matrix.target }} 61 | toolchain: ${{ matrix.toolchain }} 62 | args: "--locked" 63 | # Skip testing for specific targets that have issues 64 | if: | 65 | !contains(matrix.target, 'android') && 66 | !contains(matrix.target, 'bsd') && 67 | !contains(matrix.target, 'solaris') && 68 | matrix.target != 'armv5te-unknown-linux-musleabi' && 69 | matrix.target != 'sparc64-unknown-linux-gnu' 70 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 7 | workflow_dispatch: 8 | 9 | env: 10 | BIN: agrind 11 | 12 | jobs: 13 | release: 14 | name: Release - ${{ matrix.platform.name }} 15 | strategy: 16 | matrix: 17 | platform: 18 | - name: Linux-x86_64-gnu 19 | runs-on: ubuntu-24.04 20 | target: x86_64-unknown-linux-gnu 21 | 22 | - name: Linux-x86_64-musl 23 | runs-on: ubuntu-24.04 24 | target: x86_64-unknown-linux-musl 25 | 26 | - name: macOS-x86_64 27 | runs-on: macos-latest 28 | target: x86_64-apple-darwin 29 | 30 | # Uncomment and add additional targets as needed 31 | # - name: Windows-x86_64 32 | # runs-on: windows-latest 33 | # target: x86_64-pc-windows-msvc 34 | 35 | # - name: Linux-aarch64 36 | # runs-on: ubuntu-24.04 37 | # target: aarch64-unknown-linux-musl 38 | 39 | runs-on: ${{ matrix.platform.runs-on }} 40 | steps: 41 | - name: Checkout 42 | uses: actions/checkout@v4 43 | 44 | - name: Build binary 45 | uses: houseabsolute/actions-rust-cross@v1 46 | with: 47 | command: build 48 | target: ${{ matrix.platform.target }} 49 | args: "--locked --release" 50 | strip: true 51 | 52 | - name: Prepare assets 53 | shell: bash 54 | run: | 55 | cd target/${{ matrix.platform.target }}/release 56 | if [[ "${{ matrix.platform.target }}" == *"windows"* ]]; then 57 | 7z a ../../../${{ env.BIN }}-${{ matrix.platform.target }}.zip ${{ env.BIN }}.exe 58 | else 59 | tar -czf ../../../${{ env.BIN }}-${{ matrix.platform.target }}.tar.gz ${{ env.BIN }} 60 | fi 61 | 62 | - name: Upload Release 63 | uses: softprops/action-gh-release@v2 64 | with: 65 | files: | 66 | ${{ env.BIN }}-${{ matrix.platform.target }}.* 67 | generate_release_notes: true 68 | fail_on_unmatched_files: true 69 | env: 70 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target/ 3 | **/*.rs.bk 4 | .idea/ 5 | angle-grinder.iml 6 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ag" 3 | version = "0.19.6" 4 | authors = ["Russell Cohen "] 5 | description = "CLI App to slice and dice logfiles" 6 | license = "MIT" 7 | repository = "https://github.com/rcoh/angle-grinder" 8 | readme = "README.md" 9 | edition = "2018" 10 | include = ["src/**/*", "benches/**", "LICENSE", "README.md", "aliases/*"] 11 | [package.metadata.deb] 12 | extended-description = """Angle-grinder allows you to parse, aggregate, sum, average, percentile, and sort your data. \ 13 | You can see it, live-updating, in your terminal. Angle grinder is designed for when, for \ 14 | whatever reason, you don't have your data in graphite/honeycomb/kibana/sumologic/splunk/etc. \ 15 | but still want to be able to do sophisticated analytics. \ 16 | Angle grinder can process about a million rows per second, so it's usable for fairly meaty \ 17 | aggregation. The results will live update in your terminal as data is processed. \ 18 | Angle grinder is a bare bones functional programming language coupled with a pretty terminal UI.""" 19 | 20 | [features] 21 | default = [] 22 | self-update = ["self_update"] 23 | 24 | [target.'cfg(not(target_env = "msvc"))'.dependencies] 25 | tikv-jemallocator = "0.6" 26 | 27 | [dependencies] 28 | serde_json = "1.0.33" 29 | itertools = "0.14" 30 | nom = "7.1.1" 31 | nom_locate = "4.0.0" 32 | nom-supreme = "0.8.0" 33 | strsim = "0.11" 34 | regex = "1.5.5" 35 | terminal_size = "0.4" 36 | quantiles = "0.7.1" 37 | crossbeam-channel = "0.5.15" 38 | ordered-float = "5" 39 | thiserror = "2" 40 | anyhow = "1" 41 | human-panic = "2" 42 | self_update = { version = "0.32.0", features = ["rustls"], default-features = false, optional = true } 43 | annotate-snippets = { version = "0.9.0", features = ["color"] } 44 | lazy_static = "1.2.0" 45 | im = "15.1.0" 46 | logfmt = "0.0.2" 47 | strfmt = "0.2.2" 48 | include_dir = "0.7.3" 49 | toml = "0.8" 50 | serde = { version = "1.0", features = ["derive"] } 51 | chrono = "0.4" 52 | dtparse = "2" 53 | clap = { version = "4.0.18", features = ["derive"] } 54 | 55 | [dev-dependencies] 56 | assert_cmd = "2.0.5" 57 | cool_asserts = "2.0.3" 58 | expect-test = "1.1.0" 59 | predicates = "2.1.1" 60 | pulldown-cmark = "0.13.0" 61 | criterion = "0.5" 62 | maplit = "1.0.1" 63 | test-generator = "0.3.0" 64 | [dev-dependencies.cargo-husky] 65 | version = "1" 66 | default-features = true 67 | features = ["run-cargo-fmt", "precommit-hook"] 68 | 69 | [[bench]] 70 | name = "e2e" 71 | harness = false 72 | 73 | [[bench]] 74 | name = "parse" 75 | harness = false 76 | 77 | [profile.release] 78 | debug = true 79 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Russell Cohen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /aliases/apache.toml: -------------------------------------------------------------------------------- 1 | keyword = "apache" 2 | template = """ 3 | parse "* - * [*] \\"* * *\\" * *" as ip, name, timestamp, method, url, protocol, status, contentlength 4 | """ 5 | -------------------------------------------------------------------------------- /aliases/k8s-ingress-nginx.toml: -------------------------------------------------------------------------------- 1 | keyword = "k8singressnginx" 2 | template = """ 3 | parse "* - * [*] \\"* * *\\" * * \\"*\\" \\"*\\" * * [*] [*] * * * * *" as remote_addr, remote_user, timestamp, method, url, protocol, status, body_bytes_sent, http_referer, http_user_agent, request_length, request_time, proxy_upstream_name, proxy_alternative_upstream_name, upstream_addr, upstream_response_length, upstream_response_time, upstream_status, req_id 4 | """ 5 | -------------------------------------------------------------------------------- /aliases/multi-operator.toml: -------------------------------------------------------------------------------- 1 | keyword = "testmultioperator" 2 | template = """ 3 | json | count 4 | """ 5 | -------------------------------------------------------------------------------- /aliases/nginx.toml: -------------------------------------------------------------------------------- 1 | keyword = "nginx" 2 | template = """ 3 | parse "* - * [*] \\"* * *\\" * * \\"*\\" \\"*\\" \\"*\\"" as addr, user, timestamp, method, url, protocol, status, bytes_sent, http_referer, http_user_agent, gzip_ratio 4 | """ 5 | -------------------------------------------------------------------------------- /benches/1k.inp: -------------------------------------------------------------------------------- 1 | y 2 | y 3 | y 4 | y 5 | y 6 | y 7 | y 8 | y 9 | y 10 | y 11 | y 12 | y 13 | y 14 | y 15 | y 16 | y 17 | y 18 | y 19 | y 20 | y 21 | y 22 | y 23 | y 24 | y 25 | y 26 | y 27 | y 28 | y 29 | y 30 | y 31 | y 32 | y 33 | y 34 | y 35 | y 36 | y 37 | y 38 | y 39 | y 40 | y 41 | y 42 | y 43 | y 44 | y 45 | y 46 | y 47 | y 48 | y 49 | y 50 | y 51 | y 52 | y 53 | y 54 | y 55 | y 56 | y 57 | y 58 | y 59 | y 60 | y 61 | y 62 | y 63 | y 64 | y 65 | y 66 | y 67 | y 68 | y 69 | y 70 | y 71 | y 72 | y 73 | y 74 | y 75 | y 76 | y 77 | y 78 | y 79 | y 80 | y 81 | y 82 | y 83 | y 84 | y 85 | y 86 | y 87 | y 88 | y 89 | y 90 | y 91 | y 92 | y 93 | y 94 | y 95 | y 96 | y 97 | y 98 | y 99 | y 100 | y 101 | y 102 | y 103 | y 104 | y 105 | y 106 | y 107 | y 108 | y 109 | y 110 | y 111 | y 112 | y 113 | y 114 | y 115 | y 116 | y 117 | y 118 | y 119 | y 120 | y 121 | y 122 | y 123 | y 124 | y 125 | y 126 | y 127 | y 128 | y 129 | y 130 | y 131 | y 132 | y 133 | y 134 | y 135 | y 136 | y 137 | y 138 | y 139 | y 140 | y 141 | y 142 | y 143 | y 144 | y 145 | y 146 | y 147 | y 148 | y 149 | y 150 | y 151 | y 152 | y 153 | y 154 | y 155 | y 156 | y 157 | y 158 | y 159 | y 160 | y 161 | y 162 | y 163 | y 164 | y 165 | y 166 | y 167 | y 168 | y 169 | y 170 | y 171 | y 172 | y 173 | y 174 | y 175 | y 176 | y 177 | y 178 | y 179 | y 180 | y 181 | y 182 | y 183 | y 184 | y 185 | y 186 | y 187 | y 188 | y 189 | y 190 | y 191 | y 192 | y 193 | y 194 | y 195 | y 196 | y 197 | y 198 | y 199 | y 200 | y 201 | y 202 | y 203 | y 204 | y 205 | y 206 | y 207 | y 208 | y 209 | y 210 | y 211 | y 212 | y 213 | y 214 | y 215 | y 216 | y 217 | y 218 | y 219 | y 220 | y 221 | y 222 | y 223 | y 224 | y 225 | y 226 | y 227 | y 228 | y 229 | y 230 | y 231 | y 232 | y 233 | y 234 | y 235 | y 236 | y 237 | y 238 | y 239 | y 240 | y 241 | y 242 | y 243 | y 244 | y 245 | y 246 | y 247 | y 248 | y 249 | y 250 | y 251 | y 252 | y 253 | y 254 | y 255 | y 256 | y 257 | y 258 | y 259 | y 260 | y 261 | y 262 | y 263 | y 264 | y 265 | y 266 | y 267 | y 268 | y 269 | y 270 | y 271 | y 272 | y 273 | y 274 | y 275 | y 276 | y 277 | y 278 | y 279 | y 280 | y 281 | y 282 | y 283 | y 284 | y 285 | y 286 | y 287 | y 288 | y 289 | y 290 | y 291 | y 292 | y 293 | y 294 | y 295 | y 296 | y 297 | y 298 | y 299 | y 300 | y 301 | y 302 | y 303 | y 304 | y 305 | y 306 | y 307 | y 308 | y 309 | y 310 | y 311 | y 312 | y 313 | y 314 | y 315 | y 316 | y 317 | y 318 | y 319 | y 320 | y 321 | y 322 | y 323 | y 324 | y 325 | y 326 | y 327 | y 328 | y 329 | y 330 | y 331 | y 332 | y 333 | y 334 | y 335 | y 336 | y 337 | y 338 | y 339 | y 340 | y 341 | y 342 | y 343 | y 344 | y 345 | y 346 | y 347 | y 348 | y 349 | y 350 | y 351 | y 352 | y 353 | y 354 | y 355 | y 356 | y 357 | y 358 | y 359 | y 360 | y 361 | y 362 | y 363 | y 364 | y 365 | y 366 | y 367 | y 368 | y 369 | y 370 | y 371 | y 372 | y 373 | y 374 | y 375 | y 376 | y 377 | y 378 | y 379 | y 380 | y 381 | y 382 | y 383 | y 384 | y 385 | y 386 | y 387 | y 388 | y 389 | y 390 | y 391 | y 392 | y 393 | y 394 | y 395 | y 396 | y 397 | y 398 | y 399 | y 400 | y 401 | y 402 | y 403 | y 404 | y 405 | y 406 | y 407 | y 408 | y 409 | y 410 | y 411 | y 412 | y 413 | y 414 | y 415 | y 416 | y 417 | y 418 | y 419 | y 420 | y 421 | y 422 | y 423 | y 424 | y 425 | y 426 | y 427 | y 428 | y 429 | y 430 | y 431 | y 432 | y 433 | y 434 | y 435 | y 436 | y 437 | y 438 | y 439 | y 440 | y 441 | y 442 | y 443 | y 444 | y 445 | y 446 | y 447 | y 448 | y 449 | y 450 | y 451 | y 452 | y 453 | y 454 | y 455 | y 456 | y 457 | y 458 | y 459 | y 460 | y 461 | y 462 | y 463 | y 464 | y 465 | y 466 | y 467 | y 468 | y 469 | y 470 | y 471 | y 472 | y 473 | y 474 | y 475 | y 476 | y 477 | y 478 | y 479 | y 480 | y 481 | y 482 | y 483 | y 484 | y 485 | y 486 | y 487 | y 488 | y 489 | y 490 | y 491 | y 492 | y 493 | y 494 | y 495 | y 496 | y 497 | y 498 | y 499 | y 500 | y 501 | y 502 | y 503 | y 504 | y 505 | y 506 | y 507 | y 508 | y 509 | y 510 | y 511 | y 512 | y 513 | y 514 | y 515 | y 516 | y 517 | y 518 | y 519 | y 520 | y 521 | y 522 | y 523 | y 524 | y 525 | y 526 | y 527 | y 528 | y 529 | y 530 | y 531 | y 532 | y 533 | y 534 | y 535 | y 536 | y 537 | y 538 | y 539 | y 540 | y 541 | y 542 | y 543 | y 544 | y 545 | y 546 | y 547 | y 548 | y 549 | y 550 | y 551 | y 552 | y 553 | y 554 | y 555 | y 556 | y 557 | y 558 | y 559 | y 560 | y 561 | y 562 | y 563 | y 564 | y 565 | y 566 | y 567 | y 568 | y 569 | y 570 | y 571 | y 572 | y 573 | y 574 | y 575 | y 576 | y 577 | y 578 | y 579 | y 580 | y 581 | y 582 | y 583 | y 584 | y 585 | y 586 | y 587 | y 588 | y 589 | y 590 | y 591 | y 592 | y 593 | y 594 | y 595 | y 596 | y 597 | y 598 | y 599 | y 600 | y 601 | y 602 | y 603 | y 604 | y 605 | y 606 | y 607 | y 608 | y 609 | y 610 | y 611 | y 612 | y 613 | y 614 | y 615 | y 616 | y 617 | y 618 | y 619 | y 620 | y 621 | y 622 | y 623 | y 624 | y 625 | y 626 | y 627 | y 628 | y 629 | y 630 | y 631 | y 632 | y 633 | y 634 | y 635 | y 636 | y 637 | y 638 | y 639 | y 640 | y 641 | y 642 | y 643 | y 644 | y 645 | y 646 | y 647 | y 648 | y 649 | y 650 | y 651 | y 652 | y 653 | y 654 | y 655 | y 656 | y 657 | y 658 | y 659 | y 660 | y 661 | y 662 | y 663 | y 664 | y 665 | y 666 | y 667 | y 668 | y 669 | y 670 | y 671 | y 672 | y 673 | y 674 | y 675 | y 676 | y 677 | y 678 | y 679 | y 680 | y 681 | y 682 | y 683 | y 684 | y 685 | y 686 | y 687 | y 688 | y 689 | y 690 | y 691 | y 692 | y 693 | y 694 | y 695 | y 696 | y 697 | y 698 | y 699 | y 700 | y 701 | y 702 | y 703 | y 704 | y 705 | y 706 | y 707 | y 708 | y 709 | y 710 | y 711 | y 712 | y 713 | y 714 | y 715 | y 716 | y 717 | y 718 | y 719 | y 720 | y 721 | y 722 | y 723 | y 724 | y 725 | y 726 | y 727 | y 728 | y 729 | y 730 | y 731 | y 732 | y 733 | y 734 | y 735 | y 736 | y 737 | y 738 | y 739 | y 740 | y 741 | y 742 | y 743 | y 744 | y 745 | y 746 | y 747 | y 748 | y 749 | y 750 | y 751 | y 752 | y 753 | y 754 | y 755 | y 756 | y 757 | y 758 | y 759 | y 760 | y 761 | y 762 | y 763 | y 764 | y 765 | y 766 | y 767 | y 768 | y 769 | y 770 | y 771 | y 772 | y 773 | y 774 | y 775 | y 776 | y 777 | y 778 | y 779 | y 780 | y 781 | y 782 | y 783 | y 784 | y 785 | y 786 | y 787 | y 788 | y 789 | y 790 | y 791 | y 792 | y 793 | y 794 | y 795 | y 796 | y 797 | y 798 | y 799 | y 800 | y 801 | y 802 | y 803 | y 804 | y 805 | y 806 | y 807 | y 808 | y 809 | y 810 | y 811 | y 812 | y 813 | y 814 | y 815 | y 816 | y 817 | y 818 | y 819 | y 820 | y 821 | y 822 | y 823 | y 824 | y 825 | y 826 | y 827 | y 828 | y 829 | y 830 | y 831 | y 832 | y 833 | y 834 | y 835 | y 836 | y 837 | y 838 | y 839 | y 840 | y 841 | y 842 | y 843 | y 844 | y 845 | y 846 | y 847 | y 848 | y 849 | y 850 | y 851 | y 852 | y 853 | y 854 | y 855 | y 856 | y 857 | y 858 | y 859 | y 860 | y 861 | y 862 | y 863 | y 864 | y 865 | y 866 | y 867 | y 868 | y 869 | y 870 | y 871 | y 872 | y 873 | y 874 | y 875 | y 876 | y 877 | y 878 | y 879 | y 880 | y 881 | y 882 | y 883 | y 884 | y 885 | y 886 | y 887 | y 888 | y 889 | y 890 | y 891 | y 892 | y 893 | y 894 | y 895 | y 896 | y 897 | y 898 | y 899 | y 900 | y 901 | y 902 | y 903 | y 904 | y 905 | y 906 | y 907 | y 908 | y 909 | y 910 | y 911 | y 912 | y 913 | y 914 | y 915 | y 916 | y 917 | y 918 | y 919 | y 920 | y 921 | y 922 | y 923 | y 924 | y 925 | y 926 | y 927 | y 928 | y 929 | y 930 | y 931 | y 932 | y 933 | y 934 | y 935 | y 936 | y 937 | y 938 | y 939 | y 940 | y 941 | y 942 | y 943 | y 944 | y 945 | y 946 | y 947 | y 948 | y 949 | y 950 | y 951 | y 952 | y 953 | y 954 | y 955 | y 956 | y 957 | y 958 | y 959 | y 960 | y 961 | y 962 | y 963 | y 964 | y 965 | y 966 | y 967 | y 968 | y 969 | y 970 | y 971 | y 972 | y 973 | y 974 | y 975 | y 976 | y 977 | y 978 | y 979 | y 980 | y 981 | y 982 | y 983 | y 984 | y 985 | y 986 | y 987 | y 988 | y 989 | y 990 | y 991 | y 992 | y 993 | y 994 | y 995 | y 996 | y 997 | y 998 | y 999 | y 1000 | y 1001 | -------------------------------------------------------------------------------- /benches/e2e.rs: -------------------------------------------------------------------------------- 1 | use ag::alias::AliasCollection; 2 | use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer}; 3 | use annotate_snippets::snippet::Snippet; 4 | use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; 5 | use std::fs::File; 6 | use std::io::{BufRead, BufReader, Write}; 7 | use std::time::Duration; 8 | 9 | /// An ErrorReporter that writes errors related to the query string to the terminal 10 | struct NopErrorReporter {} 11 | 12 | impl ErrorReporter for NopErrorReporter { 13 | fn handle_error(&self, _snippet: Snippet) {} 14 | } 15 | 16 | struct NopWriter {} 17 | 18 | impl Write for NopWriter { 19 | fn write(&mut self, buf: &[u8]) -> Result { 20 | Ok(buf.len()) 21 | } 22 | 23 | fn flush(&mut self) -> Result<(), std::io::Error> { 24 | Ok(()) 25 | } 26 | } 27 | 28 | struct E2eTest { 29 | name: String, 30 | query: String, 31 | file: String, 32 | } 33 | 34 | pub fn criterion_benchmark(c: &mut Criterion) { 35 | let tests = vec![ 36 | E2eTest { 37 | name: "star".to_owned(), 38 | query: "*".to_owned(), 39 | file: "benches/10k.inp".to_owned(), 40 | }, 41 | E2eTest { 42 | name: "star-parse-count".to_owned(), 43 | query: "* | parse '*' as k | count by k".to_owned(), 44 | file: "benches/10k.inp".to_owned(), 45 | }, 46 | E2eTest { 47 | name: "star-count-parse-count".to_owned(), 48 | query: "* | count | parse '*' as k | count by k".to_owned(), 49 | file: "benches/1k.inp".to_owned(), 50 | }, 51 | ]; 52 | tests.into_iter().for_each(|test| { 53 | let query_container = QueryContainer::new_with_aliases( 54 | test.query, 55 | Box::new(NopErrorReporter {}), 56 | AliasCollection::default(), 57 | ); 58 | let mut group = c.benchmark_group("e2e_query"); 59 | let num_elems = BufReader::new(File::open(&test.file).unwrap()) 60 | .lines() 61 | .count(); 62 | group.measurement_time(Duration::from_secs(25)); 63 | group.throughput(Throughput::Elements(num_elems as u64)); 64 | 65 | let name = test.name; 66 | let file = &test.file; 67 | group.bench_function(name, |b| { 68 | b.iter(|| { 69 | let pipeline = 70 | Pipeline::new(&query_container, NopWriter {}, OutputMode::Legacy).unwrap(); 71 | let f = File::open(file).unwrap(); 72 | pipeline.process(black_box(BufReader::new(f))) 73 | }) 74 | }); 75 | group.finish(); 76 | }) 77 | } 78 | 79 | criterion_group!(benches, criterion_benchmark); 80 | criterion_main!(benches); 81 | -------------------------------------------------------------------------------- /benches/parse.rs: -------------------------------------------------------------------------------- 1 | use ag::data::Record; 2 | use ag::lang::Keyword; 3 | use ag::operator::UnaryPreAggFunction; 4 | 5 | use ag::operator::parse::{Parse, ParseOptions}; 6 | use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; 7 | 8 | pub fn criterion_benchmark(c: &mut Criterion) { 9 | let parser = Parse::new( 10 | Keyword::new_wildcard("IP * > \"*\": * length *".to_string()).to_regex(), 11 | vec![ 12 | "sender".to_string(), 13 | "recip".to_string(), 14 | "ignore".to_string(), 15 | "length".to_string(), 16 | ], 17 | None, 18 | ParseOptions { 19 | drop_nonmatching: true, 20 | no_conversion: false, 21 | }, 22 | ); 23 | let mut group = c.benchmark_group("parse_operator"); 24 | group.throughput(Throughput::Elements(1)); 25 | group.bench_function("ip query", |b| { 26 | b.iter_batched( 27 | || { 28 | Record::new( 29 | "17:12:14.214111 IP 10.0.2.243.53938 > \"taotie.canonical.com.http\": \ 30 | Flags [.], ack 56575, win 2375, options [nop,nop,TS val 13651369 ecr 169698010], \ 31 | length 99", 32 | ) 33 | }, 34 | |rec| parser.process(rec), 35 | BatchSize::SmallInput, 36 | ) 37 | }); 38 | group.finish(); 39 | } 40 | 41 | criterion_group!(benches, criterion_benchmark); 42 | criterion_main!(benches); 43 | -------------------------------------------------------------------------------- /ci/build.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Script for building your rust projects. 3 | set -e 4 | 5 | source ci/common.bash 6 | 7 | # $1 {path} = Path to cross/cargo executable 8 | CROSS=$1 9 | # $1 {string} = e.g. x86_64-pc-windows-msvc 10 | TARGET_TRIPLE=$2 11 | # $3 {boolean} = Are we building for deployment? 12 | RELEASE_BUILD=$3 13 | 14 | required_arg $CROSS 'CROSS' 15 | required_arg $TARGET_TRIPLE '' 16 | 17 | if [ -z "$RELEASE_BUILD" ]; then 18 | $CROSS build --target $TARGET_TRIPLE 19 | $CROSS build --target $TARGET_TRIPLE --all-features 20 | else 21 | $CROSS build --target $TARGET_TRIPLE --all-features --release 22 | fi 23 | 24 | -------------------------------------------------------------------------------- /ci/common.bash: -------------------------------------------------------------------------------- 1 | required_arg() { 2 | if [ -z "$1" ]; then 3 | echo "Required argument $2 missing" 4 | exit 1 5 | fi 6 | } 7 | -------------------------------------------------------------------------------- /ci/set_rust_version.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | rustup default $1 4 | rustup target add $2 5 | -------------------------------------------------------------------------------- /ci/test.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Script for building your rust projects. 3 | set -e 4 | 5 | source ci/common.bash 6 | 7 | # $1 {path} = Path to cross/cargo executable 8 | CROSS=$1 9 | # $1 {string} = 10 | TARGET_TRIPLE=$2 11 | 12 | required_arg $CROSS 'CROSS' 13 | required_arg $TARGET_TRIPLE '' 14 | 15 | $CROSS test --target $TARGET_TRIPLE 16 | $CROSS test --target $TARGET_TRIPLE --all-features 17 | -------------------------------------------------------------------------------- /pkg/brew/agrind-bin.rb: -------------------------------------------------------------------------------- 1 | class AgrindBin < Formula 2 | version 'v0.7.3' 3 | desc "Slice and dice log files on the command-line" 4 | homepage "https://github.com/rcoh/angle-grinder" 5 | 6 | if OS.mac? 7 | url "https://github.com/rcoh/angle-grinder/releases/download/#{version}/angle_grinder-#{version}-x86_64-apple-darwin.tar.gz" 8 | sha256 "d0682656294bc5d764d4110ae93add1442f6420bff49dcc4d49cb635950014fb" 9 | elsif OS.linux? 10 | url "https://github.com/rcoh/angle-grinder/releases/download/#{version}/angle_grinder-#{version}-x86_64-unknown-linux-musl.tar.gz" 11 | sha256 "6dc5438443e653e5d893c6858d0f9279205728314292636c7b5b18706a9aa759" 12 | end 13 | 14 | 15 | def install 16 | bin.install "agrind" 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /screen_shots/count.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/screen_shots/count.gif -------------------------------------------------------------------------------- /screen_shots/filter.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/screen_shots/filter.gif -------------------------------------------------------------------------------- /screen_shots/json.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/screen_shots/json.gif -------------------------------------------------------------------------------- /screen_shots/overview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/screen_shots/overview.gif -------------------------------------------------------------------------------- /screen_shots/parse.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/screen_shots/parse.gif -------------------------------------------------------------------------------- /src/alias.rs: -------------------------------------------------------------------------------- 1 | //! Instructions on adding a new alias: 2 | //! 1. Create a new file for the alias in `aliases`. 3 | //! 1a. The filename is the string to be replaced. 4 | //! 1b. The string inside the file is the replacement. 5 | //! 2. Create a new test config inside `tests/structured_tests/aliases`. 6 | //! 3. Add the test config to the `test_aliases()` test. 7 | 8 | use std::borrow::Cow; 9 | use std::path::{Path, PathBuf}; 10 | 11 | use lazy_static::lazy_static; 12 | 13 | use crate::errors::{QueryContainer, TermErrorReporter}; 14 | use crate::lang::{pipeline_template, Operator}; 15 | use include_dir::Dir; 16 | use serde::Deserialize; 17 | 18 | const ALIASES_DIR: Dir = include_dir!("aliases"); 19 | 20 | lazy_static! { 21 | pub static ref LOADED_ALIASES: Vec = ALIASES_DIR 22 | .files() 23 | .map(|file| { 24 | parse_alias( 25 | file.contents_utf8().expect("invalid utf-8"), 26 | file.path(), 27 | &[], 28 | ) 29 | .expect("invalid toml") 30 | }) 31 | .collect(); 32 | pub static ref LOADED_KEYWORDS: Vec<&'static str> = 33 | LOADED_ALIASES.iter().map(|a| a.keyword.as_str()).collect(); 34 | } 35 | 36 | #[derive(Debug)] 37 | pub struct InvalidAliasError { 38 | pub path: PathBuf, 39 | pub cause: anyhow::Error, 40 | pub keyword: Option, 41 | pub contents: Option, 42 | } 43 | 44 | fn parse_alias( 45 | contents: &str, 46 | path: &Path, 47 | aliases: &[AliasPipeline], 48 | ) -> Result { 49 | let config: AliasConfig = toml::from_str(contents).map_err(|err| InvalidAliasError { 50 | path: path.to_owned(), 51 | cause: err.into(), 52 | keyword: None, 53 | contents: Some(contents.to_string()), 54 | })?; 55 | let reporter = Box::new(TermErrorReporter {}); 56 | let aliases = AliasCollection { 57 | aliases: Cow::Borrowed(aliases), 58 | }; 59 | let qc = QueryContainer::new_with_aliases(config.template, reporter, aliases); 60 | let keyword = config.keyword; 61 | let pipeline = pipeline_template(&qc).map_err(|err| InvalidAliasError { 62 | path: path.to_owned(), 63 | cause: err.into(), 64 | keyword: Some(keyword.clone()), 65 | contents: Some(contents.to_string()), 66 | })?; 67 | 68 | Ok(AliasPipeline { keyword, pipeline }) 69 | } 70 | 71 | #[derive(Debug, Deserialize, PartialEq, Eq)] 72 | pub struct AliasConfig { 73 | keyword: String, 74 | template: String, 75 | } 76 | 77 | #[derive(Debug, Clone)] 78 | pub struct AliasPipeline { 79 | keyword: String, 80 | pipeline: Vec, 81 | } 82 | 83 | #[derive(Default)] 84 | pub struct AliasCollection<'a> { 85 | aliases: Cow<'a, [AliasPipeline]>, 86 | } 87 | 88 | #[derive(Default)] 89 | struct AliasAccum { 90 | valid_aliases: Vec, 91 | invalid_aliases: Vec, 92 | } 93 | 94 | impl AliasCollection<'_> { 95 | pub fn get_alias(&self, name: &str) -> Option<&AliasPipeline> { 96 | self.aliases 97 | .iter() 98 | .find(|alias| alias.keyword == name) 99 | .or_else(|| AliasPipeline::matching_string(name)) 100 | } 101 | 102 | pub fn valid_aliases(&self) -> impl Iterator { 103 | self.aliases.iter().map(|a| a.keyword.as_str()) 104 | } 105 | } 106 | 107 | impl AliasCollection<'static> { 108 | pub fn load_aliases_ancestors( 109 | path: Option, 110 | ) -> anyhow::Result<(AliasCollection<'static>, Vec)> { 111 | let path = match path { 112 | Some(path) => path, 113 | None => std::env::current_dir()?, 114 | }; 115 | let (valid, invalid) = find_all_aliases(path)?; 116 | Ok(( 117 | AliasCollection { 118 | aliases: Cow::Owned(valid), 119 | }, 120 | invalid, 121 | )) 122 | } 123 | 124 | pub fn load_aliases_from_dir( 125 | path: &Path, 126 | ) -> anyhow::Result<(AliasCollection<'static>, Vec)> { 127 | let mut aliases = AliasAccum::default(); 128 | aliases_from_dir(path, &mut aliases)?; 129 | Ok(( 130 | AliasCollection { 131 | aliases: Cow::Owned(aliases.valid_aliases), 132 | }, 133 | aliases.invalid_aliases, 134 | )) 135 | } 136 | } 137 | 138 | fn find_local_aliases(dir: &Path, aliases: &mut AliasAccum) -> anyhow::Result<()> { 139 | if let Some(alias_dir) = dir.read_dir()?.find_map(|file| match file { 140 | Ok(entry) if entry.file_name() == ".agrind-aliases" => Some(entry), 141 | _else => None, 142 | }) { 143 | aliases_from_dir(&alias_dir.path(), aliases)?; 144 | } 145 | Ok(()) 146 | } 147 | 148 | fn aliases_from_dir(dir: &Path, pipelines: &mut AliasAccum) -> anyhow::Result<()> { 149 | for entry in dir.read_dir()? { 150 | let entry = entry?; 151 | let path = entry.path(); 152 | let contents = match std::fs::read_to_string(&path) { 153 | Ok(s) => s, 154 | Err(e) => { 155 | pipelines.invalid_aliases.push(InvalidAliasError { 156 | keyword: None, 157 | path, 158 | cause: e.into(), 159 | contents: None, 160 | }); 161 | continue; 162 | } 163 | }; 164 | match parse_alias(&contents, &path, &pipelines.valid_aliases) { 165 | Ok(alias) => pipelines.valid_aliases.push(alias), 166 | Err(e) => pipelines.invalid_aliases.push(e), 167 | } 168 | } 169 | Ok(()) 170 | } 171 | 172 | fn find_all_aliases(path: PathBuf) -> anyhow::Result<(Vec, Vec)> { 173 | let mut accum = AliasAccum::default(); 174 | for path in path.ancestors() { 175 | find_local_aliases(path, &mut accum)?; 176 | } 177 | Ok((accum.valid_aliases, accum.invalid_aliases)) 178 | } 179 | 180 | impl AliasPipeline { 181 | pub fn matching_string(s: &str) -> Option<&'static AliasPipeline> { 182 | LOADED_ALIASES.iter().find(|alias| alias.keyword == s) 183 | } 184 | 185 | /// Render the alias as a string that should parse into a valid operator. 186 | pub fn render(&self) -> Vec { 187 | self.pipeline.clone() 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/bin/agrind.rs: -------------------------------------------------------------------------------- 1 | use ag::alias::AliasCollection; 2 | use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer, TermErrorReporter}; 3 | use annotate_snippets::display_list::FormatOptions; 4 | use annotate_snippets::snippet::{Annotation, AnnotationType, Slice, Snippet}; 5 | use human_panic::setup_panic; 6 | 7 | use clap::Parser; 8 | #[cfg(feature = "self_update")] 9 | use self_update; 10 | use std::fs::File; 11 | use std::io; 12 | use std::io::{stdout, BufReader}; 13 | use std::path::PathBuf; 14 | use thiserror::Error; 15 | 16 | #[cfg(not(target_env = "msvc"))] 17 | #[global_allocator] 18 | static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; 19 | 20 | use crate::InvalidArgs::{CantSupplyBoth, InvalidFormatString, InvalidOutputMode}; 21 | 22 | #[derive(Debug, Parser)] 23 | #[command( 24 | version, 25 | after_help = "For more details + docs, see https://github.com/rcoh/angle-grinder" 26 | )] 27 | struct Cli { 28 | /// The query 29 | #[arg(group = "main")] 30 | query: Option, 31 | 32 | #[cfg(feature = "self_update")] 33 | /// Update agrind to the latest published version Github (https://github.com/rcoh/angle-grinder) 34 | #[arg(long = "self-update", group = "main")] 35 | update: bool, 36 | 37 | /// Optionally reads from a file instead of Stdin 38 | #[arg(long = "file", short = 'f')] 39 | file: Option, 40 | 41 | /// DEPRECATED. Use -o format=... instead. Provide a Rust std::fmt string to format output 42 | #[arg(long = "format", short = 'm')] 43 | format: Option, 44 | 45 | /// Set output format. One of (json|legacy|format=|logfmt) 46 | #[arg( 47 | long = "output", 48 | short = 'o', 49 | long_help = "Set output format. Options: \n\ 50 | - `json`,\n\ 51 | - `logfmt`\n\ 52 | - `format=` (eg. -o format='{src} => {dst}'\n\ 53 | - `legacy` The original output format, auto aligning [k=v]" 54 | )] 55 | output: Option, 56 | 57 | #[arg( 58 | long = "alias-dir", 59 | short = 'a', 60 | long_help = "Specifies an alternative directory to use for aliases. Defaults to `.agrind-aliases` in all parent directories." 61 | )] 62 | alias_dir: Option, 63 | 64 | #[arg(long = "no-alias", long_help = "Disables aliases")] 65 | no_alias: bool, 66 | } 67 | 68 | #[derive(Debug, Error)] 69 | pub enum InvalidArgs { 70 | #[error("Query was missing. Usage: `agrind 'query'`")] 71 | MissingQuery, 72 | 73 | #[error("Invalid output mode {}. Valid choices: {}", choice, choices)] 74 | InvalidOutputMode { choice: String, choices: String }, 75 | 76 | #[error("Invalid format string. Expected something like `-o format='{{src}} => {{dst}}'`")] 77 | InvalidFormatString, 78 | 79 | #[error("Can't supply a format string and an output mode")] 80 | CantSupplyBoth, 81 | 82 | #[error("Can't disable aliases and also set a directory")] 83 | CantDisableAndOverride, 84 | } 85 | 86 | fn main() -> Result<(), Box> { 87 | setup_panic!(); 88 | let args = Cli::parse(); 89 | #[cfg(feature = "self_update")] 90 | if args.update { 91 | return update(); 92 | } 93 | let (aliases, errors) = match (args.alias_dir, args.no_alias) { 94 | (Some(dir), false) => AliasCollection::load_aliases_from_dir(&dir)?, 95 | (None, false) => AliasCollection::load_aliases_ancestors(None)?, 96 | (Some(_), true) => return Err(InvalidArgs::CantDisableAndOverride.into()), 97 | (None, true) => (AliasCollection::default(), vec![]), 98 | }; 99 | let error_reporter = Box::new(TermErrorReporter {}); 100 | for error in errors { 101 | error_reporter.handle_error(Snippet { 102 | title: Some(Annotation { 103 | id: None, 104 | label: Some(&format!("invalid alias: {}", error.cause)), 105 | annotation_type: AnnotationType::Warning, 106 | }), 107 | footer: vec![], 108 | slices: vec![Slice { 109 | source: "", 110 | line_start: 0, 111 | origin: Some(error.path.to_str().unwrap()), 112 | annotations: vec![], 113 | fold: true, 114 | }], 115 | opt: FormatOptions::default(), 116 | }); 117 | } 118 | let query = QueryContainer::new_with_aliases( 119 | args.query.ok_or(InvalidArgs::MissingQuery)?, 120 | error_reporter, 121 | aliases, 122 | ); 123 | let output_mode = match (args.output, args.format) { 124 | (Some(_output), Some(_format)) => Err(CantSupplyBoth), 125 | (Some(output), None) => parse_output(&output), 126 | (None, Some(format)) => Ok(OutputMode::Format(format)), 127 | (None, None) => parse_output("legacy"), 128 | }?; 129 | let pipeline = Pipeline::new(&query, stdout(), output_mode)?; 130 | match args.file { 131 | Some(file_name) => { 132 | let f = File::open(file_name)?; 133 | pipeline.process(BufReader::new(f)) 134 | } 135 | None => { 136 | let stdin = io::stdin(); 137 | let locked = stdin.lock(); 138 | pipeline.process(locked) 139 | } 140 | }; 141 | Ok(()) 142 | } 143 | 144 | fn parse_output(output_param: &str) -> Result { 145 | // for some args, we split on `=` first 146 | let (arg, val) = match output_param.find('=') { 147 | None => (output_param, "="), 148 | Some(idx) => output_param.split_at(idx), 149 | }; 150 | let val = &val[1..]; 151 | 152 | match (arg, val) { 153 | ("legacy", "") => Ok(OutputMode::Legacy), 154 | ("json", "") => Ok(OutputMode::Json), 155 | ("logfmt", "") => Ok(OutputMode::Logfmt), 156 | ("format", v) if !v.is_empty() => Ok(OutputMode::Format(v.to_owned())), 157 | ("format", "") => Err(InvalidFormatString), 158 | (other, _v) => Err(InvalidOutputMode { 159 | choice: other.to_owned(), 160 | choices: "legacy, json, logfmt, format".to_owned(), 161 | }), 162 | } 163 | } 164 | 165 | #[cfg(feature = "self_update")] 166 | fn update() -> Result<(), Box> { 167 | let crate_version = self_update::cargo_crate_version!(); 168 | let status = self_update::backends::github::Update::configure() 169 | .repo_owner("rcoh") 170 | .repo_name("angle-grinder") 171 | .bin_name("agrind") 172 | .show_download_progress(true) 173 | .current_version(crate_version) 174 | .build()? 175 | .update()?; 176 | 177 | if crate_version == status.version() { 178 | println!( 179 | "Currently running the latest version publicly available ({}). No changes", 180 | status.version() 181 | ); 182 | } else { 183 | println!("Updated to version: {}", status.version()); 184 | } 185 | Ok(()) 186 | } 187 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | use crate::alias::AliasCollection; 2 | use crate::lang::{query, Positioned, Query}; 3 | use crate::pipeline::CompileError; 4 | use annotate_snippets::snippet::{Annotation, AnnotationType, Slice, Snippet, SourceAnnotation}; 5 | use std::env; 6 | use std::io::IsTerminal; 7 | use std::ops::Range; 8 | use std::sync::atomic::{AtomicUsize, Ordering}; 9 | use strsim::normalized_levenshtein; 10 | 11 | /// Container for the query string that can be used to parse and report errors. 12 | pub struct QueryContainer<'a> { 13 | pub query: String, 14 | pub reporter: Box, 15 | pub error_count: AtomicUsize, 16 | pub aliases: AliasCollection<'a>, 17 | } 18 | 19 | /// Trait that can be used to report errors by the parser and other layers. 20 | pub trait ErrorBuilder { 21 | /// Create a SnippetBuilder for the given error 22 | fn report_error_for(&self, error: E) -> SnippetBuilder; 23 | 24 | fn get_error_count(&self) -> usize; 25 | } 26 | 27 | impl<'a> QueryContainer<'a> { 28 | pub fn new_with_aliases( 29 | query: String, 30 | reporter: Box, 31 | aliases: AliasCollection<'a>, 32 | ) -> Self { 33 | QueryContainer { 34 | query, 35 | reporter, 36 | error_count: AtomicUsize::new(0), 37 | aliases, 38 | } 39 | } 40 | } 41 | 42 | impl QueryContainer<'static> { 43 | /* 44 | pub fn new(query: String, reporter: Box) -> QueryContainer<'static> { 45 | let (aliases, errs) = AliasCollection::load_aliases_ancestors(None); 46 | for err in errs { 47 | reporter.handle_error(Snippet { 48 | title: Some(Annotation { 49 | id: None, 50 | label: Some(&format!("{err:?}")), 51 | annotation_type: AnnotationType::Warning, 52 | }), 53 | footer: vec![], 54 | slices: vec![], 55 | opt: FormatOptions::default(), 56 | }); 57 | } 58 | Self::new_with_aliases(query, reporter, aliases) 59 | }*/ 60 | 61 | /// Parse the contained query string. 62 | pub fn parse(&self) -> Result { 63 | query(self).map_err(|_| CompileError::Parse) 64 | } 65 | } 66 | 67 | impl ErrorBuilder for QueryContainer<'_> { 68 | /// Create a SnippetBuilder for the given error 69 | fn report_error_for(&self, error: E) -> SnippetBuilder { 70 | self.error_count.fetch_add(1, Ordering::Relaxed); 71 | 72 | SnippetBuilder { 73 | query: self, 74 | data: SnippetData { 75 | error: error.to_string(), 76 | source: self.query.to_string(), 77 | ..Default::default() 78 | }, 79 | } 80 | } 81 | 82 | fn get_error_count(&self) -> usize { 83 | self.error_count.load(Ordering::Relaxed) 84 | } 85 | } 86 | 87 | pub fn did_you_mean<'a>(input: &str, choices: impl Iterator) -> Option { 88 | let similarities = choices.map(|choice| (choice, normalized_levenshtein(choice, input))); 89 | let mut candidates: Vec<_> = similarities.filter(|(_op, score)| *score > 0.6).collect(); 90 | candidates.sort_by_key(|(_op, score)| (score * 100_f64) as u16); 91 | candidates.first().map(|(choice, _scoe)| choice.to_string()) 92 | } 93 | 94 | /// Callback for handling error Snippets. 95 | pub trait ErrorReporter { 96 | fn handle_error(&self, _snippet: Snippet) {} 97 | } 98 | 99 | /// An ErrorReporter that writes errors related to the query string to the terminal 100 | pub struct TermErrorReporter {} 101 | 102 | impl ErrorReporter for TermErrorReporter { 103 | fn handle_error(&self, mut snippet: Snippet) { 104 | snippet.opt.color = env::var("NO_COLOR").is_err() && std::io::stderr().is_terminal(); 105 | let dl = annotate_snippets::display_list::DisplayList::from(snippet); 106 | 107 | eprintln!("{}", dl); 108 | } 109 | } 110 | 111 | /// Container for data that will be used to construct a Snippet 112 | #[derive(Default)] 113 | pub struct SnippetData { 114 | error: String, 115 | source: String, 116 | annotations: Vec<((usize, usize), String)>, 117 | resolution: Vec, 118 | } 119 | 120 | #[must_use = "the send_report() method must eventually be called for this builder"] 121 | pub struct SnippetBuilder<'a> { 122 | query: &'a QueryContainer<'a>, 123 | data: SnippetData, 124 | } 125 | 126 | impl SnippetBuilder<'_> { 127 | /// Adds an annotation to a portion of the query string. The given position will be 128 | /// highlighted with the accompanying label. 129 | pub fn with_code_pointer(mut self, pos: &Positioned, label: S) -> Self { 130 | self.data 131 | .annotations 132 | .push(((pos.range.start, pos.range.end), label.to_string())); 133 | self 134 | } 135 | 136 | /// Adds an annotation to a portion of the query string. The given position will be 137 | /// highlighted with the accompanying label. 138 | pub fn with_code_range(mut self, range: Range, label: S) -> Self { 139 | self.data 140 | .annotations 141 | .push(((range.start, range.end), label.to_string())); 142 | self 143 | } 144 | 145 | /// Add a message to help the user resolve the error. 146 | pub fn with_resolution(mut self, resolution: T) -> Self { 147 | self.data.resolution.push(resolution.to_string()); 148 | self 149 | } 150 | 151 | /// Build and send the Snippet to the ErrorReporter in the QueryContainer. 152 | pub fn send_report(self) { 153 | self.query.reporter.handle_error(Snippet { 154 | title: Some(Annotation { 155 | label: Some(self.data.error.as_str()), 156 | id: None, 157 | annotation_type: AnnotationType::Error, 158 | }), 159 | slices: vec![Slice { 160 | source: self.data.source.as_str(), 161 | line_start: 1, 162 | origin: None, 163 | fold: false, 164 | annotations: self 165 | .data 166 | .annotations 167 | .iter() 168 | .map(|anno| SourceAnnotation { 169 | range: anno.0, 170 | label: anno.1.as_str(), 171 | annotation_type: AnnotationType::Error, 172 | }) 173 | .collect(), 174 | }], 175 | footer: self 176 | .data 177 | .resolution 178 | .iter() 179 | .map(|res| Annotation { 180 | label: Some(res), 181 | id: None, 182 | annotation_type: AnnotationType::Help, 183 | }) 184 | .collect(), 185 | opt: Default::default(), 186 | }); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/filter.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | pub enum Filter { 3 | And(Vec), 4 | Or(Vec), 5 | Not(Box), 6 | Keyword(regex::Regex), 7 | } 8 | 9 | impl Filter { 10 | pub fn matches(&self, inp: &str) -> bool { 11 | match self { 12 | Filter::Keyword(regex) => regex.is_match(inp), 13 | Filter::And(clauses) => clauses.iter().all(|clause| clause.matches(inp)), 14 | Filter::Or(clauses) => clauses.iter().any(|clause| clause.matches(inp)), 15 | Filter::Not(clause) => !clause.matches(inp), 16 | } 17 | } 18 | } 19 | 20 | #[cfg(test)] 21 | mod tests { 22 | use super::*; 23 | use crate::lang::Keyword; 24 | 25 | pub fn from_str(inp: &str) -> Filter { 26 | Filter::Keyword(Keyword::new_exact(inp.to_owned()).to_regex()) 27 | } 28 | 29 | #[test] 30 | fn filter_and() { 31 | let filt = Filter::And(vec![from_str("abc"), from_str("cde")]); 32 | assert!(filt.matches("abc cde")); 33 | assert!(!filt.matches("abc")); 34 | } 35 | 36 | #[test] 37 | fn filter_or() { 38 | let filt = Filter::Or(vec![from_str("abc"), from_str("cde")]); 39 | assert!(filt.matches("abc cde")); 40 | assert!(filt.matches("abc")); 41 | assert!(filt.matches("cde")); 42 | assert!(!filt.matches("def")); 43 | } 44 | 45 | #[test] 46 | fn filter_wildcard() { 47 | let filt = Filter::And(vec![]); 48 | assert!(filt.matches("abc")); 49 | } 50 | 51 | #[test] 52 | fn filter_not() { 53 | let filt = Filter::And(vec![from_str("abc"), from_str("cde")]); 54 | let filt = Filter::Not(Box::new(filt)); 55 | assert!(!filt.matches("abc cde")); 56 | assert!(filt.matches("abc")); 57 | } 58 | 59 | #[test] 60 | fn filter_complex() { 61 | let filt_letters = Filter::And(vec![from_str("abc"), from_str("cde")]); 62 | let filt_numbers = Filter::And(vec![from_str("123"), from_str("456")]); 63 | let filt = Filter::Or(vec![filt_letters, filt_numbers]); 64 | assert!(filt.matches("abc cde")); 65 | assert!(!filt.matches("abc")); 66 | assert!(filt.matches("123 456")); 67 | assert!(!filt.matches("123 cde")); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/funcs.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::convert::{TryFrom, TryInto}; 3 | use std::fmt; 4 | use std::fmt::{Debug, Formatter}; 5 | 6 | use chrono::{DateTime, FixedOffset, Utc}; 7 | use itertools::Itertools; 8 | use lazy_static::lazy_static; 9 | 10 | use crate::data; 11 | use crate::operator::EvalError; 12 | 13 | /// Enum used to capture a static function that can be called by the expression language. 14 | #[derive(Clone, Copy)] 15 | pub enum FunctionWrapper { 16 | Float1(fn(f64) -> f64), 17 | Float2(fn(f64, f64) -> f64), 18 | String1(fn(&str) -> Result), 19 | String2(fn(&str, &str) -> Result), 20 | Generic(fn(&[data::Value]) -> Result), 21 | } 22 | 23 | /// Struct used to capture the name of the function in the expression language and a pointer 24 | /// to the implementation. 25 | #[derive(Clone, Copy)] 26 | pub struct FunctionContainer { 27 | name: &'static str, 28 | func: FunctionWrapper, 29 | } 30 | 31 | impl Debug for FunctionContainer { 32 | fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { 33 | f.write_str(self.name) 34 | } 35 | } 36 | 37 | impl FunctionContainer { 38 | pub fn new(name: &'static str, func: FunctionWrapper) -> Self { 39 | FunctionContainer { name, func } 40 | } 41 | 42 | fn eval1<'v, A: TryFrom<&'v data::Value, Error = EvalError>, O: Into>( 43 | &self, 44 | f: fn(A) -> O, 45 | args: &'v [data::Value], 46 | ) -> Result { 47 | match args { 48 | [arg0] => { 49 | let arg0_a = arg0.try_into()?; 50 | Ok(f(arg0_a).into()) 51 | } 52 | _ => Err(EvalError::InvalidFunctionArguments { 53 | name: self.name, 54 | expected: 1, 55 | found: args.len(), 56 | }), 57 | } 58 | } 59 | 60 | fn eval2<'v, A: TryFrom<&'v data::Value, Error = EvalError>, O: Into>( 61 | &self, 62 | f: fn(A, A) -> O, 63 | args: &'v [data::Value], 64 | ) -> Result { 65 | match args { 66 | [arg0, arg1] => { 67 | let arg0_a = arg0.try_into()?; 68 | let arg1_a = arg1.try_into()?; 69 | Ok(f(arg0_a, arg1_a).into()) 70 | } 71 | _ => Err(EvalError::InvalidFunctionArguments { 72 | name: self.name, 73 | expected: 2, 74 | found: args.len(), 75 | }), 76 | } 77 | } 78 | 79 | pub fn eval_func(&self, args: &[data::Value]) -> Result { 80 | match self.func { 81 | FunctionWrapper::Float1(func) => self.eval1(func, args), 82 | FunctionWrapper::Float2(func) => self.eval2(func, args), 83 | FunctionWrapper::String1(func) => { 84 | if let [arg0] = args { 85 | func(arg0.to_string().as_str()) 86 | } else { 87 | Err(EvalError::InvalidFunctionArguments { 88 | name: self.name, 89 | expected: 1, 90 | found: args.len(), 91 | }) 92 | } 93 | } 94 | FunctionWrapper::String2(func) => { 95 | if let [arg0, arg1] = args { 96 | func(arg0.to_string().as_str(), arg1.to_string().as_str()) 97 | } else { 98 | Err(EvalError::InvalidFunctionArguments { 99 | name: self.name, 100 | expected: 2, 101 | found: args.len(), 102 | }) 103 | } 104 | } 105 | FunctionWrapper::Generic(func) => func(args), 106 | } 107 | } 108 | } 109 | 110 | // The functions below are exported to the query language 111 | // TODO add some macro magic to extract doc attributes so the function reference 112 | // docs can be generated automatically 113 | 114 | fn concat(args: &[data::Value]) -> Result { 115 | Ok(data::Value::Str( 116 | args.iter().map(|arg| arg.to_string()).join(""), 117 | )) 118 | } 119 | 120 | fn contains(left: &str, right: &str) -> Result { 121 | Ok(data::Value::from_bool(left.contains(right))) 122 | } 123 | 124 | fn length(args: &[data::Value]) -> Result { 125 | match args { 126 | [data::Value::Array(vec)] => Ok(data::Value::Int(vec.len() as i64)), 127 | [data::Value::Obj(map)] => Ok(data::Value::Int(map.len() as i64)), 128 | [arg0] => Ok(data::Value::Int(arg0.to_string().chars().count() as i64)), 129 | _ => Err(EvalError::InvalidFunctionArguments { 130 | name: "length", 131 | expected: 1, 132 | found: args.len(), 133 | }), 134 | } 135 | } 136 | 137 | fn parse_date(date_str: &str) -> Result { 138 | dtparse::parse(date_str) 139 | .map(|pair| { 140 | data::Value::DateTime( 141 | DateTime::::from_naive_utc_and_offset( 142 | pair.0, 143 | pair.1.unwrap_or_else(|| FixedOffset::west_opt(0).unwrap()), 144 | ) 145 | .into(), 146 | ) 147 | }) 148 | .map_err(|parse_err| EvalError::FunctionFailed { 149 | name: "parseDate", 150 | msg: format!("{}", parse_err), 151 | }) 152 | } 153 | 154 | fn parse_hex(num_str: &str) -> Result { 155 | i64::from_str_radix(num_str.trim().trim_start_matches("0x"), 16) 156 | .map(data::Value::Int) 157 | .map_err(|_| EvalError::FunctionFailed { 158 | name: "parseHex", 159 | msg: format!("invalid hex string -- {}", num_str), 160 | }) 161 | } 162 | 163 | fn substring(args: &[data::Value]) -> Result { 164 | match args { 165 | [arg0, arg1, arg2] => { 166 | let src_str = arg0.to_string(); 167 | let start_off: usize = arg1.try_into()?; 168 | let end_off: usize = arg2.try_into()?; 169 | 170 | if end_off < start_off { 171 | return Err(EvalError::FunctionFailed { 172 | name: "substring", 173 | msg: format!( 174 | "end offset ({}) is less than the start offset ({})", 175 | end_off, start_off 176 | ), 177 | }); 178 | } 179 | 180 | Ok(data::Value::Str( 181 | src_str 182 | .chars() 183 | .skip(start_off) 184 | .take(end_off - start_off) 185 | .collect(), 186 | )) 187 | } 188 | [arg0, arg1] => { 189 | let src_str = arg0.to_string(); 190 | let start_off: usize = arg1.try_into()?; 191 | 192 | Ok(data::Value::Str(src_str.chars().skip(start_off).collect())) 193 | } 194 | _ => Err(EvalError::InvalidFunctionArguments { 195 | name: "substring", 196 | expected: 2, 197 | found: args.len(), 198 | }), 199 | } 200 | } 201 | 202 | fn to_lower_case(s: &str) -> Result { 203 | Ok(data::Value::from_string(s.to_lowercase())) 204 | } 205 | 206 | fn to_upper_case(s: &str) -> Result { 207 | Ok(data::Value::from_string(s.to_uppercase())) 208 | } 209 | 210 | fn is_null(args: &[data::Value]) -> Result { 211 | match args { 212 | [data::Value::None] => Ok(data::Value::Bool(true)), 213 | [_arg] => Ok(data::Value::Bool(false)), 214 | _ => Err(EvalError::InvalidFunctionArguments { 215 | name: "isNull", 216 | expected: 1, 217 | found: args.len(), 218 | }), 219 | } 220 | } 221 | 222 | fn is_empty(args: &[data::Value]) -> Result { 223 | match args { 224 | [data::Value::None] => Ok(data::Value::Bool(true)), 225 | [data::Value::Str(ref s)] => Ok(data::Value::Bool(s.is_empty())), 226 | [_arg0] => Ok(data::Value::Bool(false)), 227 | _ => Err(EvalError::InvalidFunctionArguments { 228 | name: "isEmpty", 229 | expected: 1, 230 | found: args.len(), 231 | }), 232 | } 233 | } 234 | 235 | fn is_blank(args: &[data::Value]) -> Result { 236 | match args { 237 | [data::Value::None] => Ok(data::Value::Bool(true)), 238 | [data::Value::Str(ref s)] => Ok(data::Value::Bool(s.trim().is_empty())), 239 | [_arg0] => Ok(data::Value::Bool(false)), 240 | _ => Err(EvalError::InvalidFunctionArguments { 241 | name: "isBlank", 242 | expected: 1, 243 | found: args.len(), 244 | }), 245 | } 246 | } 247 | 248 | fn is_numeric(args: &[data::Value]) -> Result { 249 | match args { 250 | [arg0] => Ok(data::Value::Bool( 251 | >::try_from(arg0).is_ok(), 252 | )), 253 | _ => Err(EvalError::InvalidFunctionArguments { 254 | name: "isNumeric", 255 | expected: 1, 256 | found: args.len(), 257 | }), 258 | } 259 | } 260 | 261 | fn num(value: f64) -> f64 { 262 | value 263 | } 264 | 265 | fn now(args: &[data::Value]) -> Result { 266 | match args { 267 | [] => Ok(data::Value::DateTime(Utc::now())), 268 | _ => Err(EvalError::InvalidFunctionArguments { 269 | name: "now", 270 | expected: 0, 271 | found: args.len(), 272 | }), 273 | } 274 | } 275 | 276 | lazy_static! { 277 | pub static ref FUNC_MAP: HashMap<&'static str, FunctionContainer> = { 278 | [ 279 | // numeric 280 | FunctionContainer::new("abs", FunctionWrapper::Float1(f64::abs)), 281 | FunctionContainer::new("acos", FunctionWrapper::Float1(f64::acos)), 282 | FunctionContainer::new("asin", FunctionWrapper::Float1(f64::asin)), 283 | FunctionContainer::new("atan", FunctionWrapper::Float1(f64::atan)), 284 | FunctionContainer::new("atan2", FunctionWrapper::Float2(f64::atan2)), 285 | FunctionContainer::new("cbrt", FunctionWrapper::Float1(f64::cbrt)), 286 | FunctionContainer::new("ceil", FunctionWrapper::Float1(f64::ceil)), 287 | FunctionContainer::new("cos", FunctionWrapper::Float1(f64::cos)), 288 | FunctionContainer::new("cosh", FunctionWrapper::Float1(f64::cosh)), 289 | FunctionContainer::new("exp", FunctionWrapper::Float1(f64::exp)), 290 | FunctionContainer::new("expm1", FunctionWrapper::Float1(f64::exp_m1)), 291 | FunctionContainer::new("floor", FunctionWrapper::Float1(f64::floor)), 292 | FunctionContainer::new("hypot", FunctionWrapper::Float2(f64::hypot)), 293 | FunctionContainer::new("log", FunctionWrapper::Float1(f64::ln)), 294 | FunctionContainer::new("log10", FunctionWrapper::Float1(f64::log10)), 295 | FunctionContainer::new("log1p", FunctionWrapper::Float1(f64::ln_1p)), 296 | FunctionContainer::new("round", FunctionWrapper::Float1(f64::round)), 297 | FunctionContainer::new("sin", FunctionWrapper::Float1(f64::sin)), 298 | FunctionContainer::new("sinh", FunctionWrapper::Float1(f64::sinh)), 299 | FunctionContainer::new("sqrt", FunctionWrapper::Float1(f64::sqrt)), 300 | FunctionContainer::new("tan", FunctionWrapper::Float1(f64::tan)), 301 | FunctionContainer::new("tanh", FunctionWrapper::Float1(f64::tanh)), 302 | FunctionContainer::new("toDegrees", FunctionWrapper::Float1(f64::to_degrees)), 303 | FunctionContainer::new("toRadians", FunctionWrapper::Float1(f64::to_radians)), 304 | // string 305 | FunctionContainer::new("concat", FunctionWrapper::Generic(concat)), 306 | FunctionContainer::new("contains", FunctionWrapper::String2(contains)), 307 | FunctionContainer::new("length", FunctionWrapper::Generic(length)), 308 | FunctionContainer::new("parseDate", FunctionWrapper::String1(parse_date)), 309 | FunctionContainer::new("parseHex", FunctionWrapper::String1(parse_hex)), 310 | FunctionContainer::new("substring", FunctionWrapper::Generic(substring)), 311 | FunctionContainer::new("toLowerCase", FunctionWrapper::String1(to_lower_case)), 312 | FunctionContainer::new("toUpperCase", FunctionWrapper::String1(to_upper_case)), 313 | FunctionContainer::new("isNull", FunctionWrapper::Generic(is_null)), 314 | FunctionContainer::new("isEmpty", FunctionWrapper::Generic(is_empty)), 315 | FunctionContainer::new("isBlank", FunctionWrapper::Generic(is_blank)), 316 | FunctionContainer::new("isNumeric", FunctionWrapper::Generic(is_numeric)), 317 | FunctionContainer::new("num", FunctionWrapper::Float1(num)), 318 | 319 | FunctionContainer::new("now", FunctionWrapper::Generic(now)), 320 | ] 321 | .iter() 322 | .map(|wrap| (wrap.name, *wrap)) 323 | .collect() 324 | }; 325 | } 326 | 327 | #[cfg(test)] 328 | mod tests { 329 | use super::*; 330 | 331 | #[test] 332 | fn unicode_length() { 333 | assert_eq!( 334 | data::Value::Int(1), 335 | length(&[data::Value::Str("\u{2603}".to_string())]).unwrap() 336 | ); 337 | } 338 | 339 | #[test] 340 | fn array_length() { 341 | assert_eq!( 342 | Ok(data::Value::Int(3)), 343 | length(&[data::Value::Array(vec!( 344 | data::Value::Int(0), 345 | data::Value::Int(1), 346 | data::Value::Int(2) 347 | ))]) 348 | ); 349 | } 350 | 351 | #[test] 352 | fn int_length() { 353 | assert_eq!(Ok(data::Value::Int(3)), length(&[data::Value::Int(123)])); 354 | } 355 | 356 | #[test] 357 | fn object_length() { 358 | let mut map = im::HashMap::new(); 359 | map.insert("abc".to_string(), data::Value::from_bool(true)); 360 | assert_eq!(Ok(data::Value::Int(1)), length(&[data::Value::Obj(map)])); 361 | } 362 | 363 | #[test] 364 | fn parse_hex_str() { 365 | assert_eq!(Ok(data::Value::Int(123)), parse_hex("0x7b")); 366 | assert_eq!(Ok(data::Value::Int(123)), parse_hex("7b")); 367 | assert_eq!( 368 | Err(EvalError::FunctionFailed { 369 | name: "parseHex", 370 | msg: "invalid hex string -- not a hex".to_string() 371 | }), 372 | parse_hex("not a hex") 373 | ); 374 | } 375 | 376 | #[test] 377 | fn case_funcs() { 378 | assert_eq!(Ok(data::Value::from_string("ABC")), to_upper_case("abc")); 379 | assert_eq!(Ok(data::Value::from_string("def")), to_lower_case("DEF")); 380 | } 381 | 382 | #[test] 383 | fn does_not_contain() { 384 | assert_eq!( 385 | data::Value::from_bool(false), 386 | contains("abc", "def").unwrap() 387 | ); 388 | } 389 | 390 | #[test] 391 | fn unicode_contains() { 392 | assert_eq!( 393 | data::Value::from_bool(true), 394 | contains("abc \u{2603} def", "\u{2603}").unwrap() 395 | ); 396 | } 397 | 398 | #[test] 399 | fn substring_no_args() { 400 | assert_eq!( 401 | Err(EvalError::InvalidFunctionArguments { 402 | name: "substring", 403 | expected: 2, 404 | found: 0, 405 | }), 406 | substring(&Vec::new()) 407 | ); 408 | } 409 | 410 | #[test] 411 | fn substring_of_num() { 412 | assert_eq!( 413 | Ok(data::Value::Str("12".to_string())), 414 | substring(&[ 415 | data::Value::Int(123), 416 | data::Value::Int(0), 417 | data::Value::Int(2) 418 | ]) 419 | ); 420 | } 421 | 422 | #[test] 423 | fn substring_end_lt_start() { 424 | assert_eq!( 425 | Err(EvalError::FunctionFailed { 426 | name: "substring", 427 | msg: "end offset (0) is less than the start offset (2)".to_string() 428 | }), 429 | substring(&[ 430 | data::Value::Int(123), 431 | data::Value::Str("2".to_string()), 432 | data::Value::Int(0) 433 | ]) 434 | ); 435 | } 436 | 437 | #[test] 438 | fn value_predicates() { 439 | assert_eq!(Ok(data::Value::Bool(true)), is_null(&[data::Value::None])); 440 | assert_eq!( 441 | Ok(data::Value::Bool(false)), 442 | is_null(&[data::Value::Str("".to_string())]) 443 | ); 444 | 445 | assert_eq!(Ok(data::Value::Bool(true)), is_empty(&[data::Value::None])); 446 | assert_eq!( 447 | Ok(data::Value::Bool(true)), 448 | is_empty(&[data::Value::Str("".to_string())]) 449 | ); 450 | assert_eq!( 451 | Ok(data::Value::Bool(false)), 452 | is_empty(&[data::Value::Str(" ".to_string())]) 453 | ); 454 | 455 | assert_eq!(Ok(data::Value::Bool(true)), is_blank(&[data::Value::None])); 456 | assert_eq!( 457 | Ok(data::Value::Bool(true)), 458 | is_blank(&[data::Value::Str("".to_string())]) 459 | ); 460 | assert_eq!( 461 | Ok(data::Value::Bool(true)), 462 | is_blank(&[data::Value::Str(" ".to_string())]) 463 | ); 464 | assert_eq!( 465 | Ok(data::Value::Bool(false)), 466 | is_blank(&[data::Value::Str("abc".to_string())]) 467 | ); 468 | 469 | assert_eq!( 470 | Ok(data::Value::Bool(true)), 471 | is_numeric(&[data::Value::Str("123".to_string())]) 472 | ); 473 | assert_eq!( 474 | Ok(data::Value::Bool(true)), 475 | is_numeric(&[data::Value::Str("1.23".to_string())]) 476 | ); 477 | assert_eq!( 478 | Ok(data::Value::Bool(true)), 479 | is_numeric(&[data::Value::Str("1e3".to_string())]) 480 | ); 481 | assert_eq!( 482 | Ok(data::Value::Bool(false)), 483 | is_numeric(&[data::Value::Str("abc".to_string())]) 484 | ); 485 | } 486 | } 487 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate include_dir; 3 | 4 | pub mod alias; 5 | pub mod data; 6 | mod errors; 7 | mod filter; 8 | mod funcs; 9 | pub mod lang; 10 | pub mod operator; 11 | mod printer; 12 | mod render; 13 | mod typecheck; 14 | 15 | pub mod pipeline { 16 | use crate::data::{DisplayConfig, Record, Row}; 17 | pub use crate::errors::{ErrorReporter, QueryContainer, TermErrorReporter}; 18 | use crate::filter; 19 | use crate::lang::*; 20 | use crate::operator; 21 | use crate::operator::sort; 22 | use crate::printer::{agg_printer, raw_printer}; 23 | use crate::render::{RenderConfig, Renderer, TerminalConfig}; 24 | use crate::typecheck::{TypeCheck, TypeError}; 25 | use anyhow::Error; 26 | use crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender}; 27 | use std::collections::VecDeque; 28 | use std::io::{BufRead, Write}; 29 | use std::thread; 30 | use std::time::Duration; 31 | use thiserror::Error; 32 | 33 | #[derive(Debug, Error)] 34 | pub enum CompileError { 35 | #[error("Failed to parse query")] 36 | Parse, 37 | 38 | #[error("Non aggregate operators can't follow aggregate operators")] 39 | NonAggregateAfterAggregate, 40 | 41 | #[error("Unexpected failure: {}", message)] 42 | Unexpected { message: String }, 43 | } 44 | 45 | #[derive(Clone, PartialEq, Eq)] 46 | pub enum OutputMode { 47 | Legacy, 48 | Logfmt, 49 | Format(String), 50 | Json, 51 | } 52 | 53 | pub struct Pipeline { 54 | filter: filter::Filter, 55 | pre_aggregates: Vec>, 56 | aggregators: Vec>, 57 | renderer: Renderer, 58 | } 59 | 60 | fn convert_filter(filter: Search) -> filter::Filter { 61 | match filter { 62 | Search::And(vec) => filter::Filter::And(vec.into_iter().map(convert_filter).collect()), 63 | Search::Or(vec) => filter::Filter::Or(vec.into_iter().map(convert_filter).collect()), 64 | Search::Not(search) => filter::Filter::Not(Box::new(convert_filter(*search))), 65 | Search::Keyword(keyword) => filter::Filter::Keyword(keyword.to_regex()), 66 | } 67 | } 68 | 69 | impl Pipeline { 70 | fn convert_sort( 71 | op: SortOperator, 72 | pipeline: &QueryContainer, 73 | ) -> Result, TypeError> { 74 | let mode = match op.direction { 75 | SortMode::Ascending => sort::SortDirection::Ascending, 76 | SortMode::Descending => sort::SortDirection::Descending, 77 | }; 78 | let sort_cols: Vec = op 79 | .sort_cols 80 | .into_iter() 81 | .map(|expr| expr.type_check(pipeline)) 82 | .collect::, _>>()?; 83 | Ok(Box::new(sort::Sorter::new(sort_cols, mode))) 84 | } 85 | 86 | fn convert_multi_agg( 87 | op: MultiAggregateOperator, 88 | pipeline: &QueryContainer, 89 | ) -> Result, TypeError> { 90 | let mut agg_functions = Vec::with_capacity(op.aggregate_functions.len()); 91 | 92 | for agg in op.aggregate_functions { 93 | let operator_function = agg.1.type_check(pipeline)?; 94 | agg_functions.push((agg.0, operator_function)); 95 | } 96 | let key_cols: Vec = op 97 | .key_cols 98 | .into_iter() 99 | .map(|expr| expr.type_check(pipeline)) 100 | .collect::, _>>()?; 101 | Ok(Box::new(operator::MultiGrouper::new( 102 | &(key_cols)[..], 103 | op.key_col_headers, 104 | agg_functions, 105 | ))) 106 | } 107 | 108 | fn implicit_sort(multi_agg: &MultiAggregateOperator) -> SortOperator { 109 | let timeslice_col = Expr::column("_timeslice"); 110 | let (opt_timeslice, direction) = if multi_agg.key_cols.contains(×lice_col) { 111 | (Some(timeslice_col), SortMode::Ascending) 112 | } else { 113 | (None, SortMode::Descending) 114 | }; 115 | 116 | let sort_cols: Vec = opt_timeslice 117 | .into_iter() 118 | .chain( 119 | multi_agg 120 | .aggregate_functions 121 | .iter() 122 | .map(|(k, _)| Expr::column(k)), 123 | ) 124 | .collect(); 125 | 126 | SortOperator { 127 | sort_cols, 128 | direction, 129 | } 130 | } 131 | 132 | pub fn new( 133 | pipeline: &QueryContainer<'static>, 134 | output: W, 135 | output_mode: OutputMode, 136 | ) -> Result { 137 | let query = pipeline.parse()?; 138 | let filters = convert_filter(query.search); 139 | let mut in_agg = false; 140 | let mut pre_agg: Vec> = Vec::new(); 141 | let mut post_agg: Vec> = Vec::new(); 142 | let mut op_deque = query.operators.into_iter().collect::>(); 143 | let mut has_errors = false; 144 | while let Some(op) = op_deque.pop_front() { 145 | match op { 146 | Operator::Error => {} 147 | Operator::RenderedAlias(rendered_alias) => { 148 | rendered_alias 149 | .into_iter() 150 | .rev() 151 | .for_each(|op| op_deque.push_front(op)); 152 | } 153 | Operator::Inline(inline_op) => { 154 | let op_builder = inline_op.type_check(pipeline)?; 155 | 156 | if !in_agg { 157 | pre_agg.push(op_builder.build()); 158 | } else { 159 | post_agg.push(Box::new(operator::PreAggAdapter::new(op_builder))); 160 | } 161 | } 162 | Operator::MultiAggregate(agg_op) => { 163 | in_agg = true; 164 | let sorter = Pipeline::implicit_sort(&agg_op); 165 | if let Ok(op) = Pipeline::convert_multi_agg(agg_op, pipeline) { 166 | post_agg.push(op); 167 | 168 | let needs_sort = matches!( 169 | op_deque.front(), 170 | Some(Operator::Inline(Positioned { 171 | value: InlineOperator::Limit { .. }, 172 | .. 173 | })) | None 174 | ); 175 | if needs_sort { 176 | post_agg.push(Pipeline::convert_sort(sorter, pipeline)?); 177 | } 178 | } else { 179 | has_errors = true; 180 | } 181 | } 182 | Operator::Sort(sort_op) => { 183 | post_agg.push(Pipeline::convert_sort(sort_op, pipeline)?) 184 | } 185 | } 186 | } 187 | if has_errors { 188 | return Err(CompileError::Parse.into()); 189 | } 190 | let render_config = RenderConfig { 191 | display_config: DisplayConfig { floating_points: 2 }, 192 | min_buffer: 4, 193 | max_buffer: 8, 194 | }; 195 | let raw_printer = 196 | raw_printer(&output_mode, render_config.clone(), TerminalConfig::load())?; 197 | let agg_printer = agg_printer(&output_mode, render_config, TerminalConfig::load())?; 198 | Ok(Pipeline { 199 | filter: filters, 200 | pre_aggregates: pre_agg, 201 | aggregators: post_agg, 202 | renderer: Renderer::new( 203 | RenderConfig { 204 | display_config: DisplayConfig { floating_points: 2 }, 205 | min_buffer: 4, 206 | max_buffer: 8, 207 | }, 208 | Duration::from_millis(50), 209 | raw_printer, 210 | agg_printer, 211 | Box::new(output), 212 | ), 213 | }) 214 | } 215 | 216 | fn render_noagg(mut renderer: Renderer, rx: &Receiver) { 217 | loop { 218 | let next = rx.recv_timeout(Duration::from_millis(50)); 219 | match next { 220 | Ok(row) => { 221 | let result = renderer.render(&row, false); 222 | 223 | if let Err(e) = result { 224 | eprintln!("error: {}", e); 225 | break; 226 | } 227 | } 228 | Err(RecvTimeoutError::Timeout) => {} 229 | Err(RecvTimeoutError::Disconnected) => break, 230 | } 231 | } 232 | } 233 | 234 | fn render_aggregate( 235 | mut head: Box, 236 | mut rest: Vec>, 237 | mut renderer: Renderer, 238 | rx: &Receiver, 239 | ) { 240 | loop { 241 | let next = rx.recv_timeout(Duration::from_millis(50)); 242 | match next { 243 | Ok(row) => (*head).process(row), 244 | Err(RecvTimeoutError::Timeout) => {} 245 | Err(RecvTimeoutError::Disconnected) => break, 246 | } 247 | 248 | if renderer.should_print() { 249 | let result = 250 | renderer.render(&Pipeline::run_agg_pipeline(&*head, &mut rest), false); 251 | 252 | if let Err(e) = result { 253 | eprintln!("error: {}", e); 254 | return; 255 | } 256 | } 257 | } 258 | let result = renderer.render(&Pipeline::run_agg_pipeline(&*head, &mut rest), true); 259 | 260 | if let Err(e) = result { 261 | eprintln!("error: {}", e); 262 | } 263 | } 264 | 265 | pub fn process(self, mut buf: T) { 266 | let (tx, rx) = bounded(1000); 267 | let mut aggregators = self.aggregators; 268 | let mut preaggs = self.pre_aggregates; 269 | let renderer = self.renderer; 270 | let t = if !aggregators.is_empty() { 271 | let head = aggregators.remove(0); 272 | thread::spawn(move || Pipeline::render_aggregate(head, aggregators, renderer, &rx)) 273 | } else { 274 | thread::spawn(move || Pipeline::render_noagg(renderer, &rx)) 275 | }; 276 | 277 | // This is pretty slow in practice. We could move line splitting until after 278 | // we find a match. Another option is moving the transformation to String until 279 | // after we match (staying as Vec until then) 280 | let mut line = Vec::with_capacity(1024); 281 | loop { 282 | let ct = buf.read_until(b'\n', &mut line).unwrap(); 283 | if ct == 0 { 284 | break; 285 | } 286 | let data = String::from_utf8_lossy(&line[..ct]); 287 | if self.filter.matches(data.as_ref()) 288 | && !Pipeline::proc_preagg(Record::new(data), &mut preaggs, &tx) 289 | { 290 | break; 291 | } 292 | line.clear(); 293 | } 294 | 295 | // Drain any remaining records from the operators. 296 | while !preaggs.is_empty() { 297 | let preagg = preaggs.remove(0); 298 | 299 | for rec in preagg.drain() { 300 | if !Pipeline::proc_preagg(rec, &mut preaggs, &tx) { 301 | break; 302 | } 303 | } 304 | } 305 | 306 | // Drop tx when causes the thread to exit. 307 | drop(tx); 308 | match t.join() { 309 | Ok(_) => (), 310 | Err(e) => println!("Error: {:?}", e), 311 | } 312 | } 313 | 314 | /// Process a record using the pre-agg operators. The output of the last operator will be 315 | /// sent to `tx`. 316 | fn proc_preagg( 317 | mut rec: Record, 318 | pre_aggs: &mut [Box], 319 | tx: &Sender, 320 | ) -> bool { 321 | for pre_agg in pre_aggs { 322 | match (*pre_agg).process_mut(rec) { 323 | Ok(Some(next_rec)) => rec = next_rec, 324 | Ok(None) => return true, 325 | Err(err) => { 326 | eprintln!("error: {}", err); 327 | return true; 328 | } 329 | } 330 | } 331 | 332 | tx.send(Row::Record(rec)).is_ok() 333 | } 334 | 335 | pub fn run_agg_pipeline( 336 | head: &dyn operator::AggregateOperator, 337 | rest: &mut [Box], 338 | ) -> Row { 339 | let mut row = Row::Aggregate((*head).emit()); 340 | for agg in (*rest).iter_mut() { 341 | (*agg).process(row); 342 | row = Row::Aggregate((*agg).emit()); 343 | } 344 | row 345 | } 346 | } 347 | } 348 | -------------------------------------------------------------------------------- /src/operator/average.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate, Expr}; 3 | 4 | pub struct Average { 5 | total: f64, 6 | count: i64, 7 | column: Expr, 8 | } 9 | 10 | impl Average { 11 | pub fn empty>(column: T) -> Average { 12 | Average { 13 | total: 0.0, 14 | count: 0, 15 | column: column.into(), 16 | } 17 | } 18 | } 19 | 20 | impl AggregateFunction for Average { 21 | fn process(&mut self, data: &Data) -> Result<(), EvalError> { 22 | let value: f64 = self.column.eval(data)?; 23 | self.total += value; 24 | self.count += 1; 25 | Ok(()) 26 | } 27 | 28 | fn emit(&self) -> data::Value { 29 | data::Value::from_float(self.total / self.count as f64) 30 | } 31 | 32 | fn empty_box(&self) -> Box { 33 | Box::new(Average::empty(self.column.clone())) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/operator/count.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate}; 3 | 4 | #[derive(Default)] 5 | pub struct Count { 6 | count: i64, 7 | condition: Option, 8 | } 9 | 10 | impl Count { 11 | pub fn new(condition: Option) -> Count { 12 | Count { 13 | count: 0, 14 | condition, 15 | } 16 | } 17 | } 18 | 19 | impl> AggregateFunction for Count { 20 | fn process(&mut self, data: &Data) -> Result<(), EvalError> { 21 | let count_record = match &self.condition { 22 | Some(cond) => cond.eval(data)?, 23 | None => true, 24 | }; 25 | if count_record { 26 | self.count += 1; 27 | } 28 | Ok(()) 29 | } 30 | 31 | fn emit(&self) -> data::Value { 32 | data::Value::Int(self.count) 33 | } 34 | 35 | fn empty_box(&self) -> Box { 36 | Box::new(Count::new(self.condition.clone())) 37 | } 38 | } 39 | 40 | #[cfg(test)] 41 | mod test { 42 | use super::Count; 43 | use crate::operator::expr::Expr; 44 | impl Count { 45 | pub fn unconditional() -> Count { 46 | Count { 47 | count: 0, 48 | condition: None, 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/operator/count_distinct.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::expr::Expr; 3 | use crate::operator::{AggregateFunction, Data, EvalError}; 4 | use std::collections::HashSet; 5 | 6 | pub struct CountDistinct { 7 | state: HashSet, 8 | column: Expr, 9 | } 10 | 11 | impl CountDistinct { 12 | pub fn empty>(column: T) -> Self { 13 | CountDistinct { 14 | state: HashSet::new(), 15 | column: column.into(), 16 | } 17 | } 18 | } 19 | 20 | impl AggregateFunction for CountDistinct { 21 | fn process(&mut self, rec: &Data) -> Result<(), EvalError> { 22 | let value = self.column.eval_value(rec)?.into_owned(); 23 | self.state.insert(value); 24 | Ok(()) 25 | } 26 | 27 | fn emit(&self) -> data::Value { 28 | data::Value::Int(self.state.len() as i64) 29 | } 30 | 31 | fn empty_box(&self) -> Box { 32 | Box::new(CountDistinct::empty(self.column.clone())) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/operator/expr.rs: -------------------------------------------------------------------------------- 1 | use crate::data::DisplayConfig; 2 | use crate::operator::{Data, EvalError, Evaluate}; 3 | use crate::{data, funcs}; 4 | use std::borrow::Cow; 5 | use std::collections::HashMap; 6 | use std::convert::TryInto; 7 | 8 | #[derive(Debug, Clone)] 9 | pub enum Expr { 10 | // First record can only be a String, after that, you can do things like `[0]` in addition to `.foo` 11 | NestedColumn { 12 | head: String, 13 | rest: Vec, 14 | }, 15 | BoolUnary(UnaryExpr), 16 | Comparison(BinaryExpr), 17 | Arithmetic(BinaryExpr), 18 | Logical(BinaryExpr), 19 | FunctionCall { 20 | func: &'static funcs::FunctionContainer, 21 | args: Vec, 22 | }, 23 | IfOp { 24 | cond: Box, 25 | value_if_true: Box, 26 | value_if_false: Box, 27 | }, 28 | Value(&'static data::Value), 29 | } 30 | 31 | impl Expr { 32 | pub fn column(key: &str) -> Expr { 33 | Expr::NestedColumn { 34 | head: key.to_owned(), 35 | rest: vec![], 36 | } 37 | } 38 | } 39 | 40 | #[derive(Debug, Clone)] 41 | pub struct UnaryExpr { 42 | pub operator: T, 43 | pub operand: Box, 44 | } 45 | 46 | #[derive(Debug, Clone)] 47 | pub struct BinaryExpr { 48 | pub operator: T, 49 | pub left: Box, 50 | pub right: Box, 51 | } 52 | 53 | impl Evaluate for BinaryExpr { 54 | fn eval(&self, record: &HashMap) -> Result { 55 | let l = self.left.eval_value(record)?; 56 | let r = self.right.eval_value(record)?; 57 | let result = match self.operator { 58 | BoolExpr::Eq => l == r, 59 | BoolExpr::Neq => l != r, 60 | BoolExpr::Gt => l > r, 61 | BoolExpr::Lt => l < r, 62 | BoolExpr::Gte => l >= r, 63 | BoolExpr::Lte => l <= r, 64 | }; 65 | Ok(result) 66 | } 67 | } 68 | 69 | impl Evaluate for BinaryExpr { 70 | fn eval(&self, record: &HashMap) -> Result { 71 | let l = self.left.eval_value(record)?.into_owned(); 72 | let r = self.right.eval_value(record)?.into_owned(); 73 | match self.operator { 74 | ArithmeticExpr::Add => l + r, 75 | ArithmeticExpr::Subtract => l - r, 76 | ArithmeticExpr::Multiply => l * r, 77 | ArithmeticExpr::Divide => l / r, 78 | } 79 | } 80 | } 81 | 82 | impl Evaluate for BinaryExpr { 83 | fn eval(&self, record: &HashMap) -> Result { 84 | let l: bool = self.left.eval(record)?; 85 | match self.operator { 86 | LogicalExpr::And => { 87 | if l { 88 | self.right.eval_value(record).map(|v| v.into_owned()) 89 | } else { 90 | Ok(data::Value::Bool(false)) 91 | } 92 | } 93 | LogicalExpr::Or => { 94 | if l { 95 | Ok(data::Value::Bool(true)) 96 | } else { 97 | self.right.eval_value(record).map(|v| v.into_owned()) 98 | } 99 | } 100 | } 101 | } 102 | } 103 | 104 | impl Evaluate for UnaryExpr { 105 | fn eval(&self, record: &HashMap) -> Result { 106 | let bool_res = self.operand.eval_value(record)?; 107 | 108 | match bool_res.as_ref() { 109 | data::Value::Bool(true) => Ok(false), 110 | data::Value::Bool(false) => Ok(true), 111 | _ => Err(EvalError::ExpectedBoolean { 112 | found: bool_res.to_string(), 113 | }), 114 | } 115 | } 116 | } 117 | 118 | impl Evaluate for Expr { 119 | fn eval(&self, record: &HashMap) -> Result { 120 | match self.eval_value(record)?.as_ref() { 121 | data::Value::Bool(bool_value) => Ok(*bool_value), 122 | other => Err(EvalError::ExpectedBoolean { 123 | found: other.to_string(), 124 | }), 125 | } 126 | } 127 | } 128 | 129 | impl Evaluate for Expr { 130 | fn eval(&self, record: &HashMap) -> Result { 131 | let value = self.eval_value(record)?; 132 | match value.as_ref() { 133 | data::Value::Int(i) => Ok(*i as f64), 134 | data::Value::Float(f) => Ok(f.into_inner()), 135 | data::Value::Str(s) => data::Value::aggressively_to_num(s), 136 | other => Err(EvalError::ExpectedNumber { 137 | found: format!("{}", other), 138 | }), 139 | } 140 | } 141 | } 142 | 143 | impl Expr { 144 | pub(crate) fn eval_str<'a>(&self, record: &'a Data) -> Result, EvalError> { 145 | let as_value = self.eval_value(record)?; 146 | match as_value { 147 | v if v.as_ref() == &data::Value::None => Err(EvalError::UnexpectedNone { 148 | tpe: "String".to_string(), 149 | }), 150 | Cow::Owned(data::Value::Str(s)) => Ok(Cow::Owned(s)), 151 | Cow::Borrowed(data::Value::Str(s)) => Ok(Cow::Borrowed(s)), 152 | _ => Err(EvalError::ExpectedString { 153 | found: "other".to_string(), 154 | }), 155 | } 156 | } 157 | 158 | pub(crate) fn eval_value<'a>( 159 | &self, 160 | record: &'a HashMap, 161 | ) -> Result, EvalError> { 162 | match *self { 163 | Expr::NestedColumn { ref head, ref rest } => { 164 | let mut root_record: &data::Value = record 165 | .get(head) 166 | .ok_or_else(|| EvalError::NoValueForKey { key: head.clone() })?; 167 | 168 | // TODO: probably a nice way to do this with a fold 169 | for value_reference in rest.iter() { 170 | match (value_reference, root_record) { 171 | (ValueRef::Field(ref key), data::Value::Obj(map)) => { 172 | root_record = map 173 | .get(key) 174 | .ok_or_else(|| EvalError::NoValueForKey { key: key.clone() })? 175 | } 176 | (ValueRef::Field(_), other) => { 177 | return Err(EvalError::ExpectedXYZ { 178 | expected: "object".to_string(), 179 | found: other.render(&DisplayConfig::default()), 180 | }); 181 | } 182 | (ValueRef::IndexAt(index), data::Value::Array(vec)) => { 183 | let vec_len: i64 = vec.len().try_into().unwrap(); 184 | let real_index = if *index < 0 { *index + vec_len } else { *index }; 185 | 186 | if real_index < 0 || real_index >= vec_len { 187 | return Err(EvalError::IndexOutOfRange { index: *index }); 188 | } 189 | root_record = &vec[real_index as usize]; 190 | } 191 | (ValueRef::IndexAt(_), other) => { 192 | return Err(EvalError::ExpectedXYZ { 193 | expected: "array".to_string(), 194 | found: other.render(&DisplayConfig::default()), 195 | }); 196 | } 197 | } 198 | } 199 | Ok(Cow::Borrowed(root_record)) 200 | } 201 | Expr::BoolUnary( 202 | ref unary_op @ UnaryExpr { 203 | operator: BoolUnaryExpr::Not, 204 | .. 205 | }, 206 | ) => { 207 | let bool_res = unary_op.eval(record)?; 208 | Ok(Cow::Owned(data::Value::from_bool(bool_res))) 209 | } 210 | Expr::Comparison(ref binary_expr) => { 211 | let bool_res = binary_expr.eval(record)?; 212 | Ok(Cow::Owned(data::Value::from_bool(bool_res))) 213 | } 214 | Expr::Arithmetic(ref binary_expr) => binary_expr.eval(record).map(Cow::Owned), 215 | Expr::Logical(ref logical_expr) => logical_expr.eval(record).map(Cow::Owned), 216 | Expr::FunctionCall { func, ref args } => { 217 | let evaluated_args: Result, EvalError> = args 218 | .iter() 219 | .map(|expr| expr.eval_value(record).map(|v| v.into_owned())) 220 | .collect(); 221 | 222 | func.eval_func(&evaluated_args?).map(Cow::Owned) 223 | } 224 | Expr::IfOp { 225 | ref cond, 226 | ref value_if_true, 227 | ref value_if_false, 228 | } => { 229 | let evaluated_cond: bool = (*cond).eval(record)?; 230 | 231 | if evaluated_cond { 232 | (*value_if_true).eval_value(record) 233 | } else { 234 | (*value_if_false).eval_value(record) 235 | } 236 | } 237 | Expr::Value(v) => Ok(Cow::Borrowed(v)), 238 | } 239 | } 240 | } 241 | 242 | #[derive(Clone, Debug)] 243 | pub enum BoolUnaryExpr { 244 | Not, 245 | } 246 | 247 | #[derive(Clone, Debug)] 248 | pub enum BoolExpr { 249 | Eq, 250 | Neq, 251 | Gt, 252 | Lt, 253 | Gte, 254 | Lte, 255 | } 256 | 257 | #[derive(Clone, Debug)] 258 | pub enum ArithmeticExpr { 259 | Add, 260 | Subtract, 261 | Multiply, 262 | Divide, 263 | } 264 | 265 | #[derive(Clone, Debug)] 266 | pub enum LogicalExpr { 267 | And, 268 | Or, 269 | } 270 | 271 | #[derive(Debug, Clone)] 272 | pub enum ValueRef { 273 | Field(String), 274 | IndexAt(i64), 275 | } 276 | -------------------------------------------------------------------------------- /src/operator/fields.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Record; 2 | use crate::operator::{EvalError, Expr, UnaryPreAggFunction}; 3 | use std::collections::HashSet; 4 | use std::iter::FromIterator; 5 | 6 | #[derive(Clone)] 7 | pub struct FieldExpressionDef { 8 | value: Expr, 9 | name: String, 10 | } 11 | 12 | impl FieldExpressionDef { 13 | pub fn new(value: Expr, name: String) -> FieldExpressionDef { 14 | FieldExpressionDef { value, name } 15 | } 16 | } 17 | 18 | impl UnaryPreAggFunction for FieldExpressionDef { 19 | fn process(&self, rec: Record) -> Result, EvalError> { 20 | let res = self.value.eval_value(&rec.data)?.into_owned(); 21 | 22 | Ok(Some(rec.put(&self.name, res))) 23 | } 24 | } 25 | 26 | impl UnaryPreAggFunction for Fields { 27 | fn process(&self, rec: Record) -> Result, EvalError> { 28 | let mut rec = rec; 29 | match self.mode { 30 | FieldMode::Only => { 31 | rec.data.retain(|k, _| self.columns.contains(k)); 32 | } 33 | FieldMode::Except => { 34 | rec.data.retain(|k, _| !self.columns.contains(k)); 35 | } 36 | } 37 | if rec.data.is_empty() { 38 | Ok(None) 39 | } else { 40 | Ok(Some(rec)) 41 | } 42 | } 43 | } 44 | 45 | #[derive(Clone)] 46 | pub struct Fields { 47 | columns: HashSet, 48 | mode: FieldMode, 49 | } 50 | 51 | impl Fields { 52 | pub fn new(columns: &[String], mode: FieldMode) -> Self { 53 | let columns = HashSet::from_iter(columns.iter().cloned()); 54 | Fields { columns, mode } 55 | } 56 | } 57 | 58 | #[derive(Clone)] 59 | pub enum FieldMode { 60 | Only, 61 | Except, 62 | } 63 | -------------------------------------------------------------------------------- /src/operator/limit.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Record; 2 | use crate::operator::{EvalError, OperatorBuilder, UnaryPreAggOperator}; 3 | use std::collections::VecDeque; 4 | use std::iter; 5 | 6 | /// The state for a limit operator 7 | pub enum Limit { 8 | Head { 9 | /// The current index into the input stream. 10 | index: u64, 11 | /// The number of rows to pass through before aborting. 12 | limit: u64, 13 | }, 14 | Tail { 15 | /// A circular queue to keep track of the tail of the input stream. 16 | queue: VecDeque, 17 | /// The size of the circular buffer. 18 | /// XXX Might be better to use a separate type. 19 | limit: usize, 20 | }, 21 | } 22 | 23 | impl UnaryPreAggOperator for Limit { 24 | fn process_mut(&mut self, rec: Record) -> Result, EvalError> { 25 | match self { 26 | Limit::Head { 27 | ref mut index, 28 | limit, 29 | } => { 30 | (*index) += 1; 31 | 32 | if index <= limit { 33 | Ok(Some(rec)) 34 | } else { 35 | Ok(None) 36 | } 37 | } 38 | Limit::Tail { 39 | ref mut queue, 40 | limit, 41 | } => { 42 | if queue.len() == *limit { 43 | queue.pop_front(); 44 | } 45 | queue.push_back(rec); 46 | 47 | Ok(None) 48 | } 49 | } 50 | } 51 | 52 | fn drain(self: Box) -> Box> { 53 | match *self { 54 | Limit::Head { .. } => Box::new(iter::empty()), 55 | Limit::Tail { queue, .. } => Box::new(queue.into_iter()), 56 | } 57 | } 58 | } 59 | 60 | impl LimitDef { 61 | pub fn new(limit: i64) -> Self { 62 | LimitDef { limit } 63 | } 64 | } 65 | 66 | /// The definition for a limit operator, which is a positive number used to specify whether 67 | /// the first N rows should be passed through to the downstream operators. Negative limits are 68 | /// not supported at this time. 69 | #[derive(Debug, PartialEq, Eq, Clone)] 70 | pub struct LimitDef { 71 | limit: i64, 72 | } 73 | 74 | impl OperatorBuilder for LimitDef { 75 | fn build(&self) -> Box { 76 | Box::new(if self.limit > 0 { 77 | Limit::Head { 78 | index: 0, 79 | limit: self.limit as u64, 80 | } 81 | } else { 82 | Limit::Tail { 83 | queue: VecDeque::with_capacity(-self.limit as usize), 84 | limit: -self.limit as usize, 85 | } 86 | }) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/operator/max.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate, Expr}; 3 | 4 | pub struct Max { 5 | max: f64, 6 | column: Expr, 7 | } 8 | 9 | impl Max { 10 | pub fn empty>(column: T) -> Max { 11 | Max { 12 | max: f64::NEG_INFINITY, 13 | column: column.into(), 14 | } 15 | } 16 | } 17 | 18 | impl AggregateFunction for Max { 19 | fn process(&mut self, data: &Data) -> Result<(), EvalError> { 20 | let value: f64 = self.column.eval(data)?; 21 | if value > self.max { 22 | self.max = value; 23 | } 24 | Ok(()) 25 | } 26 | 27 | fn emit(&self) -> data::Value { 28 | if self.max.is_finite() { 29 | data::Value::from_float(self.max) 30 | } else { 31 | data::Value::None 32 | } 33 | } 34 | 35 | fn empty_box(&self) -> Box { 36 | Box::new(Max::empty(self.column.clone())) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/operator/min.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate, Expr}; 3 | 4 | pub struct Min { 5 | min: f64, 6 | column: Expr, 7 | } 8 | 9 | impl Min { 10 | pub fn empty>(column: T) -> Min { 11 | Min { 12 | min: f64::INFINITY, 13 | column: column.into(), 14 | } 15 | } 16 | } 17 | 18 | impl AggregateFunction for Min { 19 | fn process(&mut self, data: &Data) -> Result<(), EvalError> { 20 | let value: f64 = self.column.eval(data)?; 21 | if value < self.min { 22 | self.min = value; 23 | } 24 | Ok(()) 25 | } 26 | 27 | fn emit(&self) -> data::Value { 28 | if self.min.is_finite() { 29 | data::Value::from_float(self.min) 30 | } else { 31 | data::Value::None 32 | } 33 | } 34 | 35 | fn empty_box(&self) -> Box { 36 | Box::new(Min::empty(self.column.clone())) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/operator/parse.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Record; 2 | use crate::operator::expr::Expr; 3 | use crate::operator::{EvalError, UnaryPreAggFunction}; 4 | use crate::{data, operator}; 5 | use serde_json::Value as JsonValue; 6 | use std::collections::HashMap; 7 | 8 | #[derive(Clone)] 9 | pub struct Parse { 10 | regex: regex::Regex, 11 | fields: Vec, 12 | input_column: Option, 13 | options: ParseOptions, 14 | } 15 | 16 | impl Parse { 17 | pub fn new( 18 | pattern: regex::Regex, 19 | fields: Vec, 20 | input_column: Option, 21 | options: ParseOptions, 22 | ) -> Self { 23 | Parse { 24 | regex: pattern, 25 | fields, 26 | input_column, 27 | options, 28 | } 29 | } 30 | 31 | fn matches(&self, rec: &Record) -> Result>, EvalError> { 32 | let inp = operator::get_input(rec, &self.input_column)?; 33 | match self.regex.captures_iter(inp.trim()).next() { 34 | None => Ok(None), 35 | Some(capture) => { 36 | let mut values: Vec = Vec::with_capacity(self.fields.len()); 37 | for item in capture.iter().skip(1) { 38 | // the first capture is the entire string 39 | match item { 40 | None => values.push(data::Value::None), 41 | Some(match_) => { 42 | let value = match self.options.no_conversion { 43 | true => data::Value::Str(match_.as_str().to_owned()), 44 | false => data::Value::from_string(match_.as_str()), 45 | }; 46 | values.push(value) 47 | } 48 | }; 49 | } 50 | Ok(Some(values)) 51 | } 52 | } 53 | } 54 | } 55 | 56 | impl UnaryPreAggFunction for Parse { 57 | fn process(&self, rec: Record) -> Result, EvalError> { 58 | let matches = self.matches(&rec)?; 59 | match (matches, self.options.drop_nonmatching) { 60 | (None, true) => Ok(None), 61 | (None, false) => { 62 | let new_fields: Vec<_> = self 63 | .fields 64 | .iter() 65 | .filter(|f| !rec.data.contains_key(*f)) 66 | .collect(); 67 | 68 | let mut rec = rec; 69 | for field in new_fields { 70 | rec = rec.put(field, data::Value::None); 71 | } 72 | Ok(Some(rec)) 73 | } 74 | (Some(matches), _) => { 75 | let mut rec = rec; 76 | for (field, value) in self.fields.iter().zip(matches.into_iter()) { 77 | rec = rec.put(field, value); 78 | } 79 | Ok(Some(rec)) 80 | } 81 | } 82 | } 83 | } 84 | 85 | #[derive(Clone)] 86 | pub struct ParseOptions { 87 | pub drop_nonmatching: bool, 88 | pub no_conversion: bool, 89 | } 90 | 91 | #[derive(Clone)] 92 | pub struct ParseJson { 93 | input_column: Option, 94 | } 95 | 96 | impl ParseJson { 97 | pub fn new(input_column: Option) -> ParseJson { 98 | ParseJson { input_column } 99 | } 100 | } 101 | 102 | impl UnaryPreAggFunction for ParseJson { 103 | fn process(&self, rec: Record) -> Result, EvalError> { 104 | fn json_to_value(v: JsonValue) -> data::Value { 105 | match v { 106 | JsonValue::Number(num) => { 107 | if num.is_i64() { 108 | data::Value::Int(num.as_i64().unwrap()) 109 | } else { 110 | data::Value::from_float(num.as_f64().unwrap()) 111 | } 112 | } 113 | JsonValue::String(s) => data::Value::Str(s), 114 | JsonValue::Null => data::Value::None, 115 | JsonValue::Bool(b) => data::Value::Bool(b), 116 | JsonValue::Object(map) => data::Value::Obj( 117 | map.into_iter() 118 | .map(|(k, v)| (k, json_to_value(v))) 119 | .collect::>() 120 | .into(), 121 | ), 122 | JsonValue::Array(vec) => data::Value::Array( 123 | vec.into_iter() 124 | .map(json_to_value) 125 | .collect::>(), 126 | ), 127 | } 128 | } 129 | let json: JsonValue = { 130 | let inp = operator::get_input(&rec, &self.input_column)?; 131 | serde_json::from_str(&inp).map_err(|_| EvalError::ExpectedJson { 132 | found: inp.trim_end().to_string(), 133 | })? 134 | }; 135 | let res = match json { 136 | JsonValue::Object(map) => { 137 | let mut rec = rec; 138 | rec.data.reserve(map.len()); 139 | for (k, v) in map { 140 | rec.put_mut(k, json_to_value(v)); 141 | } 142 | rec 143 | } 144 | // TODO: we'll implicitly drop non-object root values. Maybe we should produce an EvalError here 145 | _other => rec, 146 | }; 147 | Ok(Some(res)) 148 | } 149 | } 150 | 151 | #[derive(Clone)] 152 | pub struct ParseLogfmt { 153 | input_column: Option, 154 | } 155 | 156 | impl ParseLogfmt { 157 | pub fn new(input_column: Option) -> ParseLogfmt { 158 | ParseLogfmt { input_column } 159 | } 160 | } 161 | 162 | impl UnaryPreAggFunction for ParseLogfmt { 163 | fn process(&self, rec: Record) -> Result, EvalError> { 164 | let pairs = { 165 | let inp = operator::get_input(&rec, &self.input_column)?; 166 | // Record includes the trailing newline, while logfmt considers that part of the 167 | // message if present. Trim any trailing whitespace. 168 | logfmt::parse(inp.trim_end()) 169 | }; 170 | let res = { 171 | pairs.into_iter().fold(rec, |record, pair| match pair.val { 172 | None => record.put(&pair.key, data::Value::None), 173 | Some(val) => record.put(&pair.key, data::Value::from_string(val)), 174 | }) 175 | }; 176 | Ok(Some(res)) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/operator/percentile.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate, Expr}; 3 | use quantiles::ckms::CKMS; 4 | 5 | impl Percentile { 6 | pub fn empty>(column: T, percentile: f64) -> Self { 7 | if percentile >= 1.0 { 8 | panic!("Percentiles must be < 1"); 9 | } 10 | 11 | Percentile { 12 | ckms: CKMS::::new(0.001), 13 | column: column.into(), 14 | percentile, 15 | } 16 | } 17 | } 18 | 19 | pub struct Percentile { 20 | ckms: CKMS, 21 | column: Expr, 22 | percentile: f64, 23 | } 24 | 25 | impl AggregateFunction for Percentile { 26 | fn process(&mut self, data: &Data) -> Result<(), EvalError> { 27 | let value: f64 = self.column.eval(data)?; 28 | self.ckms.insert(value); 29 | Ok(()) 30 | } 31 | 32 | fn emit(&self) -> data::Value { 33 | let pct_opt = self.ckms.query(self.percentile); 34 | pct_opt 35 | .map(|(_usize, pct_float)| data::Value::from_float(pct_float)) 36 | .unwrap_or(data::Value::None) 37 | } 38 | 39 | fn empty_box(&self) -> Box { 40 | Box::new(Percentile::empty(self.column.clone(), self.percentile)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/operator/sort.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::data::{Aggregate, Record, Row}; 3 | use crate::operator::{AggregateOperator, Data, EvalError, Expr}; 4 | use std::cmp::Ordering; 5 | use std::collections::HashMap; 6 | 7 | #[derive(PartialEq, Eq)] 8 | pub enum SortDirection { 9 | Ascending, 10 | Descending, 11 | } 12 | 13 | pub struct Sorter { 14 | columns: Vec, 15 | state: Vec, 16 | #[allow(clippy::type_complexity)] 17 | ordering: Box Result + Send + Sync>, 18 | direction: SortDirection, 19 | } 20 | 21 | impl Sorter { 22 | pub fn new(exprs: Vec, direction: SortDirection) -> Self { 23 | let ordering = Box::new(Record::ordering(exprs)); 24 | Sorter { 25 | state: Vec::new(), 26 | columns: Vec::new(), 27 | direction, 28 | ordering, 29 | } 30 | } 31 | 32 | fn new_columns(&self, data: &HashMap) -> Vec { 33 | let mut new_keys: Vec = data 34 | .keys() 35 | .filter(|key| !self.columns.contains(key)) 36 | .cloned() 37 | .collect(); 38 | new_keys.sort(); 39 | new_keys 40 | } 41 | } 42 | 43 | impl AggregateOperator for Sorter { 44 | fn emit(&self) -> data::Aggregate { 45 | let mut sorted_data = self.state.to_vec(); 46 | let order = &self.ordering; 47 | // To produce a deterministic sort, we should also sort by the non-key columns 48 | let second_ordering = Record::ordering_ref(&self.columns); 49 | 50 | // TODO: output errors here 51 | if self.direction == SortDirection::Ascending { 52 | sorted_data.sort_by(|l, r| { 53 | ((order)(l, r)) 54 | .unwrap_or(Ordering::Less) 55 | .then(second_ordering(l, r)) 56 | }); 57 | } else { 58 | sorted_data.sort_by(|l, r| { 59 | ((order)(r, l)) 60 | .unwrap_or(Ordering::Less) 61 | .then(second_ordering(l, r)) 62 | }); 63 | } 64 | Aggregate { 65 | data: sorted_data, 66 | columns: self.columns.clone(), 67 | } 68 | } 69 | 70 | fn process(&mut self, row: Row) { 71 | let order = &self.ordering; 72 | match row { 73 | Row::Aggregate(agg) => { 74 | self.columns = agg.columns; 75 | self.state = agg.data; 76 | } 77 | Row::Record(rec) => { 78 | let new_cols = self.new_columns(&rec.data); 79 | self.state.push(rec.data); 80 | self.state 81 | .sort_by(|l, r| ((order)(l, r)).unwrap_or(Ordering::Less)); 82 | self.columns.extend(new_cols); 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/operator/split.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Record; 2 | use crate::operator::{EvalError, Expr, UnaryPreAggFunction}; 3 | use crate::{data, operator}; 4 | use lazy_static::lazy_static; 5 | use std::collections::HashMap; 6 | 7 | lazy_static! { 8 | pub static ref DEFAULT_DELIMITERS: HashMap<&'static str, &'static str> = { 9 | let mut h = HashMap::new(); 10 | h.insert("\"", "\""); 11 | h.insert("'", "'"); 12 | h 13 | }; 14 | } 15 | 16 | /// Find the character before the character beginning ad idx 17 | fn prev_char(s: &str, idx: usize) -> Option<&str> { 18 | if idx == 0 { 19 | None 20 | } else { 21 | let chr_end = idx; 22 | let mut char_start = chr_end - 1; 23 | while !s.is_char_boundary(char_start) { 24 | char_start -= 1; 25 | } 26 | Some(&s[char_start..chr_end]) 27 | } 28 | } 29 | 30 | /// Given a slice and start and end terminators, find a closing terminator while respecting escaped values. 31 | /// If a closing terminator is found, starting and ending terminators will be removed. 32 | /// If no closing terminator exists, the starting terminator will not be removed. 33 | fn find_close_delimiter<'a>( 34 | s: &'a str, 35 | term_start: &'a str, 36 | term_end: &'a str, 37 | ) -> (&'a str, &'a str) { 38 | let mut pos = term_start.len(); 39 | while pos < s.len() { 40 | match s[pos..].find(term_end).map(|index| index + pos) { 41 | None => break, 42 | Some(i) if i == 0 || prev_char(s, i) != Some("\\") => { 43 | return (&s[term_start.len()..i], &s[i + term_end.len()..]); 44 | } 45 | Some(other) => pos = other + 1, 46 | } 47 | } 48 | // We end up here if we never found a close quote. In that case, don't strip the leading quote. 49 | (s, &s[0..0]) 50 | } 51 | 52 | fn split_once<'a>(s: &'a str, p: &'a str) -> (&'a str, &'a str) { 53 | let mut split_iter = s.splitn(2, p); 54 | (split_iter.next().unwrap(), split_iter.next().unwrap_or("")) 55 | } 56 | 57 | /// split function that respects delimiters and strips whitespace 58 | pub fn split_with_delimiters<'a>( 59 | input: &'a str, 60 | separator: &'a str, 61 | delimiters: &HashMap<&'static str, &'static str>, 62 | ) -> Vec<&'a str> { 63 | let mut wip = input; 64 | let mut ret: Vec<&'a str> = vec![]; 65 | 66 | while !wip.is_empty() { 67 | // Look for a leading quote 68 | let leading_delimiter = delimiters 69 | .iter() 70 | .flat_map(|(k, v)| { 71 | if wip.starts_with(k) { 72 | Some((k, v)) 73 | } else { 74 | None 75 | } 76 | }) 77 | .next(); 78 | 79 | // If we're left with a quoted string, consume it, otherwise read until the next separator 80 | let (token, rest) = match leading_delimiter { 81 | Some((term_start, term_end)) => find_close_delimiter(wip, term_start, term_end), 82 | None => split_once(wip, separator), 83 | }; 84 | let token = token.trim(); 85 | if !token.is_empty() { 86 | ret.push(token); 87 | } 88 | wip = rest; 89 | } 90 | ret 91 | } 92 | 93 | #[cfg(test)] 94 | mod tests { 95 | use super::*; 96 | 97 | #[test] 98 | fn test_find_terminator() { 99 | assert_eq!( 100 | find_close_delimiter(r#""foo \" bar" cde"#, "\"", "\""), 101 | (r#"foo \" bar"#, " cde") 102 | ); 103 | // Only strip the quote if a terminator is actually found 104 | assert_eq!( 105 | find_close_delimiter(r#""'foo "#, "\"", "\""), 106 | ("\"'foo ", "") 107 | ); 108 | } 109 | 110 | #[test] 111 | fn test_split_once() { 112 | assert_eq!(split_once("abc cde", " "), ("abc", "cde")); 113 | assert_eq!(split_once(" cde", " "), ("", "cde")); 114 | assert_eq!(split_once("", " "), ("", "")); 115 | } 116 | 117 | #[test] 118 | fn split_works() { 119 | assert_eq!( 120 | split_with_delimiters("power hello", " ", &DEFAULT_DELIMITERS), 121 | vec!["power", "hello"], 122 | ); 123 | assert_eq!( 124 | split_with_delimiters("morecomplicated", "ecomp", &DEFAULT_DELIMITERS), 125 | vec!["mor", "licated"], 126 | ); 127 | assert_eq!( 128 | split_with_delimiters("owmmowmow", "ow", &DEFAULT_DELIMITERS), 129 | vec!["mm", "m"], 130 | ); 131 | assert_eq!( 132 | split_with_delimiters( 133 | r#"Oct 09 20:22:21 web-001 influxd[188053]: 127.0.0.1 "POST /write \"escaped\" HTTP/1.0" 204"#, 134 | " ", 135 | &DEFAULT_DELIMITERS 136 | ), 137 | vec![ 138 | "Oct", 139 | "09", 140 | "20:22:21", 141 | "web-001", 142 | "influxd[188053]:", 143 | "127.0.0.1", 144 | "POST /write \\\"escaped\\\" HTTP/1.0", 145 | "204" 146 | ], 147 | ); 148 | } 149 | 150 | #[test] 151 | fn split_with_closures_works() { 152 | assert_eq!( 153 | split_with_delimiters("power hello \"good bye\"", " ", &DEFAULT_DELIMITERS), 154 | vec!["power", "hello", "good bye"], 155 | ); 156 | assert_eq!( 157 | split_with_delimiters("more'ecomp'licated", "ecomp", &DEFAULT_DELIMITERS), 158 | vec!["more'", "'licated"], 159 | ); 160 | assert_eq!( 161 | split_with_delimiters("ow\"mm\"ow'\"mow\"'", "ow", &DEFAULT_DELIMITERS), 162 | vec!["mm", "\"mow\""], 163 | ); 164 | } 165 | 166 | // see https://github.com/rcoh/angle-grinder/issues/138 167 | #[test] 168 | fn split_with_wide_characters() { 169 | assert_eq!( 170 | split_with_delimiters( 171 | r#""Bourgogne-Franche-Comté" hello"#, 172 | " ", 173 | &DEFAULT_DELIMITERS 174 | ), 175 | vec!["Bourgogne-Franche-Comté", "hello"] 176 | ); 177 | } 178 | } 179 | 180 | #[derive(Clone)] 181 | pub struct Split { 182 | separator: String, 183 | input_column: Option, 184 | output_column: Option, 185 | } 186 | 187 | impl Split { 188 | pub fn new(separator: String, input_column: Option, output_column: Option) -> Self { 189 | Self { 190 | separator, 191 | input_column, 192 | output_column, 193 | } 194 | } 195 | } 196 | 197 | impl UnaryPreAggFunction for Split { 198 | fn process(&self, rec: Record) -> Result, EvalError> { 199 | let inp = operator::get_input(&rec, &self.input_column)?; 200 | let array = split_with_delimiters(&inp, &self.separator, &DEFAULT_DELIMITERS) 201 | .into_iter() 202 | .map(data::Value::from_string) 203 | .collect(); 204 | let rec = if let Some(output_column) = &self.output_column { 205 | rec.put_expr(output_column, data::Value::Array(array))? 206 | } else { 207 | rec.put("_split", data::Value::Array(array)) 208 | }; 209 | Ok(Some(rec)) 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/operator/sum.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::operator::expr::Expr; 3 | use crate::operator::{AggregateFunction, Data, EvalError, Evaluate}; 4 | 5 | pub struct Sum { 6 | total: f64, 7 | column: Expr, 8 | } 9 | 10 | impl Sum { 11 | pub fn empty>(column: T) -> Self { 12 | Sum { 13 | total: 0.0, 14 | column: column.into(), 15 | } 16 | } 17 | } 18 | 19 | impl AggregateFunction for Sum { 20 | fn process(&mut self, rec: &Data) -> Result<(), EvalError> { 21 | let value: f64 = self.column.eval(rec)?; 22 | self.total += value; 23 | Ok(()) 24 | } 25 | 26 | fn emit(&self) -> data::Value { 27 | data::Value::from_float(self.total) 28 | } 29 | 30 | fn empty_box(&self) -> Box { 31 | Box::new(Sum::empty(self.column.clone())) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/operator/timeslice.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::data::Record; 3 | use crate::operator::{EvalError, Expr, UnaryPreAggFunction}; 4 | use chrono::DurationRound; 5 | 6 | #[derive(Clone)] 7 | pub struct Timeslice { 8 | input_column: Expr, 9 | duration: chrono::Duration, 10 | output_column: Option, 11 | } 12 | 13 | impl Timeslice { 14 | pub fn new( 15 | input_column: Expr, 16 | duration: chrono::Duration, 17 | output_column: Option, 18 | ) -> Self { 19 | Self { 20 | input_column, 21 | duration, 22 | output_column, 23 | } 24 | } 25 | } 26 | 27 | impl UnaryPreAggFunction for Timeslice { 28 | fn process(&self, rec: Record) -> Result, EvalError> { 29 | let inp = self.input_column.eval_value(&rec.data)?; 30 | 31 | match inp.as_ref() { 32 | data::Value::DateTime(dt) => { 33 | let rounded = 34 | dt.duration_trunc(self.duration) 35 | .map_err(|e| EvalError::InvalidDuration { 36 | error: format!("{:?}", e), 37 | })?; 38 | let rec = rec.put( 39 | self.output_column 40 | .clone() 41 | .unwrap_or_else(|| "_timeslice".to_string()), 42 | data::Value::DateTime(rounded), 43 | ); 44 | 45 | Ok(Some(rec)) 46 | } 47 | _ => Err(EvalError::ExpectedDate { 48 | found: inp.to_string(), 49 | }), 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/operator/total.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use crate::data::Record; 3 | use crate::operator::expr::Expr; 4 | use crate::operator::{EvalError, Evaluate, OperatorBuilder, UnaryPreAggOperator}; 5 | 6 | pub struct TotalDef { 7 | column: Expr, 8 | output_column: String, 9 | } 10 | 11 | impl TotalDef { 12 | pub fn new(column: Expr, output_column: String) -> Self { 13 | TotalDef { 14 | column, 15 | output_column, 16 | } 17 | } 18 | } 19 | 20 | impl OperatorBuilder for TotalDef { 21 | fn build(&self) -> Box { 22 | Box::new(Total::new(self.column.clone(), self.output_column.clone())) 23 | } 24 | } 25 | 26 | pub struct Total { 27 | column: Expr, 28 | total: f64, 29 | output_column: String, 30 | } 31 | 32 | impl Total { 33 | pub fn new>(column: T, output_column: String) -> Total { 34 | Total { 35 | column: column.into(), 36 | total: 0.0, 37 | output_column, 38 | } 39 | } 40 | } 41 | 42 | impl UnaryPreAggOperator for Total { 43 | fn process_mut(&mut self, rec: Record) -> Result, EvalError> { 44 | // I guess this means there are cases when you need to both emit a warning _and_ a row, TODO 45 | // for now, we'll just emit the row 46 | let val: f64 = self.column.eval(&rec.data).unwrap_or(0.0); 47 | self.total += val; 48 | let rec = rec.put(&self.output_column, data::Value::from_float(self.total)); 49 | Ok(Some(rec)) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/operator/where_op.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Record; 2 | use crate::operator::{EvalError, Evaluate, UnaryPreAggFunction}; 3 | 4 | #[derive(Clone)] 5 | pub struct Where { 6 | expr: T, 7 | } 8 | 9 | impl Where { 10 | pub fn new(expr: T) -> Self { 11 | Where { expr } 12 | } 13 | } 14 | 15 | impl> UnaryPreAggFunction for Where { 16 | fn process(&self, rec: Record) -> Result, EvalError> { 17 | if self.expr.eval(&rec.data)? { 18 | Ok(Some(rec)) 19 | } else { 20 | Ok(None) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/render.rs: -------------------------------------------------------------------------------- 1 | use crate::data; 2 | use anyhow::Error; 3 | use std::io::Write; 4 | 5 | use crate::data::DisplayConfig; 6 | use crate::printer::{AggregatePrinter, RecordPrinter}; 7 | use std::time::{Duration, Instant}; 8 | use terminal_size::{terminal_size, Height, Width}; 9 | 10 | #[derive(Clone)] 11 | pub struct RenderConfig { 12 | pub display_config: DisplayConfig, 13 | pub min_buffer: usize, 14 | pub max_buffer: usize, 15 | } 16 | 17 | impl Default for RenderConfig { 18 | fn default() -> Self { 19 | RenderConfig { 20 | display_config: data::DisplayConfig { floating_points: 2 }, 21 | min_buffer: 1, 22 | max_buffer: 4, 23 | } 24 | } 25 | } 26 | 27 | pub struct TerminalConfig { 28 | pub size: Option, 29 | pub is_tty: bool, 30 | pub color_enabled: bool, 31 | } 32 | 33 | impl TerminalConfig { 34 | pub fn load() -> Self { 35 | let tsize_opt = 36 | terminal_size().map(|(Width(width), Height(height))| TerminalSize { width, height }); 37 | let is_tty = tsize_opt.is_some(); 38 | TerminalConfig { 39 | size: tsize_opt, 40 | is_tty, 41 | color_enabled: is_tty, 42 | } 43 | } 44 | } 45 | 46 | #[derive(PartialEq, Eq)] 47 | pub struct TerminalSize { 48 | pub height: u16, 49 | pub width: u16, 50 | } 51 | 52 | pub struct Renderer { 53 | raw_printer: Box, 54 | agg_printer: Box, 55 | update_interval: Duration, 56 | stdout: Box, 57 | config: RenderConfig, 58 | 59 | reset_sequence: String, 60 | is_tty: bool, 61 | last_print: Option, 62 | } 63 | 64 | impl Renderer { 65 | pub fn new( 66 | config: RenderConfig, 67 | update_interval: Duration, 68 | raw_printer: Box, 69 | agg_printer: Box, 70 | output: Box, 71 | ) -> Self { 72 | let tsize_opt = 73 | terminal_size().map(|(Width(width), Height(height))| TerminalSize { width, height }); 74 | Renderer { 75 | is_tty: tsize_opt.is_some(), 76 | raw_printer, 77 | agg_printer, 78 | config, 79 | stdout: output, 80 | reset_sequence: "".to_string(), 81 | last_print: None, 82 | update_interval, 83 | } 84 | } 85 | 86 | pub fn render(&mut self, row: &data::Row, last_row: bool) -> Result<(), Error> { 87 | match *row { 88 | data::Row::Aggregate(ref aggregate) => { 89 | if !self.is_tty { 90 | if last_row { 91 | let output = self 92 | .agg_printer 93 | .final_print(aggregate, &self.config.display_config); 94 | write!(self.stdout, "{}", output)?; 95 | } 96 | } else if self.should_print() || last_row { 97 | let output = if !last_row { 98 | self.agg_printer 99 | .print(aggregate, &self.config.display_config) 100 | } else { 101 | self.agg_printer 102 | .final_print(aggregate, &self.config.display_config) 103 | }; 104 | let num_lines = output.matches('\n').count(); 105 | write!(self.stdout, "{}{}", self.reset_sequence, output)?; 106 | self.reset_sequence = "\x1b[2K\x1b[1A".repeat(num_lines); 107 | self.last_print = Some(Instant::now()); 108 | } 109 | 110 | Ok(()) 111 | } 112 | data::Row::Record(ref record) => { 113 | self.raw_printer 114 | .print(&mut self.stdout, record, &self.config.display_config)?; 115 | writeln!(&mut self.stdout)?; 116 | Ok(()) 117 | } 118 | } 119 | } 120 | 121 | pub fn should_print(&self) -> bool { 122 | if !self.is_tty { 123 | return false; 124 | } 125 | self.last_print 126 | .map(|instant| instant.elapsed() > self.update_interval) 127 | .unwrap_or(true) 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/typecheck.rs: -------------------------------------------------------------------------------- 1 | use crate::data::Value; 2 | use crate::errors::ErrorBuilder; 3 | use crate::lang; 4 | use crate::operator::{ 5 | average, count, count_distinct, expr, fields, limit, max, min, parse, percentile, split, sum, 6 | timeslice, total, where_op, 7 | }; 8 | use crate::{funcs, operator}; 9 | use thiserror::Error; 10 | 11 | #[derive(Debug, Error)] 12 | pub enum TypeError { 13 | #[error("Expected boolean expression, found {}", found)] 14 | ExpectedBool { found: String }, 15 | 16 | #[error("Expected an expression")] 17 | ExpectedExpr, 18 | 19 | #[error( 20 | "Wrong number of patterns for parse. Pattern has {} but {} were extracted", 21 | pattern, 22 | extracted 23 | )] 24 | ParseNumPatterns { pattern: usize, extracted: usize }, 25 | 26 | #[error("Two `from` clauses were provided")] 27 | DoubleFromClause, 28 | 29 | #[error("Limit must be a non-zero integer, found {}", limit)] 30 | InvalidLimit { limit: f64 }, 31 | 32 | #[error("Unknown function {}", name)] 33 | UnknownFunction { name: String }, 34 | 35 | #[error("Expected a duration for the timeslice (e.g. 1h)")] 36 | ExpectedDuration, 37 | } 38 | 39 | pub trait TypeCheck { 40 | fn type_check(self, error_builder: &E) -> Result; 41 | } 42 | 43 | impl TypeCheck for lang::ComparisonOp { 44 | fn type_check(self, _error_builder: &E) -> Result { 45 | match self { 46 | lang::ComparisonOp::Eq => Ok(expr::BoolExpr::Eq), 47 | lang::ComparisonOp::Neq => Ok(expr::BoolExpr::Neq), 48 | lang::ComparisonOp::Gt => Ok(expr::BoolExpr::Gt), 49 | lang::ComparisonOp::Lt => Ok(expr::BoolExpr::Lt), 50 | lang::ComparisonOp::Gte => Ok(expr::BoolExpr::Gte), 51 | lang::ComparisonOp::Lte => Ok(expr::BoolExpr::Lte), 52 | } 53 | } 54 | } 55 | 56 | impl TypeCheck for lang::ArithmeticOp { 57 | fn type_check( 58 | self, 59 | _error_builder: &E, 60 | ) -> Result { 61 | match self { 62 | lang::ArithmeticOp::Add => Ok(expr::ArithmeticExpr::Add), 63 | lang::ArithmeticOp::Subtract => Ok(expr::ArithmeticExpr::Subtract), 64 | lang::ArithmeticOp::Multiply => Ok(expr::ArithmeticExpr::Multiply), 65 | lang::ArithmeticOp::Divide => Ok(expr::ArithmeticExpr::Divide), 66 | } 67 | } 68 | } 69 | 70 | impl TypeCheck for lang::LogicalOp { 71 | fn type_check( 72 | self, 73 | _error_builder: &E, 74 | ) -> Result { 75 | match self { 76 | lang::LogicalOp::And => Ok(expr::LogicalExpr::And), 77 | lang::LogicalOp::Or => Ok(expr::LogicalExpr::Or), 78 | } 79 | } 80 | } 81 | 82 | impl TypeCheck for lang::Expr { 83 | fn type_check(self, error_builder: &E) -> Result { 84 | match self { 85 | lang::Expr::Column { head, rest } => { 86 | let head = match head { 87 | lang::DataAccessAtom::Key(s) => s, 88 | lang::DataAccessAtom::Index(_) => return Err(TypeError::ExpectedExpr), 89 | }; 90 | let rest = rest 91 | .iter() 92 | .map(|s| match s { 93 | lang::DataAccessAtom::Key(s) => expr::ValueRef::Field(s.to_string()), 94 | lang::DataAccessAtom::Index(i) => expr::ValueRef::IndexAt(*i), 95 | }) 96 | .collect(); 97 | 98 | Ok(operator::Expr::NestedColumn { head, rest }) 99 | } 100 | lang::Expr::Unary { op, operand } => match op { 101 | lang::UnaryOp::Not => Ok(operator::Expr::BoolUnary(expr::UnaryExpr { 102 | operator: expr::BoolUnaryExpr::Not, 103 | operand: Box::new((*operand).type_check(error_builder)?), 104 | })), 105 | }, 106 | lang::Expr::Binary { op, left, right } => match op { 107 | lang::BinaryOp::Comparison(com_op) => { 108 | Ok(operator::Expr::Comparison(expr::BinaryExpr::< 109 | expr::BoolExpr, 110 | > { 111 | left: Box::new((*left).type_check(error_builder)?), 112 | right: Box::new((*right).type_check(error_builder)?), 113 | operator: com_op.type_check(error_builder)?, 114 | })) 115 | } 116 | lang::BinaryOp::Arithmetic(arith_op) => { 117 | Ok(operator::Expr::Arithmetic(expr::BinaryExpr::< 118 | expr::ArithmeticExpr, 119 | > { 120 | left: Box::new((*left).type_check(error_builder)?), 121 | right: Box::new((*right).type_check(error_builder)?), 122 | operator: arith_op.type_check(error_builder)?, 123 | })) 124 | } 125 | lang::BinaryOp::Logical(logical_op) => { 126 | Ok(operator::Expr::Logical(expr::BinaryExpr::< 127 | expr::LogicalExpr, 128 | > { 129 | left: Box::new((*left).type_check(error_builder)?), 130 | right: Box::new((*right).type_check(error_builder)?), 131 | operator: logical_op.type_check(error_builder)?, 132 | })) 133 | } 134 | }, 135 | lang::Expr::FunctionCall { name, args } => { 136 | let converted_args: Result, TypeError> = args 137 | .into_iter() 138 | .map(|arg| arg.type_check(error_builder)) 139 | .collect(); 140 | if let Some(func) = funcs::FUNC_MAP.get(name.as_str()) { 141 | Ok(operator::Expr::FunctionCall { 142 | func, 143 | args: converted_args?, 144 | }) 145 | } else { 146 | Err(TypeError::UnknownFunction { name }) 147 | } 148 | } 149 | lang::Expr::IfOp { 150 | cond, 151 | value_if_true, 152 | value_if_false, 153 | } => Ok(operator::Expr::IfOp { 154 | cond: Box::new(cond.type_check(error_builder)?), 155 | value_if_true: Box::new(value_if_true.type_check(error_builder)?), 156 | value_if_false: Box::new(value_if_false.type_check(error_builder)?), 157 | }), 158 | lang::Expr::Value(value) => { 159 | let boxed = Box::new(value); 160 | let static_value: &'static mut Value = Box::leak(boxed); 161 | Ok(operator::Expr::Value(static_value)) 162 | } 163 | lang::Expr::Error => Err(TypeError::ExpectedExpr), 164 | } 165 | } 166 | } 167 | 168 | const DEFAULT_LIMIT: i64 = 10; 169 | 170 | impl TypeCheck> 171 | for lang::Positioned 172 | { 173 | /// Convert the operator syntax to a builder that can instantiate an operator for the 174 | /// pipeline. Any semantic errors in the operator syntax should be detected here. 175 | fn type_check( 176 | self, 177 | error_builder: &T, 178 | ) -> Result, TypeError> { 179 | match self.value { 180 | lang::InlineOperator::Json { input_column } => Ok(Box::new(parse::ParseJson::new( 181 | input_column 182 | .map(|e| e.type_check(error_builder)) 183 | .transpose()?, 184 | ))), 185 | lang::InlineOperator::Logfmt { input_column } => Ok(Box::new(parse::ParseLogfmt::new( 186 | input_column 187 | .map(|e| e.type_check(error_builder)) 188 | .transpose()?, 189 | ))), 190 | lang::InlineOperator::Parse { 191 | pattern, 192 | fields, 193 | input_column, 194 | no_drop, 195 | no_convert, 196 | } => { 197 | let regex = pattern.to_regex(); 198 | 199 | let input_column = match input_column { 200 | (Some(from), None) | (None, Some(from)) => Some(from.value), 201 | (None, None) => None, 202 | (Some(l), Some(r)) => { 203 | let e = TypeError::DoubleFromClause; 204 | error_builder 205 | .report_error_for(&e) 206 | .with_code_pointer(&l, "") 207 | .with_code_pointer(&r, "") 208 | .with_resolution("Only one from clause is allowed") 209 | .send_report(); 210 | return Err(e); 211 | } 212 | }; 213 | 214 | if (regex.captures_len() - 1) != fields.len() { 215 | Err(TypeError::ParseNumPatterns { 216 | pattern: regex.captures_len() - 1, 217 | extracted: fields.len(), 218 | }) 219 | } else { 220 | Ok(Box::new(parse::Parse::new( 221 | regex, 222 | fields, 223 | input_column 224 | .map(|e| e.type_check(error_builder)) 225 | .transpose()?, 226 | parse::ParseOptions { 227 | drop_nonmatching: !no_drop, 228 | no_conversion: no_convert, 229 | }, 230 | ))) 231 | } 232 | } 233 | lang::InlineOperator::Fields { fields, mode } => { 234 | let omode = match mode { 235 | lang::FieldMode::Except => fields::FieldMode::Except, 236 | lang::FieldMode::Only => fields::FieldMode::Only, 237 | }; 238 | Ok(Box::new(fields::Fields::new(&fields, omode))) 239 | } 240 | lang::InlineOperator::Where { expr: Some(expr) } => match expr 241 | .value 242 | .type_check(error_builder)? 243 | { 244 | operator::Expr::Value(constant) => { 245 | if let Value::Bool(bool_value) = constant { 246 | Ok(Box::new(where_op::Where::new(*bool_value))) 247 | } else { 248 | let e = TypeError::ExpectedBool { 249 | found: format!("{:?}", constant), 250 | }; 251 | 252 | error_builder 253 | .report_error_for(&e) 254 | .with_code_range(expr.range, "This is constant") 255 | .with_resolution("Perhaps you meant to compare a field to this value?") 256 | .with_resolution(format!("example: where field1 == {}", constant)) 257 | .send_report(); 258 | 259 | Err(e) 260 | } 261 | } 262 | generic_expr => Ok(Box::new(where_op::Where::new(generic_expr))), 263 | }, 264 | lang::InlineOperator::Where { expr: None } => { 265 | let e = TypeError::ExpectedExpr; 266 | 267 | error_builder 268 | .report_error_for(&e) 269 | .with_code_pointer(&self, "No condition provided for this 'where'") 270 | .with_resolution( 271 | "Insert an expression whose result determines whether a record should be \ 272 | passed downstream", 273 | ) 274 | .with_resolution("example: where duration > 100") 275 | .send_report(); 276 | 277 | Err(e) 278 | } 279 | lang::InlineOperator::Limit { count: Some(count) } => match count.value { 280 | limit if limit.trunc() == 0.0 || limit.fract() != 0.0 => { 281 | let e = TypeError::InvalidLimit { limit }; 282 | 283 | error_builder 284 | .report_error_for(e.to_string()) 285 | .with_code_pointer( 286 | &count, 287 | if limit.fract() != 0.0 { 288 | "Fractional limits are not allowed" 289 | } else { 290 | "Zero is not allowed" 291 | }, 292 | ) 293 | .with_resolution("Use a positive integer to select the first N rows") 294 | .with_resolution("Use a negative integer to select the last N rows") 295 | .send_report(); 296 | 297 | Err(e) 298 | } 299 | limit => Ok(Box::new(limit::LimitDef::new(limit as i64))), 300 | }, 301 | lang::InlineOperator::Limit { count: None } => { 302 | Ok(Box::new(limit::LimitDef::new(DEFAULT_LIMIT))) 303 | } 304 | lang::InlineOperator::Split { 305 | separator, 306 | input_column, 307 | output_column, 308 | } => Ok(Box::new(split::Split::new( 309 | separator, 310 | input_column 311 | .map(|e| e.type_check(error_builder)) 312 | .transpose()?, 313 | output_column 314 | .map(|e| e.type_check(error_builder)) 315 | .transpose()?, 316 | ))), 317 | lang::InlineOperator::Timeslice { duration: None, .. } => { 318 | Err(TypeError::ExpectedDuration) 319 | } 320 | lang::InlineOperator::Timeslice { 321 | input_column, 322 | duration: Some(duration), 323 | output_column, 324 | } => Ok(Box::new(timeslice::Timeslice::new( 325 | input_column.type_check(error_builder)?, 326 | duration, 327 | output_column, 328 | ))), 329 | lang::InlineOperator::Total { 330 | input_column, 331 | output_column, 332 | } => Ok(Box::new(total::TotalDef::new( 333 | input_column.type_check(error_builder)?, 334 | output_column, 335 | ))), 336 | lang::InlineOperator::FieldExpression { value, name } => Ok(Box::new( 337 | fields::FieldExpressionDef::new(value.type_check(error_builder)?, name), 338 | )), 339 | } 340 | } 341 | } 342 | 343 | impl TypeCheck> for lang::Positioned { 344 | fn type_check( 345 | self, 346 | error_builder: &T, 347 | ) -> Result, TypeError> { 348 | match self.value { 349 | lang::AggregateFunction::Count { condition } => { 350 | let expr = condition.map(|c| c.type_check(error_builder)).transpose()?; 351 | Ok(Box::new(count::Count::new(expr))) 352 | } 353 | lang::AggregateFunction::Min { column } => { 354 | Ok(Box::new(min::Min::empty(column.type_check(error_builder)?))) 355 | } 356 | lang::AggregateFunction::Average { column } => Ok(Box::new(average::Average::empty( 357 | column.type_check(error_builder)?, 358 | ))), 359 | lang::AggregateFunction::Max { column } => { 360 | Ok(Box::new(max::Max::empty(column.type_check(error_builder)?))) 361 | } 362 | lang::AggregateFunction::Sum { column } => { 363 | Ok(Box::new(sum::Sum::empty(column.type_check(error_builder)?))) 364 | } 365 | lang::AggregateFunction::Percentile { 366 | column, percentile, .. 367 | } => Ok(Box::new(percentile::Percentile::empty( 368 | column.type_check(error_builder)?, 369 | percentile, 370 | ))), 371 | lang::AggregateFunction::CountDistinct { column: Some(pos) } => { 372 | match pos.value.as_slice() { 373 | [column] => Ok(Box::new(count_distinct::CountDistinct::empty( 374 | column.clone().type_check(error_builder)?, 375 | ))), 376 | _ => { 377 | error_builder 378 | .report_error_for("Expecting a single expression to count") 379 | .with_code_pointer( 380 | &pos, 381 | match pos.value.len() { 382 | 0 => "No expression given", 383 | _ => "Only a single expression can be given", 384 | }, 385 | ) 386 | .with_resolution("example: count_distinct(field_to_count)") 387 | .send_report(); 388 | 389 | Err(TypeError::ExpectedExpr) 390 | } 391 | } 392 | } 393 | lang::AggregateFunction::CountDistinct { column: None } => { 394 | error_builder 395 | .report_error_for("Expecting an expression to count") 396 | .with_code_pointer(&self, "No field argument given") 397 | .with_resolution("example: count_distinct(field_to_count)") 398 | .send_report(); 399 | 400 | Err(TypeError::ExpectedExpr) 401 | } 402 | lang::AggregateFunction::Error => unreachable!(), 403 | } 404 | } 405 | } 406 | -------------------------------------------------------------------------------- /test_files/basic: -------------------------------------------------------------------------------- 1 | ok 2 | -------------------------------------------------------------------------------- /test_files/binary_data.bin: -------------------------------------------------------------------------------- 1 | line of regular text k=v2 2 | �K���@�V ���+P��Q�Þ�0����S��oU�!+��X-!����޴A�� �r��[�OC��j_kΕ����V����5; �x1rYuV��� 3 | line of regular text k=v 4 | -------------------------------------------------------------------------------- /test_files/empty: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rcoh/angle-grinder/f1f5619c8dc14cc2c8ae2c90c072888efaccdc71/test_files/empty -------------------------------------------------------------------------------- /test_files/filter_test.log: -------------------------------------------------------------------------------- 1 | [INFO] I am a log! 2 | [WARN] Uh oh, danger ahead! 3 | [ERROR] Oh no! 4 | [INFO] more logs! 5 | [INFO] Match a *STAR*! 6 | [INFO] Not a STAR! -------------------------------------------------------------------------------- /test_files/gen_logs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -u 2 | import time 3 | import random 4 | import json 5 | import os 6 | import sys 7 | # reopen stdout file descriptor with write mode 8 | # and 0 as the buffer size (unbuffered) 9 | # sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) 10 | def main(): 11 | try: 12 | status_codes = [200, 200, 200, 200, 200, 200, 500] 13 | urls = ['/login', '/posts', '/submit', '/'] 14 | total_outputs = 0 15 | while True: 16 | url = random.choice(urls) 17 | status_code = random.choice(status_codes) 18 | data = dict(status_code=status_code, response_ms=random.randint(1, len(url)*5), url=url) 19 | sys.stdout.write(json.dumps(data) + '\n') 20 | total_outputs += 1 21 | time.sleep(random.randint(1,50) / 1000.0) 22 | except KeyboardInterrupt: 23 | sys.stderr.write("Total rows: {}".format(total_outputs)) 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /test_files/long_lines.log: -------------------------------------------------------------------------------- 1 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fieldsabcdefghijklmnopqrstuvvwxyz fieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyz"} 2 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 3 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 4 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 5 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 6 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 7 | -------------------------------------------------------------------------------- /test_files/multiline: -------------------------------------------------------------------------------- 1 | abc 2 | cde 3 | efg 4 | -------------------------------------------------------------------------------- /test_files/test_json.log: -------------------------------------------------------------------------------- 1 | {"level": "info", "message": "A thing happened", "num_things": 1102} 2 | {"level": "error", "message": "Oh now an error!"} 3 | {"level": "error", "message": "So many more errors!"} 4 | {"level": "info", "message": "A thing happened", "num_things": 12} 5 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 6 | {"level": null} 7 | -------------------------------------------------------------------------------- /test_files/test_logfmt.log: -------------------------------------------------------------------------------- 1 | level=info msg="Stopping all fetchers" tag=stopping_fetchers id=ConsumerFetcherManager-1382721708341 module=kafka.consumer.ConsumerFetcherManager 2 | level=info msg="Starting all fetchers" tag=starting_fetchers id=ConsumerFetcherManager-1382721708342 module=kafka.consumer.ConsumerFetcherManager 3 | level=warn msg="Fetcher failed to start" tag=errored_fetchers id=ConsumerFetcherManager-1382721708342 module=kafka.consumer.ConsumerFetcherManager 4 | -------------------------------------------------------------------------------- /test_files/test_nested_logfmt.log: -------------------------------------------------------------------------------- 1 | {"key": "blah", "nested_key": "some=logfmt data=more"} 2 | -------------------------------------------------------------------------------- /test_files/test_parse.log: -------------------------------------------------------------------------------- 1 | INFO Server db-1 loaded response in 500ms 2 | INFO Server db-2 loaded response in 394ms 3 | INFO Server db-1 loaded request in 394ms 4 | WARN Server failed to load response 5 | INFO Server db-1 loaded response in 100ms 6 | INFO Server db-1 loaded response in 102ms 7 | INFO Server db-1 loaded response in 102ms 8 | INFO Server db-3 loaded response in 103ms 9 | INFO Server db-1 loaded response in 100ms 10 | INFO Server db-3 loaded response in 109ms 11 | INFO Server db-1 loaded response in 100ms 12 | INFO Server db-3 loaded response in 104ms 13 | INFO Server db-1 loaded response in 100ms 14 | INFO Server db-3 loaded response in 100ms 15 | INFO Server db-1 loaded response in 122ms 16 | INFO Server db-2 loaded response in 119ms 17 | -------------------------------------------------------------------------------- /test_files/test_partial_json.log: -------------------------------------------------------------------------------- 1 | INFO {"level": "info", "message": "A thing happened", "num_things": 1102} 2 | INFO {"level": "error", "message": "Oh now an error!"} 3 | ERROR {"level": "error", "message": "So many more errors!"} 4 | INFO {"level": "info", "message": "A thing happened", "num_things": 12} 5 | WARN {"level": "info", "message": "A different event", "event_duration": 1002.5} 6 | WARN {"level": null} 7 | -------------------------------------------------------------------------------- /tests/code_blocks.rs: -------------------------------------------------------------------------------- 1 | use pulldown_cmark::{Event, Tag}; 2 | pub struct CodeBlock { 3 | pub flag: String, 4 | pub code: String, 5 | } 6 | pub fn code_blocks(inp: &str) -> Vec { 7 | let markdown = pulldown_cmark::Parser::new(inp); 8 | extract_blocks(markdown) 9 | } 10 | 11 | fn extract_blocks<'md, I: Iterator>>(md_events: I) -> Vec { 12 | let mut current_block = "".to_string(); 13 | let mut blocks = Vec::new(); 14 | let mut current_flag = "".to_string(); 15 | let mut in_block = false; 16 | for event in md_events { 17 | match (event, in_block) { 18 | (Event::Start(Tag::CodeBlock(flags)), _) => { 19 | current_flag = format!("{:?}", flags); 20 | current_block.clear(); 21 | in_block = true; 22 | } 23 | (Event::Text(code), true) => { 24 | current_block += &code; 25 | } 26 | (Event::End(pulldown_cmark::TagEnd::CodeBlock), true) => { 27 | blocks.push(CodeBlock { 28 | flag: current_flag.to_string(), 29 | code: current_block.to_string(), 30 | }); 31 | in_block = false; 32 | } 33 | _ => {} 34 | } 35 | } 36 | blocks 37 | } 38 | -------------------------------------------------------------------------------- /tests/integration.rs: -------------------------------------------------------------------------------- 1 | #![cfg(test)] 2 | use serde::Deserialize; 3 | use test_generator::test_resources; 4 | 5 | mod code_blocks; 6 | 7 | #[derive(Deserialize, Debug)] 8 | struct TestDefinition { 9 | query: String, 10 | input: String, 11 | output: String, 12 | error: Option, 13 | #[allow(dead_code)] 14 | notes: Option, 15 | succeeds: Option, 16 | enabled: Option, 17 | #[serde(default)] 18 | flags: Vec, 19 | } 20 | 21 | #[cfg(test)] 22 | mod integration { 23 | use super::*; 24 | use ag::alias::AliasCollection; 25 | use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer}; 26 | use assert_cmd::Command; 27 | use predicates::prelude::predicate; 28 | 29 | use std::fs; 30 | use std::io::stdout; 31 | 32 | const _TOUCH: usize = 3; 33 | 34 | pub struct EmptyErrorReporter; 35 | 36 | impl ErrorReporter for EmptyErrorReporter {} 37 | 38 | fn run() -> Command { 39 | Command::cargo_bin("agrind").unwrap() 40 | } 41 | 42 | #[test_resources("tests/structured_tests/*.toml")] 43 | fn integration(path: &str) { 44 | structured_test(path) 45 | } 46 | 47 | #[test_resources("tests/structured_tests/aliases/*.toml")] 48 | fn alias(path: &str) { 49 | structured_test(path) 50 | } 51 | 52 | fn structured_test(path: &str) { 53 | let contents = fs::read_to_string(path).unwrap(); 54 | let conf: TestDefinition = toml::from_str(&contents).unwrap(); 55 | let err = conf.error.unwrap_or("".to_string()); 56 | 57 | if !conf.enabled.unwrap_or(true) { 58 | return; 59 | } 60 | 61 | let asserter = run() 62 | .env("RUST_BACKTRACE", "0") 63 | .write_stdin(conf.input) 64 | .arg(&conf.query) 65 | .args(conf.flags) 66 | .arg("--no-alias") 67 | .assert(); 68 | 69 | let asserter = asserter.stdout(conf.output).stderr(err); 70 | 71 | if conf.succeeds.unwrap_or(true) { 72 | asserter.code(0); 73 | } else { 74 | asserter.failure(); 75 | } 76 | } 77 | 78 | #[test] 79 | fn no_args() { 80 | run() 81 | .assert() 82 | .failure() 83 | .stderr(predicate::str::contains("MissingQuery")); 84 | } 85 | 86 | #[test] 87 | fn parse_failure() { 88 | run() 89 | .args(["* | pasres"]) 90 | .assert() 91 | .failure() 92 | .stderr(predicate::str::contains("Failed to parse query")); 93 | } 94 | 95 | #[test] 96 | fn test_where_typecheck() { 97 | run() 98 | .args(["* | where 5"]) 99 | .assert() 100 | .failure() 101 | .stderr(predicate::str::contains( 102 | "Expected boolean expression, found", 103 | )); 104 | } 105 | 106 | #[test] 107 | fn test_limit_typecheck() { 108 | run() 109 | .args(["* | limit 0"]) 110 | .assert() 111 | .failure() 112 | .stderr(predicate::str::contains( 113 | "Error: Limit must be a non-zero integer, found 0", 114 | )); 115 | run() 116 | .args(["* | limit 0.1"]) 117 | .assert() 118 | .failure() 119 | .stderr(predicate::str::contains( 120 | "Error: Limit must be a non-zero integer, found 0.1", 121 | )); 122 | } 123 | 124 | #[test] 125 | fn basic_count() { 126 | run() 127 | .write_stdin("1\n2\n3\n") 128 | .args(["* | count"]) 129 | .assert() 130 | .stdout("_count\n--------------\n3\n"); 131 | } 132 | 133 | #[test] 134 | fn file_input() { 135 | run() 136 | .args([ 137 | "* | json | count by level", 138 | "--file", 139 | "test_files/test_json.log", 140 | ]) 141 | .assert() 142 | .stdout( 143 | "level _count 144 | --------------------------- 145 | info 3 146 | error 2 147 | None 1\n", 148 | ); 149 | } 150 | 151 | #[test] 152 | fn binary_input() { 153 | run() 154 | .args([ 155 | "* | parse 'k=*' as k", 156 | "--file", 157 | "test_files/binary_data.bin", 158 | ]) 159 | .assert() 160 | .stdout("[k=v2]\n[k=v]\n"); 161 | } 162 | 163 | #[test] 164 | fn filter_wildcard() { 165 | run() 166 | .args([r#""*STAR*""#, "--file", "test_files/filter_test.log"]) 167 | .assert() 168 | .stdout("[INFO] Match a *STAR*!\n"); 169 | run() 170 | .args([r#"*STAR*"#, "--file", "test_files/filter_test.log"]) 171 | .assert() 172 | .stdout("[INFO] Match a *STAR*!\n[INFO] Not a STAR!\n"); 173 | } 174 | 175 | #[test] 176 | fn test_limit() { 177 | run() 178 | .args([r#"* | limit 2"#, "--file", "test_files/filter_test.log"]) 179 | .assert() 180 | .stdout("[INFO] I am a log!\n[WARN] Uh oh, danger ahead!\n"); 181 | } 182 | 183 | #[test] 184 | fn custom_format_backcompat() { 185 | run() 186 | .args([ 187 | "* | logfmt", 188 | "--format", 189 | "{level} | {msg:<30} module={module}", 190 | "--file", 191 | "test_files/test_logfmt.log", 192 | ]) 193 | .assert() 194 | .stdout( 195 | "info | Stopping all fetchers module=kafka.consumer.ConsumerFetcherManager 196 | info | Starting all fetchers module=kafka.consumer.ConsumerFetcherManager 197 | warn | Fetcher failed to start module=kafka.consumer.ConsumerFetcherManager\n", 198 | ); 199 | } 200 | 201 | #[test] 202 | fn custom_format() { 203 | run() 204 | .args([ 205 | "-o", 206 | "format={level} | {msg:<30} module={module}", 207 | "--file", 208 | "test_files/test_logfmt.log", 209 | "* | logfmt", 210 | ]) 211 | .assert() 212 | .stdout( 213 | "info | Stopping all fetchers module=kafka.consumer.ConsumerFetcherManager 214 | info | Starting all fetchers module=kafka.consumer.ConsumerFetcherManager 215 | warn | Fetcher failed to start module=kafka.consumer.ConsumerFetcherManager\n", 216 | ); 217 | } 218 | 219 | fn ensure_parses(query: &str) { 220 | let query_container = QueryContainer::new_with_aliases( 221 | query.to_string(), 222 | Box::new(EmptyErrorReporter), 223 | AliasCollection::default(), 224 | ); 225 | Pipeline::new(&query_container, stdout(), OutputMode::Legacy).unwrap_or_else(|err| { 226 | panic!( 227 | "Query: `{}` from the README should have parsed {}", 228 | query, err 229 | ) 230 | }); 231 | println!("validated {}", query); 232 | } 233 | 234 | #[test] 235 | fn validate_readme_examples() { 236 | let blocks = code_blocks::code_blocks(include_str!("../README.md")); 237 | for code_block in blocks { 238 | if code_block.flag == "agrind" { 239 | ensure_parses(&code_block.code); 240 | } 241 | } 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /tests/structured_tests/agg_of_agg.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count by level | count" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!"} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | _count 12 | -------------- 13 | 3 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/aliases/alias_with_failure.toml: -------------------------------------------------------------------------------- 1 | query = """* | apache | foo""" 2 | input = "" 3 | output = "" 4 | error = """ 5 | error: Expected an operator 6 | | 7 | 1 | * | apache | foo 8 | | ^^^ 9 | | 10 | = help: foo is not a valid operator 11 | Error: Failed to parse query 12 | """ 13 | succeeds = false 14 | -------------------------------------------------------------------------------- /tests/structured_tests/aliases/apache.toml: -------------------------------------------------------------------------------- 1 | query = """* | apache""" 2 | input = """ 3 | 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 4 | """ 5 | output = """ 6 | [contentlength=2326] [ip=127.0.0.1] [method=GET] [name=frank] [protocol=HTTP/1.0] [status=200] [timestamp=10/Oct/2000:13:55:36 -0700] [url=/apache_pb.gif] 7 | """ 8 | -------------------------------------------------------------------------------- /tests/structured_tests/aliases/k8s-ingress_nginx.toml: -------------------------------------------------------------------------------- 1 | query = """* | k8singressnginx""" 2 | input = """ 3 | 172.70.127.35 - - [22/Feb/2023:22:02:59 +0000] "POST /twirp/example.v1.ServiceAPI/TestJob HTTP/1.1" 200 16 "-" "tasks/testing" 902 0.247 [test-grpc] [] 10.0.74.255:8080 16 0.248 200 89f3c824055b4d87942831d74343fb9a 4 | """ 5 | output = """ 6 | [body_bytes_sent=16] [http_referer=-] [http_user_agent=tasks/testing] [method=POST] [protocol=HTTP/1.1] [proxy_alternative_upstream_name=] [proxy_upstream_name=test-grpc] [remote_addr=172.70.127.35] [remote_user=-] [req_id=89f3c824055b4d87942831d74343fb9a] [request_length=902] [request_time=0.25] [status=200] [timestamp=22/Feb/2023:22:02:59 +0000] [upstream_addr=10.0.74.255:8080] [upstream_response_length=16] [upstream_response_time=0.25] [upstream_status=200] [url=/twirp/example.v1.ServiceAPI/TestJob] 7 | """ 8 | -------------------------------------------------------------------------------- /tests/structured_tests/aliases/multi-operator.toml: -------------------------------------------------------------------------------- 1 | query = """* | testmultioperator""" 2 | input = """ 3 | { "abc": 5, "xyz": "hello" } 4 | """ 5 | output = """ 6 | _count 7 | -------------- 8 | 1 9 | """ 10 | -------------------------------------------------------------------------------- /tests/structured_tests/aliases/nginx.toml: -------------------------------------------------------------------------------- 1 | query = """* | nginx""" 2 | input = """ 3 | 127.0.0.1 - - [23/Feb/2023:17:05:13 +0000] "GET / HTTP/1.1" 200 615 "-" "curl/7.77.0" "-" 4 | """ 5 | output = """ 6 | [addr=127.0.0.1] [bytes_sent=615] [gzip_ratio=-] [http_referer=-] [http_user_agent=curl/7.77.0] [method=GET] [protocol=HTTP/1.1] [status=200] [timestamp=23/Feb/2023:17:05:13 +0000] [url=/] [user=-] 7 | """ 8 | -------------------------------------------------------------------------------- /tests/structured_tests/arrays_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | sum(thing_a[0])""" 2 | input = """ 3 | {"thing_a": [0, 1, 2]} 4 | {"thing_a": [5]} 5 | {"thing_a": []} 6 | {} 7 | {"thing_a": ["foo"]} 8 | """ 9 | output = """ 10 | _sum 11 | ------------ 12 | 5 13 | """ 14 | error = """ 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/count_conditional.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | count(level == "error") as num_errors, count(level == "info") as num_info""" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!"} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | num_errors num_info 12 | ---------------------------------- 13 | 2 3 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/count_distinct.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count, count_distinct(message) by level" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!"} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | level _count _countDistinct 12 | ------------------------------------------------- 13 | info 3 2 14 | error 2 2 15 | None 1 0 16 | """ 17 | -------------------------------------------------------------------------------- /tests/structured_tests/count_distinct_error.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count, count_distinct" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: Expecting an expression to count 9 | | 10 | 1 | * | json | count, count_distinct 11 | | ^^^^^^^^^^^^^^ No field argument given 12 | | 13 | = help: example: count_distinct(field_to_count) 14 | Error: Failed to parse query 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/count_distinct_error_2.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | countdistinct(a)" 2 | input = "" 3 | output = "" 4 | error = """ 5 | error: Expected an operator 6 | | 7 | 1 | * | json | countdistinct(a) 8 | | ^^^^^^^^^^^^^ 9 | | 10 | = help: countdistinct is not a valid operator 11 | = help: Did you mean "count_distinct"? 12 | Error: Failed to parse query 13 | """ 14 | succeeds = false 15 | -------------------------------------------------------------------------------- /tests/structured_tests/escaped_ident.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | json | count by ["grpc.method"], ["start time"], nested.["user.name"] 3 | """ 4 | input = """ 5 | {"start time": "today", "grpc.method": "Foo", "nested": {"user.name": "user1"}} 6 | {"start time": "today", "grpc.method": "Bar", "nested": {"user.name": "user1"}} 7 | """ 8 | output = """ 9 | ["grpc.method"] ["start time"] nested.["user.name"] _count 10 | --------------------------------------------------------------------------------------- 11 | Bar today user1 1 12 | Foo today user1 1 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/field_expr-1.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | num_things * 2 + 6 / 3 as num_things_x2" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1.5} 4 | {"level": "info", "message": "A thing happened", "num_things": 1103} 5 | {"level": "info", "message": "A thing happened", "num_things": 1105} 6 | {"level": "info", "message": "A thing happened", "num_things": "not_a_number"} 7 | {"level": "info", "message": "A thing happened"} 8 | {"level": "info", "message": "A thing happened", "num_things": 1105} 9 | {"level": "info", "message": "A thing happened", "num_things": 1105} 10 | {"level": "info", "message": "A thing happened", "num_things": 1105} 11 | {"level": "info", "message": "A thing happened", "num_things": 1105.5} 12 | {"level": "info", "message": "A thing happened", "num_things": "1105.5"} 13 | """ 14 | output = """ 15 | [level=info] [message=A thing happened] [num_things=1.50] [num_things_x2=5] 16 | [level=info] [message=A thing happened] [num_things=1103] [num_things_x2=2208] 17 | [level=info] [message=A thing happened] [num_things=1105] [num_things_x2=2212] 18 | [level=info] [message=A thing happened] [num_things=1105] [num_things_x2=2212] 19 | [level=info] [message=A thing happened] [num_things=1105] [num_things_x2=2212] 20 | [level=info] [message=A thing happened] [num_things=1105] [num_things_x2=2212] 21 | [level=info] [message=A thing happened] [num_things=1105.50] [num_things_x2=2213] 22 | [level=info] [message=A thing happened] [num_things=1105.5] [num_things_x2=2213] 23 | """ 24 | error = """ 25 | error: Expected numeric operands, found not_a_number * 2 26 | error: No value for key "num_things" 27 | """ 28 | -------------------------------------------------------------------------------- /tests/structured_tests/fields_after_agg.toml: -------------------------------------------------------------------------------- 1 | query = ' * | parse ":* (" as port | count by port | fields port' 2 | input = ": 45 (" 3 | output = """ 4 | port 5 | ------------ 6 | 45 7 | """ 8 | error = """ 9 | """ 10 | succeeds = true 11 | -------------------------------------------------------------------------------- /tests/structured_tests/fields_except.toml: -------------------------------------------------------------------------------- 1 | query = """* "error" | parse "* *" as lev, js 2 | | json from js 3 | | fields except js, lev""" 4 | input = """ 5 | INFO {"level": "info", "message": "A thing happened", "num_things": 1102} 6 | INFO {"level": "error", "message": "Oh now an error!"} 7 | ERROR {"level": "error", "message": "So many more errors!"} 8 | INFO {"level": "info", "message": "A thing happened", "num_things": 12} 9 | WARN {"level": "info", "message": "A different event", "event_duration": 1002.5} 10 | WARN {"level": null} 11 | """ 12 | output = """ 13 | [level=error] [message=Oh now an error!] 14 | [level=error] [message=So many more errors!] 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/filters.toml: -------------------------------------------------------------------------------- 1 | query = """(error OR warn) AND NOT hide | count""" 2 | input = """ 3 | ERROR this is bad 4 | ERROR 5 | WARN 6 | WARN hide 7 | ERROR hide 8 | INFO everything is fine 9 | """ 10 | output = """ 11 | _count 12 | -------------- 13 | 3 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/func_arg_error_1.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | abs(1, 2) as a" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: The 'abs' function expects 1 arguments, found 2 9 | error: The 'abs' function expects 1 arguments, found 2 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/func_arg_error_2.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | atan2(1) as a" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: The 'atan2' function expects 2 arguments, found 1 9 | error: The 'atan2' function expects 2 arguments, found 1 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/if-1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | if(kind == 1, v1, v2) as vany""" 2 | input = """ 3 | {"kind": 1, "v1": 5} 4 | {"kind": 1, "v1": 6} 5 | {"kind": 2, "v2": 7} 6 | {"kind": 1, "v1": 8} 7 | """ 8 | output = """ 9 | [kind=1] [v1=5] [vany=5] 10 | [kind=1] [v1=6] [vany=6] 11 | [kind=2] [vany=7] [v2=7] 12 | [kind=1] [v1=8] [vany=8] 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/if-2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | if(kind, v1, v2) as vany""" 2 | input = """ 3 | {"kind": 1, "v1": 5} 4 | {"kind": 1, "v1": 6} 5 | {"kind": 2, "v2": 7} 6 | {"kind": 1, "v1": 8} 7 | """ 8 | output = """ 9 | """ 10 | error = """ 11 | error: Expected boolean, found 1 12 | error: Expected boolean, found 1 13 | error: Expected boolean, found 2 14 | error: Expected boolean, found 1 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/json_from.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "* *" as lev, js | json from js | count by level""" 2 | input = """ 3 | INFO {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | INFO {"level": "error", "message": "Oh now an error!"} 5 | ERROR {"level": "error", "message": "So many more errors!"} 6 | INFO {"level": "info", "message": "A thing happened", "num_things": 12} 7 | WARN {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | WARN {"level": null} 9 | """ 10 | output = """ 11 | level _count 12 | --------------------------- 13 | info 3 14 | error 2 15 | None 1 16 | """ 17 | -------------------------------------------------------------------------------- /tests/structured_tests/json_output.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "thing_a:* thing_b:*" as thing_a, thing_b""" 2 | input = """ 3 | thing_a:5 thing_b:red 4 | thing_a:6 thing_b:yellow 5 | thing_a:7 thing_b:blue 6 | """ 7 | flags = ["--output", "json"] 8 | output = """ 9 | {"thing_a":5,"thing_b":"red"} 10 | {"thing_a":6,"thing_b":"yellow"} 11 | {"thing_a":7,"thing_b":"blue"} 12 | """ 13 | -------------------------------------------------------------------------------- /tests/structured_tests/json_output_sorted.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "thing_a:* thing_b:*" as thing_a, thing_b | sort by thing_b""" 2 | input = """ 3 | thing_a:5 thing_b:red 4 | thing_a:6 thing_b:yellow 5 | thing_a:7 thing_b:blue 6 | """ 7 | flags = ["--output", "json"] 8 | output = """ 9 | [{"thing_a":7,"thing_b":"blue"},{"thing_a":5,"thing_b":"red"},{"thing_a":6,"thing_b":"yellow"}] 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/limit.toml: -------------------------------------------------------------------------------- 1 | query = "* | limit 2" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | {"level": "info", "message": "A thing happened", "num_things": 1102} 12 | {"level": "error", "message": "Oh now an error!"} 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/limit_agg.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count by level | limit 1" 2 | input = """ 3 | {"level": "error", "message": "Oh now an error!"} 4 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 5 | {"level": "info", "message": "A thing happened", "num_things": 12} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A thing happened", "num_things": 12} 8 | {"level": "info", "message": "A thing happened", "num_things": 12} 9 | {"level": "info", "message": "A thing happened", "num_things": 12} 10 | {"level": "info", "message": "A thing happened", "num_things": 12} 11 | {"level": "info", "message": "A thing happened", "num_things": 12} 12 | {"level": "info", "message": "A thing happened", "num_things": 12} 13 | {"level": "info", "message": "A thing happened", "num_things": 12} 14 | {"level": "info", "message": "A thing happened", "num_things": 12} 15 | {"level": "info", "message": "A thing happened", "num_things": 12} 16 | {"level": "info", "message": "A thing happened", "num_things": 12} 17 | {"level": "info", "message": "A thing happened", "num_things": 12} 18 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 19 | {"level": "info", "message": "A thing happened", "num_things": 1102} 20 | {"level": null} 21 | """ 22 | output = """ 23 | level _count 24 | --------------------------- 25 | info 15 26 | """ 27 | notes = "If the implicit sort operator isn't added after the count, the output will be `error` and not `info`" 28 | -------------------------------------------------------------------------------- /tests/structured_tests/limit_agg_tail.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count by level | limit -1" 2 | input = """ 3 | {"level": "error", "message": "Oh now an error!"} 4 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 5 | {"level": "info", "message": "A thing happened", "num_things": 12} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A thing happened", "num_things": 12} 8 | {"level": "info", "message": "A thing happened", "num_things": 12} 9 | {"level": "info", "message": "A thing happened", "num_things": 12} 10 | {"level": "info", "message": "A thing happened", "num_things": 12} 11 | {"level": "info", "message": "A thing happened", "num_things": 12} 12 | {"level": "info", "message": "A thing happened", "num_things": 12} 13 | {"level": "info", "message": "A thing happened", "num_things": 12} 14 | {"level": "info", "message": "A thing happened", "num_things": 12} 15 | {"level": "info", "message": "A thing happened", "num_things": 12} 16 | {"level": "info", "message": "A thing happened", "num_things": 12} 17 | {"level": "info", "message": "A thing happened", "num_things": 12} 18 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 19 | {"level": "info", "message": "A thing happened", "num_things": 1102} 20 | {"level": null} 21 | """ 22 | output = """ 23 | level _count 24 | --------------------------- 25 | None 1 26 | """ 27 | notes = "If the implicit sort operator isn't added after the count, the output will be `error` and not `info`" 28 | -------------------------------------------------------------------------------- /tests/structured_tests/limit_error.toml: -------------------------------------------------------------------------------- 1 | query = "* | limt 5" 2 | input = """ 3 | """ 4 | output = "" 5 | error = """ 6 | error: Expected an operator 7 | | 8 | 1 | * | limt 5 9 | | ^^^^ 10 | | 11 | = help: limt is not a valid operator 12 | = help: Did you mean "limit"? 13 | Error: Failed to parse query 14 | """ 15 | succeeds = false 16 | -------------------------------------------------------------------------------- /tests/structured_tests/limit_error_2.toml: -------------------------------------------------------------------------------- 1 | query = "* | limitt 5" 2 | input = """ 3 | """ 4 | output = "" 5 | error = """ 6 | error: Expected an operator 7 | | 8 | 1 | * | limitt 5 9 | | ^^^^^^ 10 | | 11 | = help: limitt is not a valid operator 12 | = help: Did you mean "limit"? 13 | Error: Failed to parse query 14 | """ 15 | succeeds = false 16 | notes = "This test validates that prefixes are handled properly" 17 | -------------------------------------------------------------------------------- /tests/structured_tests/limit_tail.toml: -------------------------------------------------------------------------------- 1 | query = "* | limit -2" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 12 | {"level": null} 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/logfmt.toml: -------------------------------------------------------------------------------- 1 | query = """* | logfmt | fields thing_a, thing_b""" 2 | input = """ 3 | thing_a=5 thing_b=red 4 | thing_a=6 thing_b=yellow 5 | thing_a=7 thing_b=blue 6 | """ 7 | output = """ 8 | [thing_a=5] [thing_b=red] 9 | [thing_a=6] [thing_b=yellow] 10 | [thing_a=7] [thing_b=blue] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/logfmt_output.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "thing_a:* thing_b:*" as thing_a, thing_b""" 2 | input = """ 3 | thing_a:5 thing_b:red 4 | thing_a:6 thing_b:yellow 5 | thing_a:7 thing_b:blue 6 | """ 7 | flags = ["--output", "logfmt"] 8 | output = """ 9 | thing_a=5 thing_b=red 10 | thing_a=6 thing_b=yellow 11 | thing_a=7 thing_b=blue 12 | """ 13 | -------------------------------------------------------------------------------- /tests/structured_tests/logfmt_output_with_sort.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "thing_a:* thing_b:*" as thing_a, thing_b | sort by thing_a""" 2 | input = """ 3 | thing_a:5 thing_b:red 4 | thing_a:6 thing_b:yellow 5 | thing_a:7 thing_b:blue 6 | """ 7 | flags = ["--output", "logfmt"] 8 | output = """ 9 | thing_a=5 thing_b=red 10 | thing_a=6 thing_b=yellow 11 | thing_a=7 thing_b=blue 12 | """ 13 | -------------------------------------------------------------------------------- /tests/structured_tests/logical-expr-1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | if(!isNull(a), a, b) as value""" 2 | input = """ 3 | {"a": null, "b": "goodbye"} 4 | {"a": "hello", "b": "abc"} 5 | {"a": "world", "b": "def"} 6 | """ 7 | output = """ 8 | [a=None] [b=goodbye] [value=goodbye] 9 | [a=hello] [b=abc] [value=hello] 10 | [a=world] [b=def] [value=world] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/logical-expr-2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where a and b""" 2 | input = """ 3 | {"a": false} 4 | {"a": true, "b": false} 5 | {"a": true, "b": true} 6 | """ 7 | output = """ 8 | [a=true] [b=true] 9 | """ 10 | -------------------------------------------------------------------------------- /tests/structured_tests/longlines.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count by event" 2 | input = """ 3 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fieldsabcdefghijklmnopqrstuvvwxyz fieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwxyz"} 4 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 5 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 6 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 7 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 8 | {"env": "dev", "timestamp": "2018-03-09T18:52:16.611872Z", "level": "info", "schema_name": "db_shard", "hostname": "127.0.0.1", "module": "angle.grinder.logs:146", "key": 3, "greenlet_id": 99191, "event": "wow thats a alot of fields"} 9 | """ 10 | output = """ 11 | event _count 12 | -------------------------------------------------------------------------------------------------------------------------------------- 13 | wow thats a alot of fields 5 14 | wow thats a alot of fieldsabcdefghijklmnopqrstuvvwxyz fieldsabcdefghijklmnopqrstuvvwxyzfieldsabcdefghijklmnopqrstuvvwx… 1 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/math_funcs_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | abs(num) as num_abs | num / other_num as division_result | round(num / 100) as round_num""" 2 | input = """ 3 | {"num": 123, "other_num": 456} 4 | {"num": -456, "other_num": 123.0} 5 | {"num": 789, "other_num": "cow"} 6 | {"num": "321", "other_num": -10000} 7 | """ 8 | output = """ 9 | [division_result=0.27] [num=123] [num_abs=123] [other_num=456] [round_num=1] 10 | [division_result=-3.71] [num=-456] [num_abs=456] [other_num=123] [round_num=-5] 11 | [division_result=-0.03] [num=321] [num_abs=321] [other_num=-10000] [round_num=3] 12 | """ 13 | error = """ 14 | error: Expected numeric operands, found 789 / cow 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/min_max.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | min(num_things), max(num_things)" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | _min _max 12 | ------------------------ 13 | 0.10 1102 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/min_max_none.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | min(num_things), max(num_things)" 2 | input = """ 3 | {"level": "error", "message": "Oh now an error!"} 4 | """ 5 | output = """ 6 | _min _max 7 | ------------------------ 8 | None None 9 | """ 10 | -------------------------------------------------------------------------------- /tests/structured_tests/min_max_rounded.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | min(floor(num_things)), max(ceil(num_things))" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102.5} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | _min _max 12 | ------------------------ 13 | 0 1103 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/missing_optional_match.toml: -------------------------------------------------------------------------------- 1 | query = """* |parse regex "(?P.+)(?P\\?.+)?"""" 2 | input = """/track/?verbose=1&ip=1&_=1716389413340""" 3 | output = """ 4 | [query=None] [url=/track/?verbose=1&ip=1&_=1716389413340] 5 | """ 6 | error = """ 7 | """ 8 | -------------------------------------------------------------------------------- /tests/structured_tests/nested_values_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where thing_a.x == thing_b.y[-1]""" 2 | input = """ 3 | {"thing_a": {"x": 5 }, "thing_b": {"y": [6, 7, 5]}} 4 | {"thing_a": {"x": 6 }, "thing_b": {"y": [5]}} 5 | {"thing_a": {"z": 6 }, "thing_b": {"z": 5}} 6 | {"thing_a": {"x": {"a": 5} }, "thing_b": {"y": {"a": 5}}} 7 | {"thing_a": {"x": "blue" }, "thing_b": {"y": ["blue"]}} 8 | """ 9 | output = """ 10 | [thing_a={x:5}] [thing_b={y:[6, 7, 5]}] 11 | [thing_a={x:blue}] [thing_b={y:[blue]}] 12 | """ 13 | 14 | error = """ 15 | error: No value for key "x" 16 | error: Expected array, found {a:5} 17 | """ 18 | -------------------------------------------------------------------------------- /tests/structured_tests/nested_values_2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | sum(thing_a.x)""" 2 | input = """ 3 | {"thing_a": {"x": 5 }, "thing_b": {"y": 5}} 4 | {"thing_a": {"x": 6 }, "thing_b": {"y": 5}} 5 | {"thing_a": {"z": 6 }, "thing_b": {"z": 5}} 6 | {"thing_a": {"x": {"a": 5} }, "thing_b": {"y": {"a": 5}}} 7 | {"thing_a": {"x": "blue" }, "thing_b": {"y": "blue"}} 8 | """ 9 | output = """ 10 | _sum 11 | ------------ 12 | 11 13 | """ 14 | 15 | error = """ 16 | """ -------------------------------------------------------------------------------- /tests/structured_tests/nested_values_3.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | sum(thing_a.x) by thing_b.y""" 2 | input = """ 3 | {"thing_a": {"x": 5 }, "thing_b": {"y": 5}} 4 | {"thing_a": {"x": 6 }, "thing_b": {"y": 5}} 5 | {"thing_a": {"z": 6 }, "thing_b": {"z": 5}} 6 | {"thing_a": {"x": {"a": 5} }, "thing_b": {"y": {"a": 5}}} 7 | {"thing_a": {"x": "blue" }, "thing_b": {"y": "blue"}} 8 | """ 9 | output = """ 10 | thing_b.y _sum 11 | ----------------------------- 12 | 5 11 13 | None 0 14 | blue 0 15 | {a:5} 0 16 | """ 17 | 18 | error = """ 19 | """ -------------------------------------------------------------------------------- /tests/structured_tests/not_an_agg.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count, parse by foo" 2 | input = "" 3 | output = "" 4 | error = """ 5 | error: Not an aggregate operator 6 | | 7 | 1 | * | json | count, parse by foo 8 | | ^^^^^ 9 | | 10 | = help: parse is not a valid aggregate operator 11 | = help: parse is an inline operator, but only aggregate operators (count, average, etc.) are valid here 12 | Error: Failed to parse query 13 | """ 14 | succeeds = false 15 | -------------------------------------------------------------------------------- /tests/structured_tests/not_an_agg_2.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count, averag(c) by foo" 2 | input = "" 3 | output = "" 4 | error = """ 5 | error: Not an aggregate operator 6 | | 7 | 1 | * | json | count, averag(c) by foo 8 | | ^^^^^^ 9 | | 10 | = help: averag is not a valid aggregate operator 11 | = help: Did you mean "average"? 12 | Error: Failed to parse query 13 | """ 14 | succeeds = false 15 | -------------------------------------------------------------------------------- /tests/structured_tests/parseDate_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | parseDate(ts) as pts""" 2 | input = """ 3 | {"ts": "2014-01-01T10:20:30Z"} 4 | {"ts": "21/Jan/2014 14:18:17"} 5 | {"ts": "abc"} 6 | {"ts": false} 7 | """ 8 | output = """ 9 | [pts=2014-01-01 10:20:30 UTC] [ts=2014-01-01T10:20:30Z] 10 | [pts=2014-01-21 14:18:17 UTC] [ts=21/Jan/2014 14:18:17] 11 | """ 12 | error = """ 13 | error: The 'parseDate' function failed with the error: UnrecognizedToken("abc") 14 | error: The 'parseDate' function failed with the error: UnrecognizedToken("false") 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/parseDate_2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | parseDate(ts, 123) as pts""" 2 | input = """ 3 | {"ts": "2014-01-01T10:20:30Z"} 4 | """ 5 | output = """ 6 | """ 7 | error = """ 8 | error: The 'parseDate' function expects 1 arguments, found 2 9 | """ 10 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_drop.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "name=* says=*" as name, words""" 2 | input = """ 3 | name=Jarrad says=Hi 4 | name=Ella says=Bye 5 | name=Jim 6 | """ 7 | output = """ 8 | [name=Jarrad] [words=Hi] 9 | [name=Ella] [words=Bye] 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_error_double_from.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | parse \"a * b\" from col as x from col" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: Two `from` clauses were provided 9 | | 10 | 1 | * | json | parse "a * b" from col as x from col 11 | | ^^^^^^^^ 12 | | ^^^^^^^^ 13 | | 14 | = help: Only one from clause is allowed 15 | Error: Two `from` clauses were provided 16 | """ 17 | succeeds = false 18 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_error_missing_field_name.toml: -------------------------------------------------------------------------------- 1 | query = "* | 1 as 1" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: expecting a name for the field 9 | | 10 | 1 | * | 1 as 1 11 | | ^ 12 | | 13 | = help: Give the value a name 14 | Error: Failed to parse query 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_error_missing_operand.toml: -------------------------------------------------------------------------------- 1 | query = "* | where response_ms *" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: expecting an operand for binary operator 9 | | 10 | 1 | * | where response_ms * 11 | | ^ dangling binary operator 12 | | 13 | = help: Add the operand or delete the operator 14 | Error: Failed to parse query 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_error_unterminated.toml: -------------------------------------------------------------------------------- 1 | query = "* | parse \"abc" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: unterminated double quoted string 9 | | 10 | 1 | * | parse "abc 11 | | ^^^^ 12 | | 13 | = help: Insert a double quote (") to terminate this string 14 | Error: Failed to parse query 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_error_unterminated_sq.toml: -------------------------------------------------------------------------------- 1 | query = "* | parse 'abc" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | """ 6 | output = "" 7 | error = """ 8 | error: unterminated single quoted string 9 | | 10 | 1 | * | parse 'abc 11 | | ^^^^ 12 | | 13 | = help: Insert a single quote (') to terminate this string 14 | Error: Failed to parse query 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_literal_tab.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "name=* says=*" as name, words""" 2 | input = """ 3 | name=Jarrad\tsays=Hi 4 | name=Ella says=Bye 5 | name=Jim 6 | """ 7 | output = """ 8 | [name=Jarrad] [words=Hi] 9 | [name=Ella] [words=Bye] 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_nodrop.toml: -------------------------------------------------------------------------------- 1 | query = """* | parse "name=* says=*" as name, words nodrop""" 2 | input = """ 3 | name=Jarrad says=Hi 4 | name=Ella says=Bye 5 | name=Jim 6 | """ 7 | output = """ 8 | [name=Jarrad] [words=Hi] 9 | [name=Ella] [words=Bye] 10 | [name=None] [words=None] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_plain.toml: -------------------------------------------------------------------------------- 1 | query = """db-1 response | parse "in *ms" as duration""" 2 | input = """ 3 | INFO Server db-1 loaded response in 500ms 4 | INFO Server db-2 loaded response in 394ms 5 | INFO Server db-1 loaded request in 394ms 6 | WARN Server failed to load response 7 | INFO Server db-1 loaded response in 100ms 8 | INFO Server db-1 loaded response in 102ms 9 | INFO Server db-1 loaded response in 102ms 10 | INFO Server db-3 loaded response in 103ms 11 | INFO Server db-1 loaded response in 100ms 12 | INFO Server db-3 loaded response in 109ms 13 | INFO Server db-1 loaded response in 100ms 14 | INFO Server db-3 loaded response in 104ms 15 | INFO Server db-1 loaded response in 100ms 16 | INFO Server db-3 loaded response in 100ms 17 | INFO Server db-1 loaded response in 122ms 18 | INFO Server db-2 loaded response in 119ms 19 | """ 20 | output = """ 21 | [duration=500] 22 | [duration=100] 23 | [duration=102] 24 | [duration=102] 25 | [duration=100] 26 | [duration=100] 27 | [duration=100] 28 | [duration=122] 29 | """ 30 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_plain_no_convert.toml: -------------------------------------------------------------------------------- 1 | query = """db-1 response | parse "in *ms" as duration noconvert""" 2 | input = """ 3 | INFO Server db-1 loaded response in 000ms 4 | INFO Server db-1 loaded response in 001ms 5 | """ 6 | output = """ 7 | [duration=000] 8 | [duration=001] 9 | """ 10 | -------------------------------------------------------------------------------- /tests/structured_tests/parse_regex.toml: -------------------------------------------------------------------------------- 1 | query = 'db-1 response | parse regex "in (?P\d+)ms"' 2 | input = """ 3 | INFO Server db-1 loaded response in 500ms 4 | INFO Server db-2 loaded response in 394ms 5 | INFO Server db-1 loaded request in 394ms 6 | WARN Server failed to load response 7 | INFO Server db-1 loaded response in 100ms 8 | INFO Server db-1 loaded response in 102ms 9 | INFO Server db-1 loaded response in 102ms 10 | INFO Server db-3 loaded response in 103ms 11 | INFO Server db-1 loaded response in 100ms 12 | INFO Server db-3 loaded response in 109ms 13 | INFO Server db-1 loaded response in 100ms 14 | INFO Server db-3 loaded response in 104ms 15 | INFO Server db-1 loaded response in 100ms 16 | INFO Server db-3 loaded response in 100ms 17 | INFO Server db-1 loaded response in 122ms 18 | INFO Server db-2 loaded response in 119ms 19 | """ 20 | output = """ 21 | [duration=500] 22 | [duration=100] 23 | [duration=102] 24 | [duration=102] 25 | [duration=100] 26 | [duration=100] 27 | [duration=100] 28 | [duration=122] 29 | """ 30 | -------------------------------------------------------------------------------- /tests/structured_tests/percentile_1.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | p10(num_things), p90(num_things)" 2 | input = """ 3 | {"level": "info", "num_things": 100} 4 | {"level": "info", "num_things": 200} 5 | {"level": "info", "num_things": 100} 6 | {"level": "info", "num_things": 10} 7 | {"level": "info", "num_things": 50} 8 | {"level": "info", "num_things": 105} 9 | {"level": "info", "num_things": 1000} 10 | {"level": "info", "num_things": 1100} 11 | {"level": "info", "num_things": 150} 12 | {"level": "info", "num_things": 175} 13 | """ 14 | output = """ 15 | p10 p90 16 | ----------------------- 17 | 10 1000 18 | """ 19 | error = """ 20 | """ 21 | -------------------------------------------------------------------------------- /tests/structured_tests/sort_by_expr.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | sort by length(message), num_things.k" 2 | input = """ 3 | {"level": "info", "message": "A thing", "num_things": { "k": 1102} } 4 | {"level": "info", "message": "A thing", "num_things": { "k": 1103} } 5 | {"level": "info", "message": "A thing ha", "num_things": 2} 6 | {"level": "info", "message": "A different", "num_things": 2.000001} 7 | {"level": "info", "message": "A different e", "num_things": 0.2000001} 8 | {"level": "info", "message": "A different ev", "num_things": "whoops not a number"} 9 | {"level": null} 10 | """ 11 | output = """ 12 | level message num_things 13 | ------------------------------------------------------------- 14 | info A thing {k:1102} 15 | info A thing {k:1103} 16 | info A thing ha 2 17 | info A different 2.00 18 | info A different e 0.20 19 | info A different ev whoops not a number 20 | None None None 21 | """ 22 | -------------------------------------------------------------------------------- /tests/structured_tests/sort_order.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | count by message, num_things | sort by num_things" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "info", "message": "A thing happened", "num_things": 12} 5 | {"level": "info", "message": "A thing happened", "num_things": 2} 6 | {"level": "info", "message": "A different event", "num_things": 2.000001} 7 | {"level": "info", "message": "A different event", "num_things": 0.2000001} 8 | {"level": "info", "message": "A different event", "num_things": "whoops not a number"} 9 | {"level": null} 10 | """ 11 | output = """ 12 | message num_things _count 13 | ------------------------------------------------------------------ 14 | None None 1 15 | A different event 0.20 1 16 | A thing happened 2 1 17 | A different event 2.00 1 18 | A thing happened 12 1 19 | A thing happened 1102 1 20 | A different event whoops not a number 1 21 | """ 22 | -------------------------------------------------------------------------------- /tests/structured_tests/split_1.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | split on " " 3 | """ 4 | input = ''' 5 | Oct 09 20:22:21 web-001 influxd[188053]: 127.0.0.1 "POST /write \"escaped\" HTTP/1.0" 204 6 | ''' 7 | output = ''' 8 | [_split=[Oct, 9, 20:22:21, web-001, influxd[188053]:, 127.0.0.1, POST /write \"escaped\" HTTP/1.0, 204]] 9 | ''' 10 | -------------------------------------------------------------------------------- /tests/structured_tests/split_10.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | json | split(level) on "-" as nested.nested.boo 3 | """ 4 | input = """ 5 | {"level": "INFO", "nested": { "nested": "invalid" } } 6 | {"level": "INFO", "nested": { "nested": 10 } } 7 | {"level": "INFO", "nested": { "nested": {} } } 8 | {"level": "INFO", "nested": { "nested": [] } } 9 | {"level": "INFO", "nested": { "nested": {"boo": "foo"} } } 10 | {"level": "INFO", "nested": { "nested": {"abc": 10} } } 11 | """ 12 | output = """ 13 | [level=INFO] [nested={nested:{boo:[INFO]}}] 14 | [level=INFO] [nested={nested:{boo:[INFO]}}] 15 | [level=INFO] [nested={nested:{abc:10, boo:[INFO]}}] 16 | """ 17 | error = """ 18 | error: Expected object, found invalid 19 | error: Expected object, found 10 20 | error: Expected object, found [] 21 | """ 22 | -------------------------------------------------------------------------------- /tests/structured_tests/split_2.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | parse "* *" as level, rest | split(rest) on '.' 3 | """ 4 | input = """ 5 | INFO name.power.over_9000 6 | DEBUG pavel.5000.false 7 | DEBUG darren.10000.true 8 | """ 9 | output = """ 10 | [level=INFO] [rest=[name, power, over_9000]] 11 | [level=DEBUG] [rest=[pavel, 5000, false]] 12 | [level=DEBUG] [rest=[darren, 10000, true]] 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/split_3.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | parse "* *" as level, rest | split(rest) on '...' as new_rest 3 | """ 4 | input = """ 5 | INFO name...power...over_9000 6 | DEBUG pavel5000...false 7 | DEBUG darren...10000..true 8 | """ 9 | output = """ 10 | [level=INFO] [new_rest=[name, power, over_9000]] [rest=name...power...over_9000] 11 | [level=DEBUG] [new_rest=[pavel5000, false]] [rest=pavel5000...false] 12 | [level=DEBUG] [new_rest=[darren, 10000..true]] [rest=darren...10000..true] 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/split_4.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | parse "* *" as level, rest | split(rest) on '.' | count by rest[1] 3 | """ 4 | input = """ 5 | INFO name.power.over_9000 6 | DEBUG pavel.5000.false 7 | DEBUG darren..10000.true 8 | WARN jonathon...400..false 9 | """ 10 | output = """ 11 | rest[1] _count 12 | ----------------------------- 13 | 400 1 14 | 5000 1 15 | 10000 1 16 | power 1 17 | """ 18 | -------------------------------------------------------------------------------- /tests/structured_tests/split_5.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | parse "* *" as level, rest | split(rest) on '.' | sum(rest[1]) 3 | """ 4 | input = """ 5 | INFO name.power.over_9000 6 | DEBUG pavel.5000.false 7 | DEBUG darren..10000.true 8 | WARN jonathon...400..false 9 | """ 10 | output = """ 11 | _sum 12 | ------------- 13 | 15400 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/split_6.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | logfmt | split(raw) on "blah" as tokens | sum(tokens[-2]) 3 | """ 4 | input = """ 5 | level=INFO raw="ablah10blahcblah" 6 | level=WARN raw="ablahbblahcblah" 7 | level=DEBUG raw="ablah55.9blahcblah" 8 | level=DEBUG raw="apple" 9 | level=TRACE raw="blahb" 10 | """ 11 | output = """ 12 | _sum 13 | ------------- 14 | 65.90 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/split_7.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | logfmt | split on " " 3 | """ 4 | input = """ 5 | level=INFO raw="ablah10blahcblah" 6 | level=WARN raw="ablahbblahcblah" 7 | level=DEBUG raw="ablah55.9blahcblah" 8 | """ 9 | output = """ 10 | [_split=[level=INFO, raw="ablah10blahcblah"]] [level=INFO] [raw=ablah10blahcblah] 11 | [_split=[level=WARN, raw="ablahbblahcblah"]] [level=WARN] [raw=ablahbblahcblah] 12 | [_split=[level=DEBUG, raw="ablah55.9blahcblah"]] [level=DEBUG] [raw=ablah55.9blahcblah] 13 | """ 14 | -------------------------------------------------------------------------------- /tests/structured_tests/split_8.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | json | split(raw.nested) on "-" 3 | """ 4 | input = """ 5 | {"level": "INFO", "raw": { "nested": "a---b---c" }} 6 | {"level": "WARN", "raw": { "nested": "ab---c-", "foo": "bar"}, "hello": "world"} 7 | {"level": "DEBUG" } 8 | """ 9 | output = """ 10 | [level=INFO] [raw={nested:[a, b, c]}] 11 | [level=WARN] [raw={foo:bar, nested:[ab, c]}] [hello=world] 12 | """ 13 | error = """ 14 | error: No value for key "raw" 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/split_9.toml: -------------------------------------------------------------------------------- 1 | query = """ 2 | * | json | split(raw.nested) on "-" as oh_boy[0].abc 3 | """ 4 | input = """ 5 | {"level": "INFO", "raw": { "nested": "a---b---c" }, "oh_boy": [{"abc": "xyz", "def": "gg"}, 10]} 6 | {"level": "WARN", "raw": { "nested": "ab---c-", "foo": "bar"}, "hello": "world"} 7 | {"level": "DEBUG", "oh_boy": [] } 8 | {"level": "WARN", "raw": { "nested": "a-c" }, "oh_boy": [{"abc": 10}] } 9 | {"level": "TRACE", "raw": { "nested": "a-c" }, "oh_boy": [] } 10 | """ 11 | output = """ 12 | [level=INFO] [oh_boy=[{abc:[a, b, c], def:gg}, 10]] [raw={nested:a---b---c}] 13 | [level=WARN] [oh_boy=[{abc:[a, c]}]] [raw={nested:a-c}] 14 | """ 15 | error = """ 16 | error: No value for key "oh_boy" 17 | error: No value for key "raw" 18 | error: Index 0 out of range 19 | """ 20 | -------------------------------------------------------------------------------- /tests/structured_tests/string_funcs_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | concat('123', ts) as msg | length(msg) as msg_len""" 2 | input = """ 3 | {"ts": "2014-01-01T10:20:30Z"} 4 | {"ts": "21/Jan/2014 14:18:17"} 5 | {"ts": "abc"} 6 | {"ts": false} 7 | """ 8 | output = """ 9 | [msg=1232014-01-01T10:20:30Z] [msg_len=23] [ts=2014-01-01T10:20:30Z] 10 | [msg=12321/Jan/2014 14:18:17] [msg_len=23] [ts=21/Jan/2014 14:18:17] 11 | [msg=123abc] [msg_len=6] [ts=abc] 12 | [msg=123false] [msg_len=8] [ts=false] 13 | """ 14 | error = """ 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/string_funcs_2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | substring(f0, 1) as f1 | substring(f0, 0, 1) as f2""" 2 | input = """ 3 | {"f0": "\u2603 abc"} 4 | {"f0": "\u2603 def"} 5 | {"f0": "ghijkl"} 6 | {"f0": \"\"} 7 | """ 8 | output = """ 9 | [f0=☃ abc] [f1= abc] [f2=☃] 10 | [f0=☃ def] [f1= def] [f2=☃] 11 | [f0=ghijkl] [f1=hijkl] [f2=g] 12 | [f0=] [f1=] [f2=] 13 | """ 14 | error = """ 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/sum.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | sum(num_things)" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1102} 4 | {"level": "error", "message": "Oh now an error!"} 5 | {"level": "error", "message": "So many more errors!", "num_things": 0.1} 6 | {"level": "info", "message": "A thing happened", "num_things": 12} 7 | {"level": "info", "message": "A different event", "event_duration": 1002.5} 8 | {"level": null} 9 | """ 10 | output = """ 11 | _sum 12 | --------------- 13 | 1114.10 14 | """ 15 | -------------------------------------------------------------------------------- /tests/structured_tests/timeslice_1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | timeslice(parseDate(ts)) 5m | count by _timeslice""" 2 | input = """ 3 | {"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"} 4 | {"ts": "2013-09-06T20:00:49.124817Z", "lvl": "INFO", "msg": "Starting up service"} 5 | {"ts": "2013-09-06T22:00:49.124817Z", "lvl": "INFO", "msg": "Shutting down service", "user": "steve@example.com"} 6 | {"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG5", "msg": "Details..."} 7 | {"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG4", "msg": "Details..."} 8 | {"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG3", "msg": "Details..."} 9 | {"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG2", "msg": "Details..."} 10 | {"ts": "2013-09-06T22:00:59.124817Z", "lvl": "DEBUG", "msg": "Details..."} 11 | {"ts": "2013-09-06T22:01:49.124817Z", "lvl": "STATS", "msg": "1 beat per second"} 12 | {"ts": "2013-09-06T22:05:49.124817Z", "lvl": "WARNING", "msg": "not looking good"} 13 | {"ts": "2013-09-06T22:05:49.124817Z", "lvl": "ERROR", "msg": "looking bad"} 14 | {"ts": "2013-09-06T22:10:49.124817Z", "lvl": "CRITICAL", "msg": "sooo bad"} 15 | {"ts": "2013-09-06T23:05:49.124817Z", "lvl": "FATAL", "msg": "shoot", "obj": { "field1" : "hi", "field2": 2 }, "arr" : ["hi", {"sub1": true}]} 16 | """ 17 | output = """ 18 | _timeslice _count 19 | --------------------------------------------- 20 | 2013-09-06 20:00:00 UTC 2 21 | 2013-09-06 22:00:00 UTC 7 22 | 2013-09-06 22:05:00 UTC 2 23 | 2013-09-06 22:10:00 UTC 1 24 | 2013-09-06 23:05:00 UTC 1 25 | """ 26 | -------------------------------------------------------------------------------- /tests/structured_tests/timeslice_2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | timeslice(ts)""" 2 | input = """ 3 | {"ts": "2013-09-06T20:00:48.124817Z", "lvl": "TRACE", "msg": "trace test"} 4 | """ 5 | output = """""" 6 | error = """ 7 | Error: Expected a duration for the timeslice (e.g. 1h) 8 | """ 9 | succeeds = false 10 | -------------------------------------------------------------------------------- /tests/structured_tests/total.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | total(num_things) as _lotal" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1.5} 4 | {"level": "info", "message": "A thing happened", "num_things": 1103} 5 | {"level": "info", "message": "A thing happened", "num_things": 1105} 6 | {"level": "info", "message": "A thing happened", "num_things": "not_a_number"} 7 | {"level": "info", "message": "A thing happened"} 8 | {"level": "info", "message": "A thing happened", "num_things": 1105} 9 | {"level": "info", "message": "A thing happened", "num_things": 1105} 10 | {"level": "info", "message": "A thing happened", "num_things": 1105} 11 | {"level": "info", "message": "A thing happened", "num_things": 1105.5} 12 | """ 13 | output = """ 14 | [_lotal=1.50] [level=info] [message=A thing happened] [num_things=1.50] 15 | [_lotal=1104.50] [level=info] [message=A thing happened] [num_things=1103] 16 | [_lotal=2209.50] [level=info] [message=A thing happened] [num_things=1105] 17 | [_lotal=2209.50] [level=info] [message=A thing happened] [num_things=not_a_number] 18 | [_lotal=2209.50] [level=info] [message=A thing happened] 19 | [_lotal=3314.50] [level=info] [message=A thing happened] [num_things=1105] 20 | [_lotal=4419.50] [level=info] [message=A thing happened] [num_things=1105] 21 | [_lotal=5524.50] [level=info] [message=A thing happened] [num_things=1105] 22 | [_lotal=6630] [level=info] [message=A thing happened] [num_things=1105.50] 23 | """ 24 | -------------------------------------------------------------------------------- /tests/structured_tests/total_agg.toml: -------------------------------------------------------------------------------- 1 | query = "* | json | avg(num_things) by level | sort by _average, level | total(_average)" 2 | input = """ 3 | {"level": "info", "message": "A thing happened", "num_things": 1.5} 4 | {"level": "info", "message": "A thing happened", "num_things": 1103} 5 | {"level": "error", "message": "A thing happened", "num_things": 1105} 6 | {"level": "error", "message": "A thing happened", "num_things": "not_a_number"} 7 | {"level": "warn", "message": "A thing happened"} 8 | {"level": "warn", "message": "A thing happened", "num_things": 1105} 9 | {"level": "warn", "message": "A thing happened", "num_things": 1105} 10 | {"level": "info", "message": "A thing happened", "num_things": 1105} 11 | {"level": "debug", "message": "A thing happened", "num_things": 1105.5} 12 | """ 13 | output = """ 14 | level _average _total 15 | ------------------------------------------- 16 | info 736.50 736.50 17 | error 1105 1841.50 18 | warn 1105 2946.50 19 | debug 1105.50 4052 20 | """ 21 | -------------------------------------------------------------------------------- /tests/structured_tests/where-1.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where thing_a == thing_b""" 2 | input = """ 3 | {"thing_a": 5, "thing_b": 5} 4 | {"thing_a": 6, "thing_b": 5} 5 | {"thing_a": "hello", "thing_b": 5} 6 | {"thing_a": true, "thing_b": true} 7 | """ 8 | output = """ 9 | [thing_a=5] [thing_b=5] 10 | [thing_a=true] [thing_b=true] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/where-10.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where concat(thing_a, thing_b) == '65'""" 2 | input = """ 3 | {"thing_a": 5, "thing_b": 5} 4 | {"thing_a": 6, "thing_b": 5} 5 | {"thing_a": "hello", "thing_b": 5} 6 | {"thing_a": true, "thing_b": true} 7 | """ 8 | output = """ 9 | [thing_a=6] [thing_b=5] 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/where-11.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | parseDate(ts) as ts | where parseDate("2014-01-05T00:00:00Z") - ts < 2d""" 2 | input = """ 3 | {"ts": "2014-01-01T10:20:30Z"} 4 | {"ts": "2014-01-02T10:20:30Z"} 5 | {"ts": "2014-01-03T10:20:30Z"} 6 | {"ts": "2014-01-04T10:20:30Z"} 7 | """ 8 | output = """ 9 | [ts=2014-01-03 10:20:30 UTC] 10 | [ts=2014-01-04 10:20:30 UTC] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/where-12.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | parseDate(ts) as ts | where ts < parseDate("2015-06-01T00:00:00Z")""" 2 | input = """ 3 | {"ts": "2014-01-01T10:20:30Z"} 4 | {"ts": "2015-01-01T10:20:30Z"} 5 | {"ts": "2016-01-01T10:20:30Z"} 6 | {"ts": "2017-01-01T10:20:30Z"} 7 | """ 8 | output = """ 9 | [ts=2014-01-01 10:20:30 UTC] 10 | [ts=2015-01-01 10:20:30 UTC] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/where-2.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where thing_a == 5""" 2 | input = """ 3 | {"thing_a": 5, "thing_b": 99, "thing_c": 9999} 4 | {"thing_a": 6, "thing_b": 5} 5 | {"thing_a": "hello", "thing_b": 5} 6 | {"thing_a": 5, "thing_b": true} 7 | """ 8 | output = """ 9 | [thing_a=5] [thing_b=99] [thing_c=9999] 10 | [thing_a=5] [thing_b=true] 11 | """ 12 | -------------------------------------------------------------------------------- /tests/structured_tests/where-3.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where thing_a > thing_b""" 2 | input = """ 3 | {"thing_a": 5, "thing_b": 5} 4 | {"thing_a": 6, "thing_b": 5} 5 | {"thing_a": 0, "thing_b": 0.1} 6 | {"thing_a": 1.0, "thing_b": 0.1} 7 | {"thing_a": "hello", "thing_b": "goodbye"} 8 | {"thing_a": "goodbye", "thing_b": "hello"} 9 | {"thing_a": true, "thing_b": false} 10 | {"thing_a": false, "thing_b": true} 11 | """ 12 | output = """ 13 | [thing_a=6] [thing_b=5] 14 | [thing_a=1] [thing_b=0.10] 15 | [thing_a=hello] [thing_b=goodbye] 16 | [thing_a=true] [thing_b=false] 17 | """ 18 | -------------------------------------------------------------------------------- /tests/structured_tests/where-4.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where !bval""" 2 | input = """ 3 | {"thing_a": 5, "bval": true} 4 | {"thing_a": 6, "bval": false} 5 | {"thing_a": 0, "bval": true} 6 | {"thing_a": 0} 7 | {"thing_a": 0, "bval": 1} 8 | """ 9 | output = """ 10 | [bval=false] [thing_a=6] 11 | """ 12 | error = """ 13 | error: No value for key "bval" 14 | error: Expected boolean, found 1 15 | """ 16 | -------------------------------------------------------------------------------- /tests/structured_tests/where-5.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where bval""" 2 | input = """ 3 | {"thing_a": 5, "bval": true} 4 | {"thing_a": 6, "bval": false} 5 | {"thing_a": 0, "bval": true} 6 | """ 7 | output = """ 8 | [bval=true] [thing_a=5] 9 | [bval=true] [thing_a=0] 10 | """ 11 | -------------------------------------------------------------------------------- /tests/structured_tests/where-6.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where""" 2 | input = """ 3 | {"thing_a": 5, "bval": true} 4 | """ 5 | output = """""" 6 | error = """ 7 | error: Expected an expression 8 | | 9 | 1 | * | json | where 10 | | ^^^^^ No condition provided for this 'where' 11 | | 12 | = help: Insert an expression whose result determines whether a record should be passed downstream 13 | = help: example: where duration > 100 14 | Error: Expected an expression 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/where-7.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where 1""" 2 | input = """ 3 | {"thing_a": 5, "bval": true} 4 | """ 5 | output = """""" 6 | error = """ 7 | error: Expected boolean expression, found Int(1) 8 | | 9 | 1 | * | json | where 1 10 | | ^ This is constant 11 | | 12 | = help: Perhaps you meant to compare a field to this value? 13 | = help: example: where field1 == 1 14 | Error: Expected boolean expression, found Int(1) 15 | """ 16 | succeeds = false 17 | -------------------------------------------------------------------------------- /tests/structured_tests/where-8.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where bval""" 2 | input = """ 3 | {"thing_a": 5, "bval": true} 4 | {"thing_a": 6, "bval": false} 5 | {"thing_a": 0, "bval": true} 6 | {"thing_a": 0} 7 | {"thing_a": 0, "bval": 1} 8 | """ 9 | output = """ 10 | [bval=true] [thing_a=5] 11 | [bval=true] [thing_a=0] 12 | """ 13 | error = """ 14 | error: No value for key "bval" 15 | error: Expected boolean, found 1 16 | """ 17 | 18 | -------------------------------------------------------------------------------- /tests/structured_tests/where-9.toml: -------------------------------------------------------------------------------- 1 | query = """* | json | where (response_ms / 2) < 4""" 2 | input = """ 3 | {"response_ms": 10} 4 | {"response_ms": 6} 5 | {} 6 | {"response_ms": 2} 7 | {"response_ms": "five"} 8 | """ 9 | output = """ 10 | [response_ms=6] 11 | [response_ms=2] 12 | """ 13 | error = """ 14 | error: No value for key "response_ms" 15 | error: Expected numeric operands, found five / 2 16 | """ 17 | --------------------------------------------------------------------------------