├── .dockerignore
├── .github
    └── workflows
    │   └── main.yml
├── .gitignore
├── .rustfmt.toml
├── Cargo.lock
├── Cargo.toml
├── Dockerfile
├── LICENSE
├── Readme.md
├── src
    ├── bin
    │   ├── offline
    │   │   ├── dl.rs
    │   │   ├── extract.rs
    │   │   ├── fs.rs
    │   │   ├── learn.rs
    │   │   └── mod.rs
    │   ├── rla-offline.rs
    │   ├── rla-server.rs
    │   ├── server
    │   │   ├── mod.rs
    │   │   ├── service.rs
    │   │   └── worker.rs
    │   └── util
    │   │   └── mod.rs
    ├── ci
    │   ├── actions.rs
    │   ├── azure.rs
    │   └── mod.rs
    ├── extract.rs
    ├── github.rs
    ├── index
    │   ├── mod.rs
    │   ├── storage.rs
    │   └── table.rs
    ├── lib.rs
    ├── log_variables.rs
    └── sanitize.rs
└── tests
    ├── test.rs
    └── test_missing_line.txt


/.dockerignore:
--------------------------------------------------------------------------------
1 | /target
2 | 


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | on: [push, pull_request]
 3 | 
 4 | jobs:
 5 |   ci:
 6 |     name: CI
 7 |     runs-on: ubuntu-latest
 8 |     steps:
 9 | 
10 |       - name: Checkout the source code
11 |         uses: actions/checkout@v2
12 |         with:
13 |           fetch-depth: 1
14 | 
15 |       - name: Install Rust
16 |         run: rustup update stable && rustup default stable
17 | 
18 |       - name: Check formatting
19 |         run: cargo fmt -- --check
20 | 
21 |       - name: Run the test suite
22 |         run: cargo test
23 | 
24 |       - name: Build the Docker container
25 |         run: docker build -t rust-log-analyzer .
26 | 
27 |       - name: Deploy to production
28 |         uses: rust-lang/simpleinfra/github-actions/upload-docker-image@master
29 |         with:
30 |           image: rust-log-analyzer
31 |           repository: rust-log-analyzer
32 |           region: us-west-1
33 |           redeploy_ecs_cluster: rust-ecs-prod
34 |           redeploy_ecs_service: rust-log-analyzer
35 |           aws_access_key_id: "${{ secrets.AWS_ACCESS_KEY_ID }}"
36 |           aws_secret_access_key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
37 |         if: github.ref == 'refs/heads/master'
38 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | 
2 | /target
3 | **/*.rs.bk
4 | 


--------------------------------------------------------------------------------
/.rustfmt.toml:
--------------------------------------------------------------------------------
1 | reorder_imports = true
2 | 


--------------------------------------------------------------------------------
/Cargo.lock:
--------------------------------------------------------------------------------
   1 | # This file is automatically @generated by Cargo.
   2 | # It is not intended for manual editing.
   3 | version = 4
   4 | 
   5 | [[package]]
   6 | name = "addr2line"
   7 | version = "0.21.0"
   8 | source = "registry+https://github.com/rust-lang/crates.io-index"
   9 | checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
  10 | dependencies = [
  11 |  "gimli",
  12 | ]
  13 | 
  14 | [[package]]
  15 | name = "adler"
  16 | version = "1.0.2"
  17 | source = "registry+https://github.com/rust-lang/crates.io-index"
  18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
  19 | 
  20 | [[package]]
  21 | name = "aho-corasick"
  22 | version = "1.1.3"
  23 | source = "registry+https://github.com/rust-lang/crates.io-index"
  24 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
  25 | dependencies = [
  26 |  "memchr",
  27 | ]
  28 | 
  29 | [[package]]
  30 | name = "alloc-no-stdlib"
  31 | version = "2.0.4"
  32 | source = "registry+https://github.com/rust-lang/crates.io-index"
  33 | checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
  34 | 
  35 | [[package]]
  36 | name = "alloc-stdlib"
  37 | version = "0.2.2"
  38 | source = "registry+https://github.com/rust-lang/crates.io-index"
  39 | checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
  40 | dependencies = [
  41 |  "alloc-no-stdlib",
  42 | ]
  43 | 
  44 | [[package]]
  45 | name = "anstream"
  46 | version = "0.6.13"
  47 | source = "registry+https://github.com/rust-lang/crates.io-index"
  48 | checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb"
  49 | dependencies = [
  50 |  "anstyle",
  51 |  "anstyle-parse",
  52 |  "anstyle-query",
  53 |  "anstyle-wincon",
  54 |  "colorchoice",
  55 |  "utf8parse",
  56 | ]
  57 | 
  58 | [[package]]
  59 | name = "anstyle"
  60 | version = "1.0.6"
  61 | source = "registry+https://github.com/rust-lang/crates.io-index"
  62 | checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
  63 | 
  64 | [[package]]
  65 | name = "anstyle-parse"
  66 | version = "0.2.3"
  67 | source = "registry+https://github.com/rust-lang/crates.io-index"
  68 | checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
  69 | dependencies = [
  70 |  "utf8parse",
  71 | ]
  72 | 
  73 | [[package]]
  74 | name = "anstyle-query"
  75 | version = "1.0.2"
  76 | source = "registry+https://github.com/rust-lang/crates.io-index"
  77 | checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
  78 | dependencies = [
  79 |  "windows-sys 0.52.0",
  80 | ]
  81 | 
  82 | [[package]]
  83 | name = "anstyle-wincon"
  84 | version = "3.0.2"
  85 | source = "registry+https://github.com/rust-lang/crates.io-index"
  86 | checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
  87 | dependencies = [
  88 |  "anstyle",
  89 |  "windows-sys 0.52.0",
  90 | ]
  91 | 
  92 | [[package]]
  93 | name = "anyhow"
  94 | version = "1.0.81"
  95 | source = "registry+https://github.com/rust-lang/crates.io-index"
  96 | checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
  97 | dependencies = [
  98 |  "backtrace",
  99 | ]
 100 | 
 101 | [[package]]
 102 | name = "atomicwrites"
 103 | version = "0.4.3"
 104 | source = "registry+https://github.com/rust-lang/crates.io-index"
 105 | checksum = "fc7b2dbe9169059af0f821e811180fddc971fc210c776c133c7819ccd6e478db"
 106 | dependencies = [
 107 |  "rustix",
 108 |  "tempfile",
 109 |  "windows-sys 0.52.0",
 110 | ]
 111 | 
 112 | [[package]]
 113 | name = "atty"
 114 | version = "0.2.14"
 115 | source = "registry+https://github.com/rust-lang/crates.io-index"
 116 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
 117 | dependencies = [
 118 |  "hermit-abi 0.1.19",
 119 |  "libc",
 120 |  "winapi",
 121 | ]
 122 | 
 123 | [[package]]
 124 | name = "autocfg"
 125 | version = "1.1.0"
 126 | source = "registry+https://github.com/rust-lang/crates.io-index"
 127 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 128 | 
 129 | [[package]]
 130 | name = "aws-config"
 131 | version = "0.56.1"
 132 | source = "registry+https://github.com/rust-lang/crates.io-index"
 133 | checksum = "fc6b3804dca60326e07205179847f17a4fce45af3a1106939177ad41ac08a6de"
 134 | dependencies = [
 135 |  "aws-credential-types",
 136 |  "aws-http",
 137 |  "aws-sdk-sso",
 138 |  "aws-sdk-sts",
 139 |  "aws-smithy-async",
 140 |  "aws-smithy-client",
 141 |  "aws-smithy-http",
 142 |  "aws-smithy-http-tower",
 143 |  "aws-smithy-json",
 144 |  "aws-smithy-types",
 145 |  "aws-types",
 146 |  "bytes",
 147 |  "fastrand",
 148 |  "hex",
 149 |  "http",
 150 |  "hyper",
 151 |  "ring 0.16.20",
 152 |  "time",
 153 |  "tokio",
 154 |  "tower",
 155 |  "tracing",
 156 |  "zeroize",
 157 | ]
 158 | 
 159 | [[package]]
 160 | name = "aws-credential-types"
 161 | version = "0.56.1"
 162 | source = "registry+https://github.com/rust-lang/crates.io-index"
 163 | checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201"
 164 | dependencies = [
 165 |  "aws-smithy-async",
 166 |  "aws-smithy-types",
 167 |  "fastrand",
 168 |  "tokio",
 169 |  "tracing",
 170 |  "zeroize",
 171 | ]
 172 | 
 173 | [[package]]
 174 | name = "aws-http"
 175 | version = "0.56.1"
 176 | source = "registry+https://github.com/rust-lang/crates.io-index"
 177 | checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900"
 178 | dependencies = [
 179 |  "aws-credential-types",
 180 |  "aws-smithy-http",
 181 |  "aws-smithy-types",
 182 |  "aws-types",
 183 |  "bytes",
 184 |  "http",
 185 |  "http-body",
 186 |  "lazy_static",
 187 |  "percent-encoding",
 188 |  "pin-project-lite",
 189 |  "tracing",
 190 | ]
 191 | 
 192 | [[package]]
 193 | name = "aws-runtime"
 194 | version = "0.56.1"
 195 | source = "registry+https://github.com/rust-lang/crates.io-index"
 196 | checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07"
 197 | dependencies = [
 198 |  "aws-credential-types",
 199 |  "aws-http",
 200 |  "aws-sigv4",
 201 |  "aws-smithy-async",
 202 |  "aws-smithy-eventstream",
 203 |  "aws-smithy-http",
 204 |  "aws-smithy-runtime-api",
 205 |  "aws-smithy-types",
 206 |  "aws-types",
 207 |  "fastrand",
 208 |  "http",
 209 |  "percent-encoding",
 210 |  "tracing",
 211 |  "uuid",
 212 | ]
 213 | 
 214 | [[package]]
 215 | name = "aws-sdk-s3"
 216 | version = "0.34.0"
 217 | source = "registry+https://github.com/rust-lang/crates.io-index"
 218 | checksum = "42f7a233b27af6e70094eafd43d9ee11da6e78eb2c1a31e5a7de737b782c627d"
 219 | dependencies = [
 220 |  "aws-credential-types",
 221 |  "aws-http",
 222 |  "aws-runtime",
 223 |  "aws-sigv4",
 224 |  "aws-smithy-async",
 225 |  "aws-smithy-checksums",
 226 |  "aws-smithy-client",
 227 |  "aws-smithy-eventstream",
 228 |  "aws-smithy-http",
 229 |  "aws-smithy-json",
 230 |  "aws-smithy-runtime",
 231 |  "aws-smithy-runtime-api",
 232 |  "aws-smithy-types",
 233 |  "aws-smithy-xml",
 234 |  "aws-types",
 235 |  "bytes",
 236 |  "http",
 237 |  "http-body",
 238 |  "once_cell",
 239 |  "percent-encoding",
 240 |  "regex",
 241 |  "tokio-stream",
 242 |  "tracing",
 243 |  "url",
 244 | ]
 245 | 
 246 | [[package]]
 247 | name = "aws-sdk-sso"
 248 | version = "0.30.0"
 249 | source = "registry+https://github.com/rust-lang/crates.io-index"
 250 | checksum = "903f888ff190e64f6f5c83fb0f8d54f9c20481f1dc26359bb8896f5d99908949"
 251 | dependencies = [
 252 |  "aws-credential-types",
 253 |  "aws-http",
 254 |  "aws-runtime",
 255 |  "aws-smithy-async",
 256 |  "aws-smithy-client",
 257 |  "aws-smithy-http",
 258 |  "aws-smithy-json",
 259 |  "aws-smithy-runtime",
 260 |  "aws-smithy-runtime-api",
 261 |  "aws-smithy-types",
 262 |  "aws-types",
 263 |  "bytes",
 264 |  "http",
 265 |  "regex",
 266 |  "tokio-stream",
 267 |  "tracing",
 268 | ]
 269 | 
 270 | [[package]]
 271 | name = "aws-sdk-sts"
 272 | version = "0.30.0"
 273 | source = "registry+https://github.com/rust-lang/crates.io-index"
 274 | checksum = "a47ad6bf01afc00423d781d464220bf69fb6a674ad6629cbbcb06d88cdc2be82"
 275 | dependencies = [
 276 |  "aws-credential-types",
 277 |  "aws-http",
 278 |  "aws-runtime",
 279 |  "aws-smithy-async",
 280 |  "aws-smithy-client",
 281 |  "aws-smithy-http",
 282 |  "aws-smithy-json",
 283 |  "aws-smithy-query",
 284 |  "aws-smithy-runtime",
 285 |  "aws-smithy-runtime-api",
 286 |  "aws-smithy-types",
 287 |  "aws-smithy-xml",
 288 |  "aws-types",
 289 |  "http",
 290 |  "regex",
 291 |  "tracing",
 292 | ]
 293 | 
 294 | [[package]]
 295 | name = "aws-sigv4"
 296 | version = "0.56.1"
 297 | source = "registry+https://github.com/rust-lang/crates.io-index"
 298 | checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03"
 299 | dependencies = [
 300 |  "aws-smithy-eventstream",
 301 |  "aws-smithy-http",
 302 |  "bytes",
 303 |  "form_urlencoded",
 304 |  "hex",
 305 |  "hmac",
 306 |  "http",
 307 |  "once_cell",
 308 |  "percent-encoding",
 309 |  "regex",
 310 |  "sha2",
 311 |  "time",
 312 |  "tracing",
 313 | ]
 314 | 
 315 | [[package]]
 316 | name = "aws-smithy-async"
 317 | version = "0.56.1"
 318 | source = "registry+https://github.com/rust-lang/crates.io-index"
 319 | checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af"
 320 | dependencies = [
 321 |  "futures-util",
 322 |  "pin-project-lite",
 323 |  "tokio",
 324 |  "tokio-stream",
 325 | ]
 326 | 
 327 | [[package]]
 328 | name = "aws-smithy-checksums"
 329 | version = "0.56.1"
 330 | source = "registry+https://github.com/rust-lang/crates.io-index"
 331 | checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed"
 332 | dependencies = [
 333 |  "aws-smithy-http",
 334 |  "aws-smithy-types",
 335 |  "bytes",
 336 |  "crc32c",
 337 |  "crc32fast",
 338 |  "hex",
 339 |  "http",
 340 |  "http-body",
 341 |  "md-5",
 342 |  "pin-project-lite",
 343 |  "sha1",
 344 |  "sha2",
 345 |  "tracing",
 346 | ]
 347 | 
 348 | [[package]]
 349 | name = "aws-smithy-client"
 350 | version = "0.56.1"
 351 | source = "registry+https://github.com/rust-lang/crates.io-index"
 352 | checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a"
 353 | dependencies = [
 354 |  "aws-smithy-async",
 355 |  "aws-smithy-http",
 356 |  "aws-smithy-http-tower",
 357 |  "aws-smithy-types",
 358 |  "bytes",
 359 |  "fastrand",
 360 |  "http",
 361 |  "http-body",
 362 |  "hyper",
 363 |  "hyper-rustls",
 364 |  "lazy_static",
 365 |  "pin-project-lite",
 366 |  "rustls",
 367 |  "tokio",
 368 |  "tower",
 369 |  "tracing",
 370 | ]
 371 | 
 372 | [[package]]
 373 | name = "aws-smithy-eventstream"
 374 | version = "0.56.1"
 375 | source = "registry+https://github.com/rust-lang/crates.io-index"
 376 | checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b"
 377 | dependencies = [
 378 |  "aws-smithy-types",
 379 |  "bytes",
 380 |  "crc32fast",
 381 | ]
 382 | 
 383 | [[package]]
 384 | name = "aws-smithy-http"
 385 | version = "0.56.1"
 386 | source = "registry+https://github.com/rust-lang/crates.io-index"
 387 | checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199"
 388 | dependencies = [
 389 |  "aws-smithy-eventstream",
 390 |  "aws-smithy-types",
 391 |  "bytes",
 392 |  "bytes-utils",
 393 |  "futures-core",
 394 |  "http",
 395 |  "http-body",
 396 |  "hyper",
 397 |  "once_cell",
 398 |  "percent-encoding",
 399 |  "pin-project-lite",
 400 |  "pin-utils",
 401 |  "tokio",
 402 |  "tokio-util",
 403 |  "tracing",
 404 | ]
 405 | 
 406 | [[package]]
 407 | name = "aws-smithy-http-tower"
 408 | version = "0.56.1"
 409 | source = "registry+https://github.com/rust-lang/crates.io-index"
 410 | checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd"
 411 | dependencies = [
 412 |  "aws-smithy-http",
 413 |  "aws-smithy-types",
 414 |  "bytes",
 415 |  "http",
 416 |  "http-body",
 417 |  "pin-project-lite",
 418 |  "tower",
 419 |  "tracing",
 420 | ]
 421 | 
 422 | [[package]]
 423 | name = "aws-smithy-json"
 424 | version = "0.56.1"
 425 | source = "registry+https://github.com/rust-lang/crates.io-index"
 426 | checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f"
 427 | dependencies = [
 428 |  "aws-smithy-types",
 429 | ]
 430 | 
 431 | [[package]]
 432 | name = "aws-smithy-query"
 433 | version = "0.56.1"
 434 | source = "registry+https://github.com/rust-lang/crates.io-index"
 435 | checksum = "28556a3902091c1f768a34f6c998028921bdab8d47d92586f363f14a4a32d047"
 436 | dependencies = [
 437 |  "aws-smithy-types",
 438 |  "urlencoding",
 439 | ]
 440 | 
 441 | [[package]]
 442 | name = "aws-smithy-runtime"
 443 | version = "0.56.1"
 444 | source = "registry+https://github.com/rust-lang/crates.io-index"
 445 | checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6"
 446 | dependencies = [
 447 |  "aws-smithy-async",
 448 |  "aws-smithy-client",
 449 |  "aws-smithy-http",
 450 |  "aws-smithy-runtime-api",
 451 |  "aws-smithy-types",
 452 |  "bytes",
 453 |  "fastrand",
 454 |  "http",
 455 |  "http-body",
 456 |  "once_cell",
 457 |  "pin-project-lite",
 458 |  "pin-utils",
 459 |  "tokio",
 460 |  "tracing",
 461 | ]
 462 | 
 463 | [[package]]
 464 | name = "aws-smithy-runtime-api"
 465 | version = "0.56.1"
 466 | source = "registry+https://github.com/rust-lang/crates.io-index"
 467 | checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02"
 468 | dependencies = [
 469 |  "aws-smithy-async",
 470 |  "aws-smithy-http",
 471 |  "aws-smithy-types",
 472 |  "bytes",
 473 |  "http",
 474 |  "tokio",
 475 |  "tracing",
 476 | ]
 477 | 
 478 | [[package]]
 479 | name = "aws-smithy-types"
 480 | version = "0.56.1"
 481 | source = "registry+https://github.com/rust-lang/crates.io-index"
 482 | checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728"
 483 | dependencies = [
 484 |  "base64-simd",
 485 |  "itoa",
 486 |  "num-integer",
 487 |  "ryu",
 488 |  "serde",
 489 |  "time",
 490 | ]
 491 | 
 492 | [[package]]
 493 | name = "aws-smithy-xml"
 494 | version = "0.56.1"
 495 | source = "registry+https://github.com/rust-lang/crates.io-index"
 496 | checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6"
 497 | dependencies = [
 498 |  "xmlparser",
 499 | ]
 500 | 
 501 | [[package]]
 502 | name = "aws-types"
 503 | version = "0.56.1"
 504 | source = "registry+https://github.com/rust-lang/crates.io-index"
 505 | checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe"
 506 | dependencies = [
 507 |  "aws-credential-types",
 508 |  "aws-smithy-async",
 509 |  "aws-smithy-client",
 510 |  "aws-smithy-http",
 511 |  "aws-smithy-types",
 512 |  "http",
 513 |  "rustc_version",
 514 |  "tracing",
 515 | ]
 516 | 
 517 | [[package]]
 518 | name = "backtrace"
 519 | version = "0.3.69"
 520 | source = "registry+https://github.com/rust-lang/crates.io-index"
 521 | checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
 522 | dependencies = [
 523 |  "addr2line",
 524 |  "cc",
 525 |  "cfg-if",
 526 |  "libc",
 527 |  "miniz_oxide",
 528 |  "object",
 529 |  "rustc-demangle",
 530 | ]
 531 | 
 532 | [[package]]
 533 | name = "base64"
 534 | version = "0.21.7"
 535 | source = "registry+https://github.com/rust-lang/crates.io-index"
 536 | checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 537 | 
 538 | [[package]]
 539 | name = "base64-simd"
 540 | version = "0.8.0"
 541 | source = "registry+https://github.com/rust-lang/crates.io-index"
 542 | checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195"
 543 | dependencies = [
 544 |  "outref",
 545 |  "vsimd",
 546 | ]
 547 | 
 548 | [[package]]
 549 | name = "bincode"
 550 | version = "1.3.3"
 551 | source = "registry+https://github.com/rust-lang/crates.io-index"
 552 | checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
 553 | dependencies = [
 554 |  "serde",
 555 | ]
 556 | 
 557 | [[package]]
 558 | name = "bitflags"
 559 | version = "1.3.2"
 560 | source = "registry+https://github.com/rust-lang/crates.io-index"
 561 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 562 | 
 563 | [[package]]
 564 | name = "bitflags"
 565 | version = "2.5.0"
 566 | source = "registry+https://github.com/rust-lang/crates.io-index"
 567 | checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
 568 | 
 569 | [[package]]
 570 | name = "block-buffer"
 571 | version = "0.10.4"
 572 | source = "registry+https://github.com/rust-lang/crates.io-index"
 573 | checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 574 | dependencies = [
 575 |  "generic-array",
 576 | ]
 577 | 
 578 | [[package]]
 579 | name = "brotli"
 580 | version = "3.5.0"
 581 | source = "registry+https://github.com/rust-lang/crates.io-index"
 582 | checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391"
 583 | dependencies = [
 584 |  "alloc-no-stdlib",
 585 |  "alloc-stdlib",
 586 |  "brotli-decompressor",
 587 | ]
 588 | 
 589 | [[package]]
 590 | name = "brotli-decompressor"
 591 | version = "2.5.1"
 592 | source = "registry+https://github.com/rust-lang/crates.io-index"
 593 | checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
 594 | dependencies = [
 595 |  "alloc-no-stdlib",
 596 |  "alloc-stdlib",
 597 | ]
 598 | 
 599 | [[package]]
 600 | name = "bumpalo"
 601 | version = "3.15.4"
 602 | source = "registry+https://github.com/rust-lang/crates.io-index"
 603 | checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa"
 604 | 
 605 | [[package]]
 606 | name = "bytes"
 607 | version = "1.5.0"
 608 | source = "registry+https://github.com/rust-lang/crates.io-index"
 609 | checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 610 | 
 611 | [[package]]
 612 | name = "bytes-utils"
 613 | version = "0.1.4"
 614 | source = "registry+https://github.com/rust-lang/crates.io-index"
 615 | checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
 616 | dependencies = [
 617 |  "bytes",
 618 |  "either",
 619 | ]
 620 | 
 621 | [[package]]
 622 | name = "cc"
 623 | version = "1.0.90"
 624 | source = "registry+https://github.com/rust-lang/crates.io-index"
 625 | checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
 626 | 
 627 | [[package]]
 628 | name = "cfg-if"
 629 | version = "1.0.0"
 630 | source = "registry+https://github.com/rust-lang/crates.io-index"
 631 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 632 | 
 633 | [[package]]
 634 | name = "clap"
 635 | version = "4.5.3"
 636 | source = "registry+https://github.com/rust-lang/crates.io-index"
 637 | checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813"
 638 | dependencies = [
 639 |  "clap_builder",
 640 |  "clap_derive",
 641 | ]
 642 | 
 643 | [[package]]
 644 | name = "clap_builder"
 645 | version = "4.5.2"
 646 | source = "registry+https://github.com/rust-lang/crates.io-index"
 647 | checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
 648 | dependencies = [
 649 |  "anstream",
 650 |  "anstyle",
 651 |  "clap_lex",
 652 |  "strsim",
 653 | ]
 654 | 
 655 | [[package]]
 656 | name = "clap_derive"
 657 | version = "4.5.3"
 658 | source = "registry+https://github.com/rust-lang/crates.io-index"
 659 | checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f"
 660 | dependencies = [
 661 |  "heck",
 662 |  "proc-macro2",
 663 |  "quote",
 664 |  "syn",
 665 | ]
 666 | 
 667 | [[package]]
 668 | name = "clap_lex"
 669 | version = "0.7.0"
 670 | source = "registry+https://github.com/rust-lang/crates.io-index"
 671 | checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 672 | 
 673 | [[package]]
 674 | name = "colorchoice"
 675 | version = "1.0.0"
 676 | source = "registry+https://github.com/rust-lang/crates.io-index"
 677 | checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 678 | 
 679 | [[package]]
 680 | name = "core-foundation"
 681 | version = "0.9.4"
 682 | source = "registry+https://github.com/rust-lang/crates.io-index"
 683 | checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
 684 | dependencies = [
 685 |  "core-foundation-sys",
 686 |  "libc",
 687 | ]
 688 | 
 689 | [[package]]
 690 | name = "core-foundation-sys"
 691 | version = "0.8.6"
 692 | source = "registry+https://github.com/rust-lang/crates.io-index"
 693 | checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 694 | 
 695 | [[package]]
 696 | name = "cpufeatures"
 697 | version = "0.2.12"
 698 | source = "registry+https://github.com/rust-lang/crates.io-index"
 699 | checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
 700 | dependencies = [
 701 |  "libc",
 702 | ]
 703 | 
 704 | [[package]]
 705 | name = "crc32c"
 706 | version = "0.6.5"
 707 | source = "registry+https://github.com/rust-lang/crates.io-index"
 708 | checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2"
 709 | dependencies = [
 710 |  "rustc_version",
 711 | ]
 712 | 
 713 | [[package]]
 714 | name = "crc32fast"
 715 | version = "1.4.0"
 716 | source = "registry+https://github.com/rust-lang/crates.io-index"
 717 | checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
 718 | dependencies = [
 719 |  "cfg-if",
 720 | ]
 721 | 
 722 | [[package]]
 723 | name = "crossbeam"
 724 | version = "0.8.4"
 725 | source = "registry+https://github.com/rust-lang/crates.io-index"
 726 | checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
 727 | dependencies = [
 728 |  "crossbeam-channel",
 729 |  "crossbeam-deque",
 730 |  "crossbeam-epoch",
 731 |  "crossbeam-queue",
 732 |  "crossbeam-utils",
 733 | ]
 734 | 
 735 | [[package]]
 736 | name = "crossbeam-channel"
 737 | version = "0.5.12"
 738 | source = "registry+https://github.com/rust-lang/crates.io-index"
 739 | checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95"
 740 | dependencies = [
 741 |  "crossbeam-utils",
 742 | ]
 743 | 
 744 | [[package]]
 745 | name = "crossbeam-deque"
 746 | version = "0.8.5"
 747 | source = "registry+https://github.com/rust-lang/crates.io-index"
 748 | checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
 749 | dependencies = [
 750 |  "crossbeam-epoch",
 751 |  "crossbeam-utils",
 752 | ]
 753 | 
 754 | [[package]]
 755 | name = "crossbeam-epoch"
 756 | version = "0.9.18"
 757 | source = "registry+https://github.com/rust-lang/crates.io-index"
 758 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 759 | dependencies = [
 760 |  "crossbeam-utils",
 761 | ]
 762 | 
 763 | [[package]]
 764 | name = "crossbeam-queue"
 765 | version = "0.3.11"
 766 | source = "registry+https://github.com/rust-lang/crates.io-index"
 767 | checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
 768 | dependencies = [
 769 |  "crossbeam-utils",
 770 | ]
 771 | 
 772 | [[package]]
 773 | name = "crossbeam-utils"
 774 | version = "0.8.19"
 775 | source = "registry+https://github.com/rust-lang/crates.io-index"
 776 | checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 777 | 
 778 | [[package]]
 779 | name = "crypto-common"
 780 | version = "0.1.6"
 781 | source = "registry+https://github.com/rust-lang/crates.io-index"
 782 | checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 783 | dependencies = [
 784 |  "generic-array",
 785 |  "typenum",
 786 | ]
 787 | 
 788 | [[package]]
 789 | name = "deranged"
 790 | version = "0.3.11"
 791 | source = "registry+https://github.com/rust-lang/crates.io-index"
 792 | checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
 793 | dependencies = [
 794 |  "powerfmt",
 795 | ]
 796 | 
 797 | [[package]]
 798 | name = "diff"
 799 | version = "0.1.13"
 800 | source = "registry+https://github.com/rust-lang/crates.io-index"
 801 | checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
 802 | 
 803 | [[package]]
 804 | name = "digest"
 805 | version = "0.10.7"
 806 | source = "registry+https://github.com/rust-lang/crates.io-index"
 807 | checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 808 | dependencies = [
 809 |  "block-buffer",
 810 |  "crypto-common",
 811 |  "subtle",
 812 | ]
 813 | 
 814 | [[package]]
 815 | name = "dotenv"
 816 | version = "0.15.0"
 817 | source = "registry+https://github.com/rust-lang/crates.io-index"
 818 | checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
 819 | 
 820 | [[package]]
 821 | name = "either"
 822 | version = "1.10.0"
 823 | source = "registry+https://github.com/rust-lang/crates.io-index"
 824 | checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
 825 | 
 826 | [[package]]
 827 | name = "encoding_rs"
 828 | version = "0.8.33"
 829 | source = "registry+https://github.com/rust-lang/crates.io-index"
 830 | checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
 831 | dependencies = [
 832 |  "cfg-if",
 833 | ]
 834 | 
 835 | [[package]]
 836 | name = "equivalent"
 837 | version = "1.0.1"
 838 | source = "registry+https://github.com/rust-lang/crates.io-index"
 839 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 840 | 
 841 | [[package]]
 842 | name = "errno"
 843 | version = "0.3.8"
 844 | source = "registry+https://github.com/rust-lang/crates.io-index"
 845 | checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
 846 | dependencies = [
 847 |  "libc",
 848 |  "windows-sys 0.52.0",
 849 | ]
 850 | 
 851 | [[package]]
 852 | name = "fastrand"
 853 | version = "2.0.1"
 854 | source = "registry+https://github.com/rust-lang/crates.io-index"
 855 | checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 856 | 
 857 | [[package]]
 858 | name = "fnv"
 859 | version = "1.0.7"
 860 | source = "registry+https://github.com/rust-lang/crates.io-index"
 861 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 862 | 
 863 | [[package]]
 864 | name = "foreign-types"
 865 | version = "0.3.2"
 866 | source = "registry+https://github.com/rust-lang/crates.io-index"
 867 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
 868 | dependencies = [
 869 |  "foreign-types-shared",
 870 | ]
 871 | 
 872 | [[package]]
 873 | name = "foreign-types-shared"
 874 | version = "0.1.1"
 875 | source = "registry+https://github.com/rust-lang/crates.io-index"
 876 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 877 | 
 878 | [[package]]
 879 | name = "form_urlencoded"
 880 | version = "1.2.1"
 881 | source = "registry+https://github.com/rust-lang/crates.io-index"
 882 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 883 | dependencies = [
 884 |  "percent-encoding",
 885 | ]
 886 | 
 887 | [[package]]
 888 | name = "futures"
 889 | version = "0.3.30"
 890 | source = "registry+https://github.com/rust-lang/crates.io-index"
 891 | checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
 892 | dependencies = [
 893 |  "futures-channel",
 894 |  "futures-core",
 895 |  "futures-executor",
 896 |  "futures-io",
 897 |  "futures-sink",
 898 |  "futures-task",
 899 |  "futures-util",
 900 | ]
 901 | 
 902 | [[package]]
 903 | name = "futures-channel"
 904 | version = "0.3.31"
 905 | source = "registry+https://github.com/rust-lang/crates.io-index"
 906 | checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
 907 | dependencies = [
 908 |  "futures-core",
 909 |  "futures-sink",
 910 | ]
 911 | 
 912 | [[package]]
 913 | name = "futures-core"
 914 | version = "0.3.31"
 915 | source = "registry+https://github.com/rust-lang/crates.io-index"
 916 | checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 917 | 
 918 | [[package]]
 919 | name = "futures-executor"
 920 | version = "0.3.30"
 921 | source = "registry+https://github.com/rust-lang/crates.io-index"
 922 | checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
 923 | dependencies = [
 924 |  "futures-core",
 925 |  "futures-task",
 926 |  "futures-util",
 927 | ]
 928 | 
 929 | [[package]]
 930 | name = "futures-io"
 931 | version = "0.3.31"
 932 | source = "registry+https://github.com/rust-lang/crates.io-index"
 933 | checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 934 | 
 935 | [[package]]
 936 | name = "futures-macro"
 937 | version = "0.3.31"
 938 | source = "registry+https://github.com/rust-lang/crates.io-index"
 939 | checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 940 | dependencies = [
 941 |  "proc-macro2",
 942 |  "quote",
 943 |  "syn",
 944 | ]
 945 | 
 946 | [[package]]
 947 | name = "futures-sink"
 948 | version = "0.3.31"
 949 | source = "registry+https://github.com/rust-lang/crates.io-index"
 950 | checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
 951 | 
 952 | [[package]]
 953 | name = "futures-task"
 954 | version = "0.3.31"
 955 | source = "registry+https://github.com/rust-lang/crates.io-index"
 956 | checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 957 | 
 958 | [[package]]
 959 | name = "futures-util"
 960 | version = "0.3.31"
 961 | source = "registry+https://github.com/rust-lang/crates.io-index"
 962 | checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 963 | dependencies = [
 964 |  "futures-channel",
 965 |  "futures-core",
 966 |  "futures-io",
 967 |  "futures-macro",
 968 |  "futures-sink",
 969 |  "futures-task",
 970 |  "memchr",
 971 |  "pin-project-lite",
 972 |  "pin-utils",
 973 |  "slab",
 974 | ]
 975 | 
 976 | [[package]]
 977 | name = "generic-array"
 978 | version = "0.14.7"
 979 | source = "registry+https://github.com/rust-lang/crates.io-index"
 980 | checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 981 | dependencies = [
 982 |  "typenum",
 983 |  "version_check",
 984 | ]
 985 | 
 986 | [[package]]
 987 | name = "getrandom"
 988 | version = "0.2.12"
 989 | source = "registry+https://github.com/rust-lang/crates.io-index"
 990 | checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
 991 | dependencies = [
 992 |  "cfg-if",
 993 |  "libc",
 994 |  "wasi",
 995 | ]
 996 | 
 997 | [[package]]
 998 | name = "gimli"
 999 | version = "0.28.1"
1000 | source = "registry+https://github.com/rust-lang/crates.io-index"
1001 | checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
1002 | 
1003 | [[package]]
1004 | name = "h2"
1005 | version = "0.3.26"
1006 | source = "registry+https://github.com/rust-lang/crates.io-index"
1007 | checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
1008 | dependencies = [
1009 |  "bytes",
1010 |  "fnv",
1011 |  "futures-core",
1012 |  "futures-sink",
1013 |  "futures-util",
1014 |  "http",
1015 |  "indexmap",
1016 |  "slab",
1017 |  "tokio",
1018 |  "tokio-util",
1019 |  "tracing",
1020 | ]
1021 | 
1022 | [[package]]
1023 | name = "hashbrown"
1024 | version = "0.14.3"
1025 | source = "registry+https://github.com/rust-lang/crates.io-index"
1026 | checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
1027 | 
1028 | [[package]]
1029 | name = "heck"
1030 | version = "0.5.0"
1031 | source = "registry+https://github.com/rust-lang/crates.io-index"
1032 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
1033 | 
1034 | [[package]]
1035 | name = "hermit-abi"
1036 | version = "0.1.19"
1037 | source = "registry+https://github.com/rust-lang/crates.io-index"
1038 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
1039 | dependencies = [
1040 |  "libc",
1041 | ]
1042 | 
1043 | [[package]]
1044 | name = "hermit-abi"
1045 | version = "0.3.9"
1046 | source = "registry+https://github.com/rust-lang/crates.io-index"
1047 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
1048 | 
1049 | [[package]]
1050 | name = "hex"
1051 | version = "0.4.3"
1052 | source = "registry+https://github.com/rust-lang/crates.io-index"
1053 | checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
1054 | 
1055 | [[package]]
1056 | name = "hmac"
1057 | version = "0.12.1"
1058 | source = "registry+https://github.com/rust-lang/crates.io-index"
1059 | checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
1060 | dependencies = [
1061 |  "digest",
1062 | ]
1063 | 
1064 | [[package]]
1065 | name = "http"
1066 | version = "0.2.12"
1067 | source = "registry+https://github.com/rust-lang/crates.io-index"
1068 | checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
1069 | dependencies = [
1070 |  "bytes",
1071 |  "fnv",
1072 |  "itoa",
1073 | ]
1074 | 
1075 | [[package]]
1076 | name = "http-body"
1077 | version = "0.4.6"
1078 | source = "registry+https://github.com/rust-lang/crates.io-index"
1079 | checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
1080 | dependencies = [
1081 |  "bytes",
1082 |  "http",
1083 |  "pin-project-lite",
1084 | ]
1085 | 
1086 | [[package]]
1087 | name = "httparse"
1088 | version = "1.8.0"
1089 | source = "registry+https://github.com/rust-lang/crates.io-index"
1090 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
1091 | 
1092 | [[package]]
1093 | name = "httpdate"
1094 | version = "1.0.3"
1095 | source = "registry+https://github.com/rust-lang/crates.io-index"
1096 | checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
1097 | 
1098 | [[package]]
1099 | name = "hyper"
1100 | version = "0.14.28"
1101 | source = "registry+https://github.com/rust-lang/crates.io-index"
1102 | checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
1103 | dependencies = [
1104 |  "bytes",
1105 |  "futures-channel",
1106 |  "futures-core",
1107 |  "futures-util",
1108 |  "h2",
1109 |  "http",
1110 |  "http-body",
1111 |  "httparse",
1112 |  "httpdate",
1113 |  "itoa",
1114 |  "pin-project-lite",
1115 |  "socket2",
1116 |  "tokio",
1117 |  "tower-service",
1118 |  "tracing",
1119 |  "want",
1120 | ]
1121 | 
1122 | [[package]]
1123 | name = "hyper-rustls"
1124 | version = "0.24.2"
1125 | source = "registry+https://github.com/rust-lang/crates.io-index"
1126 | checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
1127 | dependencies = [
1128 |  "futures-util",
1129 |  "http",
1130 |  "hyper",
1131 |  "log",
1132 |  "rustls",
1133 |  "rustls-native-certs",
1134 |  "tokio",
1135 |  "tokio-rustls",
1136 | ]
1137 | 
1138 | [[package]]
1139 | name = "hyper-tls"
1140 | version = "0.5.0"
1141 | source = "registry+https://github.com/rust-lang/crates.io-index"
1142 | checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
1143 | dependencies = [
1144 |  "bytes",
1145 |  "hyper",
1146 |  "native-tls",
1147 |  "tokio",
1148 |  "tokio-native-tls",
1149 | ]
1150 | 
1151 | [[package]]
1152 | name = "idna"
1153 | version = "0.5.0"
1154 | source = "registry+https://github.com/rust-lang/crates.io-index"
1155 | checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
1156 | dependencies = [
1157 |  "unicode-bidi",
1158 |  "unicode-normalization",
1159 | ]
1160 | 
1161 | [[package]]
1162 | name = "indexmap"
1163 | version = "2.2.5"
1164 | source = "registry+https://github.com/rust-lang/crates.io-index"
1165 | checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
1166 | dependencies = [
1167 |  "equivalent",
1168 |  "hashbrown",
1169 | ]
1170 | 
1171 | [[package]]
1172 | name = "ipnet"
1173 | version = "2.9.0"
1174 | source = "registry+https://github.com/rust-lang/crates.io-index"
1175 | checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
1176 | 
1177 | [[package]]
1178 | name = "itoa"
1179 | version = "1.0.10"
1180 | source = "registry+https://github.com/rust-lang/crates.io-index"
1181 | checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
1182 | 
1183 | [[package]]
1184 | name = "js-sys"
1185 | version = "0.3.69"
1186 | source = "registry+https://github.com/rust-lang/crates.io-index"
1187 | checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
1188 | dependencies = [
1189 |  "wasm-bindgen",
1190 | ]
1191 | 
1192 | [[package]]
1193 | name = "lazy_static"
1194 | version = "1.4.0"
1195 | source = "registry+https://github.com/rust-lang/crates.io-index"
1196 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
1197 | 
1198 | [[package]]
1199 | name = "libc"
1200 | version = "0.2.153"
1201 | source = "registry+https://github.com/rust-lang/crates.io-index"
1202 | checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
1203 | 
1204 | [[package]]
1205 | name = "linux-raw-sys"
1206 | version = "0.4.13"
1207 | source = "registry+https://github.com/rust-lang/crates.io-index"
1208 | checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
1209 | 
1210 | [[package]]
1211 | name = "log"
1212 | version = "0.4.21"
1213 | source = "registry+https://github.com/rust-lang/crates.io-index"
1214 | checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
1215 | 
1216 | [[package]]
1217 | name = "matchers"
1218 | version = "0.1.0"
1219 | source = "registry+https://github.com/rust-lang/crates.io-index"
1220 | checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
1221 | dependencies = [
1222 |  "regex-automata 0.1.10",
1223 | ]
1224 | 
1225 | [[package]]
1226 | name = "md-5"
1227 | version = "0.10.6"
1228 | source = "registry+https://github.com/rust-lang/crates.io-index"
1229 | checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
1230 | dependencies = [
1231 |  "cfg-if",
1232 |  "digest",
1233 | ]
1234 | 
1235 | [[package]]
1236 | name = "memchr"
1237 | version = "2.7.1"
1238 | source = "registry+https://github.com/rust-lang/crates.io-index"
1239 | checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
1240 | 
1241 | [[package]]
1242 | name = "mime"
1243 | version = "0.3.17"
1244 | source = "registry+https://github.com/rust-lang/crates.io-index"
1245 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
1246 | 
1247 | [[package]]
1248 | name = "miniz_oxide"
1249 | version = "0.7.2"
1250 | source = "registry+https://github.com/rust-lang/crates.io-index"
1251 | checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
1252 | dependencies = [
1253 |  "adler",
1254 | ]
1255 | 
1256 | [[package]]
1257 | name = "mio"
1258 | version = "0.8.11"
1259 | source = "registry+https://github.com/rust-lang/crates.io-index"
1260 | checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
1261 | dependencies = [
1262 |  "libc",
1263 |  "wasi",
1264 |  "windows-sys 0.48.0",
1265 | ]
1266 | 
1267 | [[package]]
1268 | name = "native-tls"
1269 | version = "0.2.11"
1270 | source = "registry+https://github.com/rust-lang/crates.io-index"
1271 | checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
1272 | dependencies = [
1273 |  "lazy_static",
1274 |  "libc",
1275 |  "log",
1276 |  "openssl",
1277 |  "openssl-probe",
1278 |  "openssl-sys",
1279 |  "schannel",
1280 |  "security-framework",
1281 |  "security-framework-sys",
1282 |  "tempfile",
1283 | ]
1284 | 
1285 | [[package]]
1286 | name = "nu-ansi-term"
1287 | version = "0.46.0"
1288 | source = "registry+https://github.com/rust-lang/crates.io-index"
1289 | checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
1290 | dependencies = [
1291 |  "overload",
1292 |  "winapi",
1293 | ]
1294 | 
1295 | [[package]]
1296 | name = "num-conv"
1297 | version = "0.1.0"
1298 | source = "registry+https://github.com/rust-lang/crates.io-index"
1299 | checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
1300 | 
1301 | [[package]]
1302 | name = "num-integer"
1303 | version = "0.1.46"
1304 | source = "registry+https://github.com/rust-lang/crates.io-index"
1305 | checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
1306 | dependencies = [
1307 |  "num-traits",
1308 | ]
1309 | 
1310 | [[package]]
1311 | name = "num-traits"
1312 | version = "0.2.18"
1313 | source = "registry+https://github.com/rust-lang/crates.io-index"
1314 | checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
1315 | dependencies = [
1316 |  "autocfg",
1317 | ]
1318 | 
1319 | [[package]]
1320 | name = "num_cpus"
1321 | version = "1.16.0"
1322 | source = "registry+https://github.com/rust-lang/crates.io-index"
1323 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
1324 | dependencies = [
1325 |  "hermit-abi 0.3.9",
1326 |  "libc",
1327 | ]
1328 | 
1329 | [[package]]
1330 | name = "object"
1331 | version = "0.32.2"
1332 | source = "registry+https://github.com/rust-lang/crates.io-index"
1333 | checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
1334 | dependencies = [
1335 |  "memchr",
1336 | ]
1337 | 
1338 | [[package]]
1339 | name = "once_cell"
1340 | version = "1.19.0"
1341 | source = "registry+https://github.com/rust-lang/crates.io-index"
1342 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
1343 | 
1344 | [[package]]
1345 | name = "openssl"
1346 | version = "0.10.71"
1347 | source = "registry+https://github.com/rust-lang/crates.io-index"
1348 | checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd"
1349 | dependencies = [
1350 |  "bitflags 2.5.0",
1351 |  "cfg-if",
1352 |  "foreign-types",
1353 |  "libc",
1354 |  "once_cell",
1355 |  "openssl-macros",
1356 |  "openssl-sys",
1357 | ]
1358 | 
1359 | [[package]]
1360 | name = "openssl-macros"
1361 | version = "0.1.1"
1362 | source = "registry+https://github.com/rust-lang/crates.io-index"
1363 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
1364 | dependencies = [
1365 |  "proc-macro2",
1366 |  "quote",
1367 |  "syn",
1368 | ]
1369 | 
1370 | [[package]]
1371 | name = "openssl-probe"
1372 | version = "0.1.5"
1373 | source = "registry+https://github.com/rust-lang/crates.io-index"
1374 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
1375 | 
1376 | [[package]]
1377 | name = "openssl-sys"
1378 | version = "0.9.106"
1379 | source = "registry+https://github.com/rust-lang/crates.io-index"
1380 | checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd"
1381 | dependencies = [
1382 |  "cc",
1383 |  "libc",
1384 |  "pkg-config",
1385 |  "vcpkg",
1386 | ]
1387 | 
1388 | [[package]]
1389 | name = "outref"
1390 | version = "0.5.1"
1391 | source = "registry+https://github.com/rust-lang/crates.io-index"
1392 | checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
1393 | 
1394 | [[package]]
1395 | name = "overload"
1396 | version = "0.1.1"
1397 | source = "registry+https://github.com/rust-lang/crates.io-index"
1398 | checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
1399 | 
1400 | [[package]]
1401 | name = "percent-encoding"
1402 | version = "2.3.1"
1403 | source = "registry+https://github.com/rust-lang/crates.io-index"
1404 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
1405 | 
1406 | [[package]]
1407 | name = "pin-project"
1408 | version = "1.1.5"
1409 | source = "registry+https://github.com/rust-lang/crates.io-index"
1410 | checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
1411 | dependencies = [
1412 |  "pin-project-internal",
1413 | ]
1414 | 
1415 | [[package]]
1416 | name = "pin-project-internal"
1417 | version = "1.1.5"
1418 | source = "registry+https://github.com/rust-lang/crates.io-index"
1419 | checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
1420 | dependencies = [
1421 |  "proc-macro2",
1422 |  "quote",
1423 |  "syn",
1424 | ]
1425 | 
1426 | [[package]]
1427 | name = "pin-project-lite"
1428 | version = "0.2.13"
1429 | source = "registry+https://github.com/rust-lang/crates.io-index"
1430 | checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
1431 | 
1432 | [[package]]
1433 | name = "pin-utils"
1434 | version = "0.1.0"
1435 | source = "registry+https://github.com/rust-lang/crates.io-index"
1436 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
1437 | 
1438 | [[package]]
1439 | name = "pkg-config"
1440 | version = "0.3.30"
1441 | source = "registry+https://github.com/rust-lang/crates.io-index"
1442 | checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
1443 | 
1444 | [[package]]
1445 | name = "powerfmt"
1446 | version = "0.2.0"
1447 | source = "registry+https://github.com/rust-lang/crates.io-index"
1448 | checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
1449 | 
1450 | [[package]]
1451 | name = "pretty_assertions"
1452 | version = "1.4.1"
1453 | source = "registry+https://github.com/rust-lang/crates.io-index"
1454 | checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d"
1455 | dependencies = [
1456 |  "diff",
1457 |  "yansi",
1458 | ]
1459 | 
1460 | [[package]]
1461 | name = "proc-macro2"
1462 | version = "1.0.79"
1463 | source = "registry+https://github.com/rust-lang/crates.io-index"
1464 | checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
1465 | dependencies = [
1466 |  "unicode-ident",
1467 | ]
1468 | 
1469 | [[package]]
1470 | name = "quote"
1471 | version = "1.0.35"
1472 | source = "registry+https://github.com/rust-lang/crates.io-index"
1473 | checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
1474 | dependencies = [
1475 |  "proc-macro2",
1476 | ]
1477 | 
1478 | [[package]]
1479 | name = "regex"
1480 | version = "1.10.3"
1481 | source = "registry+https://github.com/rust-lang/crates.io-index"
1482 | checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
1483 | dependencies = [
1484 |  "aho-corasick",
1485 |  "memchr",
1486 |  "regex-automata 0.4.6",
1487 |  "regex-syntax 0.8.2",
1488 | ]
1489 | 
1490 | [[package]]
1491 | name = "regex-automata"
1492 | version = "0.1.10"
1493 | source = "registry+https://github.com/rust-lang/crates.io-index"
1494 | checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
1495 | dependencies = [
1496 |  "regex-syntax 0.6.29",
1497 | ]
1498 | 
1499 | [[package]]
1500 | name = "regex-automata"
1501 | version = "0.4.6"
1502 | source = "registry+https://github.com/rust-lang/crates.io-index"
1503 | checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
1504 | dependencies = [
1505 |  "aho-corasick",
1506 |  "memchr",
1507 |  "regex-syntax 0.8.2",
1508 | ]
1509 | 
1510 | [[package]]
1511 | name = "regex-syntax"
1512 | version = "0.6.29"
1513 | source = "registry+https://github.com/rust-lang/crates.io-index"
1514 | checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
1515 | 
1516 | [[package]]
1517 | name = "regex-syntax"
1518 | version = "0.8.2"
1519 | source = "registry+https://github.com/rust-lang/crates.io-index"
1520 | checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
1521 | 
1522 | [[package]]
1523 | name = "reqwest"
1524 | version = "0.11.27"
1525 | source = "registry+https://github.com/rust-lang/crates.io-index"
1526 | checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
1527 | dependencies = [
1528 |  "base64",
1529 |  "bytes",
1530 |  "encoding_rs",
1531 |  "futures-core",
1532 |  "futures-util",
1533 |  "h2",
1534 |  "http",
1535 |  "http-body",
1536 |  "hyper",
1537 |  "hyper-tls",
1538 |  "ipnet",
1539 |  "js-sys",
1540 |  "log",
1541 |  "mime",
1542 |  "native-tls",
1543 |  "once_cell",
1544 |  "percent-encoding",
1545 |  "pin-project-lite",
1546 |  "rustls-pemfile",
1547 |  "serde",
1548 |  "serde_json",
1549 |  "serde_urlencoded",
1550 |  "sync_wrapper",
1551 |  "system-configuration",
1552 |  "tokio",
1553 |  "tokio-native-tls",
1554 |  "tower-service",
1555 |  "url",
1556 |  "wasm-bindgen",
1557 |  "wasm-bindgen-futures",
1558 |  "web-sys",
1559 |  "winreg",
1560 | ]
1561 | 
1562 | [[package]]
1563 | name = "ring"
1564 | version = "0.16.20"
1565 | source = "registry+https://github.com/rust-lang/crates.io-index"
1566 | checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
1567 | dependencies = [
1568 |  "cc",
1569 |  "libc",
1570 |  "once_cell",
1571 |  "spin 0.5.2",
1572 |  "untrusted 0.7.1",
1573 |  "web-sys",
1574 |  "winapi",
1575 | ]
1576 | 
1577 | [[package]]
1578 | name = "ring"
1579 | version = "0.17.8"
1580 | source = "registry+https://github.com/rust-lang/crates.io-index"
1581 | checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
1582 | dependencies = [
1583 |  "cc",
1584 |  "cfg-if",
1585 |  "getrandom",
1586 |  "libc",
1587 |  "spin 0.9.8",
1588 |  "untrusted 0.9.0",
1589 |  "windows-sys 0.52.0",
1590 | ]
1591 | 
1592 | [[package]]
1593 | name = "rust-log-analyzer"
1594 | version = "0.1.0"
1595 | dependencies = [
1596 |  "aho-corasick",
1597 |  "anyhow",
1598 |  "atomicwrites",
1599 |  "atty",
1600 |  "aws-config",
1601 |  "aws-sdk-s3",
1602 |  "bincode",
1603 |  "brotli",
1604 |  "clap",
1605 |  "crossbeam",
1606 |  "dotenv",
1607 |  "fnv",
1608 |  "futures",
1609 |  "hex",
1610 |  "hmac",
1611 |  "hyper",
1612 |  "lazy_static",
1613 |  "percent-encoding",
1614 |  "pretty_assertions",
1615 |  "regex",
1616 |  "reqwest",
1617 |  "serde",
1618 |  "serde_derive",
1619 |  "serde_json",
1620 |  "sha1",
1621 |  "threadpool",
1622 |  "tokio",
1623 |  "tracing",
1624 |  "tracing-subscriber",
1625 |  "walkdir",
1626 | ]
1627 | 
1628 | [[package]]
1629 | name = "rustc-demangle"
1630 | version = "0.1.23"
1631 | source = "registry+https://github.com/rust-lang/crates.io-index"
1632 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
1633 | 
1634 | [[package]]
1635 | name = "rustc_version"
1636 | version = "0.4.0"
1637 | source = "registry+https://github.com/rust-lang/crates.io-index"
1638 | checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
1639 | dependencies = [
1640 |  "semver",
1641 | ]
1642 | 
1643 | [[package]]
1644 | name = "rustix"
1645 | version = "0.38.32"
1646 | source = "registry+https://github.com/rust-lang/crates.io-index"
1647 | checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89"
1648 | dependencies = [
1649 |  "bitflags 2.5.0",
1650 |  "errno",
1651 |  "libc",
1652 |  "linux-raw-sys",
1653 |  "windows-sys 0.52.0",
1654 | ]
1655 | 
1656 | [[package]]
1657 | name = "rustls"
1658 | version = "0.21.12"
1659 | source = "registry+https://github.com/rust-lang/crates.io-index"
1660 | checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e"
1661 | dependencies = [
1662 |  "log",
1663 |  "ring 0.17.8",
1664 |  "rustls-webpki",
1665 |  "sct",
1666 | ]
1667 | 
1668 | [[package]]
1669 | name = "rustls-native-certs"
1670 | version = "0.6.3"
1671 | source = "registry+https://github.com/rust-lang/crates.io-index"
1672 | checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
1673 | dependencies = [
1674 |  "openssl-probe",
1675 |  "rustls-pemfile",
1676 |  "schannel",
1677 |  "security-framework",
1678 | ]
1679 | 
1680 | [[package]]
1681 | name = "rustls-pemfile"
1682 | version = "1.0.4"
1683 | source = "registry+https://github.com/rust-lang/crates.io-index"
1684 | checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
1685 | dependencies = [
1686 |  "base64",
1687 | ]
1688 | 
1689 | [[package]]
1690 | name = "rustls-webpki"
1691 | version = "0.101.7"
1692 | source = "registry+https://github.com/rust-lang/crates.io-index"
1693 | checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
1694 | dependencies = [
1695 |  "ring 0.17.8",
1696 |  "untrusted 0.9.0",
1697 | ]
1698 | 
1699 | [[package]]
1700 | name = "ryu"
1701 | version = "1.0.17"
1702 | source = "registry+https://github.com/rust-lang/crates.io-index"
1703 | checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
1704 | 
1705 | [[package]]
1706 | name = "same-file"
1707 | version = "1.0.6"
1708 | source = "registry+https://github.com/rust-lang/crates.io-index"
1709 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
1710 | dependencies = [
1711 |  "winapi-util",
1712 | ]
1713 | 
1714 | [[package]]
1715 | name = "schannel"
1716 | version = "0.1.23"
1717 | source = "registry+https://github.com/rust-lang/crates.io-index"
1718 | checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
1719 | dependencies = [
1720 |  "windows-sys 0.52.0",
1721 | ]
1722 | 
1723 | [[package]]
1724 | name = "sct"
1725 | version = "0.7.1"
1726 | source = "registry+https://github.com/rust-lang/crates.io-index"
1727 | checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
1728 | dependencies = [
1729 |  "ring 0.17.8",
1730 |  "untrusted 0.9.0",
1731 | ]
1732 | 
1733 | [[package]]
1734 | name = "security-framework"
1735 | version = "2.9.2"
1736 | source = "registry+https://github.com/rust-lang/crates.io-index"
1737 | checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
1738 | dependencies = [
1739 |  "bitflags 1.3.2",
1740 |  "core-foundation",
1741 |  "core-foundation-sys",
1742 |  "libc",
1743 |  "security-framework-sys",
1744 | ]
1745 | 
1746 | [[package]]
1747 | name = "security-framework-sys"
1748 | version = "2.9.1"
1749 | source = "registry+https://github.com/rust-lang/crates.io-index"
1750 | checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
1751 | dependencies = [
1752 |  "core-foundation-sys",
1753 |  "libc",
1754 | ]
1755 | 
1756 | [[package]]
1757 | name = "semver"
1758 | version = "1.0.22"
1759 | source = "registry+https://github.com/rust-lang/crates.io-index"
1760 | checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
1761 | 
1762 | [[package]]
1763 | name = "serde"
1764 | version = "1.0.197"
1765 | source = "registry+https://github.com/rust-lang/crates.io-index"
1766 | checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
1767 | dependencies = [
1768 |  "serde_derive",
1769 | ]
1770 | 
1771 | [[package]]
1772 | name = "serde_derive"
1773 | version = "1.0.197"
1774 | source = "registry+https://github.com/rust-lang/crates.io-index"
1775 | checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
1776 | dependencies = [
1777 |  "proc-macro2",
1778 |  "quote",
1779 |  "syn",
1780 | ]
1781 | 
1782 | [[package]]
1783 | name = "serde_json"
1784 | version = "1.0.114"
1785 | source = "registry+https://github.com/rust-lang/crates.io-index"
1786 | checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
1787 | dependencies = [
1788 |  "itoa",
1789 |  "ryu",
1790 |  "serde",
1791 | ]
1792 | 
1793 | [[package]]
1794 | name = "serde_urlencoded"
1795 | version = "0.7.1"
1796 | source = "registry+https://github.com/rust-lang/crates.io-index"
1797 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
1798 | dependencies = [
1799 |  "form_urlencoded",
1800 |  "itoa",
1801 |  "ryu",
1802 |  "serde",
1803 | ]
1804 | 
1805 | [[package]]
1806 | name = "sha1"
1807 | version = "0.10.6"
1808 | source = "registry+https://github.com/rust-lang/crates.io-index"
1809 | checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
1810 | dependencies = [
1811 |  "cfg-if",
1812 |  "cpufeatures",
1813 |  "digest",
1814 | ]
1815 | 
1816 | [[package]]
1817 | name = "sha2"
1818 | version = "0.10.8"
1819 | source = "registry+https://github.com/rust-lang/crates.io-index"
1820 | checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
1821 | dependencies = [
1822 |  "cfg-if",
1823 |  "cpufeatures",
1824 |  "digest",
1825 | ]
1826 | 
1827 | [[package]]
1828 | name = "sharded-slab"
1829 | version = "0.1.7"
1830 | source = "registry+https://github.com/rust-lang/crates.io-index"
1831 | checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
1832 | dependencies = [
1833 |  "lazy_static",
1834 | ]
1835 | 
1836 | [[package]]
1837 | name = "signal-hook-registry"
1838 | version = "1.4.1"
1839 | source = "registry+https://github.com/rust-lang/crates.io-index"
1840 | checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
1841 | dependencies = [
1842 |  "libc",
1843 | ]
1844 | 
1845 | [[package]]
1846 | name = "slab"
1847 | version = "0.4.9"
1848 | source = "registry+https://github.com/rust-lang/crates.io-index"
1849 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
1850 | dependencies = [
1851 |  "autocfg",
1852 | ]
1853 | 
1854 | [[package]]
1855 | name = "smallvec"
1856 | version = "1.13.1"
1857 | source = "registry+https://github.com/rust-lang/crates.io-index"
1858 | checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
1859 | 
1860 | [[package]]
1861 | name = "socket2"
1862 | version = "0.5.6"
1863 | source = "registry+https://github.com/rust-lang/crates.io-index"
1864 | checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871"
1865 | dependencies = [
1866 |  "libc",
1867 |  "windows-sys 0.52.0",
1868 | ]
1869 | 
1870 | [[package]]
1871 | name = "spin"
1872 | version = "0.5.2"
1873 | source = "registry+https://github.com/rust-lang/crates.io-index"
1874 | checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
1875 | 
1876 | [[package]]
1877 | name = "spin"
1878 | version = "0.9.8"
1879 | source = "registry+https://github.com/rust-lang/crates.io-index"
1880 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
1881 | 
1882 | [[package]]
1883 | name = "strsim"
1884 | version = "0.11.0"
1885 | source = "registry+https://github.com/rust-lang/crates.io-index"
1886 | checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
1887 | 
1888 | [[package]]
1889 | name = "subtle"
1890 | version = "2.5.0"
1891 | source = "registry+https://github.com/rust-lang/crates.io-index"
1892 | checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
1893 | 
1894 | [[package]]
1895 | name = "syn"
1896 | version = "2.0.53"
1897 | source = "registry+https://github.com/rust-lang/crates.io-index"
1898 | checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032"
1899 | dependencies = [
1900 |  "proc-macro2",
1901 |  "quote",
1902 |  "unicode-ident",
1903 | ]
1904 | 
1905 | [[package]]
1906 | name = "sync_wrapper"
1907 | version = "0.1.2"
1908 | source = "registry+https://github.com/rust-lang/crates.io-index"
1909 | checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
1910 | 
1911 | [[package]]
1912 | name = "system-configuration"
1913 | version = "0.5.1"
1914 | source = "registry+https://github.com/rust-lang/crates.io-index"
1915 | checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
1916 | dependencies = [
1917 |  "bitflags 1.3.2",
1918 |  "core-foundation",
1919 |  "system-configuration-sys",
1920 | ]
1921 | 
1922 | [[package]]
1923 | name = "system-configuration-sys"
1924 | version = "0.5.0"
1925 | source = "registry+https://github.com/rust-lang/crates.io-index"
1926 | checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
1927 | dependencies = [
1928 |  "core-foundation-sys",
1929 |  "libc",
1930 | ]
1931 | 
1932 | [[package]]
1933 | name = "tempfile"
1934 | version = "3.10.1"
1935 | source = "registry+https://github.com/rust-lang/crates.io-index"
1936 | checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
1937 | dependencies = [
1938 |  "cfg-if",
1939 |  "fastrand",
1940 |  "rustix",
1941 |  "windows-sys 0.52.0",
1942 | ]
1943 | 
1944 | [[package]]
1945 | name = "thread_local"
1946 | version = "1.1.8"
1947 | source = "registry+https://github.com/rust-lang/crates.io-index"
1948 | checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
1949 | dependencies = [
1950 |  "cfg-if",
1951 |  "once_cell",
1952 | ]
1953 | 
1954 | [[package]]
1955 | name = "threadpool"
1956 | version = "1.8.1"
1957 | source = "registry+https://github.com/rust-lang/crates.io-index"
1958 | checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
1959 | dependencies = [
1960 |  "num_cpus",
1961 | ]
1962 | 
1963 | [[package]]
1964 | name = "time"
1965 | version = "0.3.36"
1966 | source = "registry+https://github.com/rust-lang/crates.io-index"
1967 | checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
1968 | dependencies = [
1969 |  "deranged",
1970 |  "num-conv",
1971 |  "powerfmt",
1972 |  "serde",
1973 |  "time-core",
1974 |  "time-macros",
1975 | ]
1976 | 
1977 | [[package]]
1978 | name = "time-core"
1979 | version = "0.1.2"
1980 | source = "registry+https://github.com/rust-lang/crates.io-index"
1981 | checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
1982 | 
1983 | [[package]]
1984 | name = "time-macros"
1985 | version = "0.2.18"
1986 | source = "registry+https://github.com/rust-lang/crates.io-index"
1987 | checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
1988 | dependencies = [
1989 |  "num-conv",
1990 |  "time-core",
1991 | ]
1992 | 
1993 | [[package]]
1994 | name = "tinyvec"
1995 | version = "1.6.0"
1996 | source = "registry+https://github.com/rust-lang/crates.io-index"
1997 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
1998 | dependencies = [
1999 |  "tinyvec_macros",
2000 | ]
2001 | 
2002 | [[package]]
2003 | name = "tinyvec_macros"
2004 | version = "0.1.1"
2005 | source = "registry+https://github.com/rust-lang/crates.io-index"
2006 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
2007 | 
2008 | [[package]]
2009 | name = "tokio"
2010 | version = "1.36.0"
2011 | source = "registry+https://github.com/rust-lang/crates.io-index"
2012 | checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
2013 | dependencies = [
2014 |  "backtrace",
2015 |  "bytes",
2016 |  "libc",
2017 |  "mio",
2018 |  "num_cpus",
2019 |  "pin-project-lite",
2020 |  "signal-hook-registry",
2021 |  "socket2",
2022 |  "tokio-macros",
2023 |  "windows-sys 0.48.0",
2024 | ]
2025 | 
2026 | [[package]]
2027 | name = "tokio-macros"
2028 | version = "2.2.0"
2029 | source = "registry+https://github.com/rust-lang/crates.io-index"
2030 | checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
2031 | dependencies = [
2032 |  "proc-macro2",
2033 |  "quote",
2034 |  "syn",
2035 | ]
2036 | 
2037 | [[package]]
2038 | name = "tokio-native-tls"
2039 | version = "0.3.1"
2040 | source = "registry+https://github.com/rust-lang/crates.io-index"
2041 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
2042 | dependencies = [
2043 |  "native-tls",
2044 |  "tokio",
2045 | ]
2046 | 
2047 | [[package]]
2048 | name = "tokio-rustls"
2049 | version = "0.24.1"
2050 | source = "registry+https://github.com/rust-lang/crates.io-index"
2051 | checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
2052 | dependencies = [
2053 |  "rustls",
2054 |  "tokio",
2055 | ]
2056 | 
2057 | [[package]]
2058 | name = "tokio-stream"
2059 | version = "0.1.15"
2060 | source = "registry+https://github.com/rust-lang/crates.io-index"
2061 | checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
2062 | dependencies = [
2063 |  "futures-core",
2064 |  "pin-project-lite",
2065 |  "tokio",
2066 | ]
2067 | 
2068 | [[package]]
2069 | name = "tokio-util"
2070 | version = "0.7.10"
2071 | source = "registry+https://github.com/rust-lang/crates.io-index"
2072 | checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
2073 | dependencies = [
2074 |  "bytes",
2075 |  "futures-core",
2076 |  "futures-sink",
2077 |  "pin-project-lite",
2078 |  "tokio",
2079 |  "tracing",
2080 | ]
2081 | 
2082 | [[package]]
2083 | name = "tower"
2084 | version = "0.4.13"
2085 | source = "registry+https://github.com/rust-lang/crates.io-index"
2086 | checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
2087 | dependencies = [
2088 |  "futures-core",
2089 |  "futures-util",
2090 |  "pin-project",
2091 |  "pin-project-lite",
2092 |  "tokio",
2093 |  "tower-layer",
2094 |  "tower-service",
2095 |  "tracing",
2096 | ]
2097 | 
2098 | [[package]]
2099 | name = "tower-layer"
2100 | version = "0.3.2"
2101 | source = "registry+https://github.com/rust-lang/crates.io-index"
2102 | checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
2103 | 
2104 | [[package]]
2105 | name = "tower-service"
2106 | version = "0.3.2"
2107 | source = "registry+https://github.com/rust-lang/crates.io-index"
2108 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
2109 | 
2110 | [[package]]
2111 | name = "tracing"
2112 | version = "0.1.40"
2113 | source = "registry+https://github.com/rust-lang/crates.io-index"
2114 | checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
2115 | dependencies = [
2116 |  "log",
2117 |  "pin-project-lite",
2118 |  "tracing-attributes",
2119 |  "tracing-core",
2120 | ]
2121 | 
2122 | [[package]]
2123 | name = "tracing-attributes"
2124 | version = "0.1.27"
2125 | source = "registry+https://github.com/rust-lang/crates.io-index"
2126 | checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
2127 | dependencies = [
2128 |  "proc-macro2",
2129 |  "quote",
2130 |  "syn",
2131 | ]
2132 | 
2133 | [[package]]
2134 | name = "tracing-core"
2135 | version = "0.1.32"
2136 | source = "registry+https://github.com/rust-lang/crates.io-index"
2137 | checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
2138 | dependencies = [
2139 |  "once_cell",
2140 |  "valuable",
2141 | ]
2142 | 
2143 | [[package]]
2144 | name = "tracing-log"
2145 | version = "0.2.0"
2146 | source = "registry+https://github.com/rust-lang/crates.io-index"
2147 | checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
2148 | dependencies = [
2149 |  "log",
2150 |  "once_cell",
2151 |  "tracing-core",
2152 | ]
2153 | 
2154 | [[package]]
2155 | name = "tracing-subscriber"
2156 | version = "0.3.18"
2157 | source = "registry+https://github.com/rust-lang/crates.io-index"
2158 | checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
2159 | dependencies = [
2160 |  "matchers",
2161 |  "nu-ansi-term",
2162 |  "once_cell",
2163 |  "regex",
2164 |  "sharded-slab",
2165 |  "smallvec",
2166 |  "thread_local",
2167 |  "tracing",
2168 |  "tracing-core",
2169 |  "tracing-log",
2170 | ]
2171 | 
2172 | [[package]]
2173 | name = "try-lock"
2174 | version = "0.2.5"
2175 | source = "registry+https://github.com/rust-lang/crates.io-index"
2176 | checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
2177 | 
2178 | [[package]]
2179 | name = "typenum"
2180 | version = "1.17.0"
2181 | source = "registry+https://github.com/rust-lang/crates.io-index"
2182 | checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
2183 | 
2184 | [[package]]
2185 | name = "unicode-bidi"
2186 | version = "0.3.15"
2187 | source = "registry+https://github.com/rust-lang/crates.io-index"
2188 | checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
2189 | 
2190 | [[package]]
2191 | name = "unicode-ident"
2192 | version = "1.0.12"
2193 | source = "registry+https://github.com/rust-lang/crates.io-index"
2194 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
2195 | 
2196 | [[package]]
2197 | name = "unicode-normalization"
2198 | version = "0.1.23"
2199 | source = "registry+https://github.com/rust-lang/crates.io-index"
2200 | checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
2201 | dependencies = [
2202 |  "tinyvec",
2203 | ]
2204 | 
2205 | [[package]]
2206 | name = "untrusted"
2207 | version = "0.7.1"
2208 | source = "registry+https://github.com/rust-lang/crates.io-index"
2209 | checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
2210 | 
2211 | [[package]]
2212 | name = "untrusted"
2213 | version = "0.9.0"
2214 | source = "registry+https://github.com/rust-lang/crates.io-index"
2215 | checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
2216 | 
2217 | [[package]]
2218 | name = "url"
2219 | version = "2.5.0"
2220 | source = "registry+https://github.com/rust-lang/crates.io-index"
2221 | checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
2222 | dependencies = [
2223 |  "form_urlencoded",
2224 |  "idna",
2225 |  "percent-encoding",
2226 | ]
2227 | 
2228 | [[package]]
2229 | name = "urlencoding"
2230 | version = "2.1.3"
2231 | source = "registry+https://github.com/rust-lang/crates.io-index"
2232 | checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
2233 | 
2234 | [[package]]
2235 | name = "utf8parse"
2236 | version = "0.2.1"
2237 | source = "registry+https://github.com/rust-lang/crates.io-index"
2238 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
2239 | 
2240 | [[package]]
2241 | name = "uuid"
2242 | version = "1.10.0"
2243 | source = "registry+https://github.com/rust-lang/crates.io-index"
2244 | checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
2245 | 
2246 | [[package]]
2247 | name = "valuable"
2248 | version = "0.1.0"
2249 | source = "registry+https://github.com/rust-lang/crates.io-index"
2250 | checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
2251 | 
2252 | [[package]]
2253 | name = "vcpkg"
2254 | version = "0.2.15"
2255 | source = "registry+https://github.com/rust-lang/crates.io-index"
2256 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
2257 | 
2258 | [[package]]
2259 | name = "version_check"
2260 | version = "0.9.4"
2261 | source = "registry+https://github.com/rust-lang/crates.io-index"
2262 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
2263 | 
2264 | [[package]]
2265 | name = "vsimd"
2266 | version = "0.8.0"
2267 | source = "registry+https://github.com/rust-lang/crates.io-index"
2268 | checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
2269 | 
2270 | [[package]]
2271 | name = "walkdir"
2272 | version = "2.5.0"
2273 | source = "registry+https://github.com/rust-lang/crates.io-index"
2274 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
2275 | dependencies = [
2276 |  "same-file",
2277 |  "winapi-util",
2278 | ]
2279 | 
2280 | [[package]]
2281 | name = "want"
2282 | version = "0.3.1"
2283 | source = "registry+https://github.com/rust-lang/crates.io-index"
2284 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
2285 | dependencies = [
2286 |  "try-lock",
2287 | ]
2288 | 
2289 | [[package]]
2290 | name = "wasi"
2291 | version = "0.11.0+wasi-snapshot-preview1"
2292 | source = "registry+https://github.com/rust-lang/crates.io-index"
2293 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
2294 | 
2295 | [[package]]
2296 | name = "wasm-bindgen"
2297 | version = "0.2.92"
2298 | source = "registry+https://github.com/rust-lang/crates.io-index"
2299 | checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
2300 | dependencies = [
2301 |  "cfg-if",
2302 |  "wasm-bindgen-macro",
2303 | ]
2304 | 
2305 | [[package]]
2306 | name = "wasm-bindgen-backend"
2307 | version = "0.2.92"
2308 | source = "registry+https://github.com/rust-lang/crates.io-index"
2309 | checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
2310 | dependencies = [
2311 |  "bumpalo",
2312 |  "log",
2313 |  "once_cell",
2314 |  "proc-macro2",
2315 |  "quote",
2316 |  "syn",
2317 |  "wasm-bindgen-shared",
2318 | ]
2319 | 
2320 | [[package]]
2321 | name = "wasm-bindgen-futures"
2322 | version = "0.4.42"
2323 | source = "registry+https://github.com/rust-lang/crates.io-index"
2324 | checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
2325 | dependencies = [
2326 |  "cfg-if",
2327 |  "js-sys",
2328 |  "wasm-bindgen",
2329 |  "web-sys",
2330 | ]
2331 | 
2332 | [[package]]
2333 | name = "wasm-bindgen-macro"
2334 | version = "0.2.92"
2335 | source = "registry+https://github.com/rust-lang/crates.io-index"
2336 | checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
2337 | dependencies = [
2338 |  "quote",
2339 |  "wasm-bindgen-macro-support",
2340 | ]
2341 | 
2342 | [[package]]
2343 | name = "wasm-bindgen-macro-support"
2344 | version = "0.2.92"
2345 | source = "registry+https://github.com/rust-lang/crates.io-index"
2346 | checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
2347 | dependencies = [
2348 |  "proc-macro2",
2349 |  "quote",
2350 |  "syn",
2351 |  "wasm-bindgen-backend",
2352 |  "wasm-bindgen-shared",
2353 | ]
2354 | 
2355 | [[package]]
2356 | name = "wasm-bindgen-shared"
2357 | version = "0.2.92"
2358 | source = "registry+https://github.com/rust-lang/crates.io-index"
2359 | checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
2360 | 
2361 | [[package]]
2362 | name = "web-sys"
2363 | version = "0.3.69"
2364 | source = "registry+https://github.com/rust-lang/crates.io-index"
2365 | checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
2366 | dependencies = [
2367 |  "js-sys",
2368 |  "wasm-bindgen",
2369 | ]
2370 | 
2371 | [[package]]
2372 | name = "winapi"
2373 | version = "0.3.9"
2374 | source = "registry+https://github.com/rust-lang/crates.io-index"
2375 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
2376 | dependencies = [
2377 |  "winapi-i686-pc-windows-gnu",
2378 |  "winapi-x86_64-pc-windows-gnu",
2379 | ]
2380 | 
2381 | [[package]]
2382 | name = "winapi-i686-pc-windows-gnu"
2383 | version = "0.4.0"
2384 | source = "registry+https://github.com/rust-lang/crates.io-index"
2385 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
2386 | 
2387 | [[package]]
2388 | name = "winapi-util"
2389 | version = "0.1.6"
2390 | source = "registry+https://github.com/rust-lang/crates.io-index"
2391 | checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
2392 | dependencies = [
2393 |  "winapi",
2394 | ]
2395 | 
2396 | [[package]]
2397 | name = "winapi-x86_64-pc-windows-gnu"
2398 | version = "0.4.0"
2399 | source = "registry+https://github.com/rust-lang/crates.io-index"
2400 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
2401 | 
2402 | [[package]]
2403 | name = "windows-sys"
2404 | version = "0.48.0"
2405 | source = "registry+https://github.com/rust-lang/crates.io-index"
2406 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
2407 | dependencies = [
2408 |  "windows-targets 0.48.5",
2409 | ]
2410 | 
2411 | [[package]]
2412 | name = "windows-sys"
2413 | version = "0.52.0"
2414 | source = "registry+https://github.com/rust-lang/crates.io-index"
2415 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
2416 | dependencies = [
2417 |  "windows-targets 0.52.4",
2418 | ]
2419 | 
2420 | [[package]]
2421 | name = "windows-targets"
2422 | version = "0.48.5"
2423 | source = "registry+https://github.com/rust-lang/crates.io-index"
2424 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
2425 | dependencies = [
2426 |  "windows_aarch64_gnullvm 0.48.5",
2427 |  "windows_aarch64_msvc 0.48.5",
2428 |  "windows_i686_gnu 0.48.5",
2429 |  "windows_i686_msvc 0.48.5",
2430 |  "windows_x86_64_gnu 0.48.5",
2431 |  "windows_x86_64_gnullvm 0.48.5",
2432 |  "windows_x86_64_msvc 0.48.5",
2433 | ]
2434 | 
2435 | [[package]]
2436 | name = "windows-targets"
2437 | version = "0.52.4"
2438 | source = "registry+https://github.com/rust-lang/crates.io-index"
2439 | checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
2440 | dependencies = [
2441 |  "windows_aarch64_gnullvm 0.52.4",
2442 |  "windows_aarch64_msvc 0.52.4",
2443 |  "windows_i686_gnu 0.52.4",
2444 |  "windows_i686_msvc 0.52.4",
2445 |  "windows_x86_64_gnu 0.52.4",
2446 |  "windows_x86_64_gnullvm 0.52.4",
2447 |  "windows_x86_64_msvc 0.52.4",
2448 | ]
2449 | 
2450 | [[package]]
2451 | name = "windows_aarch64_gnullvm"
2452 | version = "0.48.5"
2453 | source = "registry+https://github.com/rust-lang/crates.io-index"
2454 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
2455 | 
2456 | [[package]]
2457 | name = "windows_aarch64_gnullvm"
2458 | version = "0.52.4"
2459 | source = "registry+https://github.com/rust-lang/crates.io-index"
2460 | checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
2461 | 
2462 | [[package]]
2463 | name = "windows_aarch64_msvc"
2464 | version = "0.48.5"
2465 | source = "registry+https://github.com/rust-lang/crates.io-index"
2466 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
2467 | 
2468 | [[package]]
2469 | name = "windows_aarch64_msvc"
2470 | version = "0.52.4"
2471 | source = "registry+https://github.com/rust-lang/crates.io-index"
2472 | checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
2473 | 
2474 | [[package]]
2475 | name = "windows_i686_gnu"
2476 | version = "0.48.5"
2477 | source = "registry+https://github.com/rust-lang/crates.io-index"
2478 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
2479 | 
2480 | [[package]]
2481 | name = "windows_i686_gnu"
2482 | version = "0.52.4"
2483 | source = "registry+https://github.com/rust-lang/crates.io-index"
2484 | checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
2485 | 
2486 | [[package]]
2487 | name = "windows_i686_msvc"
2488 | version = "0.48.5"
2489 | source = "registry+https://github.com/rust-lang/crates.io-index"
2490 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
2491 | 
2492 | [[package]]
2493 | name = "windows_i686_msvc"
2494 | version = "0.52.4"
2495 | source = "registry+https://github.com/rust-lang/crates.io-index"
2496 | checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
2497 | 
2498 | [[package]]
2499 | name = "windows_x86_64_gnu"
2500 | version = "0.48.5"
2501 | source = "registry+https://github.com/rust-lang/crates.io-index"
2502 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
2503 | 
2504 | [[package]]
2505 | name = "windows_x86_64_gnu"
2506 | version = "0.52.4"
2507 | source = "registry+https://github.com/rust-lang/crates.io-index"
2508 | checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
2509 | 
2510 | [[package]]
2511 | name = "windows_x86_64_gnullvm"
2512 | version = "0.48.5"
2513 | source = "registry+https://github.com/rust-lang/crates.io-index"
2514 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
2515 | 
2516 | [[package]]
2517 | name = "windows_x86_64_gnullvm"
2518 | version = "0.52.4"
2519 | source = "registry+https://github.com/rust-lang/crates.io-index"
2520 | checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
2521 | 
2522 | [[package]]
2523 | name = "windows_x86_64_msvc"
2524 | version = "0.48.5"
2525 | source = "registry+https://github.com/rust-lang/crates.io-index"
2526 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
2527 | 
2528 | [[package]]
2529 | name = "windows_x86_64_msvc"
2530 | version = "0.52.4"
2531 | source = "registry+https://github.com/rust-lang/crates.io-index"
2532 | checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
2533 | 
2534 | [[package]]
2535 | name = "winreg"
2536 | version = "0.50.0"
2537 | source = "registry+https://github.com/rust-lang/crates.io-index"
2538 | checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
2539 | dependencies = [
2540 |  "cfg-if",
2541 |  "windows-sys 0.48.0",
2542 | ]
2543 | 
2544 | [[package]]
2545 | name = "xmlparser"
2546 | version = "0.13.6"
2547 | source = "registry+https://github.com/rust-lang/crates.io-index"
2548 | checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
2549 | 
2550 | [[package]]
2551 | name = "yansi"
2552 | version = "1.0.1"
2553 | source = "registry+https://github.com/rust-lang/crates.io-index"
2554 | checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
2555 | 
2556 | [[package]]
2557 | name = "zeroize"
2558 | version = "1.7.0"
2559 | source = "registry+https://github.com/rust-lang/crates.io-index"
2560 | checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
2561 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "rust-log-analyzer"
 3 | version = "0.1.0"
 4 | authors = ["Rust Project Developers"]
 5 | license = "MIT"
 6 | edition = "2021"
 7 | 
 8 | [dependencies]
 9 | aho-corasick = "1.1"
10 | atomicwrites = "0.4"
11 | bincode = "1.0"
12 | brotli = "3.3"
13 | anyhow = { version = "1.0", features = ["backtrace"]}
14 | futures = "0.3"
15 | fnv = "1.0"
16 | hex = "0.4"
17 | hmac = "0.12"
18 | hyper = { version = "0.14.25", features = ["http1", "server", "runtime"] }
19 | lazy_static = "1.0"
20 | regex = "1.7.1"
21 | reqwest = { version = "0.11.14", features = ["json", "blocking"] }
22 | serde = "1.0"
23 | serde_derive = "1.0"
24 | serde_json = "1.0"
25 | sha1 = "0.10"
26 | walkdir = "2.1"
27 | dotenv = "0.15"
28 | crossbeam = "0.8"
29 | percent-encoding = "2.1.0"
30 | threadpool = "1.8.1"
31 | tracing = "0.1.16"
32 | tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
33 | atty = "0.2.14"
34 | clap = { version = "4.1.8", features = ["derive"] }
35 | tokio = { version = "1.26.0", features = ["rt-multi-thread", "rt", "signal", "macros"] }
36 | 
37 | # aws release of https://github.com/awslabs/aws-sdk-rust/releases/tag/release-2023-10-26
38 | aws-sdk-s3 = "0.34.0"
39 | aws-config = "0.56.1"
40 | 
41 | [dev-dependencies]
42 | pretty_assertions = "1.0"
43 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | # This Dockerfile is composed of two steps: the first one builds the release
 2 | # binary, and then the binary is copied inside another, empty image.
 3 | 
 4 | #################
 5 | #  Build image  #
 6 | #################
 7 | 
 8 | FROM ubuntu:22.04 AS build
 9 | 
10 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
11 |     ca-certificates \
12 |     curl \
13 |     build-essential \
14 |     pkg-config \
15 |     libssl-dev
16 | 
17 | # Install the currently pinned toolchain with rustup
18 | RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
19 |     chmod +x /tmp/rustup-init && \
20 |     /tmp/rustup-init -y --no-modify-path --default-toolchain stable
21 | ENV PATH=/root/.cargo/bin:$PATH
22 | 
23 | # Build the dependencies in a separate step to avoid rebuilding all of them
24 | # every time the source code changes. This takes advantage of Docker's layer
25 | # caching, and it works by copying the Cargo.{toml,lock} with dummy source code
26 | # and doing a full build with it.
27 | WORKDIR /tmp/source
28 | COPY Cargo.lock Cargo.toml /tmp/source/
29 | RUN mkdir -p /tmp/source/src && \
30 |     echo "fn main() {}" > /tmp/source/src/main.rs
31 | RUN cargo fetch
32 | RUN cargo build --release
33 | 
34 | # Dependencies are now cached, copy the actual source code and do another full
35 | # build. The touch on all the .rs files is needed, otherwise cargo assumes the
36 | # source code didn't change thanks to mtime weirdness.
37 | RUN rm -rf /tmp/source/src
38 | COPY src /tmp/source/src
39 | RUN find -name "*.rs" -exec touch {} \; && cargo build --release
40 | 
41 | ##################
42 | #  Output image  #
43 | ##################
44 | 
45 | FROM ubuntu:22.04 AS binary
46 | 
47 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
48 |     ca-certificates
49 | 
50 | COPY --from=build /tmp/source/target/release/rla-offline /usr/local/bin/
51 | COPY --from=build /tmp/source/target/release/rla-server /usr/local/bin/
52 | 
53 | CMD rla-server --bind 0.0.0.0 --port 80 --index-file $INDEX_FILE --webhook-verify --ci $CI_PROVIDER --repo $CI_REPO $EXTRA_ARGS
54 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2018 The Rust Project Developers
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Readme.md:
--------------------------------------------------------------------------------
 1 | # Rust Log Analyzer
 2 | 
 3 | The **Rust Log Analyzer** (**RLA**) is a tool that analyzes the CI build logs of the [rust-lang/rust](https://github.com/rust-lang/rust) repository with the purpose of automatically extracting error messages from failed builds.
 4 | 
 5 | This repository contains three components:
 6 | 
 7 | * The **`rust_log_analyzer`** library, which contains the analysis logic.
 8 | * The **`rla-offline`** binary, a collection of various tools to run the analyzer off-line (see the output of `rla-offline --help` for available commands).
 9 | * The **`rla-server`** binary, a web server which receives GitHub webhooks and automatically posts analysis results to the Rust repository.
10 | 
11 | ## Running RLA
12 | 
13 | RLA uses the `log` crate to write all output. By default, anything logged at `INFO` or higher will be printed. You can change this behavior by setting the `RLA_LOG` environment variable, using the syntax specified by the `env_logger` crate.
14 | 
15 | ### Secrets
16 | 
17 | To run commands which access online resources, you have to provide the required authentication information in environment variables:
18 | 
19 | * For *GitHub*, set `GITHUB_TOKEN` to a [personal access token](https://github.com/settings/tokens) with at least "repo" scope.
20 | 
21 | ### Bootstrapping an index file
22 | 
23 | To initialize a new index file, perform the following steps:
24 | 
25 | 1. Download some successful build logs using the `rla-offline dl` command.
26 |     * It is recommended that you run in `release` mode.
27 |     * I'm still gathering data, but you should probably have well over 1000 log files (this does not mean over 1000 *builds*, since one builds consists of dozens of jobs)
28 |     * Example command: `rla-offline dl --ci actions --repo rust-lang/rust -c 40 --branch auto --passed -o data/training`
29 | 2. Train on the downloaded logs using the `rla-offline learn command`.
30 |     * Example command: `rla-offline learn --ci actions -i demo.idx data/training`
31 | 
32 | ### Analyzing a specific log
33 | 
34 | 1. Download the log file you want to analyze using either the `rla-offline dl` command or manually from your CI provider.
35 |     * All tools will automatically decompress files ending in `.brotli`, or assume uncompressed data otherwise.
36 | 2. Use the `rla-offline extract-one` command analyze the log file.
37 |     * Example command: `rla-offline extract-one --ci actions -i demo.idx my-log.txt`
38 | 
39 | ### Evaluating quality while developing
40 | 
41 | *Note: This process will / should be integrated as regression tests.*
42 | 
43 | 1. Download, or otherwise curate, a set of log files you want to evaluate against.
44 |     * Example command: `rla-offline dl --ci actions --repo rust-lang/rust -c 50 --failed -o data/failed`
45 |     * *Note: Eventually, a set of test log files will be provided in the repository.*
46 | 2. Use the `rla-offline extract-dir` command to analyze all the log files and write the results to a separate directory.
47 |     * You can (temporarily) check the result directory in to the repository to see diffs.
48 |     * Example command: `rla-offline extract-dir --ci actions -i demo.idx -s data/failed -d data/err`
49 |     * *Note: Eventually, the expected results for the test log files will be provided in the repository and used as regression tests.*
50 | 
51 | ### Index file storage
52 | 
53 | The index file can be stored either in the local filesystem (by providing the
54 | absolute or relative path to the file) or in S3 (by providing a
55 | `s3://{bucket}/{key}` URL). The S3 region of the bucket is detected
56 | automatically at startup.
57 | 


--------------------------------------------------------------------------------
/src/bin/offline/dl.rs:
--------------------------------------------------------------------------------
  1 | use crate::offline;
  2 | use crate::rla::ci::{self, CiPlatform};
  3 | use std::collections::HashSet;
  4 | use std::io::{self, Write};
  5 | use std::path::Path;
  6 | 
  7 | const LOG_DL_MAX_ATTEMPTS: u32 = 3;
  8 | 
  9 | pub fn cat(input: &Path, strip_control: bool, decode_utf8: bool) -> rla::Result<()> {
 10 |     let mut data = offline::fs::load_maybe_compressed(input)?;
 11 | 
 12 |     if strip_control {
 13 |         data.retain(|&b| b == b'\n' || !b.is_ascii_control());
 14 |     }
 15 | 
 16 |     if decode_utf8 {
 17 |         let stdout = io::stdout();
 18 |         stdout
 19 |             .lock()
 20 |             .write_all(String::from_utf8_lossy(&data).as_bytes())?;
 21 |     } else {
 22 |         let stdout = io::stdout();
 23 |         stdout.lock().write_all(&data)?;
 24 |     }
 25 | 
 26 |     Ok(())
 27 | }
 28 | 
 29 | pub fn download(
 30 |     ci: &dyn CiPlatform,
 31 |     repo: &str,
 32 |     output: &Path,
 33 |     count: u32,
 34 |     offset: u32,
 35 |     filter_branches: &[String],
 36 |     only_passed: bool,
 37 |     only_failed: bool,
 38 | ) -> rla::Result<()> {
 39 |     let client = reqwest::blocking::Client::new();
 40 |     let filter_branches = filter_branches
 41 |         .iter()
 42 |         .map(|s| s.as_str())
 43 |         .collect::<HashSet<_>>();
 44 | 
 45 |     let check_outcome = |outcome: &dyn rla::ci::Outcome| {
 46 |         (!only_passed || outcome.is_passed()) && (!only_failed || outcome.is_failed())
 47 |     };
 48 |     let builds = ci.query_builds(repo, count, offset, &|build| {
 49 |         (filter_branches.is_empty() || filter_branches.contains(build.branch_name()))
 50 |             && check_outcome(build.outcome())
 51 |     })?;
 52 | 
 53 |     let compression_pool = threadpool::Builder::new().build();
 54 | 
 55 |     'job_loop: for job in builds.iter().flat_map(|b| b.jobs()) {
 56 |         if !check_outcome(job.outcome()) {
 57 |             continue;
 58 |         }
 59 | 
 60 |         let save_path = output.join(offline::fs::encode_path(&format!(
 61 |             "{}.log.brotli",
 62 |             job.log_file_name()
 63 |         )));
 64 |         if save_path.is_file() {
 65 |             warn!("Skipping log for {} because the output file exists.", job);
 66 |             continue;
 67 |         }
 68 | 
 69 |         let data;
 70 |         let mut attempt = 0;
 71 | 
 72 |         loop {
 73 |             attempt += 1;
 74 |             info!(
 75 |                 "Downloading log for {} [Attempt {}/{}]...",
 76 |                 job, attempt, LOG_DL_MAX_ATTEMPTS
 77 |             );
 78 | 
 79 |             match ci::download_log(ci, job, &client) {
 80 |                 Some(Ok(d)) => {
 81 |                     data = d;
 82 |                     break;
 83 |                 }
 84 |                 None => {
 85 |                     warn!("no log for {}", job);
 86 |                 }
 87 |                 Some(Err(e)) => {
 88 |                     if attempt >= LOG_DL_MAX_ATTEMPTS {
 89 |                         warn!("Failed to download log, skipping: {}", e);
 90 |                         continue 'job_loop;
 91 |                     }
 92 |                 }
 93 |             }
 94 |         }
 95 | 
 96 |         debug!("Compressing...");
 97 | 
 98 |         // When the pool is busy this compresses on the main thread, to avoid using a huge amount
 99 |         // of RAM to store all the queued logs.
100 |         if compression_pool.active_count() >= compression_pool.max_count() {
101 |             debug!("compression pool is busy, compressing on the main thread...");
102 |             offline::fs::save_compressed(&save_path, &data)?;
103 |         } else {
104 |             debug!("compressing on the pool...");
105 |             compression_pool.execute(move || {
106 |                 crate::util::log_and_exit_error(move || {
107 |                     offline::fs::save_compressed(&save_path, &data)
108 |                 });
109 |             });
110 |         }
111 |     }
112 | 
113 |     Ok(())
114 | }
115 | 


--------------------------------------------------------------------------------
/src/bin/offline/extract.rs:
--------------------------------------------------------------------------------
  1 | use crate::offline;
  2 | 
  3 | use rla::index::IndexStorage;
  4 | use std::fs;
  5 | use std::io::{self, Write};
  6 | use std::path::Path;
  7 | use std::time::Duration;
  8 | use std::time::Instant;
  9 | use walkdir::WalkDir;
 10 | 
 11 | struct Line<'a> {
 12 |     _original: &'a [u8],
 13 |     sanitized: Vec<u8>,
 14 | }
 15 | 
 16 | impl<'a> rla::index::IndexData for Line<'a> {
 17 |     fn sanitized(&self) -> &[u8] {
 18 |         &self.sanitized
 19 |     }
 20 | }
 21 | 
 22 | fn load_lines<'a>(ci: &dyn rla::ci::CiPlatform, log: &'a [u8]) -> Vec<Line<'a>> {
 23 |     rla::sanitize::split_lines(log)
 24 |         .iter()
 25 |         .map(|&line| Line {
 26 |             _original: line,
 27 |             sanitized: rla::sanitize::clean(ci, line),
 28 |         })
 29 |         .collect()
 30 | }
 31 | 
 32 | pub fn dir(
 33 |     ci: &dyn rla::ci::CiPlatform,
 34 |     index_file: &IndexStorage,
 35 |     src_dir: &Path,
 36 |     dst_dir: &Path,
 37 | ) -> rla::Result<()> {
 38 |     let config = rla::extract::Config::default();
 39 |     let index = rla::Index::load(index_file)?;
 40 | 
 41 |     for entry in walk_non_hidden_children(dst_dir) {
 42 |         let entry = entry?;
 43 | 
 44 |         if entry.file_type().is_dir() {
 45 |             continue;
 46 |         }
 47 | 
 48 |         fs::remove_file(entry.path())?;
 49 |     }
 50 | 
 51 |     let progress_every = Duration::from_secs(1);
 52 |     let mut last_print = Instant::now();
 53 | 
 54 |     for (count, entry) in walk_non_hidden_children(&src_dir).enumerate() {
 55 |         let entry = entry?;
 56 | 
 57 |         if entry.file_type().is_dir() {
 58 |             continue;
 59 |         }
 60 | 
 61 |         let now = Instant::now();
 62 | 
 63 |         if now - last_print >= progress_every {
 64 |             last_print = now;
 65 |             debug!(
 66 |                 "Extracting errors from {} [{}/?]...",
 67 |                 entry.path().display(),
 68 |                 count
 69 |             );
 70 |         } else {
 71 |             trace!(
 72 |                 "Extracting errors from {} [{}/?]...",
 73 |                 entry.path().display(),
 74 |                 count
 75 |             );
 76 |         }
 77 | 
 78 |         let log = offline::fs::load_maybe_compressed(entry.path())?;
 79 |         let lines = load_lines(ci, &log);
 80 |         let blocks = rla::extract::extract(&config, &index, &lines);
 81 | 
 82 |         let mut out_name = entry.file_name().to_owned();
 83 |         out_name.push(".err");
 84 | 
 85 |         write_blocks_to(
 86 |             io::BufWriter::new(fs::File::create(dst_dir.join(out_name))?),
 87 |             &blocks,
 88 |         )?;
 89 |     }
 90 | 
 91 |     Ok(())
 92 | }
 93 | 
 94 | pub fn one(
 95 |     ci: &dyn rla::ci::CiPlatform,
 96 |     index_file: &IndexStorage,
 97 |     log_file: &Path,
 98 | ) -> rla::Result<()> {
 99 |     let config = rla::extract::Config::default();
100 |     let index = rla::Index::load(index_file)?;
101 | 
102 |     let log = offline::fs::load_maybe_compressed(log_file)?;
103 |     let lines = load_lines(ci, &log);
104 |     let blocks = rla::extract::extract(&config, &index, &lines);
105 | 
106 |     let stdout = io::stdout();
107 |     write_blocks_to(stdout.lock(), &blocks)?;
108 | 
109 |     Ok(())
110 | }
111 | 
112 | fn write_blocks_to<W: Write>(mut w: W, blocks: &[Vec<&Line>]) -> rla::Result<()> {
113 |     let mut first = true;
114 | 
115 |     for block in blocks {
116 |         if !first {
117 |             writeln!(w, "---")?;
118 |         }
119 |         first = false;
120 | 
121 |         for &line in block {
122 |             w.write_all(&line.sanitized)?;
123 |             w.write_all(b"\n")?;
124 |         }
125 |     }
126 | 
127 |     Ok(())
128 | }
129 | 
130 | fn walk_non_hidden_children(
131 |     root: &Path,
132 | ) -> Box<dyn Iterator<Item = walkdir::Result<walkdir::DirEntry>>> {
133 |     fn not_hidden(entry: &walkdir::DirEntry) -> bool {
134 |         !entry
135 |             .file_name()
136 |             .to_str()
137 |             .map(|s| s.starts_with('.'))
138 |             .unwrap_or(false)
139 |     }
140 | 
141 |     Box::new(
142 |         WalkDir::new(root)
143 |             .min_depth(1)
144 |             .max_depth(1)
145 |             .into_iter()
146 |             .filter_entry(not_hidden),
147 |     )
148 | }
149 | 


--------------------------------------------------------------------------------
/src/bin/offline/fs.rs:
--------------------------------------------------------------------------------
 1 | use anyhow::Context;
 2 | use percent_encoding::{AsciiSet, CONTROLS};
 3 | use std::fs;
 4 | use std::io::{Read, Write};
 5 | use std::path::Path;
 6 | 
 7 | const BROTLI_BUFFER: usize = 4096;
 8 | 
 9 | // Defaults from the Python implementation
10 | const BROTLI_QUALITY: u32 = 11;
11 | const BROTLI_LGWIN: u32 = 22;
12 | 
13 | /// The set of characters which cannot be used in a [filename on Windows][windows].
14 | ///
15 | /// [windows]: https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file#naming-conventions
16 | const FILENAME_ENCODE_SET: AsciiSet = CONTROLS
17 |     .add(b'<')
18 |     .add(b'>')
19 |     .add(b':')
20 |     .add(b'"')
21 |     .add(b'/')
22 |     .add(b'\\')
23 |     .add(b'|')
24 |     .add(b'?')
25 |     .add(b'*');
26 | 
27 | pub fn save_compressed(out: &Path, data: &[u8]) -> rla::Result<()> {
28 |     let mut writer = brotli::CompressorWriter::new(
29 |         fs::File::create(out).with_context(|| format!("save_compressed: {:?}", out.to_owned()))?,
30 |         BROTLI_BUFFER,
31 |         BROTLI_QUALITY,
32 |         BROTLI_LGWIN,
33 |     );
34 | 
35 |     writer.write_all(data)?;
36 | 
37 |     Ok(())
38 | }
39 | 
40 | pub fn load_compressed(inp: &Path) -> rla::Result<Vec<u8>> {
41 |     let mut reader = brotli::Decompressor::new(fs::File::open(inp)?, BROTLI_BUFFER);
42 | 
43 |     let mut buf = vec![];
44 |     reader.read_to_end(&mut buf)?;
45 | 
46 |     Ok(buf)
47 | }
48 | 
49 | pub fn load_maybe_compressed(inp: &Path) -> rla::Result<Vec<u8>> {
50 |     if inp.extension().map_or(false, |e| e == "brotli") {
51 |         load_compressed(inp)
52 |     } else {
53 |         let mut buf = vec![];
54 |         fs::File::open(inp)?.read_to_end(&mut buf)?;
55 |         Ok(buf)
56 |     }
57 | }
58 | 
59 | pub(crate) fn encode_path(path: &str) -> String {
60 |     percent_encoding::percent_encode(path.as_bytes(), &FILENAME_ENCODE_SET).collect::<String>()
61 | }
62 | 


--------------------------------------------------------------------------------
/src/bin/offline/learn.rs:
--------------------------------------------------------------------------------
 1 | use crate::offline;
 2 | 
 3 | use rla::index::IndexStorage;
 4 | use std::path::PathBuf;
 5 | use std::time::Duration;
 6 | use std::time::Instant;
 7 | use walkdir::WalkDir;
 8 | 
 9 | pub fn learn(
10 |     ci: &dyn rla::ci::CiPlatform,
11 |     index_file: &IndexStorage,
12 |     inputs: &[PathBuf],
13 |     multiplier: u32,
14 | ) -> rla::Result<()> {
15 |     let mut index = rla::Index::load_or_create(index_file)?;
16 | 
17 |     let progress_every = Duration::from_secs(1);
18 |     let mut last_print = Instant::now();
19 | 
20 |     for (count, input) in inputs
21 |         .iter()
22 |         .flat_map(|i| WalkDir::new(i).into_iter().filter_entry(not_hidden))
23 |         .enumerate()
24 |     {
25 |         let input = input?;
26 |         if input.file_type().is_dir() {
27 |             continue;
28 |         }
29 | 
30 |         let now = Instant::now();
31 | 
32 |         if now - last_print >= progress_every {
33 |             last_print = now;
34 |             debug!("Learning from {} [{}/?]...", input.path().display(), count);
35 |         } else {
36 |             trace!("Learning from {} [{}/?]...", input.path().display(), count);
37 |         }
38 | 
39 |         let data = offline::fs::load_maybe_compressed(input.path())?;
40 | 
41 |         for line in rla::sanitize::split_lines(&data) {
42 |             index.learn(
43 |                 &rla::index::Sanitized(rla::sanitize::clean(ci, line)),
44 |                 multiplier,
45 |             );
46 |         }
47 |     }
48 | 
49 |     index.save(index_file)?;
50 | 
51 |     Ok(())
52 | }
53 | 
54 | fn not_hidden(entry: &walkdir::DirEntry) -> bool {
55 |     !entry
56 |         .file_name()
57 |         .to_str()
58 |         .map(|s| s.starts_with('.'))
59 |         .unwrap_or(false)
60 | }
61 | 


--------------------------------------------------------------------------------
/src/bin/offline/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod dl;
2 | pub mod extract;
3 | 
4 | pub use self::learn::learn;
5 | 
6 | mod fs;
7 | mod learn;
8 | 


--------------------------------------------------------------------------------
/src/bin/rla-offline.rs:
--------------------------------------------------------------------------------
  1 | #![deny(unused_must_use)]
  2 | #![allow(
  3 |     clippy::collapsible_if,
  4 |     clippy::needless_range_loop,
  5 |     clippy::useless_let_if_seq
  6 | )]
  7 | 
  8 | extern crate brotli;
  9 | #[macro_use]
 10 | extern crate tracing;
 11 | extern crate rust_log_analyzer as rla;
 12 | extern crate walkdir;
 13 | 
 14 | use clap::Parser;
 15 | use rla::index::IndexStorage;
 16 | use std::path::PathBuf;
 17 | 
 18 | mod offline;
 19 | mod util;
 20 | 
 21 | #[derive(Debug, Parser)]
 22 | #[command(
 23 |     name = "Rust Log Analyzer Offline Tools",
 24 |     about = "A collection of tools to run the log analyzer without starting a server."
 25 | )]
 26 | enum Cli {
 27 |     #[command(
 28 |         name = "cat",
 29 |         about = "Read, and optionally process, a previously downloaded log file, then dump it to stdout."
 30 |     )]
 31 |     Cat {
 32 |         #[arg(
 33 |             short = 's',
 34 |             long = "strip-control",
 35 |             help = "Removes all ASCII control characters, except newlines, before dumping."
 36 |         )]
 37 |         strip_control: bool,
 38 |         #[arg(
 39 |             short = 'd',
 40 |             long = "decode-utf8",
 41 |             help = "Lossily decode as UTF-8 before dumping."
 42 |         )]
 43 |         decode_utf8: bool,
 44 |         #[arg(help = "The log file to read and dump.")]
 45 |         input: PathBuf,
 46 |     },
 47 | 
 48 |     #[command(name = "learn", about = "Learn from previously downloaded log files.")]
 49 |     Learn {
 50 |         #[arg(long = "ci", help = "CI platform to download from.")]
 51 |         ci: util::CliCiPlatform,
 52 |         #[arg(
 53 |             short = 'i',
 54 |             long = "index-file",
 55 |             help = "The index file to read / write. An existing index file is updated."
 56 |         )]
 57 |         index_file: IndexStorage,
 58 |         #[arg(
 59 |             short = 'm',
 60 |             long = "multiplier",
 61 |             default_value = "1",
 62 |             help = "A multiplier to apply when learning."
 63 |         )]
 64 |         multiplier: u32,
 65 |         #[arg(
 66 |             help = "The log files to learn from.\nDirectories are traversed recursively. Hidden files are ignore."
 67 |         )]
 68 |         logs: Vec<PathBuf>,
 69 |     },
 70 | 
 71 |     #[command(
 72 |         name = "extract-dir",
 73 |         about = "Extract potential error messages from all log files in a directory, writing the results to a different directory."
 74 |     )]
 75 |     ExtractDir {
 76 |         #[arg(long = "ci", help = "CI platform to download from.")]
 77 |         ci: util::CliCiPlatform,
 78 |         #[arg(
 79 |             short = 'i',
 80 |             long = "index-file",
 81 |             help = "The index file to read / write."
 82 |         )]
 83 |         index_file: IndexStorage,
 84 |         #[arg(
 85 |             short = 's',
 86 |             long = "source",
 87 |             help = "The directory in which to (non-recursively) look for log files. Hidden files are ignored."
 88 |         )]
 89 |         source: PathBuf,
 90 |         #[arg(
 91 |             short = 'd',
 92 |             long = "destination",
 93 |             help = "The directory in which to write the results. All non-hidden will be deleted from the directory."
 94 |         )]
 95 |         dest: PathBuf,
 96 |     },
 97 | 
 98 |     #[command(
 99 |         name = "extract-one",
100 |         about = "Extract a potential error message from a single log file."
101 |     )]
102 |     ExtractOne {
103 |         #[arg(long = "ci", help = "CI platform to download from.")]
104 |         ci: util::CliCiPlatform,
105 |         #[arg(
106 |             short = 'i',
107 |             long = "index-file",
108 |             help = "The index file to read / write."
109 |         )]
110 |         index_file: IndexStorage,
111 |         #[arg(help = "The log file to analyze.")]
112 |         log: PathBuf,
113 |     },
114 | 
115 |     #[command(name = "dl", about = "Download build logs from the CI platform.")]
116 |     Dl {
117 |         #[arg(long = "ci", help = "CI platform to download from.")]
118 |         ci: util::CliCiPlatform,
119 |         #[arg(long = "repo", help = "Repository to download from.")]
120 |         repo: String,
121 |         #[arg(short = 'o', long = "output", help = "Log output directory.")]
122 |         output: PathBuf,
123 |         #[arg(short = 'c', long = "count", help = "Number of _builds_ to process.")]
124 |         count: u32,
125 |         #[arg(
126 |             short = 's',
127 |             long = "skip",
128 |             default_value = "0",
129 |             help = "Number of _builds_ to skip."
130 |         )]
131 |         skip: u32,
132 |         #[arg(short = 'b', long = "branch", help = "Branches to filter by.")]
133 |         branches: Vec<String>,
134 |         #[arg(long = "passed", help = "Only download passed builds and jobs.")]
135 |         passed: bool,
136 |         #[arg(long = "failed", help = "Only download failed builds and jobs.")]
137 |         failed: bool,
138 |     },
139 | }
140 | 
141 | #[test]
142 | fn verify_cli() {
143 |     use clap::CommandFactory;
144 |     Cli::command().debug_assert()
145 | }
146 | 
147 | fn main() {
148 |     dotenv::dotenv().ok();
149 |     util::run(|| match Cli::parse() {
150 |         Cli::Cat {
151 |             strip_control,
152 |             decode_utf8,
153 |             input,
154 |         } => offline::dl::cat(&input, strip_control, decode_utf8),
155 |         Cli::Learn {
156 |             ci,
157 |             index_file,
158 |             multiplier,
159 |             logs,
160 |         } => offline::learn(ci.get()?.as_ref(), &index_file, &logs, multiplier),
161 |         Cli::ExtractDir {
162 |             ci,
163 |             index_file,
164 |             source,
165 |             dest,
166 |         } => offline::extract::dir(ci.get()?.as_ref(), &index_file, &source, &dest),
167 |         Cli::ExtractOne {
168 |             ci,
169 |             index_file,
170 |             log,
171 |         } => offline::extract::one(ci.get()?.as_ref(), &index_file, &log),
172 |         Cli::Dl {
173 |             ci,
174 |             repo,
175 |             output,
176 |             count,
177 |             skip,
178 |             branches,
179 |             passed,
180 |             failed,
181 |         } => offline::dl::download(
182 |             ci.get()?.as_ref(),
183 |             &repo,
184 |             &output,
185 |             count,
186 |             skip,
187 |             &branches,
188 |             passed,
189 |             failed,
190 |         ),
191 |     });
192 | }
193 | 


--------------------------------------------------------------------------------
/src/bin/rla-server.rs:
--------------------------------------------------------------------------------
  1 | #![deny(unused_must_use)]
  2 | #![allow(
  3 |     clippy::collapsible_if,
  4 |     clippy::needless_range_loop,
  5 |     clippy::useless_let_if_seq
  6 | )]
  7 | 
  8 | extern crate futures;
  9 | extern crate hyper;
 10 | #[macro_use]
 11 | extern crate tracing;
 12 | extern crate crossbeam;
 13 | extern crate regex;
 14 | extern crate rust_log_analyzer as rla;
 15 | extern crate serde_json;
 16 | 
 17 | use crate::server::QueueItem;
 18 | use clap::Parser;
 19 | use crossbeam::channel::Sender;
 20 | use rla::index::IndexStorage;
 21 | use std::process;
 22 | use std::sync::Arc;
 23 | use std::thread;
 24 | 
 25 | mod server;
 26 | mod util;
 27 | 
 28 | #[derive(Parser, Debug)]
 29 | #[command(
 30 |     name = "Rust Log Analyzer WebHook Server",
 31 |     about = "A http server that listens for GitHub webhooks and posts comments with potential causes on failed builds."
 32 | )]
 33 | struct Cli {
 34 |     #[arg(
 35 |         short = 'p',
 36 |         long = "port",
 37 |         default_value = "8080",
 38 |         help = "The port to listen on for HTTP connections."
 39 |     )]
 40 |     port: u16,
 41 |     #[arg(
 42 |         short = 'b',
 43 |         long = "bind",
 44 |         default_value = "127.0.0.1",
 45 |         help = "The address to bind."
 46 |     )]
 47 |     bind: std::net::IpAddr,
 48 |     #[arg(
 49 |         short = 'i',
 50 |         long = "index-file",
 51 |         help = "The index file to read / write. An existing index file is updated."
 52 |     )]
 53 |     index_file: IndexStorage,
 54 |     #[arg(
 55 |         long = "debug-post",
 56 |         help = "Post all comments to the given issue instead of the actual PR. Format: \"user/repo#id\""
 57 |     )]
 58 |     debug_post: Option<String>,
 59 |     #[arg(
 60 |         long = "webhook-verify",
 61 |         help = "If enabled, web hooks that cannot be verified are rejected."
 62 |     )]
 63 |     webhook_verify: bool,
 64 |     #[arg(long = "ci", help = "CI platform to interact with.")]
 65 |     ci: util::CliCiPlatform,
 66 |     #[arg(long = "repo", help = "Repository to interact with.")]
 67 |     repo: String,
 68 |     #[arg(
 69 |         long = "secondary-repo",
 70 |         help = "Secondary repositories to listen for builds.",
 71 |         required = false
 72 |     )]
 73 |     secondary_repos: Vec<String>,
 74 |     #[arg(
 75 |         long = "query-builds-from-primary-repo",
 76 |         help = "Always query builds from the primary repo instead of the repo receiving them."
 77 |     )]
 78 |     query_builds_from_primary_repo: bool,
 79 | }
 80 | 
 81 | #[test]
 82 | fn verify_cli() {
 83 |     use clap::CommandFactory;
 84 |     Cli::command().debug_assert()
 85 | }
 86 | 
 87 | fn main() {
 88 |     dotenv::dotenv().ok();
 89 |     util::run(|| {
 90 |         let args = Cli::parse();
 91 | 
 92 |         let addr = std::net::SocketAddr::new(args.bind, args.port);
 93 | 
 94 |         let (queue_send, queue_recv) = crossbeam::channel::unbounded();
 95 | 
 96 |         let service = Arc::new(server::RlaService::new(
 97 |             args.webhook_verify,
 98 |             queue_send.clone(),
 99 |         )?);
100 | 
101 |         let mut worker = server::Worker::new(
102 |             args.index_file,
103 |             args.debug_post,
104 |             queue_recv,
105 |             args.ci.get()?,
106 |             args.repo,
107 |             args.secondary_repos,
108 |             args.query_builds_from_primary_repo,
109 |         )?;
110 | 
111 |         let worker_thread = thread::spawn(move || {
112 |             if let Err(e) = worker.main() {
113 |                 error!("Worker failed, exiting: {}", e);
114 |                 process::exit(1);
115 |             }
116 | 
117 |             info!("Work finished, exiting.");
118 | 
119 |             process::exit(0);
120 |         });
121 | 
122 |         tokio::runtime::Runtime::new()?.block_on(async move {
123 |             let s = service.clone();
124 |             hyper::server::Server::bind(&addr)
125 |                 .serve(hyper::service::make_service_fn(move |_| {
126 |                     let s = s.clone();
127 |                     async move {
128 |                         Ok::<_, hyper::Error>(hyper::service::service_fn(move |req| {
129 |                             let s = s.clone();
130 |                             async move { s.call(req).await }
131 |                         }))
132 |                     }
133 |                 }))
134 |                 .with_graceful_shutdown(graceful_shutdown(queue_send))
135 |                 .await
136 |         })?;
137 | 
138 |         worker_thread.join().expect("worker thread failed");
139 | 
140 |         Ok(())
141 |     });
142 | }
143 | 
144 | async fn graceful_shutdown(sender: Sender<QueueItem>) {
145 |     let ctrl_c = tokio::signal::ctrl_c();
146 | 
147 |     // ECS uses SIGTERM to signal graceful shutdown must begin.
148 |     #[cfg(unix)]
149 |     let mut sigterm =
150 |         tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap();
151 |     #[cfg(unix)]
152 |     let sigterm = sigterm.recv();
153 | 
154 |     #[cfg(not(unix))]
155 |     let sigterm = std::future::pending(); // Never resolves
156 | 
157 |     tokio::select! {
158 |         _ = ctrl_c => {}
159 |         _ = sigterm => {}
160 |     };
161 | 
162 |     info!("graceful shutdown signal received");
163 |     let _ = sender.send(QueueItem::GracefulShutdown);
164 | }
165 | 


--------------------------------------------------------------------------------
/src/bin/server/mod.rs:
--------------------------------------------------------------------------------
 1 | pub use self::service::RlaService;
 2 | pub use self::worker::Worker;
 3 | 
 4 | mod service;
 5 | mod worker;
 6 | 
 7 | pub enum QueueItem {
 8 |     GitHubStatus {
 9 |         payload: rla::github::CommitStatusEvent,
10 |         delivery_id: String,
11 |     },
12 |     GitHubCheckRun {
13 |         payload: rla::github::CheckRunEvent,
14 |         delivery_id: String,
15 |     },
16 |     GitHubPullRequest {
17 |         payload: rla::github::PullRequestEvent,
18 |         delivery_id: String,
19 |     },
20 |     GracefulShutdown,
21 | }
22 | 
23 | impl QueueItem {
24 |     fn delivery_id(&self) -> Option<&str> {
25 |         match self {
26 |             QueueItem::GitHubStatus { delivery_id, .. } => Some(&delivery_id),
27 |             QueueItem::GitHubCheckRun { delivery_id, .. } => Some(&delivery_id),
28 |             QueueItem::GitHubPullRequest { delivery_id, .. } => Some(&delivery_id),
29 |             QueueItem::GracefulShutdown => None,
30 |         }
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/src/bin/server/service.rs:
--------------------------------------------------------------------------------
  1 | use super::QueueItem;
  2 | 
  3 | use anyhow::bail;
  4 | use hyper::{Body, Method, StatusCode};
  5 | use hyper::{Request, Response};
  6 | use std::env;
  7 | 
  8 | #[derive(Clone)]
  9 | pub struct RlaService {
 10 |     github_webhook_secret: Option<Vec<u8>>,
 11 |     reject_unverified_webhooks: bool,
 12 |     queue: crossbeam::channel::Sender<QueueItem>,
 13 | }
 14 | 
 15 | impl RlaService {
 16 |     pub fn new(
 17 |         reject_unverified_webhooks: bool,
 18 |         queue: crossbeam::channel::Sender<QueueItem>,
 19 |     ) -> rla::Result<RlaService> {
 20 |         let github_webhook_secret = match env::var("GITHUB_WEBHOOK_SECRET") {
 21 |             Err(env::VarError::NotPresent) => None,
 22 |             Err(env::VarError::NotUnicode(_)) => {
 23 |                 bail!("GITHUB_WEBHOOK_SECRET contained non-UTF-8 data.")
 24 |             }
 25 |             Ok(s) => {
 26 |                 if !s.bytes().all(|b| b.is_ascii_alphanumeric()) {
 27 |                     bail!("Only alphanumeric ASCII characters are allowed in GITHUB_WEBHOOK_SECRET at this time.");
 28 |                 }
 29 | 
 30 |                 Some(s.into_bytes())
 31 |             }
 32 |         };
 33 | 
 34 |         if reject_unverified_webhooks {
 35 |             if github_webhook_secret.is_none() {
 36 |                 bail!("Web hook verification was requested but no valid GITHUB_WEBHOOK_SECRET was specified.");
 37 |             }
 38 |         }
 39 | 
 40 |         Ok(RlaService {
 41 |             github_webhook_secret,
 42 |             reject_unverified_webhooks,
 43 |             queue,
 44 |         })
 45 |     }
 46 | 
 47 |     async fn handle_webhook(
 48 |         &self,
 49 |         event: &str,
 50 |         headers: &hyper::HeaderMap,
 51 |         body: &[u8],
 52 |     ) -> Result<Response<Body>, hyper::Error> {
 53 |         if let Some(ref secret) = self.github_webhook_secret {
 54 |             let sig = headers.get("X-Hub-Signature");
 55 | 
 56 |             let sig = sig.and_then(|s| s.to_str().ok());
 57 |             if let Err(e) = rla::github::verify_webhook_signature(secret, sig, body) {
 58 |                 if self.reject_unverified_webhooks {
 59 |                     error!("Rejecting web hook with invalid signature: {}", e);
 60 |                     return reply(StatusCode::FORBIDDEN, "Invalid signature.\n");
 61 |                 }
 62 | 
 63 |                 warn!("Processing web hook with invalid signature: {}", e);
 64 |             }
 65 |         };
 66 | 
 67 |         let delivery_header = headers
 68 |             .get("X-GitHub-Delivery")
 69 |             .and_then(|s| s.to_str().ok())
 70 |             .map(|s| s.to_string());
 71 |         let delivery_id = if let Some(id) = delivery_header {
 72 |             id
 73 |         } else {
 74 |             return reply(StatusCode::BAD_REQUEST, "Missing delivery ID.\n");
 75 |         };
 76 | 
 77 |         let item = match event {
 78 |             "status" => {
 79 |                 let payload = match serde_json::from_slice(body) {
 80 |                     Ok(p) => p,
 81 |                     Err(e) => {
 82 |                         error!("Failed to decode 'status' web hook payload: {}", e);
 83 |                         return reply(StatusCode::BAD_REQUEST, "Failed to decode payload.\n");
 84 |                     }
 85 |                 };
 86 |                 QueueItem::GitHubStatus {
 87 |                     payload,
 88 |                     delivery_id,
 89 |                 }
 90 |             }
 91 |             "check_run" => {
 92 |                 let payload = match serde_json::from_slice(body) {
 93 |                     Ok(p) => p,
 94 |                     Err(e) => {
 95 |                         error!("Failed to decode 'check_run' web hook payload: {}", e);
 96 |                         return reply(StatusCode::BAD_REQUEST, "Failed to decode payload.\n");
 97 |                     }
 98 |                 };
 99 | 
100 |                 QueueItem::GitHubCheckRun {
101 |                     payload,
102 |                     delivery_id,
103 |                 }
104 |             }
105 |             "pull_request" => match serde_json::from_slice(body) {
106 |                 Ok(payload) => QueueItem::GitHubPullRequest {
107 |                     payload,
108 |                     delivery_id,
109 |                 },
110 |                 Err(err) => {
111 |                     error!("Failed to decode 'pull_request' webhook payload: {}", err);
112 |                     return reply(StatusCode::BAD_REQUEST, "Failed to decode payload\n");
113 |                 }
114 |             },
115 |             "issue_comment" => {
116 |                 debug!("Ignoring 'issue_comment' event.");
117 |                 return reply(StatusCode::OK, "Event ignored.\n");
118 |             }
119 |             _ => {
120 |                 warn!("Unexpected '{}' event.", event);
121 |                 return reply(StatusCode::BAD_REQUEST, "Unexpected event.\n");
122 |             }
123 |         };
124 | 
125 |         match self.queue.send(item) {
126 |             Ok(()) => reply(StatusCode::OK, "Event processed.\n"),
127 |             Err(e) => {
128 |                 error!("Failed to queue payload: {}", e);
129 |                 reply(
130 |                     StatusCode::INTERNAL_SERVER_ERROR,
131 |                     "Failed to process the event.\n",
132 |                 )
133 |             }
134 |         }
135 |     }
136 | }
137 | 
138 | impl RlaService {
139 |     pub async fn call(&self, req: Request<Body>) -> Result<Response<Body>, hyper::Error> {
140 |         let (req, body) = req.into_parts();
141 |         info!("request: {} {}", req.method, req.uri.path());
142 |         match (req.method.clone(), req.uri.path()) {
143 |             (Method::GET, "/") => reply(StatusCode::OK, "Rust Log Analyzer is running.\n"),
144 |             (Method::POST, "/") => {
145 |                 if let Some(ev) = req.headers.get("X-GitHub-Event").cloned() {
146 |                     let slf = self.clone();
147 |                     let body = hyper::body::to_bytes(body).await?;
148 |                     slf.handle_webhook(ev.to_str().unwrap(), &req.headers, &body)
149 |                         .await
150 |                 } else {
151 |                     reply(StatusCode::BAD_REQUEST, "Missing X-GitHub-Event header.\n")
152 |                 }
153 |             }
154 |             (_, "/") => reply(StatusCode::METHOD_NOT_ALLOWED, "Method not allowed.\n"),
155 |             _ => reply(StatusCode::NOT_FOUND, "Not found.\n"),
156 |         }
157 |     }
158 | }
159 | 
160 | fn reply(status: StatusCode, body: &'static str) -> Result<Response<Body>, hyper::Error> {
161 |     trace!("response: {} {:?}", status.as_u16(), body.trim());
162 |     let mut resp = Response::new(Body::from(body));
163 |     *resp.status_mut() = status;
164 |     Ok(resp)
165 | }
166 | 


--------------------------------------------------------------------------------
/src/bin/server/worker.rs:
--------------------------------------------------------------------------------
  1 | use super::QueueItem;
  2 | 
  3 | use crate::rla::ci::{self, BuildCommit, CiPlatform};
  4 | use anyhow::bail;
  5 | use rla::index::IndexStorage;
  6 | use std::collections::{HashSet, VecDeque};
  7 | use std::hash::Hash;
  8 | use std::str;
  9 | use std::time::{Duration, Instant};
 10 | 
 11 | const MINIMUM_DELAY_BETWEEN_INDEX_BACKUPS: Duration = Duration::from_secs(60 * 60);
 12 | const SILENCE_LABEL: &str = "rla-silenced";
 13 | 
 14 | pub struct Worker {
 15 |     debug_post: Option<(String, u32)>,
 16 |     index_file: IndexStorage,
 17 |     index: rla::Index,
 18 |     extract_config: rla::extract::Config,
 19 |     github: rla::github::Client,
 20 |     queue: crossbeam::channel::Receiver<QueueItem>,
 21 |     ci: Box<dyn CiPlatform + Send>,
 22 |     repo: String,
 23 |     secondary_repos: Vec<String>,
 24 |     query_builds_from_primary_repo: bool,
 25 | 
 26 |     recently_notified: RecentlySeen<u64>,
 27 |     recently_learned: RecentlySeen<String>,
 28 | 
 29 |     last_index_backup: Option<Instant>,
 30 | }
 31 | 
 32 | impl Worker {
 33 |     pub fn new(
 34 |         index_file: IndexStorage,
 35 |         debug_post: Option<String>,
 36 |         queue: crossbeam::channel::Receiver<QueueItem>,
 37 |         ci: Box<dyn CiPlatform + Send>,
 38 |         repo: String,
 39 |         secondary_repos: Vec<String>,
 40 |         query_builds_from_primary_repo: bool,
 41 |     ) -> rla::Result<Worker> {
 42 |         let debug_post = match debug_post {
 43 |             None => None,
 44 |             Some(v) => {
 45 |                 let parts = v.splitn(2, '#').collect::<Vec<_>>();
 46 |                 if parts.len() != 2 {
 47 |                     bail!("Invalid debug-post argument: '{}'", v);
 48 |                 }
 49 | 
 50 |                 let n = parts[1].parse()?;
 51 |                 Some((parts[0].to_owned(), n))
 52 |             }
 53 |         };
 54 | 
 55 |         Ok(Worker {
 56 |             debug_post,
 57 |             index: rla::Index::load(&index_file)?,
 58 |             index_file,
 59 |             extract_config: Default::default(),
 60 |             github: rla::github::Client::new()?,
 61 |             queue,
 62 |             ci,
 63 |             repo,
 64 |             secondary_repos,
 65 |             query_builds_from_primary_repo,
 66 | 
 67 |             recently_notified: RecentlySeen::new(32),
 68 |             recently_learned: RecentlySeen::new(256),
 69 | 
 70 |             last_index_backup: None,
 71 |         })
 72 |     }
 73 | 
 74 |     pub fn main(&mut self) -> rla::Result<()> {
 75 |         loop {
 76 |             let item = self.queue.recv()?;
 77 | 
 78 |             let span = span!(
 79 |                 tracing::Level::INFO,
 80 |                 "request",
 81 |                 delivery = item.delivery_id(),
 82 |                 build_id = tracing::field::Empty
 83 |             );
 84 |             let _enter = span.enter();
 85 | 
 86 |             match self.process(item, &span) {
 87 |                 Ok(ProcessOutcome::Continue) => (),
 88 |                 Ok(ProcessOutcome::Exit) => return Ok(()),
 89 |                 Err(e) => error!("Processing queue item failed: {}", e),
 90 |             }
 91 |         }
 92 |     }
 93 | 
 94 |     fn is_repo_valid(&self, repo: &str) -> bool {
 95 |         if repo == self.repo {
 96 |             return true;
 97 |         }
 98 |         self.secondary_repos.iter().find(|r| *r == repo).is_some()
 99 |     }
100 | 
101 |     fn process(&mut self, item: QueueItem, span: &tracing::Span) -> rla::Result<ProcessOutcome> {
102 |         let (repo, build_id, outcome) = match &item {
103 |             QueueItem::GitHubStatus { payload, .. } => {
104 |                 match self.ci.build_id_from_github_status(&payload) {
105 |                     Some(id) if self.is_repo_valid(&payload.repository.full_name) => {
106 |                         (&payload.repository.full_name, id, None)
107 |                     }
108 |                     _ => {
109 |                         info!(
110 |                             "Ignoring invalid event (ctx: {:?}, url: {:?}).",
111 |                             payload.context, payload.target_url
112 |                         );
113 |                         return Ok(ProcessOutcome::Continue);
114 |                     }
115 |                 }
116 |             }
117 |             QueueItem::GitHubCheckRun { payload, .. } => {
118 |                 match self.ci.build_id_from_github_check(&payload) {
119 |                     Some(id) if self.is_repo_valid(&payload.repository.full_name) => (
120 |                         &payload.repository.full_name,
121 |                         id,
122 |                         Some(&payload.check_run.outcome),
123 |                     ),
124 |                     _ => {
125 |                         info!(
126 |                             "Ignoring invalid event (app id: {:?}, url: {:?}).",
127 |                             payload.check_run.app.id, payload.check_run.details_url
128 |                         );
129 |                         return Ok(ProcessOutcome::Continue);
130 |                     }
131 |                 }
132 |             }
133 |             QueueItem::GitHubPullRequest { payload, .. } => {
134 |                 self.process_pr(payload)?;
135 |                 return Ok(ProcessOutcome::Continue);
136 |             }
137 | 
138 |             QueueItem::GracefulShutdown => {
139 |                 info!("persisting the index to disk before shutting down");
140 |                 self.index.save(&self.index_file)?;
141 |                 return Ok(ProcessOutcome::Exit);
142 |             }
143 |         };
144 | 
145 |         span.record("build_id", &build_id);
146 | 
147 |         info!("started processing");
148 | 
149 |         let query_from = if self.query_builds_from_primary_repo {
150 |             &self.repo
151 |         } else {
152 |             repo
153 |         };
154 |         let build = self.ci.query_build(query_from, build_id)?;
155 | 
156 |         let outcome = match outcome {
157 |             Some(outcome) if self.ci.is_build_outcome_unreliable() => &*outcome,
158 |             _ => build.outcome(),
159 |         };
160 | 
161 |         debug!("current outcome: {:?}", outcome);
162 |         debug!("PR number: {:?}", build.pr_number());
163 |         debug!("branch name: {:?}", build.branch_name());
164 | 
165 |         if !outcome.is_finished() {
166 |             info!("ignoring in-progress build");
167 |             return Ok(ProcessOutcome::Continue);
168 |         }
169 | 
170 |         // Avoid processing the same build multiple times.
171 |         if !outcome.is_passed() {
172 |             self.report_failed(build_id, build.as_ref())?;
173 |         }
174 |         if build.pr_number().is_some() || build.branch_name() == "auto" {
175 |             info!("learning from the log");
176 |             self.learn(build.as_ref())?;
177 |         } else {
178 |             info!("did not learn as it's not an auto build or a PR build");
179 |         }
180 | 
181 |         Ok(ProcessOutcome::Continue)
182 |     }
183 | 
184 |     fn report_failed(&mut self, build_id: u64, build: &dyn rla::ci::Build) -> rla::Result<()> {
185 |         if self.recently_notified.recently_witnessed(&build_id) {
186 |             info!("avoided reporting recently notified build");
187 |             return Ok(());
188 |         }
189 | 
190 |         info!("preparing report");
191 | 
192 |         let job = match build.jobs().iter().find(|j| j.outcome().is_failed()) {
193 |             Some(job) => *job,
194 |             None => bail!("No failed job found, cannot report."),
195 |         };
196 | 
197 |         let log = match ci::download_log(self.ci.as_ref(), job, self.github.internal()) {
198 |             Some(res) => res?,
199 |             None => bail!("No log for failed job"),
200 |         };
201 | 
202 |         let lines = rla::sanitize::split_lines(&log)
203 |             .iter()
204 |             .map(|l| rla::index::Sanitized(rla::sanitize::clean(self.ci.as_ref(), l)))
205 |             .collect::<Vec<_>>();
206 | 
207 |         let blocks = rla::extract::extract(&self.extract_config, &self.index, &lines);
208 | 
209 |         let blocks = blocks
210 |             .iter()
211 |             .map(|block| {
212 |                 block
213 |                     .iter()
214 |                     .map(|line| String::from_utf8_lossy(&line.0).into_owned())
215 |                     .collect::<Vec<_>>()
216 |                     .join("\n")
217 |             })
218 |             .collect::<Vec<_>>();
219 | 
220 |         let extracted = blocks.join("\n---\n");
221 | 
222 |         // Some CI providers return a merge commit instead of the head commit of the branch/PR when
223 |         // querying the build. If the provider returned a merge commit, this fetches the related
224 |         // head commit from the GitHub API.
225 |         let commit_sha = match build.commit_sha() {
226 |             BuildCommit::Head { sha } => sha.to_string(),
227 |             BuildCommit::Merge { sha } => {
228 |                 let mut commit = self.github.query_commit(&self.repo, sha)?;
229 |                 if commit.parents.len() > 1 {
230 |                     // The first parent is master, the second parent is the branch/PR.
231 |                     commit.parents.remove(1).sha
232 |                 } else {
233 |                     bail!("commit {} is not a merge commit", sha);
234 |                 }
235 |             }
236 |         };
237 | 
238 |         let commit_message = self
239 |             .github
240 |             .query_commit(&self.repo, &commit_sha)?
241 |             .commit
242 |             .message;
243 | 
244 |         let log_variables = rla::log_variables::LogVariables::extract(&lines);
245 | 
246 |         let (pr, is_bors) = if let Some(pr) = build.pr_number() {
247 |             (pr, false)
248 |         } else {
249 |             static BORS_MERGE_PREFIX: &str = "Auto merge of #";
250 | 
251 |             if commit_message.starts_with(BORS_MERGE_PREFIX) {
252 |                 let s = &commit_message[BORS_MERGE_PREFIX.len()..];
253 |                 (
254 |                     s[..s.find(' ').ok_or_else(|| {
255 |                         anyhow::format_err!("Invalid bors commit message: '{}'", commit_message)
256 |                     })?]
257 |                         .parse()?,
258 |                     true,
259 |                 )
260 |             } else if let Some(number) = log_variables.pr_number {
261 |                 (number.parse()?, false)
262 |             } else {
263 |                 bail!("Could not determine PR number, cannot report.");
264 |             }
265 |         };
266 | 
267 |         if !is_bors {
268 |             let pr_info = self.github.query_pr(&self.repo, pr)?;
269 |             if pr_info.head.sha != commit_sha {
270 |                 info!("Build results outdated, skipping report.");
271 |                 return Ok(());
272 |             }
273 |             if pr_info
274 |                 .labels
275 |                 .iter()
276 |                 .any(|label| label.name == SILENCE_LABEL)
277 |             {
278 |                 info!("PR has label `{SILENCE_LABEL}`, skipping report");
279 |                 return Ok(());
280 |             }
281 |         }
282 | 
283 |         let (repo, pr) = match self.debug_post {
284 |             Some((ref repo, pr_override)) => {
285 |                 warn!(
286 |                     "Would post to '{}#{}', debug override to '{}#{}'",
287 |                     self.repo, pr, repo, pr_override
288 |                 );
289 |                 (repo.as_str(), pr_override)
290 |             }
291 |             None => (self.repo.as_str(), pr),
292 |         };
293 | 
294 |         let opening = match log_variables.job_name {
295 |             Some(job_name) => format!("The job **`{}`**", job_name),
296 |             None => "A job".to_owned(),
297 |         };
298 |         let trailer = match log_variables.doc_url {
299 |             Some(url) => format!("\nFor more information how to resolve CI failures of this job, visit this [link]({url})."),
300 |             None => "".to_string(),
301 |         };
302 | 
303 |         let log_url = job.log_url().unwrap_or_else(|| "unknown".into());
304 |         self.github.post_comment(repo, pr, &format!(r#"
305 | {opening} failed! Check out the build log: [(web)]({html_url}) [(plain)]({log_url})
306 | 
307 | <details><summary><i>Click to see the possible cause of the failure (guessed by this bot)</i></summary>
308 | 
309 | ```plain
310 | {log}
311 | ```
312 | 
313 | </details>
314 | {trailer}"#, opening = opening, html_url = job.html_url(), log_url = log_url, log = extracted, trailer = trailer))?;
315 | 
316 |         info!("marked build {} as recently notified", build_id);
317 |         self.recently_notified.store(build_id);
318 | 
319 |         Ok(())
320 |     }
321 | 
322 |     fn learn(&mut self, build: &dyn rla::ci::Build) -> rla::Result<()> {
323 |         for job in &build.jobs() {
324 |             if !job.outcome().is_passed() {
325 |                 continue;
326 |             }
327 | 
328 |             if self.recently_learned.recently_witnessed(&job.id()) {
329 |                 trace!("Skipped already processed {}", job);
330 |                 continue;
331 |             }
332 | 
333 |             debug!("Processing {}...", job);
334 | 
335 |             match ci::download_log(self.ci.as_ref(), *job, self.github.internal()) {
336 |                 Some(Ok(log)) => {
337 |                     for line in rla::sanitize::split_lines(&log) {
338 |                         self.index.learn(
339 |                             &rla::index::Sanitized(rla::sanitize::clean(self.ci.as_ref(), line)),
340 |                             1,
341 |                         );
342 |                     }
343 |                     self.recently_learned.store(job.id());
344 |                 }
345 |                 None => {
346 |                     warn!(
347 |                         "Failed to learn from successful {}, download failed; no log",
348 |                         job
349 |                     );
350 |                 }
351 |                 Some(Err(e)) => {
352 |                     warn!(
353 |                         "Failed to learn from successful {}, download failed: {}",
354 |                         job, e
355 |                     );
356 |                 }
357 |             }
358 |         }
359 | 
360 |         // To avoid persisting the index too many times to storage, we only persist it after some
361 |         // time elapsed since the last save.
362 |         match self.last_index_backup {
363 |             Some(last) if last.elapsed() >= MINIMUM_DELAY_BETWEEN_INDEX_BACKUPS => {
364 |                 self.last_index_backup = Some(Instant::now());
365 |                 self.index.save(&self.index_file)?;
366 |             }
367 |             Some(_) => {}
368 |             None => self.last_index_backup = Some(Instant::now()),
369 |         }
370 | 
371 |         Ok(())
372 |     }
373 | 
374 |     fn process_pr(&self, e: &rla::github::PullRequestEvent) -> rla::Result<()> {
375 |         // Hide all comments by the bot when a new commit is pushed.
376 |         if let rla::github::PullRequestAction::Synchronize = e.action {
377 |             self.github
378 |                 .hide_own_comments(&e.repository.full_name, e.number)?;
379 |         }
380 |         Ok(())
381 |     }
382 | }
383 | 
384 | /// Keeps track of the recently seen IDs for both the failed build reports and the learned jobs.
385 | /// Only the most recent IDs are stored, to avoid growing the memory usage endlessly.
386 | ///
387 | /// Internally this uses both an HashSet to provide fast lookups and a VecDeque to know which old
388 | /// jobs needs to be removed.
389 | struct RecentlySeen<T: Clone + Eq + Hash> {
390 |     size: usize,
391 |     lookup: HashSet<T>,
392 |     removal: VecDeque<T>,
393 | }
394 | 
395 | impl<T: Clone + Eq + Hash> RecentlySeen<T> {
396 |     fn new(size: usize) -> Self {
397 |         Self {
398 |             size,
399 |             lookup: HashSet::with_capacity(size),
400 |             removal: VecDeque::with_capacity(size),
401 |         }
402 |     }
403 | 
404 |     fn recently_witnessed(&self, key: &T) -> bool {
405 |         self.lookup.contains(key)
406 |     }
407 | 
408 |     fn store(&mut self, key: T) {
409 |         if self.lookup.contains(&key) {
410 |             return;
411 |         }
412 |         if self.removal.len() >= self.size {
413 |             if let Some(item) = self.removal.pop_back() {
414 |                 self.lookup.remove(&item);
415 |             }
416 |         }
417 |         self.lookup.insert(key.clone());
418 |         self.removal.push_front(key);
419 |     }
420 | }
421 | 
422 | enum ProcessOutcome {
423 |     Continue,
424 |     Exit,
425 | }
426 | 
427 | #[cfg(test)]
428 | mod tests {
429 |     use super::*;
430 | 
431 |     #[test]
432 |     fn test_recently_seen() {
433 |         let mut recently = RecentlySeen::new(2);
434 | 
435 |         assert!(!recently.recently_witnessed(&0));
436 |         assert!(!recently.recently_witnessed(&1));
437 |         assert!(!recently.recently_witnessed(&2));
438 | 
439 |         recently.store(0);
440 |         assert!(recently.recently_witnessed(&0));
441 |         assert!(!recently.recently_witnessed(&1));
442 |         assert!(!recently.recently_witnessed(&2));
443 | 
444 |         recently.store(1);
445 |         assert!(recently.recently_witnessed(&0));
446 |         assert!(recently.recently_witnessed(&1));
447 |         assert!(!recently.recently_witnessed(&2));
448 | 
449 |         recently.store(2);
450 |         assert!(!recently.recently_witnessed(&0));
451 |         assert!(recently.recently_witnessed(&1));
452 |         assert!(recently.recently_witnessed(&2));
453 |     }
454 | }
455 | 


--------------------------------------------------------------------------------
/src/bin/util/mod.rs:
--------------------------------------------------------------------------------
 1 | use anyhow::Context;
 2 | use std::process;
 3 | 
 4 | #[derive(Debug, Copy, Clone)]
 5 | pub(crate) enum CliCiPlatform {
 6 |     Azure,
 7 |     Actions,
 8 | }
 9 | 
10 | impl CliCiPlatform {
11 |     pub(crate) fn get(&self) -> rla::Result<Box<dyn rla::ci::CiPlatform + Send>> {
12 |         Ok(match self {
13 |             CliCiPlatform::Azure => {
14 |                 let token = std::env::var("AZURE_DEVOPS_TOKEN")
15 |                     .with_context(|| "failed to read AZURE_DEVOPS_TOKEN env var")?;
16 |                 Box::new(rla::ci::AzurePipelines::new(&token))
17 |             }
18 |             CliCiPlatform::Actions => {
19 |                 let token = std::env::var("GITHUB_TOKEN")
20 |                     .with_context(|| "failed to read GITHUB_TOKEN env var")?;
21 |                 Box::new(rla::ci::GitHubActions::new(&token))
22 |             }
23 |         })
24 |     }
25 | }
26 | 
27 | impl std::str::FromStr for CliCiPlatform {
28 |     type Err = anyhow::Error;
29 | 
30 |     fn from_str(input: &str) -> rla::Result<Self> {
31 |         Ok(match input {
32 |             "azure" => CliCiPlatform::Azure,
33 |             "actions" => CliCiPlatform::Actions,
34 |             other => anyhow::bail!("unknown CI platform: {}", other),
35 |         })
36 |     }
37 | }
38 | 
39 | pub fn run<F: FnOnce() -> rla::Result<()>>(f: F) {
40 |     tracing_subscriber::fmt()
41 |         .with_writer(std::io::stderr)
42 |         .with_ansi(atty::is(atty::Stream::Stderr))
43 |         .with_env_filter(tracing_subscriber::EnvFilter::from_env("RLA_LOG"))
44 |         .init();
45 | 
46 |     log_and_exit_error(|| f());
47 | }
48 | 
49 | pub fn log_and_exit_error<F: FnOnce() -> rla::Result<()>>(f: F) {
50 |     if let Err(e) = f() {
51 |         if let Some(v) = e.downcast_ref::<std::io::Error>() {
52 |             if v.kind() == std::io::ErrorKind::BrokenPipe {
53 |                 // exit without printing
54 |                 process::exit(1);
55 |             }
56 |         }
57 |         error!("{}\n\n{}", e, e.backtrace());
58 |         process::exit(1);
59 |     }
60 | }
61 | 


--------------------------------------------------------------------------------
/src/ci/actions.rs:
--------------------------------------------------------------------------------
  1 | use crate::ci::{Build, BuildCommit, CiPlatform, Job, Outcome};
  2 | use crate::github::{BuildOutcome, CheckRun};
  3 | use crate::Result;
  4 | use regex::Regex;
  5 | use reqwest::blocking::{Client as ReqwestClient, RequestBuilder, Response};
  6 | use reqwest::Method;
  7 | use std::borrow::Cow;
  8 | use std::collections::HashMap;
  9 | 
 10 | #[derive(Deserialize)]
 11 | struct ActionsRun {
 12 |     id: u64,
 13 |     head_branch: String,
 14 |     head_sha: String,
 15 |     #[serde(flatten)]
 16 |     outcome: BuildOutcome,
 17 | }
 18 | 
 19 | struct GHABuild {
 20 |     run: ActionsRun,
 21 |     jobs: Vec<GHAJob>,
 22 | }
 23 | 
 24 | impl GHABuild {
 25 |     #[allow(clippy::new_ret_no_self)]
 26 |     fn new(client: &Client, repo: &str, run: ActionsRun) -> Result<Box<dyn Build>> {
 27 |         let mut jobs = Vec::new();
 28 |         client.paginated(
 29 |             Method::GET,
 30 |             &format!("repos/{}/actions/runs/{}/jobs", repo, run.id),
 31 |             &mut |resp| {
 32 |                 #[derive(Deserialize)]
 33 |                 struct JobsResult {
 34 |                     jobs: Vec<WorkflowJob>,
 35 |                 }
 36 | 
 37 |                 let mut partial_jobs: JobsResult = resp.json()?;
 38 |                 for job in partial_jobs.jobs.drain(..) {
 39 |                     jobs.push(GHAJob {
 40 |                         inner: job,
 41 |                         repo_name: repo.to_string(),
 42 |                     });
 43 |                 }
 44 |                 Ok(true)
 45 |             },
 46 |         )?;
 47 | 
 48 |         Ok(Box::new(GHABuild { run, jobs }))
 49 |     }
 50 | }
 51 | 
 52 | impl Build for GHABuild {
 53 |     fn pr_number(&self) -> Option<u32> {
 54 |         // GitHub Actions can't fetch it for us, so let's rely on the detection with log variables
 55 |         // (defined in src/log_variables.rs).
 56 |         None
 57 |     }
 58 | 
 59 |     fn branch_name(&self) -> &str {
 60 |         &self.run.head_branch
 61 |     }
 62 | 
 63 |     fn commit_sha(&self) -> BuildCommit<'_> {
 64 |         BuildCommit::Head {
 65 |             sha: &self.run.head_sha,
 66 |         }
 67 |     }
 68 | 
 69 |     fn outcome(&self) -> &dyn Outcome {
 70 |         &self.run.outcome
 71 |     }
 72 | 
 73 |     fn jobs(&self) -> Vec<&dyn Job> {
 74 |         self.jobs.iter().map(|j| j as &dyn Job).collect()
 75 |     }
 76 | }
 77 | 
 78 | #[derive(Deserialize)]
 79 | struct WorkflowJob {
 80 |     id: usize,
 81 |     name: String,
 82 |     html_url: String,
 83 |     head_sha: String,
 84 |     #[serde(flatten)]
 85 |     outcome: BuildOutcome,
 86 | }
 87 | 
 88 | struct GHAJob {
 89 |     inner: WorkflowJob,
 90 |     repo_name: String,
 91 | }
 92 | 
 93 | impl Job for GHAJob {
 94 |     fn id(&self) -> String {
 95 |         self.inner.id.to_string()
 96 |     }
 97 | 
 98 |     fn html_url(&self) -> String {
 99 |         self.inner.html_url.clone()
100 |     }
101 | 
102 |     fn log_url(&self) -> Option<String> {
103 |         Some(format!(
104 |             "https://github.com/{}/commit/{}/checks/{}/logs",
105 |             self.repo_name, self.inner.head_sha, self.inner.id
106 |         ))
107 |     }
108 | 
109 |     fn log_api_url(&self) -> Option<String> {
110 |         Some(format!(
111 |             "https://api.github.com/repos/{}/actions/jobs/{}/logs",
112 |             self.repo_name, self.inner.id
113 |         ))
114 |     }
115 | 
116 |     fn log_file_name(&self) -> String {
117 |         format!("actions-{}-{}", self.inner.id, self.inner.name)
118 |     }
119 | 
120 |     fn outcome(&self) -> &dyn Outcome {
121 |         &self.inner.outcome
122 |     }
123 | }
124 | 
125 | impl std::fmt::Display for GHAJob {
126 |     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
127 |         write!(
128 |             f,
129 |             "job {} named {} (outcome={:?})",
130 |             self.inner.id, self.inner.name, self.inner.outcome
131 |         )
132 |     }
133 | }
134 | 
135 | const GITHUB_ACTIONS_APP_ID: u64 = 15368;
136 | 
137 | pub struct Client {
138 |     http: ReqwestClient,
139 |     token: String,
140 | }
141 | 
142 | impl Client {
143 |     pub fn new(token: &str) -> Client {
144 |         Client {
145 |             http: ReqwestClient::new(),
146 |             token: token.to_string(),
147 |         }
148 |     }
149 | 
150 |     fn req(&self, method: Method, url: &str) -> Result<Response> {
151 |         Ok(self
152 |             .authenticate_request(self.http.request(
153 |                 method,
154 |                 &if url.starts_with("https://") {
155 |                     url.to_string()
156 |                 } else {
157 |                     format!("https://api.github.com/{}", url)
158 |                 },
159 |             ))
160 |             .send()?)
161 |     }
162 | 
163 |     fn paginated(
164 |         &self,
165 |         method: Method,
166 |         url: &str,
167 |         handle: &mut dyn FnMut(Response) -> Result<bool>,
168 |     ) -> Result<()> {
169 |         let mut next_url = Some(url.to_string());
170 |         while let Some(url) = next_url {
171 |             let resp = self.req(method.clone(), &url)?.error_for_status()?;
172 | 
173 |             // Try to extract the next page URL from the Link header.
174 |             if let Some(Ok(link)) = resp.headers().get("link").map(|l| l.to_str()) {
175 |                 next_url = parse_link_header(link)?.remove(&LinkRel::Next);
176 |             } else {
177 |                 next_url = None;
178 |             }
179 | 
180 |             if !handle(resp)? {
181 |                 break;
182 |             }
183 |         }
184 |         Ok(())
185 |     }
186 | }
187 | 
188 | impl CiPlatform for Client {
189 |     fn build_id_from_github_check(&self, e: &crate::github::CheckRunEvent) -> Option<u64> {
190 |         if e.check_run.app.id != GITHUB_ACTIONS_APP_ID {
191 |             return None;
192 |         }
193 | 
194 |         match fetch_workflow_run_id_from_check_run(self, &e.repository.full_name, &e.check_run) {
195 |             Ok(id) => Some(id),
196 |             Err(err) => {
197 |                 debug!("failed to fetch GHA build ID: {}", err);
198 |                 None
199 |             }
200 |         }
201 |     }
202 | 
203 |     fn build_id_from_github_status(&self, _e: &crate::github::CommitStatusEvent) -> Option<u64> {
204 |         None
205 |     }
206 | 
207 |     fn query_builds(
208 |         &self,
209 |         repo: &str,
210 |         count: u32,
211 |         _offset: u32,
212 |         filter: &dyn Fn(&dyn Build) -> bool,
213 |     ) -> Result<Vec<Box<dyn Build>>> {
214 |         #[derive(Deserialize)]
215 |         struct AllRuns {
216 |             workflow_runs: Vec<ActionsRun>,
217 |         }
218 | 
219 |         let mut builds = Vec::new();
220 |         self.paginated(
221 |             Method::GET,
222 |             &format!("repos/{}/actions/runs", repo),
223 |             &mut |resp| {
224 |                 let mut partial_runs: AllRuns = resp.json()?;
225 |                 for run in partial_runs.workflow_runs.drain(..) {
226 |                     if !run.outcome.is_finished() {
227 |                         continue;
228 |                     }
229 | 
230 |                     let build = GHABuild::new(self, repo, run)?;
231 |                     if filter(build.as_ref()) {
232 |                         builds.push(build);
233 |                     }
234 |                 }
235 | 
236 |                 Ok(builds.len() <= count as usize)
237 |             },
238 |         )?;
239 | 
240 |         Ok(builds)
241 |     }
242 | 
243 |     fn query_build(&self, repo: &str, id: u64) -> Result<Box<dyn Build>> {
244 |         let run: ActionsRun = self
245 |             .req(Method::GET, &format!("repos/{}/actions/runs/{}", repo, id))?
246 |             .error_for_status()?
247 |             .json()?;
248 |         Ok(GHABuild::new(self, repo, run)?)
249 |     }
250 | 
251 |     fn remove_timestamp_from_log_line<'a>(&self, line: &'a [u8]) -> Cow<'a, [u8]> {
252 |         // GitHub Actions log lines are always prefixed by the timestamp.
253 |         Cow::Borrowed(line.splitn(2, |c| *c == b' ').last().unwrap_or(line))
254 |     }
255 | 
256 |     fn authenticate_request(&self, request: RequestBuilder) -> RequestBuilder {
257 |         request
258 |             .header(
259 |                 reqwest::header::AUTHORIZATION,
260 |                 format!("token {}", self.token),
261 |             )
262 |             .header(reqwest::header::USER_AGENT, format!("rust-log-analyzer"))
263 |     }
264 | 
265 |     fn is_build_outcome_unreliable(&self) -> bool {
266 |         true
267 |     }
268 | }
269 | 
270 | fn fetch_workflow_run_id_from_check_run(
271 |     client: &Client,
272 |     repo: &str,
273 |     run: &CheckRun,
274 | ) -> Result<u64> {
275 |     #[derive(Deserialize)]
276 |     struct ResponseRuns {
277 |         total_count: usize,
278 |         workflow_runs: Vec<ResponseRun>,
279 |     }
280 | 
281 |     #[derive(Deserialize)]
282 |     struct ResponseRun {
283 |         id: u64,
284 |         check_suite_url: String,
285 |     }
286 | 
287 |     trace!("starting to fetch workflow run IDs for the {} repo", repo);
288 | 
289 |     let runs: ResponseRuns = client
290 |         .req(
291 |             Method::GET,
292 |             &format!("repos/{}/actions/runs?per_page=100", repo),
293 |         )?
294 |         .error_for_status()?
295 |         .json()?;
296 | 
297 |     trace!("received {} workflow runs", runs.total_count);
298 | 
299 |     for workflow_run in &runs.workflow_runs {
300 |         if workflow_run.check_suite_url == run.check_suite.url {
301 |             trace!("found a matching workflow run");
302 |             return Ok(workflow_run.id);
303 |         }
304 |     }
305 | 
306 |     anyhow::bail!("can't find the Workflow Run ID from the Check Run");
307 | }
308 | 
309 | #[derive(Debug, Eq, PartialEq, Hash)]
310 | enum LinkRel {
311 |     First,
312 |     Previous,
313 |     Next,
314 |     Last,
315 |     Other(String),
316 | }
317 | 
318 | fn parse_link_header(content: &str) -> Result<HashMap<LinkRel, String>> {
319 |     lazy_static! {
320 |         static ref REGEX: Regex = Regex::new(r#"<([^>]+)>; *rel="([^"]+)""#).unwrap();
321 |     }
322 | 
323 |     let mut result = HashMap::new();
324 |     for entry in content.split(',') {
325 |         if let Some(captures) = REGEX.captures(entry.trim()) {
326 |             let rel = match &captures[2] {
327 |                 "first" => LinkRel::First,
328 |                 "previous" => LinkRel::Previous,
329 |                 "next" => LinkRel::Next,
330 |                 "last" => LinkRel::Last,
331 |                 other => LinkRel::Other(other.into()),
332 |             };
333 |             result.insert(rel, captures[1].into());
334 |         } else {
335 |             anyhow::bail!("invalid link header entry: {}", entry);
336 |         }
337 |     }
338 |     Ok(result)
339 | }
340 | 
341 | #[cfg(test)]
342 | mod tests {
343 |     use super::*;
344 |     use std::collections::HashMap;
345 | 
346 |     #[test]
347 |     fn test_parse_link_header() {
348 |         let mut expected = HashMap::new();
349 |         expected.insert(LinkRel::Previous, "https://example.com/1".into());
350 |         expected.insert(LinkRel::Next, "https://example.com/3".into());
351 |         expected.insert(
352 |             LinkRel::Other("docs".into()),
353 |             "https://docs.example.com".into(),
354 |         );
355 | 
356 |         assert_eq!(
357 |             expected,
358 |             parse_link_header(
359 |                 "<https://example.com/1>;  rel=\"previous\",
360 |                  <https://example.com/3>; rel=\"next\",
361 |                  <https://docs.example.com>; rel=\"docs\""
362 |             )
363 |             .unwrap(),
364 |         );
365 |     }
366 | }
367 | 


--------------------------------------------------------------------------------
/src/ci/azure.rs:
--------------------------------------------------------------------------------
  1 | #![allow(unused)]
  2 | use crate::ci::{Build, BuildCommit, CiPlatform, Job, Outcome};
  3 | use crate::Result;
  4 | use anyhow::Context;
  5 | use reqwest::blocking::{Client as ReqwestClient, Response};
  6 | use reqwest::{Method, StatusCode};
  7 | use std::borrow::Cow;
  8 | use std::fmt;
  9 | use std::io::Read;
 10 | 
 11 | #[derive(Debug, Eq, PartialEq, Deserialize)]
 12 | #[serde(rename_all = "camelCase")]
 13 | enum BuildResult {
 14 |     Canceled,
 15 |     Failed,
 16 |     None,
 17 |     PartiallySucceeded,
 18 |     Succeeded,
 19 |     Skipped,
 20 |     Abandoned,
 21 |     SucceededWithIssues,
 22 | }
 23 | 
 24 | #[derive(Debug, Eq, PartialEq, Deserialize)]
 25 | #[serde(rename_all = "camelCase")]
 26 | enum BuildStatus {
 27 |     All,
 28 |     Cancelling,
 29 |     Completed,
 30 |     InProgress,
 31 |     None,
 32 |     NotStarted,
 33 |     Postponed,
 34 | }
 35 | 
 36 | #[derive(Debug, Deserialize)]
 37 | struct BuildOutcome {
 38 |     result: Option<BuildResult>,
 39 |     status: Option<BuildStatus>,
 40 | }
 41 | 
 42 | impl Outcome for BuildOutcome {
 43 |     fn is_finished(&self) -> bool {
 44 |         // TimelineRecord of type Job does not have a status
 45 |         self.status == Some(BuildStatus::Completed) || self.status.is_none()
 46 |     }
 47 | 
 48 |     fn is_passed(&self) -> bool {
 49 |         self.is_finished() && self.result == Some(BuildResult::Succeeded)
 50 |     }
 51 | 
 52 |     fn is_failed(&self) -> bool {
 53 |         self.is_finished() && self.result == Some(BuildResult::Failed)
 54 |     }
 55 | }
 56 | 
 57 | #[derive(Debug, Deserialize)]
 58 | struct TimelineLog {
 59 |     url: String,
 60 | }
 61 | 
 62 | #[derive(Debug, Deserialize)]
 63 | struct TimelineRecord {
 64 |     #[serde(rename = "type")]
 65 |     type_: String,
 66 |     id: String,
 67 |     name: String,
 68 |     log: Option<TimelineLog>,
 69 |     #[serde(flatten)]
 70 |     outcome: BuildOutcome,
 71 | }
 72 | 
 73 | #[derive(Debug, Deserialize)]
 74 | struct TaskReference {
 75 |     id: String,
 76 |     name: String,
 77 |     version: String,
 78 | }
 79 | 
 80 | impl TimelineRecord {
 81 |     fn log(&self) -> &TimelineLog {
 82 |         self.log.as_ref().unwrap_or_else(|| {
 83 |             panic!("log field = None for {:?}", self);
 84 |         })
 85 |     }
 86 | }
 87 | 
 88 | #[derive(Debug)]
 89 | struct AzureJob {
 90 |     record: TimelineRecord,
 91 |     build: u64,
 92 |     repo: String,
 93 | }
 94 | 
 95 | impl Job for AzureJob {
 96 |     fn id(&self) -> String {
 97 |         self.record.id.clone()
 98 |     }
 99 | 
100 |     fn html_url(&self) -> String {
101 |         format!(
102 |             "https://dev.azure.com/{repo}/_build/results?buildId={build}&view=logs&jobId={job}",
103 |             repo = self.repo,
104 |             build = self.build,
105 |             job = self.record.id
106 |         )
107 |     }
108 | 
109 |     fn log_url(&self) -> Option<String> {
110 |         self.record.log.as_ref().map(|l| l.url.clone())
111 |     }
112 | 
113 |     fn log_file_name(&self) -> String {
114 |         format!("azure-{}-{}", self.id(), self.build)
115 |     }
116 | 
117 |     fn outcome(&self) -> &dyn Outcome {
118 |         &self.record.outcome
119 |     }
120 | }
121 | 
122 | impl fmt::Display for AzureJob {
123 |     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
124 |         write!(
125 |             f,
126 |             "job {} of build {} (outcome={:?})",
127 |             self.record.name, self.build, self.record.outcome
128 |         )
129 |     }
130 | }
131 | 
132 | #[derive(Debug, Deserialize)]
133 | struct Timeline {
134 |     records: Vec<TimelineRecord>,
135 | }
136 | 
137 | #[derive(Debug, Deserialize)]
138 | struct TriggerInfo {
139 |     #[serde(rename = "pr.number")]
140 |     pr_number: Option<String>,
141 |     #[serde(rename = "pr.sourceBranch")]
142 |     pr_branch: Option<String>,
143 | }
144 | 
145 | #[derive(Debug, Deserialize)]
146 | struct Link {
147 |     href: String,
148 | }
149 | 
150 | #[derive(Debug, Deserialize)]
151 | struct AzureBuildLinks {
152 |     timeline: Link,
153 |     #[allow(unused)]
154 |     web: Link,
155 | }
156 | 
157 | #[derive(Debug, Deserialize)]
158 | #[serde(rename_all = "camelCase")]
159 | struct AzureBuildData {
160 |     #[serde(rename = "_links")]
161 |     links: AzureBuildLinks,
162 |     id: u64,
163 |     trigger_info: TriggerInfo,
164 |     source_branch: String,
165 |     source_version: String,
166 |     build_number: String,
167 |     #[serde(flatten)]
168 |     outcome: BuildOutcome,
169 | }
170 | 
171 | #[derive(Debug)]
172 | struct AzureBuild {
173 |     data: AzureBuildData,
174 |     jobs: Vec<AzureJob>,
175 | }
176 | 
177 | impl AzureBuild {
178 |     fn new(client: &Client, repo: &str, data: AzureBuildData) -> Result<Option<Self>> {
179 |         let mut resp = client
180 |             .req(Method::GET, repo, &data.links.timeline.href)?
181 |             .error_for_status()?;
182 |         // this means that the build didn't parse from the yaml,
183 |         // or at least that's the one case we've hit so far
184 |         if resp.status() == StatusCode::NO_CONTENT {
185 |             return Ok(None);
186 |         }
187 |         let dbg = format!("{:?}", resp);
188 |         let timeline: Timeline = resp.json().with_context(|| dbg)?;
189 |         Ok(Some(AzureBuild {
190 |             jobs: timeline
191 |                 .records
192 |                 .into_iter()
193 |                 .map(|record| AzureJob {
194 |                     build: data.id,
195 |                     repo: repo.to_string(),
196 |                     record,
197 |                 })
198 |                 .collect(),
199 |             data,
200 |         }))
201 |     }
202 | }
203 | 
204 | impl Build for AzureBuild {
205 |     fn pr_number(&self) -> Option<u32> {
206 |         self.data
207 |             .trigger_info
208 |             .pr_number
209 |             .as_ref()
210 |             .and_then(|num| num.parse().ok())
211 |     }
212 | 
213 |     fn branch_name(&self) -> &str {
214 |         const HEAD_PREFIX: &str = "refs/heads/";
215 |         if let Some(branch) = &self.data.trigger_info.pr_branch {
216 |             &branch
217 |         } else if self.data.source_branch.starts_with(HEAD_PREFIX) {
218 |             &self.data.source_branch[HEAD_PREFIX.len()..]
219 |         } else {
220 |             &self.data.source_branch
221 |         }
222 |     }
223 | 
224 |     fn commit_sha(&self) -> BuildCommit<'_> {
225 |         // Azure Pipelines returns merge commits for PRs and head commits for branches
226 |         if self.data.trigger_info.pr_number.is_some() {
227 |             BuildCommit::Merge {
228 |                 sha: &self.data.source_version,
229 |             }
230 |         } else {
231 |             BuildCommit::Head {
232 |                 sha: &self.data.source_version,
233 |             }
234 |         }
235 |     }
236 | 
237 |     fn outcome(&self) -> &dyn Outcome {
238 |         &self.data.outcome
239 |     }
240 | 
241 |     fn jobs(&self) -> Vec<&dyn Job> {
242 |         self.jobs
243 |             .iter()
244 |             .filter(|job| job.record.type_ == "Job")
245 |             // Azure does not properly publish logs for canceled builds. These builds are the ones
246 |             // that cancelbot killed manually, vs. the "failed" builds, so we don't care too much
247 |             // about them for now, and just ignore them here
248 |             .filter(|job| job.record.outcome.result != Some(BuildResult::Canceled))
249 |             .map(|job| job as &dyn Job)
250 |             .collect()
251 |     }
252 | }
253 | 
254 | #[derive(Debug, Deserialize)]
255 | struct AzureBuilds {
256 |     count: usize,
257 |     value: Vec<AzureBuildData>,
258 | }
259 | 
260 | pub struct Client {
261 |     http: ReqwestClient,
262 |     token: String,
263 | }
264 | 
265 | impl Client {
266 |     pub fn new(token: &str) -> Client {
267 |         Client {
268 |             http: ReqwestClient::new(),
269 |             token: token.to_string(),
270 |         }
271 |     }
272 | 
273 |     fn req(&self, method: Method, repo: &str, url: &str) -> Result<Response> {
274 |         Ok(self
275 |             .http
276 |             .request(
277 |                 method,
278 |                 &if url.starts_with("https://") {
279 |                     url.to_owned()
280 |                 } else {
281 |                     format!("https://dev.azure.com/{}/_apis/{}", repo, url)
282 |                 },
283 |             )
284 |             .basic_auth("", Some(self.token.clone()))
285 |             .send()?)
286 |     }
287 | }
288 | 
289 | const AZURE_API_ID: u64 = 9426;
290 | 
291 | impl CiPlatform for Client {
292 |     fn build_id_from_github_check(&self, e: &crate::github::CheckRunEvent) -> Option<u64> {
293 |         if e.check_run.app.id != AZURE_API_ID {
294 |             return None;
295 |         }
296 |         e.check_run
297 |             .external_id
298 |             .split('|')
299 |             .nth(1)
300 |             .and_then(|id| id.parse().ok())
301 |     }
302 | 
303 |     fn build_id_from_github_status(&self, e: &crate::github::CommitStatusEvent) -> Option<u64> {
304 |         None
305 |     }
306 | 
307 |     fn query_builds(
308 |         &self,
309 |         repo: &str,
310 |         count: u32,
311 |         offset: u32,
312 |         filter: &dyn Fn(&dyn Build) -> bool,
313 |     ) -> Result<Vec<Box<dyn Build>>> {
314 |         let resp = self.req(
315 |             Method::GET,
316 |             repo,
317 |             &format!("build/builds?api-version=5.0&$top={}", count),
318 |         )?;
319 |         let mut resp = resp.error_for_status()?;
320 |         let builds: AzureBuilds = resp.json()?;
321 |         let mut ret = Vec::new();
322 |         for build in builds.value.into_iter() {
323 |             if build.outcome.status == Some(BuildStatus::InProgress) {
324 |                 continue;
325 |             }
326 |             if let Some(build) = AzureBuild::new(&self, repo, build)? {
327 |                 println!(
328 |                     "id={} pr={:?} branch_name={}, commit={}, status={:?}",
329 |                     build.data.id,
330 |                     build.pr_number(),
331 |                     build.branch_name(),
332 |                     build.data.source_version,
333 |                     build.data.outcome,
334 |                 );
335 |                 if filter(&build) {
336 |                     ret.push(Box::new(build) as Box<dyn Build>);
337 |                 }
338 |             }
339 |         }
340 | 
341 |         eprintln!("pushed {:?}", ret.len());
342 | 
343 |         Ok(ret)
344 |     }
345 | 
346 |     fn query_build(&self, repo: &str, id: u64) -> Result<Box<dyn Build>> {
347 |         let resp = self.req(
348 |             Method::GET,
349 |             repo,
350 |             &format!("build/builds/{}?api-version=5.0", id),
351 |         )?;
352 |         let mut resp = resp.error_for_status()?;
353 |         let data: AzureBuildData = resp.json()?;
354 |         if let Some(build) = AzureBuild::new(&self, repo, data)? {
355 |             Ok(Box::new(build))
356 |         } else {
357 |             Err(anyhow::anyhow!("no build results"))
358 |         }
359 |     }
360 | 
361 |     fn remove_timestamp_from_log_line<'a>(&self, line: &'a [u8]) -> Cow<'a, [u8]> {
362 |         // Azure Pipelines log lines are always prefixed by the timestamp.
363 |         Cow::Borrowed(line.splitn(2, |c| *c == b' ').last().unwrap_or(line))
364 |     }
365 | }
366 | 


--------------------------------------------------------------------------------
/src/ci/mod.rs:
--------------------------------------------------------------------------------
 1 | use anyhow::anyhow;
 2 | use reqwest::blocking::RequestBuilder;
 3 | use std::borrow::Cow;
 4 | use std::io::Read;
 5 | 
 6 | mod actions;
 7 | mod azure;
 8 | 
 9 | pub use actions::Client as GitHubActions;
10 | pub use azure::Client as AzurePipelines;
11 | 
12 | use crate::Result;
13 | 
14 | #[derive(Debug)]
15 | pub enum BuildCommit<'a> {
16 |     Merge { sha: &'a str },
17 |     Head { sha: &'a str },
18 | }
19 | 
20 | pub trait Outcome: std::fmt::Debug {
21 |     fn is_finished(&self) -> bool;
22 |     fn is_passed(&self) -> bool;
23 |     fn is_failed(&self) -> bool;
24 | }
25 | 
26 | pub trait Build {
27 |     fn pr_number(&self) -> Option<u32>;
28 |     fn branch_name(&self) -> &str;
29 |     fn commit_sha(&self) -> BuildCommit;
30 |     fn outcome(&self) -> &dyn Outcome;
31 |     fn jobs(&self) -> Vec<&dyn Job>;
32 | }
33 | 
34 | pub trait Job: std::fmt::Display {
35 |     fn id(&self) -> String;
36 |     fn html_url(&self) -> String;
37 |     fn log_url(&self) -> Option<String>; // sometimes we just don't have log URLs
38 |     fn log_file_name(&self) -> String;
39 |     fn outcome(&self) -> &dyn Outcome;
40 | 
41 |     fn log_api_url(&self) -> Option<String> {
42 |         self.log_url()
43 |     }
44 | }
45 | 
46 | pub trait CiPlatform {
47 |     fn build_id_from_github_check(&self, e: &crate::github::CheckRunEvent) -> Option<u64>;
48 |     fn build_id_from_github_status(&self, e: &crate::github::CommitStatusEvent) -> Option<u64>;
49 | 
50 |     fn query_builds(
51 |         &self,
52 |         repo: &str,
53 |         count: u32,
54 |         offset: u32,
55 |         filter: &dyn Fn(&dyn Build) -> bool,
56 |     ) -> Result<Vec<Box<dyn Build>>>;
57 |     fn query_build(&self, repo: &str, id: u64) -> Result<Box<dyn Build>>;
58 | 
59 |     fn remove_timestamp_from_log_line<'a>(&self, line: &'a [u8]) -> Cow<'a, [u8]> {
60 |         Cow::Borrowed(line)
61 |     }
62 | 
63 |     fn authenticate_request(&self, request: RequestBuilder) -> RequestBuilder {
64 |         request
65 |     }
66 | 
67 |     /// Some CI providers return mismatched data in the API compared to the webhook. Those
68 |     /// providers should return `true` from this method.
69 |     fn is_build_outcome_unreliable(&self) -> bool {
70 |         false
71 |     }
72 | }
73 | 
74 | pub fn download_log(
75 |     ci: &dyn CiPlatform,
76 |     job: &dyn Job,
77 |     client: &reqwest::blocking::Client,
78 | ) -> Option<Result<Vec<u8>>> {
79 |     if let Some(url) = job.log_api_url() {
80 |         let mut resp = match ci.authenticate_request(client.get(&url)).send() {
81 |             Ok(v) => v,
82 |             Err(e) => return Some(Err(e.into())),
83 |         };
84 | 
85 |         if !resp.status().is_success() {
86 |             return Some(Err(anyhow!("Downloading log failed: {:?}", resp)));
87 |         }
88 | 
89 |         let mut bytes: Vec<u8> = vec![];
90 |         if let Err(err) = resp.read_to_end(&mut bytes) {
91 |             return Some(Err(err.into()));
92 |         }
93 | 
94 |         return Some(Ok(bytes));
95 |     }
96 | 
97 |     None
98 | }
99 | 


--------------------------------------------------------------------------------
/src/extract.rs:
--------------------------------------------------------------------------------
  1 | use crate::index::{Index, IndexData};
  2 | use aho_corasick::AhoCorasick;
  3 | use std::iter;
  4 | use std::mem;
  5 | 
  6 | /// Plaintext patterns which, if found in a line, cause all remaining lines to be ignored until the
  7 | /// corresponding pattern (second tuple element) is found in a line.
  8 | static IGNORE_BLOCK: &[(&str, &str)] = &[
  9 |     // Skip environment varialbes
 10 |     (
 11 |         "##[group]Run src/ci/scripts/dump-environment.sh",
 12 |         "##[group]",
 13 |     ),
 14 |     // Skip initialization as it involves submodules, which can change often
 15 |     (
 16 |         "##[group]Run src/ci/scripts/checkout-submodules.sh",
 17 |         "##[group]",
 18 |     ),
 19 |     // Skip AWS cli installation, it involves network (so changes)
 20 |     ("##[group]Run src/ci/scripts/install-awscli.sh", "##[group]"),
 21 |     // Cargo downloads crates in unpredictable order
 22 |     ("Downloading crates ...", "Compiling"),
 23 |     // Skip all groups invoking git commands
 24 |     ("[command]/usr/bin/git", "##[endgroup]"),
 25 |     // Skip clock drift checks
 26 |     ("#[group]Clock drift check", "##[endgroup]"),
 27 |     // Skip environment variable dumps, as these can contain e.g. a SHA which is different in every
 28 |     // build.
 29 |     ("env:", "##[endgroup]"),
 30 |     // See src/ci/scripts/dump-environment.sh in rust-lang/rust
 31 |     (
 32 |         "environment variables:",
 33 |         "biggest files in the working dir:",
 34 |     ),
 35 | ];
 36 | 
 37 | lazy_static! {
 38 |     static ref IGNORE_BLOCK_START: AhoCorasick =
 39 |         AhoCorasick::new(IGNORE_BLOCK.iter().map(|x| &x.0).cloned()).unwrap();
 40 | }
 41 | 
 42 | lazy_static! {
 43 |     static ref IGNORE_BLOCK_END: Vec<AhoCorasick> = IGNORE_BLOCK
 44 |         .iter()
 45 |         .map(|&s| AhoCorasick::new(iter::once(s.1)).unwrap())
 46 |         .collect();
 47 | }
 48 | 
 49 | pub struct Config {
 50 |     pub unique_5gram_max_index: u32,
 51 |     pub block_merge_distance: usize,
 52 |     pub block_separator_max_score: u32,
 53 |     pub unique_line_min_score: u32,
 54 |     pub block_max_lines: usize,
 55 |     pub context_lines: usize,
 56 | }
 57 | 
 58 | impl Default for Config {
 59 |     fn default() -> Self {
 60 |         Config {
 61 |             unique_5gram_max_index: 10,
 62 |             block_merge_distance: 8,
 63 |             block_separator_max_score: 0,
 64 |             unique_line_min_score: 50,
 65 |             block_max_lines: 500,
 66 |             context_lines: 4,
 67 |         }
 68 |     }
 69 | }
 70 | 
 71 | pub fn score<I: IndexData>(config: &Config, index: &Index, line: &I) -> u32 {
 72 |     index
 73 |         .scores(line)
 74 |         .filter(|&val| val <= config.unique_5gram_max_index)
 75 |         .map(|val| config.unique_5gram_max_index - val)
 76 |         .sum()
 77 | }
 78 | 
 79 | enum State<'a> {
 80 |     SearchingSectionStart,
 81 |     SearchingOutlier,
 82 |     Printing,
 83 |     Ignoring(&'a AhoCorasick),
 84 | }
 85 | 
 86 | #[derive(Copy, Clone)]
 87 | struct Line<'i, I: IndexData + 'i> {
 88 |     score: u32,
 89 |     line: &'i I,
 90 | }
 91 | 
 92 | pub fn extract<'i, I: IndexData + 'i>(
 93 |     config: &Config,
 94 |     index: &Index,
 95 |     lines: &'i [I],
 96 | ) -> Vec<Vec<&'i I>> {
 97 |     assert!(config.context_lines < config.block_merge_distance);
 98 | 
 99 |     let lines: Vec<Line<_>> = lines
100 |         .iter()
101 |         .map(|line| Line {
102 |             line,
103 |             score: score(config, index, line),
104 |         })
105 |         .collect();
106 | 
107 |     let mut i = 0;
108 |     let mut state = State::SearchingSectionStart;
109 |     let mut section_start = 0;
110 |     let mut prev_section_end = 0;
111 | 
112 |     let mut active_block = vec![];
113 |     let mut blocks = vec![];
114 | 
115 |     let mut trailing_context = 0;
116 | 
117 |     while i < lines.len() {
118 |         if let Some(m) = IGNORE_BLOCK_START.find(lines[i].line.sanitized()) {
119 |             trailing_context = 0;
120 | 
121 |             if let State::Printing = state {
122 |                 if !active_block.is_empty() {
123 |                     blocks.push(mem::replace(&mut active_block, vec![]));
124 |                 }
125 |             }
126 | 
127 |             state = State::Ignoring(&IGNORE_BLOCK_END[m.pattern()]);
128 |             i += 1;
129 |             continue;
130 |         }
131 | 
132 |         match state {
133 |             State::Ignoring(a) => {
134 |                 if a.find(lines[i].line.sanitized()).is_some() {
135 |                     state = State::SearchingSectionStart;
136 |                 }
137 | 
138 |                 i += 1;
139 |                 continue;
140 |             }
141 | 
142 |             State::SearchingSectionStart => {
143 |                 if lines[i].score > config.block_separator_max_score {
144 |                     state = State::SearchingOutlier;
145 |                     section_start = i;
146 |                 } else {
147 |                     if trailing_context > 0 {
148 |                         trailing_context -= 1;
149 |                         blocks.last_mut().unwrap().push(lines[i].line);
150 |                         prev_section_end = i;
151 |                     }
152 | 
153 |                     i += 1;
154 |                     continue;
155 |                 }
156 |             }
157 | 
158 |             State::SearchingOutlier => {
159 |                 if lines[i].score <= config.block_separator_max_score {
160 |                     if trailing_context > 0 {
161 |                         trailing_context -= 1;
162 |                         blocks.last_mut().unwrap().push(lines[i].line);
163 |                         prev_section_end = i;
164 |                     }
165 | 
166 |                     state = State::SearchingSectionStart;
167 |                     i += 1;
168 |                     continue;
169 |                 }
170 | 
171 |                 if lines[i].score >= config.unique_line_min_score {
172 |                     trailing_context = 0;
173 | 
174 |                     let start_printing;
175 | 
176 |                     if prev_section_end + config.block_merge_distance >= section_start {
177 |                         if !blocks.is_empty() {
178 |                             let last_idx = blocks.len() - 1;
179 |                             active_block = blocks.remove(last_idx);
180 |                         }
181 |                         // prev_section_end' line already contained in some block, so start
182 |                         // from next one. (Except from case from State::Printing and empty active_block,
183 |                         // idk?).
184 |                         start_printing = prev_section_end + 1;
185 |                     } else {
186 |                         start_printing = section_start.saturating_sub(config.context_lines);
187 |                     }
188 | 
189 |                     for j in start_printing..i {
190 |                         active_block.push(lines[j].line);
191 |                     }
192 | 
193 |                     state = State::Printing;
194 |                 } else {
195 |                     if trailing_context > 0 {
196 |                         trailing_context -= 1;
197 |                         blocks.last_mut().unwrap().push(lines[i].line);
198 |                         prev_section_end = i;
199 | 
200 |                         // No need to update section_start since we'll trigger the `merge` case above
201 |                         // anyway (prev_section_end >= section_start).
202 |                     }
203 | 
204 |                     i += 1;
205 |                     continue;
206 |                 }
207 |             }
208 | 
209 |             State::Printing => {
210 |                 if lines[i].score <= config.block_separator_max_score {
211 |                     if !active_block.is_empty() {
212 |                         active_block.push(lines[i].line);
213 |                         blocks.push(mem::replace(&mut active_block, vec![]));
214 |                     }
215 |                     prev_section_end = i;
216 |                     state = State::SearchingSectionStart;
217 | 
218 |                     trailing_context = config.context_lines;
219 |                 } else {
220 |                     active_block.push(lines[i].line);
221 |                 }
222 | 
223 |                 i += 1;
224 |                 continue;
225 |             }
226 |         }
227 |     }
228 | 
229 |     if !active_block.is_empty() {
230 |         blocks.push(active_block);
231 |     }
232 | 
233 |     blocks.retain(|block| !block.is_empty());
234 |     blocks
235 |         .iter_mut()
236 |         .for_each(|block| block.truncate(config.block_max_lines));
237 | 
238 |     blocks
239 | }
240 | 


--------------------------------------------------------------------------------
/src/github.rs:
--------------------------------------------------------------------------------
  1 | use crate::ci::Outcome;
  2 | use crate::Result;
  3 | use anyhow::{bail, format_err};
  4 | use hyper::header;
  5 | use serde::{de::DeserializeOwned, Serialize};
  6 | use std::env;
  7 | use std::str;
  8 | use std::time::Duration;
  9 | 
 10 | const TIMEOUT_SECS: u64 = 15;
 11 | static ACCEPT_VERSION: &str = "application/vnd.github.v3+json";
 12 | static API_BASE: &str = "https://api.github.com";
 13 | 
 14 | #[derive(Deserialize, Debug, PartialEq, Eq)]
 15 | #[serde(rename_all = "snake_case")]
 16 | pub enum BuildStatus {
 17 |     Queued,
 18 |     InProgress,
 19 |     Completed,
 20 | }
 21 | 
 22 | #[derive(Deserialize, Debug, PartialEq, Eq)]
 23 | #[serde(rename_all = "snake_case")]
 24 | pub enum BuildConclusion {
 25 |     Success,
 26 |     Failure,
 27 |     Neutral,
 28 |     Cancelled,
 29 |     TimedOut,
 30 |     ActionRequired,
 31 |     Skipped,
 32 | }
 33 | 
 34 | #[derive(Deserialize, Debug)]
 35 | pub struct BuildOutcome {
 36 |     status: BuildStatus,
 37 |     conclusion: Option<BuildConclusion>,
 38 | }
 39 | 
 40 | impl Outcome for BuildOutcome {
 41 |     fn is_finished(&self) -> bool {
 42 |         self.status == BuildStatus::Completed
 43 |     }
 44 | 
 45 |     fn is_passed(&self) -> bool {
 46 |         self.is_finished() && self.conclusion == Some(BuildConclusion::Success)
 47 |     }
 48 | 
 49 |     fn is_failed(&self) -> bool {
 50 |         self.is_finished() && self.conclusion == Some(BuildConclusion::Failure)
 51 |     }
 52 | }
 53 | 
 54 | #[derive(Deserialize)]
 55 | pub struct CommitStatusEvent {
 56 |     pub target_url: String,
 57 |     pub context: String,
 58 |     pub repository: Repository,
 59 | }
 60 | 
 61 | #[derive(Deserialize)]
 62 | pub struct Pr {
 63 |     pub head: PrCommitRef,
 64 |     pub labels: Vec<Label>,
 65 | }
 66 | 
 67 | #[derive(Deserialize)]
 68 | pub struct PrCommitRef {
 69 |     pub sha: String,
 70 | }
 71 | 
 72 | #[derive(Deserialize)]
 73 | pub struct Label {
 74 |     pub name: String,
 75 | }
 76 | 
 77 | #[derive(Deserialize)]
 78 | pub struct CommitMeta {
 79 |     pub commit: Commit,
 80 |     pub parents: Vec<CommitParent>,
 81 | }
 82 | 
 83 | #[derive(Deserialize)]
 84 | pub struct Commit {
 85 |     pub message: String,
 86 | }
 87 | 
 88 | #[derive(Deserialize)]
 89 | pub struct CommitParent {
 90 |     pub sha: String,
 91 | }
 92 | 
 93 | #[derive(Serialize)]
 94 | struct Comment<'a> {
 95 |     body: &'a str,
 96 | }
 97 | 
 98 | #[derive(Deserialize)]
 99 | pub struct CheckRunEvent {
100 |     pub check_run: CheckRun,
101 |     pub repository: Repository,
102 | }
103 | 
104 | #[derive(Deserialize)]
105 | pub struct CheckRun {
106 |     pub url: String,
107 |     pub external_id: String,
108 |     pub details_url: String,
109 |     pub app: App,
110 |     pub check_suite: CheckSuite,
111 |     #[serde(flatten)]
112 |     pub outcome: BuildOutcome,
113 | }
114 | 
115 | #[derive(Deserialize)]
116 | pub struct CheckSuite {
117 |     pub id: u64,
118 |     pub url: String,
119 | }
120 | 
121 | #[derive(Deserialize)]
122 | pub struct App {
123 |     pub id: u64,
124 | }
125 | 
126 | #[derive(Deserialize)]
127 | pub struct Repository {
128 |     pub full_name: String,
129 | }
130 | 
131 | #[derive(Deserialize)]
132 | #[serde(rename_all = "snake_case")]
133 | pub enum PullRequestAction {
134 |     Opened,
135 |     Edited,
136 |     Closed,
137 |     Assigned,
138 |     Unassigned,
139 |     ReviewRequested,
140 |     ReviewRequestRemoved,
141 |     ReadyForReview,
142 |     Labeled,
143 |     Unlabeled,
144 |     Synchronize,
145 |     Locked,
146 |     Unlocked,
147 |     Reopened,
148 | }
149 | 
150 | #[derive(Deserialize)]
151 | pub struct PullRequestEvent {
152 |     pub action: PullRequestAction,
153 |     pub number: u32,
154 |     pub repository: Repository,
155 | }
156 | 
157 | #[derive(Deserialize)]
158 | struct GraphResponse<T> {
159 |     data: T,
160 |     #[serde(default)]
161 |     errors: Vec<GraphError>,
162 | }
163 | 
164 | #[derive(Debug, Deserialize)]
165 | struct GraphError {
166 |     message: String,
167 | }
168 | 
169 | #[derive(Debug, Deserialize)]
170 | #[serde(rename_all = "camelCase")]
171 | struct GraphPageInfo {
172 |     end_cursor: Option<String>,
173 | }
174 | 
175 | pub struct Client {
176 |     internal: reqwest::blocking::Client,
177 | }
178 | 
179 | impl Client {
180 |     pub fn new() -> Result<Client> {
181 |         let token = env::var("GITHUB_TOKEN")
182 |             .map_err(|e| format_err!("Could not read GITHUB_TOKEN: {}", e))?;
183 | 
184 |         let mut headers = header::HeaderMap::new();
185 |         headers.insert(
186 |             header::ACCEPT,
187 |             header::HeaderValue::from_static(ACCEPT_VERSION),
188 |         );
189 |         headers.insert(
190 |             header::USER_AGENT,
191 |             header::HeaderValue::from_static(super::USER_AGENT),
192 |         );
193 |         headers.insert(
194 |             header::AUTHORIZATION,
195 |             header::HeaderValue::from_str(&format!("token {}", token))?,
196 |         );
197 | 
198 |         let client = reqwest::blocking::Client::builder()
199 |             .default_headers(headers)
200 |             .referer(false)
201 |             .timeout(Duration::from_secs(TIMEOUT_SECS))
202 |             .build()?;
203 | 
204 |         Ok(Client { internal: client })
205 |     }
206 | 
207 |     pub fn query_pr(&self, repo: &str, pr_id: u32) -> Result<Pr> {
208 |         let resp = self
209 |             .internal
210 |             .get(format!("{}/repos/{}/pulls/{}", API_BASE, repo, pr_id).as_str())
211 |             .send()?;
212 | 
213 |         if !resp.status().is_success() {
214 |             bail!("Querying PR failed: {:?}", resp);
215 |         }
216 | 
217 |         Ok(resp.json()?)
218 |     }
219 | 
220 |     pub fn query_commit(&self, repo: &str, sha: &str) -> Result<CommitMeta> {
221 |         let resp = self
222 |             .internal
223 |             .get(format!("{}/repos/{}/commits/{}", API_BASE, repo, sha).as_str())
224 |             .send()?;
225 | 
226 |         if !resp.status().is_success() {
227 |             bail!("Querying commit failed: {:?}", resp);
228 |         }
229 | 
230 |         Ok(resp.json()?)
231 |     }
232 | 
233 |     pub fn post_comment(&self, repo: &str, issue_id: u32, comment: &str) -> Result<()> {
234 |         let resp = self
235 |             .internal
236 |             .post(format!("{}/repos/{}/issues/{}/comments", API_BASE, repo, issue_id).as_str())
237 |             .json(&Comment { body: comment })
238 |             .send()?;
239 |         if !resp.status().is_success() {
240 |             bail!("Posting comment failed: {:?}", resp);
241 |         }
242 | 
243 |         Ok(())
244 |     }
245 | 
246 |     pub fn hide_own_comments(&self, repo: &str, pull_request_id: u32) -> Result<()> {
247 |         const QUERY: &str = "query($owner: String!, $repo: String!, $pr: Int!, $cursor: String) {
248 |                 repository(owner: $owner, name: $repo) {
249 |                     pullRequest(number: $pr) {
250 |                         comments(first: 100, after: $cursor) {
251 |                             nodes {
252 |                                 id
253 |                                 isMinimized
254 |                                 viewerDidAuthor
255 |                             }
256 |                             pageInfo {
257 |                                 endCursor
258 |                             }
259 |                         }
260 |                     }
261 |                 }
262 |             }";
263 | 
264 |         #[derive(Debug, Deserialize)]
265 |         struct Response {
266 |             repository: ResponseRepo,
267 |         }
268 |         #[derive(Debug, Deserialize)]
269 |         #[serde(rename_all = "camelCase")]
270 |         struct ResponseRepo {
271 |             pull_request: ResponsePR,
272 |         }
273 |         #[derive(Debug, Deserialize)]
274 |         struct ResponsePR {
275 |             comments: ResponseComments,
276 |         }
277 |         #[derive(Debug, Deserialize)]
278 |         #[serde(rename_all = "camelCase")]
279 |         struct ResponseComments {
280 |             nodes: Vec<ResponseComment>,
281 |             page_info: GraphPageInfo,
282 |         }
283 |         #[derive(Debug, Deserialize)]
284 |         #[serde(rename_all = "camelCase")]
285 |         struct ResponseComment {
286 |             id: String,
287 |             is_minimized: bool,
288 |             viewer_did_author: bool,
289 |         }
290 | 
291 |         debug!("started hiding comments in {}#{}", repo, pull_request_id);
292 | 
293 |         let (owner, repo) = if let Some(mid) = repo.find('/') {
294 |             let split = repo.split_at(mid);
295 |             (split.0, split.1.trim_start_matches('/'))
296 |         } else {
297 |             bail!("invalid repository name: {}", repo);
298 |         };
299 | 
300 |         let mut comments = Vec::new();
301 |         let mut cursor = None;
302 |         loop {
303 |             let mut resp: Response = self.graphql(
304 |                 QUERY,
305 |                 serde_json::json!({
306 |                     "owner": owner,
307 |                     "repo": repo,
308 |                     "pr": pull_request_id,
309 |                     "cursor": cursor,
310 |                 }),
311 |             )?;
312 |             cursor = resp.repository.pull_request.comments.page_info.end_cursor;
313 |             comments.append(&mut resp.repository.pull_request.comments.nodes);
314 | 
315 |             if cursor.is_none() {
316 |                 break;
317 |             }
318 |         }
319 | 
320 |         for comment in &comments {
321 |             if comment.viewer_did_author && !comment.is_minimized {
322 |                 self.hide_comment(&comment.id, "OUTDATED")?;
323 |             }
324 |         }
325 |         Ok(())
326 |     }
327 | 
328 |     fn hide_comment(&self, node_id: &str, reason: &str) -> Result<()> {
329 |         #[derive(Deserialize)]
330 |         struct MinimizeData {}
331 | 
332 |         const MINIMIZE: &str = "mutation($node_id: ID!, $reason: ReportedContentClassifiers!) {
333 |             minimizeComment(input: {subjectId: $node_id, classifier: $reason}) {
334 |                 __typename
335 |             }
336 |         }";
337 | 
338 |         trace!("hiding comment {}", node_id);
339 | 
340 |         self.graphql::<Option<MinimizeData>, _>(
341 |             MINIMIZE,
342 |             serde_json::json!({
343 |                 "node_id": node_id,
344 |                 "reason": reason,
345 |             }),
346 |         )?;
347 |         Ok(())
348 |     }
349 | 
350 |     pub fn internal(&self) -> &reqwest::blocking::Client {
351 |         &self.internal
352 |     }
353 | 
354 |     fn graphql<T: DeserializeOwned, V: Serialize>(&self, query: &str, variables: V) -> Result<T> {
355 |         #[derive(Serialize)]
356 |         struct GraphPayload<'a, V> {
357 |             query: &'a str,
358 |             variables: V,
359 |         }
360 | 
361 |         let response: GraphResponse<T> = self
362 |             .internal
363 |             .post(&format!("{}/graphql", API_BASE))
364 |             .json(&GraphPayload { query, variables })
365 |             .send()?
366 |             .error_for_status()?
367 |             .json()?;
368 | 
369 |         if response.errors.is_empty() {
370 |             Ok(response.data)
371 |         } else {
372 |             dbg!(&response.errors);
373 |             bail!("GraphQL query failed: {}", response.errors[0].message);
374 |         }
375 |     }
376 | }
377 | 
378 | pub fn verify_webhook_signature(secret: &[u8], signature: Option<&str>, body: &[u8]) -> Result<()> {
379 |     use hmac::{Hmac, Mac};
380 |     use sha1::Sha1;
381 | 
382 |     let signature = match signature {
383 |         Some(sig) => sig,
384 |         None => bail!("Missing signature."),
385 |     };
386 | 
387 |     if !signature.starts_with("sha1=") {
388 |         bail!("Invalid signature format.");
389 |     }
390 | 
391 |     let signature = &signature[5..];
392 | 
393 |     let decoded_signature = hex::decode(signature)?;
394 | 
395 |     let mut mac = Hmac::<Sha1>::new_from_slice(secret).unwrap();
396 |     mac.update(body);
397 | 
398 |     if mac.verify_slice(&decoded_signature).is_ok() {
399 |         Ok(())
400 |     } else {
401 |         bail!("Signature mismatch.");
402 |     }
403 | }
404 | 


--------------------------------------------------------------------------------
/src/index/mod.rs:
--------------------------------------------------------------------------------
  1 | use super::Result;
  2 | use std::slice;
  3 | 
  4 | mod storage;
  5 | mod table;
  6 | 
  7 | pub use self::storage::IndexStorage;
  8 | 
  9 | pub trait IndexData {
 10 |     fn sanitized(&self) -> &[u8];
 11 | }
 12 | 
 13 | pub struct Sanitized<T: AsRef<[u8]>>(pub T);
 14 | 
 15 | impl<T: AsRef<[u8]>> IndexData for Sanitized<T> {
 16 |     fn sanitized(&self) -> &[u8] {
 17 |         self.0.as_ref()
 18 |     }
 19 | }
 20 | 
 21 | #[derive(Default, Serialize, Deserialize)]
 22 | pub struct Index {
 23 |     internal: fnv::FnvHashMap<u32, u32>,
 24 | }
 25 | 
 26 | impl Index {
 27 |     pub fn learn<I: IndexData>(&mut self, data: &I, multiplier: u32) {
 28 |         let encoded = encode(data);
 29 | 
 30 |         for id in IdIter::new(&encoded) {
 31 |             let val = self.internal.entry(id).or_insert(0);
 32 |             *val = val.saturating_add(multiplier);
 33 |         }
 34 |     }
 35 | 
 36 |     pub fn scores<I: IndexData>(&self, data: &I) -> std::vec::IntoIter<u32> {
 37 |         let encoded = encode(data);
 38 | 
 39 |         IdIter::new(&encoded)
 40 |             .map(|id| self.internal.get(&id).cloned().unwrap_or(0))
 41 |             .collect::<Vec<_>>()
 42 |             .into_iter()
 43 |     }
 44 | 
 45 |     pub fn save(&self, storage: &IndexStorage) -> Result<()> {
 46 |         debug!("Saving index to '{storage}'...");
 47 |         storage.write(self)?;
 48 |         debug!("Index saved.");
 49 |         Ok(())
 50 |     }
 51 | 
 52 |     pub fn load(storage: &IndexStorage) -> Result<Index> {
 53 |         Index::load_or_create_internal(storage, false)
 54 |     }
 55 | 
 56 |     pub fn load_or_create(storage: &IndexStorage) -> Result<Index> {
 57 |         Index::load_or_create_internal(storage, true)
 58 |     }
 59 | 
 60 |     fn load_or_create_internal(storage: &IndexStorage, create: bool) -> Result<Index> {
 61 |         info!("Loading index...");
 62 |         let index = if let Some(index) = storage.read()? {
 63 |             index
 64 |         } else {
 65 |             if create {
 66 |                 info!("Index missing, initializing new index...");
 67 |                 Index::default()
 68 |             } else {
 69 |                 anyhow::bail!("missing index, aborting");
 70 |             }
 71 |         };
 72 | 
 73 |         info!("Index ready ({} keys).", index.internal.len());
 74 | 
 75 |         Ok(index)
 76 |     }
 77 | 
 78 |     fn deserialize(reader: &mut dyn std::io::Read) -> Result<Self> {
 79 |         Ok(bincode::deserialize_from(reader)?)
 80 |     }
 81 | 
 82 |     fn serialize(
 83 |         &self,
 84 |         writer: &mut dyn std::io::Write,
 85 |     ) -> std::result::Result<(), bincode::Error> {
 86 |         bincode::serialize_into(writer, self)?;
 87 |         Ok(())
 88 |     }
 89 | }
 90 | 
 91 | pub fn encode<I: IndexData>(data: &I) -> Vec<u8> {
 92 |     data.sanitized()
 93 |         .iter()
 94 |         .map(|&b| table::ASCII_ID_MAP[b as usize])
 95 |         .filter(|&b| b != 0xFF)
 96 |         .collect()
 97 | }
 98 | 
 99 | pub fn decode(data: &[u8]) -> Vec<u8> {
100 |     data.iter()
101 |         .map(|&b| table::ID_ASCII_MAP[b as usize])
102 |         .collect()
103 | }
104 | 
105 | struct IdIter<'a> {
106 |     windows: slice::Windows<'a, u8>,
107 | }
108 | 
109 | impl<'a> IdIter<'a> {
110 |     fn new(encoded_data: &'a [u8]) -> IdIter<'a> {
111 |         IdIter {
112 |             windows: encoded_data.windows(5),
113 |         }
114 |     }
115 | }
116 | 
117 | impl<'a> Iterator for IdIter<'a> {
118 |     type Item = u32;
119 | 
120 |     #[allow(clippy::complexity)]
121 |     fn next(&mut self) -> Option<<Self as Iterator>::Item> {
122 |         self.windows.next().map(|w| {
123 |             0u32 + w[0] as u32
124 |                 + w[1] as u32 * 64
125 |                 + w[2] as u32 * 64 * 64
126 |                 + w[3] as u32 * 64 * 64 * 64
127 |                 + w[4] as u32 * 64 * 64 * 64 * 64
128 |         })
129 |     }
130 | }
131 | 


--------------------------------------------------------------------------------
/src/index/storage.rs:
--------------------------------------------------------------------------------
  1 | use crate::{Index, Result};
  2 | use anyhow::anyhow;
  3 | use atomicwrites::{AtomicFile, OverwriteBehavior};
  4 | use aws_sdk_s3::config::Region;
  5 | use aws_sdk_s3::error::SdkError;
  6 | use aws_sdk_s3::operation::get_object::GetObjectError;
  7 | use aws_sdk_s3::Client as S3Client;
  8 | use hyper::body::Buf;
  9 | use std::fs::File;
 10 | use std::path::PathBuf;
 11 | use std::str::FromStr;
 12 | use std::sync::Arc;
 13 | use tokio::runtime::Runtime;
 14 | 
 15 | #[derive(Debug, Clone)]
 16 | pub enum IndexStorage {
 17 |     FileSystem(FileSystemStorage),
 18 |     S3(Arc<S3Storage>),
 19 | }
 20 | 
 21 | impl IndexStorage {
 22 |     pub fn new(path: &str) -> Result<Self> {
 23 |         if let Some(s3_url) = path.strip_prefix("s3://") {
 24 |             let (bucket, key) = s3_url
 25 |                 .split_once('/')
 26 |                 .ok_or_else(|| anyhow!("invalid s3 url: {path}"))?;
 27 |             Ok(IndexStorage::S3(Arc::new(S3Storage::new(bucket, key)?)))
 28 |         } else {
 29 |             Ok(IndexStorage::FileSystem(FileSystemStorage {
 30 |                 path: path.into(),
 31 |             }))
 32 |         }
 33 |     }
 34 | 
 35 |     pub(super) fn read(&self) -> Result<Option<Index>> {
 36 |         match self {
 37 |             IndexStorage::FileSystem(fs) => fs.read(),
 38 |             IndexStorage::S3(s3) => s3.read(),
 39 |         }
 40 |     }
 41 | 
 42 |     pub(super) fn write(&self, index: &Index) -> Result<()> {
 43 |         match self {
 44 |             IndexStorage::FileSystem(fs) => fs.write(index),
 45 |             IndexStorage::S3(s3) => s3.write(index),
 46 |         }
 47 |     }
 48 | }
 49 | 
 50 | impl FromStr for IndexStorage {
 51 |     type Err = anyhow::Error;
 52 | 
 53 |     fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
 54 |         IndexStorage::new(s)
 55 |     }
 56 | }
 57 | 
 58 | impl std::fmt::Display for IndexStorage {
 59 |     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 60 |         match self {
 61 |             IndexStorage::FileSystem(fs) => write!(f, "{}", fs.path.display()),
 62 |             IndexStorage::S3(s3) => write!(f, "s3://{}/{}", s3.bucket, s3.key),
 63 |         }
 64 |     }
 65 | }
 66 | 
 67 | #[derive(Debug, Clone)]
 68 | pub struct FileSystemStorage {
 69 |     path: PathBuf,
 70 | }
 71 | 
 72 | impl FileSystemStorage {
 73 |     fn read(&self) -> Result<Option<Index>> {
 74 |         let mut file = match File::open(&self.path) {
 75 |             Ok(file) => file,
 76 |             Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
 77 |             Err(err) => return Err(err.into()),
 78 |         };
 79 |         Ok(Some(Index::deserialize(&mut file)?))
 80 |     }
 81 | 
 82 |     fn write(&self, index: &Index) -> Result<()> {
 83 |         let file = AtomicFile::new(&self.path, OverwriteBehavior::AllowOverwrite);
 84 |         file.write(|inner| index.serialize(inner))?;
 85 |         Ok(())
 86 |     }
 87 | }
 88 | 
 89 | #[derive(Debug)]
 90 | pub struct S3Storage {
 91 |     runtime: Runtime,
 92 |     client: S3Client,
 93 |     bucket: String,
 94 |     key: String,
 95 | }
 96 | 
 97 | impl S3Storage {
 98 |     fn new(bucket: &str, key: &str) -> Result<Self> {
 99 |         let runtime = Runtime::new()?;
100 | 
101 |         let config = runtime.block_on(async {
102 |             let global_config = aws_config::load_from_env().await;
103 |             let global_s3 = S3Client::new(&global_config);
104 | 
105 |             let location = global_s3
106 |                 .get_bucket_location()
107 |                 .bucket(bucket)
108 |                 .send()
109 |                 .await?;
110 |             let region = location
111 |                 .location_constraint()
112 |                 .map(|c| c.as_str())
113 |                 .unwrap_or("us-east-1")
114 |                 .to_string();
115 | 
116 |             info!("using S3 bucket {bucket} in region {region}");
117 | 
118 |             let regional_config = aws_config::from_env()
119 |                 .region(Region::new(region))
120 |                 .load()
121 |                 .await;
122 | 
123 |             Ok::<_, anyhow::Error>(regional_config)
124 |         })?;
125 |         let client = S3Client::new(&config);
126 | 
127 |         Ok(S3Storage {
128 |             runtime,
129 |             client,
130 |             bucket: bucket.into(),
131 |             key: key.into(),
132 |         })
133 |     }
134 | 
135 |     fn read(&self) -> Result<Option<Index>> {
136 |         self.runtime.block_on(async {
137 |             let result = self
138 |                 .client
139 |                 .get_object()
140 |                 .bucket(&self.bucket)
141 |                 .key(&self.key)
142 |                 .send()
143 |                 .await;
144 | 
145 |             match result {
146 |                 Ok(response) => Ok(Some(Index::deserialize(
147 |                     &mut response.body.collect().await?.reader(),
148 |                 )?)),
149 |                 Err(err) => {
150 |                     if let SdkError::ServiceError(service_err) = &err {
151 |                         if let GetObjectError::NoSuchKey(_) = service_err.err() {
152 |                             return Ok(None);
153 |                         }
154 |                     }
155 |                     Err(err.into())
156 |                 }
157 |             }
158 |         })
159 |     }
160 | 
161 |     fn write(&self, index: &Index) -> Result<()> {
162 |         self.runtime.block_on(async {
163 |             // FIXME: this buffers the serialized data into memory before sending it, as I'm not
164 |             // aware of a way to convert from Write to AsyncWrite.
165 |             let mut buf = Vec::new();
166 |             index.serialize(&mut buf)?;
167 | 
168 |             self.client
169 |                 .put_object()
170 |                 .bucket(&self.bucket)
171 |                 .key(&self.key)
172 |                 .body(buf.into())
173 |                 .send()
174 |                 .await?;
175 | 
176 |             Ok(())
177 |         })
178 |     }
179 | }
180 | 


--------------------------------------------------------------------------------
/src/index/table.rs:
--------------------------------------------------------------------------------
  1 | pub static ASCII_ID_MAP: [u8; 256] = [
  2 |     // Control
  3 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
  4 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
  5 |     0x00, // Space
  6 |     0x01, // !
  7 |     0x02, // "
  8 |     0x03, // #
  9 |     0x04, // $
 10 |     0x05, // %
 11 |     0x06, // &
 12 |     0x07, // '
 13 |     0x08, // (
 14 |     0x09, // )
 15 |     0x0A, // *
 16 |     0x0B, // +
 17 |     0x0C, // ,
 18 |     0x0D, // -
 19 |     0x0E, // .
 20 |     0x0F, // /
 21 |     // Digits
 22 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x10, // :
 23 |     0x11, // ;
 24 |     0x12, // <
 25 |     0x13, // =
 26 |     0x14, // >
 27 |     0x15, // ?
 28 |     0x16, // @
 29 |     0x17, // A
 30 |     0x18, // B
 31 |     0x19, // C
 32 |     0x1A, // D
 33 |     0x1B, // E
 34 |     0x1C, // F
 35 |     0x1D, // G
 36 |     0x1E, // H
 37 |     0x1F, // I
 38 |     0x20, // J
 39 |     0x21, // K
 40 |     0x22, // L
 41 |     0x23, // M
 42 |     0x24, // N
 43 |     0x25, // O
 44 |     0x26, // P
 45 |     0x27, // Q
 46 |     0x28, // R
 47 |     0x29, // S
 48 |     0x2A, // T
 49 |     0x2B, // U
 50 |     0x2C, // V
 51 |     0x2D, // W
 52 |     0x2E, // X
 53 |     0x2F, // Y
 54 |     0x30, // Z
 55 |     0x31, // [
 56 |     0x32, // \
 57 |     0x33, // ]
 58 |     0x34, // ^
 59 |     0x35, // _
 60 |     0x36, // `
 61 |     0x17, // a
 62 |     0x18, // b
 63 |     0x19, // c
 64 |     0x1A, // d
 65 |     0x1B, // e
 66 |     0x1C, // f
 67 |     0x1D, // g
 68 |     0x1E, // h
 69 |     0x1F, // i
 70 |     0x20, // j
 71 |     0x21, // k
 72 |     0x22, // l
 73 |     0x23, // m
 74 |     0x24, // n
 75 |     0x25, // o
 76 |     0x26, // p
 77 |     0x27, // q
 78 |     0x28, // r
 79 |     0x29, // s
 80 |     0x2A, // t
 81 |     0x2B, // u
 82 |     0x2C, // v
 83 |     0x2D, // w
 84 |     0x2E, // x
 85 |     0x2F, // y
 86 |     0x30, // z
 87 |     0x37, // {
 88 |     0x38, // |
 89 |     0x39, // }
 90 |     0x40, // ~
 91 |     0xFF, // DEL
 92 |     // Extended / multi-byte UTF-8
 93 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 94 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 95 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 96 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 97 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 98 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
 99 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
100 |     0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
101 | ];
102 | 
103 | lazy_static! {
104 |     pub static ref ID_ASCII_MAP: [u8; 256] = {
105 |         let mut table = [0; 256];
106 | 
107 |         for (i, &b) in ASCII_ID_MAP.iter().enumerate() {
108 |             table[b as usize] = i as u8;
109 |         }
110 | 
111 |         table
112 |     };
113 | }
114 | 


--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
 1 | #![deny(unused_must_use)]
 2 | #![allow(
 3 |     clippy::collapsible_if,
 4 |     clippy::needless_range_loop,
 5 |     clippy::useless_let_if_seq
 6 | )]
 7 | 
 8 | extern crate aho_corasick;
 9 | extern crate atomicwrites;
10 | extern crate bincode;
11 | extern crate fnv;
12 | extern crate hex;
13 | extern crate hmac;
14 | extern crate hyper;
15 | #[macro_use]
16 | extern crate lazy_static;
17 | extern crate regex;
18 | extern crate reqwest;
19 | extern crate serde;
20 | #[macro_use]
21 | extern crate serde_derive;
22 | extern crate sha1;
23 | #[macro_use]
24 | extern crate tracing;
25 | 
26 | pub mod ci;
27 | pub mod extract;
28 | pub mod github;
29 | pub mod index;
30 | pub mod log_variables;
31 | pub mod sanitize;
32 | 
33 | pub use self::index::Index;
34 | 
35 | pub static USER_AGENT: &str = concat!("rust-lang/rust-log-analyzer ", env!("CARGO_PKG_VERSION"));
36 | 
37 | pub type Result<T> = anyhow::Result<T>;
38 | 


--------------------------------------------------------------------------------
/src/log_variables.rs:
--------------------------------------------------------------------------------
 1 | const START_DELIMITER: u8 = b'[';
 2 | const END_DELIMITER: u8 = b']';
 3 | const SEPARATOR: u8 = b'=';
 4 | 
 5 | const JOB_NAME_VARIABLE: &str = "CI_JOB_NAME";
 6 | const PR_NUMBER_VARIABLE: &str = "CI_PR_NUMBER";
 7 | /// URL pointing to a documentation page about the job.
 8 | /// Added in https://github.com/rust-lang/rust/pull/136911.
 9 | const JOB_DOC_URL: &str = "CI_JOB_DOC_URL";
10 | 
11 | pub struct LogVariables<'a> {
12 |     pub job_name: Option<&'a str>,
13 |     pub pr_number: Option<&'a str>,
14 |     pub doc_url: Option<&'a str>,
15 | }
16 | 
17 | impl<'a> LogVariables<'a> {
18 |     pub fn extract<I: crate::index::IndexData>(lines: &'a [I]) -> Self {
19 |         let mut result = LogVariables {
20 |             job_name: None,
21 |             pr_number: None,
22 |             doc_url: None,
23 |         };
24 | 
25 |         for line in lines {
26 |             let sanitized = line.sanitized();
27 | 
28 |             if result.job_name.is_none() {
29 |                 result.job_name = extract_variable(sanitized, JOB_NAME_VARIABLE);
30 |             }
31 |             if result.pr_number.is_none() {
32 |                 result.pr_number = extract_variable(sanitized, PR_NUMBER_VARIABLE);
33 |             }
34 |             if result.doc_url.is_none() {
35 |                 result.doc_url = extract_variable(sanitized, JOB_DOC_URL);
36 |             }
37 | 
38 |             // Early exit if everything was found
39 |             if result.job_name.is_some() && result.pr_number.is_some() && result.doc_url.is_some() {
40 |                 break;
41 |             }
42 |         }
43 | 
44 |         result
45 |     }
46 | }
47 | 
48 | fn extract_variable<'a>(line: &'a [u8], name: &str) -> Option<&'a str> {
49 |     if line.first() != Some(&START_DELIMITER) || line.last() != Some(&END_DELIMITER) {
50 |         return None;
51 |     }
52 | 
53 |     let equals = line.iter().position(|byte| *byte == SEPARATOR)?;
54 |     if &line[1..equals] != name.as_bytes() {
55 |         return None;
56 |     }
57 |     std::str::from_utf8(&line[equals + 1..line.len() - 1]).ok()
58 | }
59 | 
60 | #[cfg(test)]
61 | mod tests {
62 |     use super::*;
63 |     use crate::index::Sanitized;
64 | 
65 |     #[test]
66 |     fn test_extract_variable() {
67 |         assert_eq!(None, extract_variable(b"[foo=bar", "foo"));
68 |         assert_eq!(None, extract_variable(b"foo=bar]", "foo"));
69 |         assert_eq!(None, extract_variable(b"[foo]", "foo"));
70 |         assert_eq!(None, extract_variable(b"[baz=bar]", "foo"));
71 |         assert_eq!(Some("bar"), extract_variable(b"[foo=bar]", "foo"));
72 |     }
73 | 
74 |     #[test]
75 |     fn test_log_variables_extract() {
76 |         const LOG: &[Sanitized<&str>] = &[
77 |             Sanitized("foo"),
78 |             Sanitized("bar"),
79 |             Sanitized("[CI_JOB_NAME=test-job]"),
80 |             Sanitized("baz"),
81 |             Sanitized("[CI_PR_NUMBER=123]"),
82 |             Sanitized("quux"),
83 |             Sanitized("[CI_JOB_DOC_URL=https://github.com/rust-lang/rust/job1]"),
84 |             Sanitized("foobar"),
85 |         ];
86 | 
87 |         let extracted = LogVariables::extract(LOG);
88 |         assert_eq!(Some("test-job"), extracted.job_name);
89 |         assert_eq!(Some("123"), extracted.pr_number);
90 |         assert_eq!(
91 |             Some("https://github.com/rust-lang/rust/job1"),
92 |             extracted.doc_url
93 |         );
94 |     }
95 | }
96 | 


--------------------------------------------------------------------------------
/src/sanitize.rs:
--------------------------------------------------------------------------------
 1 | use crate::ci::CiPlatform;
 2 | use regex::bytes::Regex;
 3 | 
 4 | pub fn split_lines(data: &[u8]) -> Vec<&[u8]> {
 5 |     lazy_static! {
 6 |         static ref LINE_BREAK: Regex = Regex::new("[\\r\\n]").unwrap();
 7 |     }
 8 | 
 9 |     LINE_BREAK
10 |         .split(data)
11 |         .filter(|line| !line.iter().all(|b| b.is_ascii_whitespace()))
12 |         .collect()
13 | }
14 | 
15 | /// Cleans up the given `data`:
16 | ///
17 | /// * Removes most ANSI escape codes from the input.
18 | /// * Replaces all (Unicode) whitespace with single spaces.
19 | /// * Removes all (Unicode) control characters.
20 | pub fn clean(ci: &dyn CiPlatform, data: &[u8]) -> Vec<u8> {
21 |     lazy_static! {
22 |         /// This catches most escape sequences. And I care about neither
23 |         ///
24 |         /// * the very-special cases ("Set Keyboard Strings") that aren't matched properly nor
25 |         /// * legitimate output (which hopefully doesn't exist) that contains `ESC`.
26 |         ///
27 |         /// Reference: http://ascii-table.com/ansi-escape-sequences.php
28 |         static ref ANSI_ESCAPES: Regex = Regex::new("\x1b.*?[a-zA-Z]").unwrap();
29 | 
30 |         static ref UNICODE_WHITESPACE: Regex = Regex::new("(?u:\\p{White_Space})").unwrap();
31 | 
32 |         static ref UNICODE_CONTROL: Regex = Regex::new("(?u:\\p{Control})").unwrap();
33 |     }
34 | 
35 |     let data = ci.remove_timestamp_from_log_line(data);
36 | 
37 |     let data = ANSI_ESCAPES.replace_all(data.as_ref(), b"".as_ref());
38 |     let data = UNICODE_WHITESPACE.replace_all(&data, b" ".as_ref());
39 |     UNICODE_CONTROL
40 |         .replace_all(&data, b"".as_ref())
41 |         .into_owned()
42 | }
43 | 


--------------------------------------------------------------------------------
/tests/test.rs:
--------------------------------------------------------------------------------
 1 | use pretty_assertions::assert_eq;
 2 | 
 3 | use std::path::Path;
 4 | 
 5 | use rust_log_analyzer as rla;
 6 | 
 7 | const TEST_LOCATION: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests");
 8 | 
 9 | #[test]
10 | fn test_missing_line() {
11 |     let log = std::fs::read(Path::new(TEST_LOCATION).join("test_missing_line.txt")).unwrap();
12 |     let lines = rla::sanitize::split_lines(&log)
13 |         .iter()
14 |         .map(|l| {
15 |             rla::index::Sanitized(rla::sanitize::clean(
16 |                 &rla::ci::GitHubActions::new("DUMMY_TOKEN"),
17 |                 l,
18 |             ))
19 |         })
20 |         .collect::<Vec<_>>();
21 |     let blocks = rla::extract::extract(
22 |         &rla::extract::Config::default(),
23 |         &rla::index::Index::default(),
24 |         &lines,
25 |     );
26 | 
27 |     let expected = r###"Current runner version: '2.320.0'
28 | Runner name: 'ubuntu-20.04-4core-16gb_f6653e6045ce'
29 | Runner group name: 'Default Larger Runners'
30 | Machine name: 'runner'
31 | ##[group]Operating System
32 | Ubuntu
33 | 20.04.6
34 | LTS
35 | ##[endgroup]
36 | ##[group]Runner Image
37 | Image: ubuntu-20.04
38 | Version: 20240922.1.0
39 | Included Software: https://github.com/actions/runner-images/blob/ubuntu20/20240922.1/images/ubuntu/Ubuntu2004-Readme.md
40 | Image Release: https://github.com/actions/runner-images/releases/tag/ubuntu20%2F20240922.1
41 | ##[endgroup]
42 | ##[group]GITHUB_TOKEN Permissions
43 | Contents: read
44 | Metadata: read
45 | Packages: read
46 | ##[endgroup]
47 | Secret source: None
48 | Prepare workflow directory
49 | Prepare all required actions
50 | Getting action download info
51 | Download action repository 'msys2/setup-msys2@v2.22.0' (SHA:cc11e9188b693c2b100158c3322424c4cc1dadea)
52 | Download action repository 'actions/checkout@v4' (SHA:eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871)
53 | Download action repository 'actions/upload-artifact@v4' (SHA:604373da6381bf24206979c74d06a550515601b9)
54 | Complete job name: PR - mingw-check
55 | ##[group]Run git config --global core.autocrlf false
56 | git config --global core.autocrlf false
57 | shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}"###;
58 | 
59 |     let actual = blocks[0]
60 |         .iter()
61 |         .map(|line| String::from_utf8_lossy(&line.0).into_owned())
62 |         .collect::<Vec<_>>()
63 |         .join("\n");
64 |     assert_eq!(actual, expected);
65 | }
66 | 


--------------------------------------------------------------------------------
/tests/test_missing_line.txt:
--------------------------------------------------------------------------------
  1 | 2024-10-08T13:24:44.9714395Z Current runner version: '2.320.0'
  2 | 2024-10-08T13:24:44.9720211Z Runner name: 'ubuntu-20.04-4core-16gb_f6653e6045ce'
  3 | 2024-10-08T13:24:44.9721030Z Runner group name: 'Default Larger Runners'
  4 | 2024-10-08T13:24:44.9721819Z Machine name: 'runner'
  5 | 2024-10-08T13:24:44.9737572Z ##[group]Operating System
  6 | 2024-10-08T13:24:44.9738426Z Ubuntu
  7 | 2024-10-08T13:24:44.9738831Z 20.04.6
  8 | 2024-10-08T13:24:44.9739117Z LTS
  9 | 2024-10-08T13:24:44.9739465Z ##[endgroup]
 10 | 2024-10-08T13:24:44.9739831Z ##[group]Runner Image
 11 | 2024-10-08T13:24:44.9740250Z Image: ubuntu-20.04
 12 | 2024-10-08T13:24:44.9740641Z Version: 20240922.1.0
 13 | 2024-10-08T13:24:44.9741642Z Included Software: https://github.com/actions/runner-images/blob/ubuntu20/20240922.1/images/ubuntu/Ubuntu2004-Readme.md
 14 | 2024-10-08T13:24:44.9743054Z Image Release: https://github.com/actions/runner-images/releases/tag/ubuntu20%2F20240922.1
 15 | 2024-10-08T13:24:44.9743919Z ##[endgroup]
 16 | 2024-10-08T13:24:44.9758386Z ##[group]GITHUB_TOKEN Permissions
 17 | 2024-10-08T13:24:44.9760178Z Contents: read
 18 | 2024-10-08T13:24:44.9760679Z Metadata: read
 19 | 2024-10-08T13:24:44.9761095Z Packages: read
 20 | 2024-10-08T13:24:44.9761458Z ##[endgroup]
 21 | 2024-10-08T13:24:44.9764576Z Secret source: None
 22 | 2024-10-08T13:24:44.9765223Z Prepare workflow directory
 23 | 2024-10-08T13:24:45.0790330Z Prepare all required actions
 24 | 2024-10-08T13:24:45.0951557Z Getting action download info
 25 | 2024-10-08T13:24:45.2809779Z Download action repository 'msys2/setup-msys2@v2.22.0' (SHA:cc11e9188b693c2b100158c3322424c4cc1dadea)
 26 | 2024-10-08T13:24:45.7529926Z Download action repository 'actions/checkout@v4' (SHA:eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871)
 27 | 2024-10-08T13:24:46.0510433Z Download action repository 'actions/upload-artifact@v4' (SHA:604373da6381bf24206979c74d06a550515601b9)
 28 | 2024-10-08T13:24:46.6497915Z Complete job name: PR - mingw-check
 29 | 2024-10-08T13:24:46.7427541Z ##[group]Run git config --global core.autocrlf false
 30 | 2024-10-08T13:24:46.7428446Z git config --global core.autocrlf false
 31 | 2024-10-08T13:24:46.7460563Z shell: /usr/bin/bash --noprofile --norc -e -o pipefail {0}
 32 | 2024-10-08T13:24:46.7461452Z env:
 33 | 2024-10-08T13:24:46.7462007Z   TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
 34 | 2024-10-08T13:24:46.7462649Z   TOOLSTATE_REPO_ACCESS_TOKEN: 
 35 | 2024-10-08T13:24:46.7463065Z   CI_JOB_NAME: mingw-check
 36 | 2024-10-08T13:24:46.7463438Z   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
 37 | 2024-10-08T13:24:46.7463988Z   HEAD_SHA: 8c5b0edbe8c46fc2cd42de2cfc4cce2790262401
 38 | 2024-10-08T13:24:46.7464778Z   DOCKER_TOKEN: ***
 39 | 2024-10-08T13:24:46.7465139Z   SCCACHE_BUCKET: rust-lang-ci-sccache2
 40 | 2024-10-08T13:24:46.7465637Z   CACHE_DOMAIN: ci-caches.rust-lang.org
 41 | 2024-10-08T13:24:46.7466055Z ##[endgroup]
 42 | 2024-10-08T13:24:47.1799712Z ##[group]Run actions/checkout@v4
 43 | 2024-10-08T13:24:47.1800134Z with:
 44 | 2024-10-08T13:24:47.1800369Z   fetch-depth: 2
 45 | 2024-10-08T13:24:47.1800641Z   repository: rust-lang/rust
 46 | 2024-10-08T13:24:47.1801140Z   token: ***
 47 | 2024-10-08T13:24:47.1801377Z   ssh-strict: true
 48 | 2024-10-08T13:24:47.1801626Z   ssh-user: git
 49 | 2024-10-08T13:24:47.1801876Z   persist-credentials: true
 50 | 2024-10-08T13:24:47.1802183Z   clean: true
 51 | 2024-10-08T13:24:47.1802438Z   sparse-checkout-cone-mode: true
 52 | 2024-10-08T13:24:47.1802751Z   fetch-tags: false
 53 | 2024-10-08T13:24:47.1802996Z   show-progress: true
 54 | 2024-10-08T13:24:47.1803253Z   lfs: false
 55 | 2024-10-08T13:24:47.1803479Z   submodules: false
 56 | 2024-10-08T13:24:47.1803724Z   set-safe-directory: true
 57 | 2024-10-08T13:24:47.1804006Z env:
 58 | 2024-10-08T13:24:47.1804400Z   TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
 59 | 2024-10-08T13:24:47.1804896Z   TOOLSTATE_REPO_ACCESS_TOKEN: 
 60 | 2024-10-08T13:24:47.1805217Z   CI_JOB_NAME: mingw-check
 61 | 2024-10-08T13:24:47.1805672Z   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
 62 | 2024-10-08T13:24:47.1806109Z   HEAD_SHA: 8c5b0edbe8c46fc2cd42de2cfc4cce2790262401
 63 | 2024-10-08T13:24:47.1806629Z   DOCKER_TOKEN: ***
 64 | 2024-10-08T13:24:47.1806904Z   SCCACHE_BUCKET: rust-lang-ci-sccache2
 65 | 2024-10-08T13:24:47.1807265Z   CACHE_DOMAIN: ci-caches.rust-lang.org
 66 | 2024-10-08T13:24:47.1807593Z ##[endgroup]
 67 | 2024-10-08T13:24:47.2929497Z Syncing repository: rust-lang/rust
 68 | 2024-10-08T13:24:47.2931117Z ##[group]Getting Git version info
 69 | 2024-10-08T13:24:47.2931684Z Working directory is '/home/runner/work/rust/rust'
 70 | 2024-10-08T13:24:47.2932381Z [command]/usr/bin/git version
 71 | 2024-10-08T13:24:47.2932678Z git version 2.46.1
 72 | 2024-10-08T13:24:47.2933764Z ##[endgroup]
 73 | 2024-10-08T13:24:47.2945100Z Copying '/home/runner/.gitconfig' to '/home/runner/work/_temp/23e68be9-1b8b-4b10-8ace-ed08cab47f1e/.gitconfig'
 74 | 2024-10-08T13:24:47.2951942Z Temporarily overriding HOME='/home/runner/work/_temp/23e68be9-1b8b-4b10-8ace-ed08cab47f1e' before making global git config changes
 75 | 2024-10-08T13:24:47.2953149Z Adding repository directory to the temporary git global config as a safe directory
 76 | 2024-10-08T13:24:47.2957354Z [command]/usr/bin/git config --global --add safe.directory /home/runner/work/rust/rust
 77 | 2024-10-08T13:24:47.2987507Z Deleting the contents of '/home/runner/work/rust/rust'
 78 | 2024-10-08T13:24:47.2990999Z ##[group]Initializing the repository
 79 | 2024-10-08T13:24:47.2995629Z [command]/usr/bin/git init /home/runner/work/rust/rust
 80 | 2024-10-08T13:24:47.3680602Z hint: Using 'master' as the name for the initial branch. This default branch name
 81 | 2024-10-08T13:24:47.3681927Z hint: is subject to change. To configure the initial branch name to use in all
 82 | 2024-10-08T13:24:47.3683101Z hint: of your new repositories, which will suppress this warning, call:
 83 | 2024-10-08T13:24:47.3683909Z hint:
 84 | 2024-10-08T13:24:47.3684561Z hint: 	git config --global init.defaultBranch <name>
 85 | 2024-10-08T13:24:47.3685224Z hint:
 86 | 2024-10-08T13:24:47.3685726Z hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
 87 | 2024-10-08T13:24:47.3686482Z hint: 'development'. The just-created branch can be renamed via this command:
 88 | 2024-10-08T13:24:47.3686984Z hint:
 89 | 2024-10-08T13:24:47.3687251Z hint: 	git branch -m <name>
 90 | 2024-10-08T13:24:47.3687907Z Initialized empty Git repository in /home/runner/work/rust/rust/.git/
 91 | 2024-10-08T13:24:47.3694337Z [command]/usr/bin/git remote add origin https://github.com/rust-lang/rust
 92 | 2024-10-08T13:24:47.3788184Z ##[endgroup]
 93 | 2024-10-08T13:24:47.3789110Z ##[group]Disabling automatic garbage collection
 94 | 2024-10-08T13:24:47.3793450Z [command]/usr/bin/git config --local gc.auto 0
 95 | 2024-10-08T13:24:47.3821131Z ##[endgroup]
 96 | 2024-10-08T13:24:47.3821981Z ##[group]Setting up auth
 97 | 2024-10-08T13:24:47.3828842Z [command]/usr/bin/git config --local --name-only --get-regexp core\.sshCommand
 98 | 2024-10-08T13:24:47.3859866Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'core\.sshCommand' && git config --local --unset-all 'core.sshCommand' || :"
 99 | 2024-10-08T13:24:47.7730825Z [command]/usr/bin/git config --local --name-only --get-regexp http\.https\:\/\/github\.com\/\.extraheader
100 | 2024-10-08T13:24:47.7758099Z [command]/usr/bin/git submodule foreach --recursive sh -c "git config --local --name-only --get-regexp 'http\.https\:\/\/github\.com\/\.extraheader' && git config --local --unset-all 'http.https://github.com/.extraheader' || :"
101 | 2024-10-08T13:24:47.7977189Z [command]/usr/bin/git config --local http.https://github.com/.extraheader AUTHORIZATION: basic ***
102 | 2024-10-08T13:24:47.8011736Z ##[endgroup]
103 | 2024-10-08T13:24:47.8012475Z ##[group]Fetching the repository
104 | 2024-10-08T13:24:47.8020846Z [command]/usr/bin/git -c protocol.version=2 fetch --no-tags --prune --no-recurse-submodules --depth=2 origin +d7844ebeec0d4ff6b742a4f303ec63e2bfe74a0e:refs/remotes/pull/131375/merge
105 | 2024-10-08T13:24:53.8717035Z From https://github.com/rust-lang/rust
106 | 2024-10-08T13:24:53.8718509Z  * [new ref]           d7844ebeec0d4ff6b742a4f303ec63e2bfe74a0e -> pull/131375/merge
107 | 2024-10-08T13:24:53.8739637Z ##[endgroup]
108 | 2024-10-08T13:24:53.8740518Z ##[group]Determining the checkout info
109 | 2024-10-08T13:24:53.8741825Z ##[endgroup]
110 | 


--------------------------------------------------------------------------------