├── .dockerignore ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── ansi-to-html ├── Cargo.lock ├── Cargo.toml ├── fuzz │ ├── Cargo.lock │ ├── Cargo.toml │ └── fuzz_targets │ │ └── render.rs └── src │ ├── ansi.rs │ ├── lib.rs │ ├── main.rs │ ├── perform.rs │ └── renderer.rs ├── cargo-download ├── Cargo.lock ├── Cargo.toml └── src │ └── main.rs ├── ci-crates ├── ci.sh ├── docker ├── Dockerfile ├── Dockerfile.base ├── Dockerfile.ci ├── build.sh ├── nextest.toml ├── packages.txt └── run.sh ├── get-args ├── Cargo.lock ├── Cargo.toml └── src │ └── main.rs ├── htmlpty ├── Cargo.lock ├── Cargo.toml └── src │ └── main.rs ├── inapty ├── Cargo.lock ├── Cargo.toml └── src │ └── main.rs └── src ├── client.rs ├── db_dump.rs ├── diagnose.rs ├── main.rs ├── render.rs ├── run.rs └── sync.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | target/ 2 | logs/ 3 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | pull_request: 6 | merge_group: 7 | branches: 8 | - main 9 | 10 | name: CI 11 | 12 | jobs: 13 | suite_matrix: 14 | strategy: 15 | matrix: 16 | suite: [style, check, build, miri, asan, fuzz] 17 | runs-on: ubuntu-latest 18 | name: ${{ matrix.suite }} 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Rustup 22 | run: | 23 | rustup self update 24 | rustup default stable 25 | rustup update 26 | - uses: actions/cache@v4 27 | with: 28 | path: | 29 | ~/.cargo/registry 30 | ~/.cargo/bin/cargo-fuzz 31 | ~/.cargo/.crates2.json 32 | ~/.cargo/.crates.toml 33 | ~/.cargo/.global-cache 34 | ~/.cargo/.package-cache 35 | ~/.cargo/.package-cache-mutate 36 | ~/.cargo/.rustc_info.json 37 | target/ 38 | key: ${{ runner.os }}-${{ matrix.suite }}-${{ hashFiles('**/Cargo.lock') }} 39 | restore-keys: ${{ runner.os }}-$(rustc --version) 40 | - name: Run 41 | env: 42 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 43 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 44 | AWS_DEFAULT_REGION: us-east-1 45 | CARGO_INCREMENTAL: 0 46 | RUSTFLAGS: -Cdebuginfo=0 47 | run: | 48 | echo "::group::Install dependencies" 49 | set -o pipefail 50 | CARGO_TARGET_DIR=target cargo install htmlpty --locked --path htmlpty 51 | set +e 52 | echo "::endgroup" 53 | htmlpty bash ci.sh ${{ matrix.suite }} 2> output.html 54 | FAILED=$? 55 | aws s3 cp --content-type "text/html;charset=utf-8" output.html s3://miri-bot-dev/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${{ matrix.suite }}.html 56 | LOG_URL=https://miri-bot-dev.s3.amazonaws.com/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${{ matrix.suite }}.html 57 | if [ $FAILED -ne 0 ] 58 | then 59 | curl -L \ 60 | -X POST \ 61 | -H "Accept: application/vnd.github+json" \ 62 | -H "Authorization: Bearer ${{ secrets.github_token }}" \ 63 | -H "X-GitHub-Api-Version: 2022-11-28" \ 64 | ${{ github.event.pull_request.comments_url }} \ 65 | -d "{\"body\":\"$LOG_URL\"}" 66 | fi 67 | exit $FAILED 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | ansi-to-html/fuzz/corpus 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "crater-at-home" 3 | version = "0.1.0" 4 | edition = "2021" 5 | default-run = "crater-at-home" 6 | 7 | [dependencies] 8 | clap = { version = "4", features = ["derive"] } 9 | env_logger = "0.10" 10 | log = "0.4.17" 11 | serde = { version = "1.0.140", features = ["derive"] } 12 | ansi-to-html = { path = "ansi-to-html" } 13 | color-eyre = "0.6.2" 14 | semver = "1.0.12" 15 | tar = "0.4.38" 16 | flate2 = "1.0.24" 17 | ureq = "2.5.0" 18 | once_cell = "1.13.0" 19 | csv = "1.1.6" 20 | fxhash = "0.2.1" 21 | regex = "1.6.0" 22 | uuid = { version = "1.1.2", features = ["v4"] } 23 | num_cpus = "1.13.1" 24 | tokio = { version = "1.21.2", features = ["full"] } 25 | futures-util = "0.3.24" 26 | aws-sdk-s3 = "0.24" 27 | aws-smithy-types-convert = { version = "0.54", features = ["convert-time"] } 28 | aws-config = "0.54" 29 | serde_json = "1.0.96" 30 | xz2 = "0.1.7" 31 | backoff = { version = "0.4.0", features = ["futures", "tokio"] } 32 | time = { version = "0.3", features = ["std"] } 33 | 34 | [profile.release] 35 | panic = "abort" 36 | debug = 1 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Ben Kimock 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # crater-at-home 2 | 3 | > Mom, can I have a crater run? 4 | > 5 | > No, we have crater at home 6 | 7 | This project exists because I tried to use [crater](https://github.com/rust-lang/crater) or [rustwide](https://crates.io/crates/rustwide) to run `cargo miri test` on all published crates, and I found it very challenging to add support for Miri, and generally very difficult to hack on those projects. 8 | 9 | ## Intentional differences with Crater 10 | * Sandboxing restricts writes to anything below the crate root, not the target directory 11 | * Build/test commands are run attached to a pseudoterminal, output is uploaded raw and converted to HTML 12 | * Default resource limits per build are 1 CPU and 8 GB memory 13 | * All state is stored in S3, never on local disk 14 | * No limits on the size of output from builds (some of the logs do get large, but this has not been a problem) 15 | 16 | The outcome seems to be significantly greater reliability than crater: We do not see any of the causes of spurious results (apart from crates with flaky test suites) that crater does, and our uploads do not occasionally crash. 17 | I do not understand exactly what causes this to be so much more reliable than crater, but if someone can figure that out it would be great to improve crater. 18 | 19 | ## Architecture 20 | 21 | We sort of have a client-server model. 22 | The server is written in Rust and distributes work one job at a time, and uploads results to S3. 23 | The client is written in bash and runs inside a simple Docker container, which is just an Ubuntu image with a lot of packages installed, a Rust toolchain, and some utilities. 24 | The server `docker run`s a bunch of clients, then writes the names and versions of crates to their stdin, and reads build logs from their stdout. 25 | 26 | It is possible to expand the client-server communication to go over the network. 27 | I am not going to implement that, but if it can be done without blowing up the complexity of the project, such a contribution would be welcome. 28 | 29 | ## Sandboxing 30 | 31 | We provide sandboxing by mounting the client containers as read-only, with exceptions for `/tmp`, the Cargo registry, and the entire directory crates are built in. 32 | Unlike crater, if a crate's build system tries to write outside of its target directory, that is fine. 33 | The build directory is cleaned out between crates without taking down the container. 34 | 35 | ## Resource limits 36 | 37 | Every container is pinned to a single CPU with a cpuset. 38 | There are some crates that for whatever reason don't respect this and try to launch 2 or 64 jobs. 39 | Lol. 40 | 41 | Memory is very different. 42 | About 10% of crates need 2 GB peak, and in Miri that number is significantly higher. 43 | We impose a memory limit of 8 GB per container to prevent runaway resource usage, but in general it is strongly advised that you run this program heavily oversubscribed. 44 | On *average*, a container needs less than 1 GB, but averages are only relevant if outliers cannot be significant. 45 | It is generally advised to run this on a large system; I normally run it on a system with 64 CPUs (my desktop or a c6.16xlarge instance). 46 | 47 | ## Usage Suggestions 48 | 49 | This crate uses `clap`; run `cargo run -- --help` for CLI options. 50 | 51 | * Install docker (`sudo apt install docker.io`, `sudo pacman -S docker`, etc.) 52 | * Add yourself to docker group (`sudo adduser $USER docker`) 53 | * Re-login or `newgrp docker` to make your shell know about docker 54 | * `cargo run -- run --tool=miri --bucket=my-bucket-here` 55 | * Have lots of patience 56 | 57 | Contributions of or suggestions for more sophisticated data processing are welcome. 58 | -------------------------------------------------------------------------------- /ansi-to-html/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ansi-to-html" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "env_logger", 10 | "fnv", 11 | "log", 12 | "vte", 13 | ] 14 | 15 | [[package]] 16 | name = "arrayvec" 17 | version = "0.7.4" 18 | source = "registry+https://github.com/rust-lang/crates.io-index" 19 | checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" 20 | 21 | [[package]] 22 | name = "env_logger" 23 | version = "0.10.0" 24 | source = "registry+https://github.com/rust-lang/crates.io-index" 25 | checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" 26 | dependencies = [ 27 | "log", 28 | ] 29 | 30 | [[package]] 31 | name = "fnv" 32 | version = "1.0.7" 33 | source = "registry+https://github.com/rust-lang/crates.io-index" 34 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 35 | 36 | [[package]] 37 | name = "log" 38 | version = "0.4.20" 39 | source = "registry+https://github.com/rust-lang/crates.io-index" 40 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 41 | 42 | [[package]] 43 | name = "proc-macro2" 44 | version = "1.0.66" 45 | source = "registry+https://github.com/rust-lang/crates.io-index" 46 | checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" 47 | dependencies = [ 48 | "unicode-ident", 49 | ] 50 | 51 | [[package]] 52 | name = "quote" 53 | version = "1.0.33" 54 | source = "registry+https://github.com/rust-lang/crates.io-index" 55 | checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" 56 | dependencies = [ 57 | "proc-macro2", 58 | ] 59 | 60 | [[package]] 61 | name = "unicode-ident" 62 | version = "1.0.11" 63 | source = "registry+https://github.com/rust-lang/crates.io-index" 64 | checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" 65 | 66 | [[package]] 67 | name = "utf8parse" 68 | version = "0.2.1" 69 | source = "registry+https://github.com/rust-lang/crates.io-index" 70 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" 71 | 72 | [[package]] 73 | name = "vte" 74 | version = "0.11.1" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" 77 | dependencies = [ 78 | "arrayvec", 79 | "utf8parse", 80 | "vte_generate_state_changes", 81 | ] 82 | 83 | [[package]] 84 | name = "vte_generate_state_changes" 85 | version = "0.1.1" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" 88 | dependencies = [ 89 | "proc-macro2", 90 | "quote", 91 | ] 92 | -------------------------------------------------------------------------------- /ansi-to-html/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ansi-to-html" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | env_logger = { version = "0.10.0", default-features = false } 8 | fnv = "1.0.7" 9 | log = "0.4.17" 10 | vte = "0.11.0" 11 | -------------------------------------------------------------------------------- /ansi-to-html/fuzz/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ansi-to-html" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "env_logger", 10 | "log", 11 | "vte", 12 | ] 13 | 14 | [[package]] 15 | name = "ansi-to-html-fuzz" 16 | version = "0.0.0" 17 | dependencies = [ 18 | "ansi-to-html", 19 | "libfuzzer-sys", 20 | ] 21 | 22 | [[package]] 23 | name = "arbitrary" 24 | version = "1.3.0" 25 | source = "registry+https://github.com/rust-lang/crates.io-index" 26 | checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" 27 | 28 | [[package]] 29 | name = "arrayvec" 30 | version = "0.7.4" 31 | source = "registry+https://github.com/rust-lang/crates.io-index" 32 | checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" 33 | 34 | [[package]] 35 | name = "cc" 36 | version = "1.0.83" 37 | source = "registry+https://github.com/rust-lang/crates.io-index" 38 | checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" 39 | dependencies = [ 40 | "jobserver", 41 | "libc", 42 | ] 43 | 44 | [[package]] 45 | name = "env_logger" 46 | version = "0.10.0" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" 49 | dependencies = [ 50 | "log", 51 | ] 52 | 53 | [[package]] 54 | name = "jobserver" 55 | version = "0.1.26" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" 58 | dependencies = [ 59 | "libc", 60 | ] 61 | 62 | [[package]] 63 | name = "libc" 64 | version = "0.2.147" 65 | source = "registry+https://github.com/rust-lang/crates.io-index" 66 | checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" 67 | 68 | [[package]] 69 | name = "libfuzzer-sys" 70 | version = "0.4.7" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" 73 | dependencies = [ 74 | "arbitrary", 75 | "cc", 76 | "once_cell", 77 | ] 78 | 79 | [[package]] 80 | name = "log" 81 | version = "0.4.20" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 84 | 85 | [[package]] 86 | name = "once_cell" 87 | version = "1.18.0" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" 90 | 91 | [[package]] 92 | name = "proc-macro2" 93 | version = "1.0.66" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" 96 | dependencies = [ 97 | "unicode-ident", 98 | ] 99 | 100 | [[package]] 101 | name = "quote" 102 | version = "1.0.33" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" 105 | dependencies = [ 106 | "proc-macro2", 107 | ] 108 | 109 | [[package]] 110 | name = "unicode-ident" 111 | version = "1.0.11" 112 | source = "registry+https://github.com/rust-lang/crates.io-index" 113 | checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" 114 | 115 | [[package]] 116 | name = "utf8parse" 117 | version = "0.2.1" 118 | source = "registry+https://github.com/rust-lang/crates.io-index" 119 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" 120 | 121 | [[package]] 122 | name = "vte" 123 | version = "0.11.1" 124 | source = "registry+https://github.com/rust-lang/crates.io-index" 125 | checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" 126 | dependencies = [ 127 | "arrayvec", 128 | "utf8parse", 129 | "vte_generate_state_changes", 130 | ] 131 | 132 | [[package]] 133 | name = "vte_generate_state_changes" 134 | version = "0.1.1" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" 137 | dependencies = [ 138 | "proc-macro2", 139 | "quote", 140 | ] 141 | -------------------------------------------------------------------------------- /ansi-to-html/fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "ansi-to-html-fuzz" 3 | version = "0.0.0" 4 | publish = false 5 | edition = "2021" 6 | 7 | [package.metadata] 8 | cargo-fuzz = true 9 | 10 | [dependencies] 11 | libfuzzer-sys = "0.4" 12 | 13 | [dependencies.ansi-to-html] 14 | path = ".." 15 | 16 | # Prevent this from interfering with workspaces 17 | [workspace] 18 | members = ["."] 19 | 20 | [profile.release] 21 | debug = 1 22 | 23 | [[bin]] 24 | name = "render" 25 | path = "fuzz_targets/render.rs" 26 | test = false 27 | doc = false 28 | -------------------------------------------------------------------------------- /ansi-to-html/fuzz/fuzz_targets/render.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | 3 | use libfuzzer_sys::fuzz_target; 4 | 5 | fuzz_target!(|data: &[u8]| { 6 | let _ = ansi_to_html::render(String::new(), data); 7 | }); 8 | -------------------------------------------------------------------------------- /ansi-to-html/src/ansi.rs: -------------------------------------------------------------------------------- 1 | /// C0 set of 7-bit control characters (from ANSI X3.4-1977). 2 | #[allow(non_snake_case)] 3 | pub mod C0 { 4 | #![allow(unused)] 5 | /// Null filler, terminal should ignore this character. 6 | pub const NUL: u8 = 0x00; 7 | /// Start of Header. 8 | pub const SOH: u8 = 0x01; 9 | /// Start of Text, implied end of header. 10 | pub const STX: u8 = 0x02; 11 | /// End of Text, causes some terminal to respond with ACK or NAK. 12 | pub const ETX: u8 = 0x03; 13 | /// End of Transmission. 14 | pub const EOT: u8 = 0x04; 15 | /// Enquiry, causes terminal to send ANSWER-BACK ID. 16 | pub const ENQ: u8 = 0x05; 17 | /// Acknowledge, usually sent by terminal in response to ETX. 18 | pub const ACK: u8 = 0x06; 19 | /// Bell, triggers the bell, buzzer, or beeper on the terminal. 20 | pub const BEL: u8 = 0x07; 21 | /// Backspace, can be used to define overstruck characters. 22 | pub const BS: u8 = 0x08; 23 | /// Horizontal Tabulation, move to next predetermined position. 24 | pub const HT: u8 = 0x09; 25 | /// Linefeed, move to same position on next line (see also NL). 26 | pub const LF: u8 = 0x0A; 27 | /// Vertical Tabulation, move to next predetermined line. 28 | pub const VT: u8 = 0x0B; 29 | /// Form Feed, move to next form or page. 30 | pub const FF: u8 = 0x0C; 31 | /// Carriage Return, move to first character of current line. 32 | pub const CR: u8 = 0x0D; 33 | /// Shift Out, switch to G1 (other half of character set). 34 | pub const SO: u8 = 0x0E; 35 | /// Shift In, switch to G0 (normal half of character set). 36 | pub const SI: u8 = 0x0F; 37 | /// Data Link Escape, interpret next control character specially. 38 | pub const DLE: u8 = 0x10; 39 | /// (DC1) Terminal is allowed to resume transmitting. 40 | pub const XON: u8 = 0x11; 41 | /// Device Control 2, causes ASR-33 to activate paper-tape reader. 42 | pub const DC2: u8 = 0x12; 43 | /// (DC2) Terminal must pause and refrain from transmitting. 44 | pub const XOFF: u8 = 0x13; 45 | /// Device Control 4, causes ASR-33 to deactivate paper-tape reader. 46 | pub const DC4: u8 = 0x14; 47 | /// Negative Acknowledge, used sometimes with ETX and ACK. 48 | pub const NAK: u8 = 0x15; 49 | /// Synchronous Idle, used to maintain timing in Sync communication. 50 | pub const SYN: u8 = 0x16; 51 | /// End of Transmission block. 52 | pub const ETB: u8 = 0x17; 53 | /// Cancel (makes VT100 abort current escape sequence if any). 54 | pub const CAN: u8 = 0x18; 55 | /// End of Medium. 56 | pub const EM: u8 = 0x19; 57 | /// Substitute (VT100 uses this to display parity errors). 58 | pub const SUB: u8 = 0x1A; 59 | /// Prefix to an escape sequence. 60 | pub const ESC: u8 = 0x1B; 61 | /// File Separator. 62 | pub const FS: u8 = 0x1C; 63 | /// Group Separator. 64 | pub const GS: u8 = 0x1D; 65 | /// Record Separator (sent by VT132 in block-transfer mode). 66 | pub const RS: u8 = 0x1E; 67 | /// Unit Separator. 68 | pub const US: u8 = 0x1F; 69 | /// Delete, should be ignored by terminal. 70 | pub const DEL: u8 = 0x7f; 71 | } 72 | 73 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 74 | pub enum Color { 75 | EightBit(u8), 76 | Rgb([u8; 7]), 77 | } 78 | 79 | const COLORS: [&str; 256] = [ 80 | "#000", "#a00", "#0a0", "#a60", "#00a", "#a0a", "#0aa", "#aaa", "#555", "#f55", "#5f5", "#ff5", 81 | "#55f", "#f5f", "#5ff", "#fff", "#000", "#00005f", "#000087", "#0000af", "#0000d7", "#00f", 82 | "#005f00", "#005f5f", "#005f87", "#005faf", "#005fd7", "#005fff", "#008700", "#00875f", 83 | "#008787", "#0087af", "#0087d7", "#0087ff", "#00af00", "#00af5f", "#00af87", "#00afaf", 84 | "#00afd7", "#00afff", "#00d700", "#00d75f", "#00d787", "#00d7af", "#00d7d7", "#00d7ff", "#0f0", 85 | "#00ff5f", "#00ff87", "#00ffaf", "#00ffd7", "#0ff", "#5f0000", "#5f005f", "#5f0087", "#5f00af", 86 | "#5f00d7", "#5f00ff", "#5f5f00", "#5f5f5f", "#5f5f87", "#5f5faf", "#5f5fd7", "#5f5fff", 87 | "#5f8700", "#5f875f", "#5f8787", "#5f87af", "#5f87d7", "#5f87ff", "#5faf00", "#5faf5f", 88 | "#5faf87", "#5fafaf", "#5fafd7", "#5fafff", "#5fd700", "#5fd75f", "#5fd787", "#5fd7af", 89 | "#5fd7d7", "#5fd7ff", "#5fff00", "#5fff5f", "#5fff87", "#5fffaf", "#5fffd7", "#5fffff", 90 | "#870000", "#87005f", "#870087", "#8700af", "#8700d7", "#8700ff", "#875f00", "#875f5f", 91 | "#875f87", "#875faf", "#875fd7", "#875fff", "#878700", "#87875f", "#878787", "#8787af", 92 | "#8787d7", "#8787ff", "#87af00", "#87af5f", "#87af87", "#87afaf", "#87afd7", "#87afff", 93 | "#87d700", "#87d75f", "#87d787", "#87d7af", "#87d7d7", "#87d7ff", "#87ff00", "#87ff5f", 94 | "#87ff87", "#87ffaf", "#87ffd7", "#87ffff", "#af0000", "#af005f", "#af0087", "#af00af", 95 | "#af00d7", "#af00ff", "#af5f00", "#af5f5f", "#af5f87", "#af5faf", "#af5fd7", "#af5fff", 96 | "#af8700", "#af875f", "#af8787", "#af87af", "#af87d7", "#af87ff", "#afaf00", "#afaf5f", 97 | "#afaf87", "#afafaf", "#afafd7", "#afafff", "#afd700", "#afd75f", "#afd787", "#afd7af", 98 | "#afd7d7", "#afd7ff", "#afff00", "#afff5f", "#afff87", "#afffaf", "#afffd7", "#afffff", 99 | "#d70000", "#d7005f", "#d70087", "#d700af", "#d700d7", "#d700ff", "#d75f00", "#d75f5f", 100 | "#d75f87", "#d75faf", "#d75fd7", "#d75fff", "#d78700", "#d7875f", "#d78787", "#d787af", 101 | "#d787d7", "#d787ff", "#d7af00", "#d7af5f", "#d7af87", "#d7afaf", "#d7afd7", "#d7afff", 102 | "#d7d700", "#d7d75f", "#d7d787", "#d7d7af", "#d7d7d7", "#d7d7ff", "#d7ff00", "#d7ff5f", 103 | "#d7ff87", "#d7ffaf", "#d7ffd7", "#d7ffff", "#f00", "#ff005f", "#ff0087", "#ff00af", "#ff00d7", 104 | "#f0f", "#ff5f00", "#ff5f5f", "#ff5f87", "#ff5faf", "#ff5fd7", "#ff5fff", "#ff8700", "#ff875f", 105 | "#ff8787", "#ff87af", "#ff87d7", "#ff87ff", "#ffaf00", "#ffaf5f", "#ffaf87", "#ffafaf", 106 | "#ffafd7", "#ffafff", "#ffd700", "#ffd75f", "#ffd787", "#ffd7af", "#ffd7d7", "#ffd7ff", "#ff0", 107 | "#ffff5f", "#ffff87", "#ffffaf", "#ffffd7", "#fff", "#080808", "#121212", "#1c1c1c", "#262626", 108 | "#303030", "#3a3a3a", "#444", "#4e4e4e", "#585858", "#626262", "#6c6c6c", "#767676", "#808080", 109 | "#8a8a8a", "#949494", "#9e9e9e", "#a8a8a8", "#b2b2b2", "#bcbcbc", "#c6c6c6", "#d0d0d0", 110 | "#dadada", "#e4e4e4", "#eee", 111 | ]; 112 | 113 | impl Color { 114 | pub fn as_str(&self) -> &str { 115 | match self { 116 | Color::EightBit(code) => COLORS[*code as usize], 117 | Color::Rgb(bytes) => std::str::from_utf8(&bytes[..]).unwrap(), 118 | } 119 | } 120 | 121 | pub fn black() -> Self { 122 | Color::EightBit(0) 123 | } 124 | 125 | pub fn red() -> Self { 126 | Color::EightBit(1) 127 | } 128 | 129 | pub fn green() -> Self { 130 | Color::EightBit(2) 131 | } 132 | 133 | pub fn yellow() -> Self { 134 | Color::EightBit(3) 135 | } 136 | 137 | pub fn blue() -> Self { 138 | Color::EightBit(4) 139 | } 140 | 141 | pub fn magenta() -> Self { 142 | Color::EightBit(5) 143 | } 144 | 145 | pub fn cyan() -> Self { 146 | Color::EightBit(6) 147 | } 148 | 149 | pub fn white() -> Self { 150 | Color::EightBit(7) 151 | } 152 | 153 | pub fn bright_black() -> Self { 154 | Color::EightBit(8) 155 | } 156 | 157 | pub fn bright_red() -> Self { 158 | Color::EightBit(9) 159 | } 160 | 161 | pub fn bright_green() -> Self { 162 | Color::EightBit(10) 163 | } 164 | 165 | pub fn bright_yellow() -> Self { 166 | Color::EightBit(11) 167 | } 168 | 169 | pub fn bright_blue() -> Self { 170 | Color::EightBit(12) 171 | } 172 | 173 | pub fn bright_magenta() -> Self { 174 | Color::EightBit(13) 175 | } 176 | 177 | pub fn bright_cyan() -> Self { 178 | Color::EightBit(14) 179 | } 180 | 181 | pub fn bright_white() -> Self { 182 | Color::EightBit(15) 183 | } 184 | 185 | pub fn parse_8bit(code: u16) -> Option { 186 | Some(match code { 187 | 0..=255 => Color::EightBit(code as u8), 188 | _ => return None, 189 | }) 190 | } 191 | 192 | pub fn parse_rgb(r: u16, b: u16, g: u16) -> Option { 193 | use std::io::Write; 194 | if r > 255 || b > 255 || g > 255 { 195 | return None; 196 | } 197 | let mut bytes = [b'#'; 7]; 198 | write!(&mut bytes[1..], "{:02x}", r).unwrap(); 199 | write!(&mut bytes[3..], "{:02x}", g).unwrap(); 200 | write!(&mut bytes[5..], "{:02x}", b).unwrap(); 201 | Some(Color::Rgb(bytes)) 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /ansi-to-html/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | 3 | mod ansi; 4 | mod perform; 5 | mod renderer; 6 | 7 | pub struct Handle { 8 | renderer: renderer::Renderer, 9 | parser: vte::Parser, 10 | } 11 | 12 | impl Handle { 13 | pub fn new(mut out: W) -> Self { 14 | out.write_all( 15 | br#" 16 | 
"#
17 |         ).unwrap();
18 |         Self {
19 |             renderer: renderer::Renderer::new(out, String::new()),
20 |             parser: vte::Parser::new(),
21 |         }
22 |     }
23 | 
24 |     pub fn finish(&mut self) -> std::io::Result<()> {
25 |         self.renderer.emit_html()?;
26 |         self.renderer.out.write_all(b"
") 29 | } 30 | } 31 | 32 | impl Write for Handle { 33 | fn write(&mut self, buf: &[u8]) -> std::io::Result { 34 | for b in buf { 35 | self.parser.advance(&mut self.renderer, *b); 36 | } 37 | Ok(buf.len()) 38 | } 39 | 40 | fn flush(&mut self) -> std::io::Result<()> { 41 | Ok(()) 42 | } 43 | } 44 | 45 | pub fn render(name: String, bytes: &[u8]) -> (String, String) { 46 | let mut html = Vec::new(); 47 | let mut handle = Handle { 48 | renderer: renderer::Renderer::new(&mut html, name), 49 | parser: vte::Parser::new(), 50 | }; 51 | handle.write_all(&bytes).unwrap(); 52 | handle.renderer.emit_html().unwrap(); 53 | 54 | let mut css = Vec::new(); 55 | handle.renderer.out = &mut css; 56 | handle.renderer.emit_css().unwrap(); 57 | 58 | ( 59 | String::from_utf8(css).unwrap(), 60 | String::from_utf8(html).unwrap(), 61 | ) 62 | } 63 | -------------------------------------------------------------------------------- /ansi-to-html/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::io::{copy, stdin, stdout, BufWriter}; 2 | 3 | fn main() { 4 | env_logger::init(); 5 | let out = BufWriter::new(stdout().lock()); 6 | let mut handle = ansi_to_html::Handle::new(out); 7 | copy(&mut stdin().lock(), &mut handle).unwrap(); 8 | handle.finish().unwrap() 9 | } 10 | -------------------------------------------------------------------------------- /ansi-to-html/src/perform.rs: -------------------------------------------------------------------------------- 1 | use crate::ansi::{Color, C0}; 2 | use crate::renderer::Renderer; 3 | use std::io::Write; 4 | use vte::{Params, ParamsIter, Perform}; 5 | 6 | impl Perform for Renderer { 7 | fn print(&mut self, c: char) { 8 | self.print(c); 9 | } 10 | 11 | #[inline] 12 | fn execute(&mut self, byte: u8) { 13 | match byte { 14 | 0 | 1 => {} // wat 15 | C0::HT => self.put_tab(), 16 | C0::BS => self.backspace(), 17 | C0::CR => self.carriage_return(), 18 | C0::LF | C0::VT | C0::FF => self.linefeed(), 19 | C0::BEL => {} 20 | C0::US => {} // Unit separator, which is for machines, ignored by terminals. 21 | //C0::SUB => self.substitute(), 22 | //C0::SI => self.set_active_charset(CharsetIndex::G0), 23 | //C0::SO => self.set_active_charset(CharsetIndex::G1), 24 | _ => { 25 | log::warn!("Unhandled execute byte={:02x} in {:?}", byte, self.name) 26 | } 27 | } 28 | } 29 | 30 | fn csi_dispatch( 31 | &mut self, 32 | params: &Params, 33 | _intermediates: &[u8], 34 | _ignore: bool, 35 | action: char, 36 | ) { 37 | if action == 'm' { 38 | let mut it = params.iter(); 39 | while let Some(p) = it.next() { 40 | match p { 41 | &[0] => { 42 | self.bold = false; 43 | self.italic = false; 44 | self.underline = false; 45 | self.dim = false; 46 | self.foreground = Color::bright_white(); 47 | self.background = Color::black(); 48 | } 49 | &[1] => self.bold = true, 50 | &[2] => self.dim = true, 51 | &[3] => self.italic = true, 52 | &[4] => self.underline = true, 53 | &[5] => { 54 | // Slow blink (we don't blink) 55 | } 56 | &[7] => { 57 | // Reverse video or invert. Inconsistent emulation. 58 | } 59 | &[22] => { 60 | self.bold = false; 61 | self.dim = false; 62 | // Set intensity to normal 63 | } 64 | &[23] => { 65 | self.italic = false; 66 | // Neither italic nor blackletter(?) 67 | } 68 | &[25] => { 69 | // Turn blinking off (we don't blink) 70 | } 71 | &[30] => self.foreground = Color::black(), 72 | &[31] => self.foreground = Color::red(), 73 | &[32] => self.foreground = Color::green(), 74 | &[33] => self.foreground = Color::yellow(), 75 | &[34] => self.foreground = Color::blue(), 76 | &[35] => self.foreground = Color::magenta(), 77 | &[36] => self.foreground = Color::cyan(), 78 | &[37] => self.foreground = Color::white(), 79 | // 8-bit foreground color 80 | &[38] => { 81 | if let Some(color) = parse_color(&mut it) { 82 | self.foreground = color; 83 | } else { 84 | log::warn!("Unhandled m 48: {:?} in {:?}", params, self.name); 85 | } 86 | } 87 | &[39] => self.foreground = Color::bright_white(), // Default foreground color 88 | &[40] => self.background = Color::black(), 89 | &[41] => self.background = Color::red(), 90 | &[42] => self.background = Color::green(), 91 | &[43] => self.background = Color::yellow(), 92 | &[44] => self.background = Color::blue(), 93 | &[45] => self.background = Color::magenta(), 94 | &[46] => self.background = Color::cyan(), 95 | &[47] => self.background = Color::white(), 96 | &[48] => { 97 | if let Some(color) = parse_color(&mut it) { 98 | self.background = color; 99 | } else { 100 | log::warn!("Unhandled m 48: {:?} in {:?}", params, self.name); 101 | } 102 | } 103 | &[49] => self.background = Color::black(), // Default foreground color 104 | &[90] => self.foreground = Color::bright_black(), 105 | &[91] => self.foreground = Color::bright_red(), 106 | &[92] => self.foreground = Color::bright_green(), 107 | &[93] => self.foreground = Color::bright_yellow(), 108 | &[94] => self.foreground = Color::bright_blue(), 109 | &[95] => self.foreground = Color::bright_magenta(), 110 | &[96] => self.foreground = Color::bright_cyan(), 111 | &[97] => self.foreground = Color::bright_white(), 112 | 113 | &[100] => self.background = Color::bright_black(), 114 | &[101] => self.background = Color::bright_red(), 115 | &[102] => self.background = Color::bright_green(), 116 | &[103] => self.background = Color::bright_yellow(), 117 | &[104] => self.background = Color::bright_blue(), 118 | &[105] => self.background = Color::bright_magenta(), 119 | &[106] => self.background = Color::bright_cyan(), 120 | &[107] => self.background = Color::bright_white(), 121 | 122 | _ => { 123 | log::warn!( 124 | "Unhandled m with unknown start: {:?} in {:?}", 125 | params, 126 | self.name 127 | ) 128 | } 129 | } 130 | } 131 | } else if action == 'H' { 132 | let mut it = params.iter(); 133 | let row = if let Some(&[row]) = it.next() { row } else { 1 }; 134 | let col = if let Some(&[col]) = it.next() { col } else { 1 }; 135 | self.handle_move(row, col); 136 | } else if action == 'A' { 137 | if let Some(&[cells]) = params.iter().next() { 138 | self.move_up_by(cells); 139 | } 140 | } else if action == 'B' { 141 | if let Some(&[cells]) = params.iter().next() { 142 | self.move_down_by(cells); 143 | } 144 | } else if action == 'C' { 145 | if let Some(&[cells]) = params.iter().next() { 146 | self.move_right_by(cells); 147 | } 148 | } else if action == 'D' { 149 | if let Some(&[cells]) = params.iter().next() { 150 | self.move_left_by(cells); 151 | } 152 | } else if action == 'J' { 153 | self.erase_in_display(params.get::<1>().map(|a| a[0])); 154 | } else if action == 'K' { 155 | self.erase_in_line(params.get::<1>().map(|a| a[0])); 156 | } else if action == 'h' || action == 'l' { 157 | // show/hide the cursor. Nothing for us to do. 158 | } else if action == 'F' { 159 | self.move_up_by(params.get::<1>().map(|a| a[0]).unwrap_or(1)); 160 | self.set_column(1); 161 | } else if action == 'G' { 162 | self.set_column(params.get::<1>().map(|a| a[0]).unwrap_or(1)); 163 | } else { 164 | log::warn!( 165 | "Unhandled dispatch {} {:?} in {:?}", 166 | action, 167 | params, 168 | self.name 169 | ); 170 | } 171 | } 172 | } 173 | 174 | #[inline] 175 | fn parse_color(it: &mut ParamsIter) -> Option { 176 | match it.next() { 177 | Some(&[5]) => { 178 | let code = it.next()?; 179 | Color::parse_8bit(code[0]) 180 | } 181 | Some(&[2]) => { 182 | let r = it.next()?; 183 | let g = it.next()?; 184 | let b = it.next()?; 185 | Color::parse_rgb(r[0], g[0], b[0]) 186 | } 187 | _ => None, 188 | } 189 | } 190 | 191 | trait ParamsExt { 192 | fn get(&self) -> Option<[u16; N]>; 193 | } 194 | 195 | impl ParamsExt for Params { 196 | fn get(&self) -> Option<[u16; N]> { 197 | if self.len() != N { 198 | return None; 199 | } 200 | let mut out = [0u16; N]; 201 | for (p, o) in self.iter().zip(out.iter_mut()) { 202 | *o = p[0]; 203 | } 204 | Some(out) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /ansi-to-html/src/renderer.rs: -------------------------------------------------------------------------------- 1 | use crate::ansi::Color; 2 | use std::cell::RefCell; 3 | use std::collections::VecDeque; 4 | use std::io::Write; 5 | use fnv::FnvHashMap as HashMap; 6 | 7 | // This is the number of rows that inapty uses, should be good enough? 8 | const MAX_ROWS: usize = 64; 9 | 10 | pub struct Renderer { 11 | pub name: String, 12 | pub bold: bool, 13 | pub italic: bool, 14 | pub underline: bool, 15 | pub dim: bool, 16 | pub foreground: Color, 17 | pub background: Color, 18 | current_row: usize, 19 | rows: VecDeque, 20 | styles: Styles, 21 | pub out: W, 22 | prev: Cell, 23 | } 24 | 25 | #[derive(Debug, Default)] 26 | pub struct Styles { 27 | known: RefCell>, 28 | } 29 | 30 | const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz"; 31 | const BASE: usize = ALPHABET.len(); 32 | 33 | impl Styles { 34 | fn with(&self, color: Color, bold: bool, mut func: impl FnMut(&str) -> T) -> T { 35 | let mut known = self.known.borrow_mut(); 36 | let mut next_idx = known.len(); 37 | let name = known.entry((color, bold)).or_insert_with(|| { 38 | let mut name = String::new(); 39 | loop { 40 | name.push(ALPHABET[next_idx % BASE] as char); 41 | next_idx /= BASE; 42 | if next_idx == 0 { 43 | break; 44 | } 45 | } 46 | name 47 | }); 48 | func(&name) 49 | } 50 | } 51 | 52 | struct Row { 53 | cells: Vec, 54 | position: usize, 55 | } 56 | 57 | impl Row { 58 | const LEN: usize = 256; 59 | 60 | fn new() -> Self { 61 | Row { 62 | cells: vec![Cell::default(); Row::LEN], 63 | position: 0, 64 | } 65 | } 66 | 67 | fn clear(&mut self) { 68 | self.cells.truncate(Row::LEN); 69 | for c in &mut self.cells { 70 | *c = Cell::default(); 71 | } 72 | self.position = 0; 73 | } 74 | 75 | fn erase(&mut self) { 76 | for c in &mut self.cells { 77 | c.text = ' '; 78 | } 79 | } 80 | 81 | fn seek(&mut self, position: usize) { 82 | self.position = position; 83 | } 84 | 85 | #[inline] 86 | fn print(&mut self, cell: Cell) { 87 | if let Some(current) = self.cells.get_mut(self.position) { 88 | *current = cell; 89 | } else { 90 | self.print_cold(cell); 91 | } 92 | self.position += 1; 93 | } 94 | 95 | #[cold] 96 | #[inline(never)] 97 | fn print_cold(&mut self, cell: Cell) { 98 | self.cells.push(cell); 99 | } 100 | } 101 | 102 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] 103 | pub struct Cell { 104 | text: char, // FIXME: totally wrong, graphmeme clusters 105 | foreground: Color, 106 | background: Color, 107 | bold: bool, 108 | italic: bool, 109 | underline: bool, 110 | dim: bool, 111 | } 112 | 113 | impl Default for Cell { 114 | fn default() -> Self { 115 | Self { 116 | text: ' ', 117 | foreground: Color::bright_white(), 118 | background: Color::black(), 119 | bold: false, 120 | italic: false, 121 | underline: false, 122 | dim: false, 123 | } 124 | } 125 | } 126 | 127 | impl Renderer { 128 | pub fn new(out: W, name: String) -> Self { 129 | Self { 130 | name, 131 | bold: false, 132 | italic: false, 133 | underline: false, 134 | dim: false, 135 | foreground: Color::bright_white(), 136 | background: Color::black(), 137 | current_row: 0, 138 | rows: vec![Row::new()].into(), 139 | styles: Styles::default(), 140 | out, 141 | prev: Cell::default(), 142 | } 143 | } 144 | 145 | pub fn print(&mut self, c: char) { 146 | let cell = Cell { 147 | text: c, 148 | background: self.background, 149 | foreground: self.foreground, 150 | bold: self.bold, 151 | italic: self.italic, 152 | underline: self.underline, 153 | dim: self.dim, 154 | }; 155 | self.current_row().print(cell); 156 | } 157 | 158 | fn current_row(&mut self) -> &mut Row { 159 | &mut self.rows[self.current_row] 160 | } 161 | 162 | pub fn put_tab(&mut self) { 163 | self.print(' '); 164 | self.print(' '); 165 | self.print(' '); 166 | self.print(' '); 167 | } 168 | 169 | pub fn backspace(&mut self) { 170 | self.current_row().position = self.current_row().position.saturating_sub(1); 171 | } 172 | 173 | pub fn carriage_return(&mut self) { 174 | self.current_row().seek(0); 175 | } 176 | 177 | pub fn linefeed(&mut self) { 178 | if self.current_row == MAX_ROWS - 1 { 179 | // Pushing something off the screen 180 | let mut row = self.rows.pop_front().unwrap(); 181 | self.render(&row).unwrap(); 182 | row.clear(); 183 | self.rows.push_back(row); 184 | } else if self.current_row == self.rows.len() - 1 { 185 | // Not pushing something off screen, but we need a new row 186 | self.rows.push_back(Row::new()); 187 | self.current_row += 1; 188 | } else { 189 | // Moving within the screen 190 | self.current_row += 1; 191 | } 192 | } 193 | 194 | pub fn erase_in_display(&mut self, mode: Option) { 195 | // Ignore attempts to clear the whole screen 196 | if mode == Some(2) || mode == Some(3) { 197 | return; 198 | } 199 | log::warn!("Unimplemented erase_in_display {:?}", mode); 200 | } 201 | 202 | pub fn erase_in_line(&mut self, mode: Option) { 203 | let row = self.current_row(); 204 | match mode.unwrap_or(0) { 205 | 0 => { 206 | row.cells.truncate(row.position + 1); 207 | } 208 | 1 => { 209 | for cell in &mut row.cells[..row.position] { 210 | *cell = Cell::default(); 211 | } 212 | } 213 | 2 => { 214 | self.current_row().erase(); 215 | } 216 | _ => {} 217 | } 218 | } 219 | 220 | pub fn handle_move(&mut self, row: u16, col: u16) { 221 | if row <= self.current_row as u16 { 222 | self.move_up_by(self.current_row as u16 - row) 223 | } else { 224 | self.move_down_by(row - self.current_row as u16); 225 | } 226 | self.set_column(col); 227 | } 228 | 229 | pub fn move_up_by(&mut self, cells: u16) { 230 | self.current_row = self.current_row.saturating_sub(cells as usize); 231 | } 232 | 233 | pub fn move_down_by(&mut self, cells: u16) { 234 | for _ in 0..cells { 235 | self.linefeed(); 236 | } 237 | } 238 | 239 | pub fn move_right_by(&mut self, cells: u16) { 240 | let pos = (self.current_row().position as u16).saturating_add(cells); 241 | self.set_column(pos); 242 | } 243 | 244 | pub fn move_left_by(&mut self, cells: u16) { 245 | self.current_row().position = self.current_row().position.saturating_sub(cells as usize); 246 | } 247 | 248 | #[inline] 249 | pub fn set_column(&mut self, cells: u16) { 250 | let row = self.current_row(); 251 | row.position = cells.saturating_sub(1) as usize; 252 | while row.cells.len() < row.position { 253 | row.cells.push(Cell::default()); 254 | } 255 | let _ = &row.cells[..row.position]; 256 | } 257 | 258 | #[inline] 259 | fn render(&mut self, row: &Row) -> std::io::Result<()> { 260 | for cell in &row.cells { 261 | // Terminal applications will often reset the style right after some formatted text 262 | // then write some whitespace then set it to something again. 263 | // So we only apply style changes if the cell is nonempty. This is a ~50% savings 264 | // in emitted HTML. 265 | let text = cell.text; 266 | if text != ' ' { 267 | if cell.bold != self.prev.bold || cell.foreground != self.prev.foreground { 268 | self.out.write_all(b"")?; 273 | } 274 | self.prev = *cell; 275 | } 276 | match text { 277 | '<' => self.out.write_all(b"<")?, 278 | '>' => self.out.write_all(b">")?, 279 | c => { 280 | let mut bytes = [0u8; 4]; 281 | let s = c.encode_utf8(&mut bytes); 282 | self.out.write_all(s.as_bytes())?; 283 | } 284 | } 285 | } 286 | self.out.write_all(&[b'\n'])?; 287 | Ok(()) 288 | } 289 | 290 | pub fn emit_html(&mut self) -> std::io::Result<()> { 291 | self.out.write_all(b"")?; 292 | for row in core::mem::take(&mut self.rows) { 293 | self.render(&row)?; 294 | } 295 | self.out.write_all(b"") 296 | } 297 | 298 | pub fn emit_css(&mut self) -> std::io::Result<()> { 299 | for ((color, bold), name) in self.styles.known.borrow().iter() { 300 | write!( 301 | self.out, 302 | ".{}{{color:{};font-weight:{}}}\n", 303 | name, 304 | color.as_str(), 305 | if *bold { "bold" } else { "normal" } 306 | )?; 307 | } 308 | 309 | Ok(()) 310 | } 311 | 312 | /* 313 | pub fn clear(&mut self) { 314 | self.rows.clear(); 315 | self.rows.push(Row::new()); 316 | self.current_row = 0; 317 | } 318 | */ 319 | } 320 | -------------------------------------------------------------------------------- /cargo-download/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "adler" 7 | version = "1.0.2" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 10 | 11 | [[package]] 12 | name = "atty" 13 | version = "0.2.14" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 16 | dependencies = [ 17 | "hermit-abi", 18 | "libc", 19 | "winapi", 20 | ] 21 | 22 | [[package]] 23 | name = "autocfg" 24 | version = "1.1.0" 25 | source = "registry+https://github.com/rust-lang/crates.io-index" 26 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 27 | 28 | [[package]] 29 | name = "base64" 30 | version = "0.13.0" 31 | source = "registry+https://github.com/rust-lang/crates.io-index" 32 | checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" 33 | 34 | [[package]] 35 | name = "bitflags" 36 | version = "1.3.2" 37 | source = "registry+https://github.com/rust-lang/crates.io-index" 38 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 39 | 40 | [[package]] 41 | name = "bumpalo" 42 | version = "3.11.0" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d" 45 | 46 | [[package]] 47 | name = "cargo-download" 48 | version = "0.1.0" 49 | dependencies = [ 50 | "clap", 51 | "flate2", 52 | "json", 53 | "tar", 54 | "ureq", 55 | ] 56 | 57 | [[package]] 58 | name = "cc" 59 | version = "1.0.73" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" 62 | 63 | [[package]] 64 | name = "cfg-if" 65 | version = "1.0.0" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 68 | 69 | [[package]] 70 | name = "chunked_transfer" 71 | version = "1.4.0" 72 | source = "registry+https://github.com/rust-lang/crates.io-index" 73 | checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" 74 | 75 | [[package]] 76 | name = "clap" 77 | version = "3.2.17" 78 | source = "registry+https://github.com/rust-lang/crates.io-index" 79 | checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b" 80 | dependencies = [ 81 | "atty", 82 | "bitflags", 83 | "clap_derive", 84 | "clap_lex", 85 | "indexmap", 86 | "once_cell", 87 | "strsim", 88 | "termcolor", 89 | "textwrap", 90 | ] 91 | 92 | [[package]] 93 | name = "clap_derive" 94 | version = "3.2.17" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "13547f7012c01ab4a0e8f8967730ada8f9fdf419e8b6c792788f39cf4e46eefa" 97 | dependencies = [ 98 | "heck", 99 | "proc-macro-error", 100 | "proc-macro2", 101 | "quote", 102 | "syn", 103 | ] 104 | 105 | [[package]] 106 | name = "clap_lex" 107 | version = "0.2.4" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" 110 | dependencies = [ 111 | "os_str_bytes", 112 | ] 113 | 114 | [[package]] 115 | name = "crc32fast" 116 | version = "1.3.2" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" 119 | dependencies = [ 120 | "cfg-if", 121 | ] 122 | 123 | [[package]] 124 | name = "filetime" 125 | version = "0.2.17" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" 128 | dependencies = [ 129 | "cfg-if", 130 | "libc", 131 | "redox_syscall", 132 | "windows-sys", 133 | ] 134 | 135 | [[package]] 136 | name = "flate2" 137 | version = "1.0.24" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" 140 | dependencies = [ 141 | "crc32fast", 142 | "miniz_oxide", 143 | ] 144 | 145 | [[package]] 146 | name = "form_urlencoded" 147 | version = "1.0.1" 148 | source = "registry+https://github.com/rust-lang/crates.io-index" 149 | checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" 150 | dependencies = [ 151 | "matches", 152 | "percent-encoding", 153 | ] 154 | 155 | [[package]] 156 | name = "hashbrown" 157 | version = "0.12.3" 158 | source = "registry+https://github.com/rust-lang/crates.io-index" 159 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 160 | 161 | [[package]] 162 | name = "heck" 163 | version = "0.4.0" 164 | source = "registry+https://github.com/rust-lang/crates.io-index" 165 | checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" 166 | 167 | [[package]] 168 | name = "hermit-abi" 169 | version = "0.1.19" 170 | source = "registry+https://github.com/rust-lang/crates.io-index" 171 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 172 | dependencies = [ 173 | "libc", 174 | ] 175 | 176 | [[package]] 177 | name = "idna" 178 | version = "0.2.3" 179 | source = "registry+https://github.com/rust-lang/crates.io-index" 180 | checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" 181 | dependencies = [ 182 | "matches", 183 | "unicode-bidi", 184 | "unicode-normalization", 185 | ] 186 | 187 | [[package]] 188 | name = "indexmap" 189 | version = "1.9.1" 190 | source = "registry+https://github.com/rust-lang/crates.io-index" 191 | checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" 192 | dependencies = [ 193 | "autocfg", 194 | "hashbrown", 195 | ] 196 | 197 | [[package]] 198 | name = "js-sys" 199 | version = "0.3.59" 200 | source = "registry+https://github.com/rust-lang/crates.io-index" 201 | checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" 202 | dependencies = [ 203 | "wasm-bindgen", 204 | ] 205 | 206 | [[package]] 207 | name = "json" 208 | version = "0.12.4" 209 | source = "registry+https://github.com/rust-lang/crates.io-index" 210 | checksum = "078e285eafdfb6c4b434e0d31e8cfcb5115b651496faca5749b88fafd4f23bfd" 211 | 212 | [[package]] 213 | name = "libc" 214 | version = "0.2.132" 215 | source = "registry+https://github.com/rust-lang/crates.io-index" 216 | checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5" 217 | 218 | [[package]] 219 | name = "log" 220 | version = "0.4.17" 221 | source = "registry+https://github.com/rust-lang/crates.io-index" 222 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 223 | dependencies = [ 224 | "cfg-if", 225 | ] 226 | 227 | [[package]] 228 | name = "matches" 229 | version = "0.1.9" 230 | source = "registry+https://github.com/rust-lang/crates.io-index" 231 | checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" 232 | 233 | [[package]] 234 | name = "miniz_oxide" 235 | version = "0.5.3" 236 | source = "registry+https://github.com/rust-lang/crates.io-index" 237 | checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" 238 | dependencies = [ 239 | "adler", 240 | ] 241 | 242 | [[package]] 243 | name = "once_cell" 244 | version = "1.13.1" 245 | source = "registry+https://github.com/rust-lang/crates.io-index" 246 | checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" 247 | 248 | [[package]] 249 | name = "os_str_bytes" 250 | version = "6.3.0" 251 | source = "registry+https://github.com/rust-lang/crates.io-index" 252 | checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" 253 | 254 | [[package]] 255 | name = "percent-encoding" 256 | version = "2.1.0" 257 | source = "registry+https://github.com/rust-lang/crates.io-index" 258 | checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" 259 | 260 | [[package]] 261 | name = "proc-macro-error" 262 | version = "1.0.4" 263 | source = "registry+https://github.com/rust-lang/crates.io-index" 264 | checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" 265 | dependencies = [ 266 | "proc-macro-error-attr", 267 | "proc-macro2", 268 | "quote", 269 | "syn", 270 | "version_check", 271 | ] 272 | 273 | [[package]] 274 | name = "proc-macro-error-attr" 275 | version = "1.0.4" 276 | source = "registry+https://github.com/rust-lang/crates.io-index" 277 | checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" 278 | dependencies = [ 279 | "proc-macro2", 280 | "quote", 281 | "version_check", 282 | ] 283 | 284 | [[package]] 285 | name = "proc-macro2" 286 | version = "1.0.43" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" 289 | dependencies = [ 290 | "unicode-ident", 291 | ] 292 | 293 | [[package]] 294 | name = "quote" 295 | version = "1.0.21" 296 | source = "registry+https://github.com/rust-lang/crates.io-index" 297 | checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" 298 | dependencies = [ 299 | "proc-macro2", 300 | ] 301 | 302 | [[package]] 303 | name = "redox_syscall" 304 | version = "0.2.16" 305 | source = "registry+https://github.com/rust-lang/crates.io-index" 306 | checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" 307 | dependencies = [ 308 | "bitflags", 309 | ] 310 | 311 | [[package]] 312 | name = "ring" 313 | version = "0.16.20" 314 | source = "registry+https://github.com/rust-lang/crates.io-index" 315 | checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" 316 | dependencies = [ 317 | "cc", 318 | "libc", 319 | "once_cell", 320 | "spin", 321 | "untrusted", 322 | "web-sys", 323 | "winapi", 324 | ] 325 | 326 | [[package]] 327 | name = "rustls" 328 | version = "0.20.6" 329 | source = "registry+https://github.com/rust-lang/crates.io-index" 330 | checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" 331 | dependencies = [ 332 | "log", 333 | "ring", 334 | "sct", 335 | "webpki", 336 | ] 337 | 338 | [[package]] 339 | name = "sct" 340 | version = "0.7.0" 341 | source = "registry+https://github.com/rust-lang/crates.io-index" 342 | checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" 343 | dependencies = [ 344 | "ring", 345 | "untrusted", 346 | ] 347 | 348 | [[package]] 349 | name = "spin" 350 | version = "0.5.2" 351 | source = "registry+https://github.com/rust-lang/crates.io-index" 352 | checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" 353 | 354 | [[package]] 355 | name = "strsim" 356 | version = "0.10.0" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 359 | 360 | [[package]] 361 | name = "syn" 362 | version = "1.0.99" 363 | source = "registry+https://github.com/rust-lang/crates.io-index" 364 | checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" 365 | dependencies = [ 366 | "proc-macro2", 367 | "quote", 368 | "unicode-ident", 369 | ] 370 | 371 | [[package]] 372 | name = "tar" 373 | version = "0.4.38" 374 | source = "registry+https://github.com/rust-lang/crates.io-index" 375 | checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" 376 | dependencies = [ 377 | "filetime", 378 | "libc", 379 | "xattr", 380 | ] 381 | 382 | [[package]] 383 | name = "termcolor" 384 | version = "1.1.3" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 387 | dependencies = [ 388 | "winapi-util", 389 | ] 390 | 391 | [[package]] 392 | name = "textwrap" 393 | version = "0.15.0" 394 | source = "registry+https://github.com/rust-lang/crates.io-index" 395 | checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" 396 | 397 | [[package]] 398 | name = "tinyvec" 399 | version = "1.6.0" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 402 | dependencies = [ 403 | "tinyvec_macros", 404 | ] 405 | 406 | [[package]] 407 | name = "tinyvec_macros" 408 | version = "0.1.0" 409 | source = "registry+https://github.com/rust-lang/crates.io-index" 410 | checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" 411 | 412 | [[package]] 413 | name = "unicode-bidi" 414 | version = "0.3.8" 415 | source = "registry+https://github.com/rust-lang/crates.io-index" 416 | checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" 417 | 418 | [[package]] 419 | name = "unicode-ident" 420 | version = "1.0.3" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" 423 | 424 | [[package]] 425 | name = "unicode-normalization" 426 | version = "0.1.21" 427 | source = "registry+https://github.com/rust-lang/crates.io-index" 428 | checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" 429 | dependencies = [ 430 | "tinyvec", 431 | ] 432 | 433 | [[package]] 434 | name = "untrusted" 435 | version = "0.7.1" 436 | source = "registry+https://github.com/rust-lang/crates.io-index" 437 | checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" 438 | 439 | [[package]] 440 | name = "ureq" 441 | version = "2.5.0" 442 | source = "registry+https://github.com/rust-lang/crates.io-index" 443 | checksum = "b97acb4c28a254fd7a4aeec976c46a7fa404eac4d7c134b30c75144846d7cb8f" 444 | dependencies = [ 445 | "base64", 446 | "chunked_transfer", 447 | "flate2", 448 | "log", 449 | "once_cell", 450 | "rustls", 451 | "url", 452 | "webpki", 453 | "webpki-roots", 454 | ] 455 | 456 | [[package]] 457 | name = "url" 458 | version = "2.2.2" 459 | source = "registry+https://github.com/rust-lang/crates.io-index" 460 | checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" 461 | dependencies = [ 462 | "form_urlencoded", 463 | "idna", 464 | "matches", 465 | "percent-encoding", 466 | ] 467 | 468 | [[package]] 469 | name = "version_check" 470 | version = "0.9.4" 471 | source = "registry+https://github.com/rust-lang/crates.io-index" 472 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 473 | 474 | [[package]] 475 | name = "wasm-bindgen" 476 | version = "0.2.82" 477 | source = "registry+https://github.com/rust-lang/crates.io-index" 478 | checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" 479 | dependencies = [ 480 | "cfg-if", 481 | "wasm-bindgen-macro", 482 | ] 483 | 484 | [[package]] 485 | name = "wasm-bindgen-backend" 486 | version = "0.2.82" 487 | source = "registry+https://github.com/rust-lang/crates.io-index" 488 | checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" 489 | dependencies = [ 490 | "bumpalo", 491 | "log", 492 | "once_cell", 493 | "proc-macro2", 494 | "quote", 495 | "syn", 496 | "wasm-bindgen-shared", 497 | ] 498 | 499 | [[package]] 500 | name = "wasm-bindgen-macro" 501 | version = "0.2.82" 502 | source = "registry+https://github.com/rust-lang/crates.io-index" 503 | checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" 504 | dependencies = [ 505 | "quote", 506 | "wasm-bindgen-macro-support", 507 | ] 508 | 509 | [[package]] 510 | name = "wasm-bindgen-macro-support" 511 | version = "0.2.82" 512 | source = "registry+https://github.com/rust-lang/crates.io-index" 513 | checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" 514 | dependencies = [ 515 | "proc-macro2", 516 | "quote", 517 | "syn", 518 | "wasm-bindgen-backend", 519 | "wasm-bindgen-shared", 520 | ] 521 | 522 | [[package]] 523 | name = "wasm-bindgen-shared" 524 | version = "0.2.82" 525 | source = "registry+https://github.com/rust-lang/crates.io-index" 526 | checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" 527 | 528 | [[package]] 529 | name = "web-sys" 530 | version = "0.3.59" 531 | source = "registry+https://github.com/rust-lang/crates.io-index" 532 | checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" 533 | dependencies = [ 534 | "js-sys", 535 | "wasm-bindgen", 536 | ] 537 | 538 | [[package]] 539 | name = "webpki" 540 | version = "0.22.0" 541 | source = "registry+https://github.com/rust-lang/crates.io-index" 542 | checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" 543 | dependencies = [ 544 | "ring", 545 | "untrusted", 546 | ] 547 | 548 | [[package]] 549 | name = "webpki-roots" 550 | version = "0.22.4" 551 | source = "registry+https://github.com/rust-lang/crates.io-index" 552 | checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" 553 | dependencies = [ 554 | "webpki", 555 | ] 556 | 557 | [[package]] 558 | name = "winapi" 559 | version = "0.3.9" 560 | source = "registry+https://github.com/rust-lang/crates.io-index" 561 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 562 | dependencies = [ 563 | "winapi-i686-pc-windows-gnu", 564 | "winapi-x86_64-pc-windows-gnu", 565 | ] 566 | 567 | [[package]] 568 | name = "winapi-i686-pc-windows-gnu" 569 | version = "0.4.0" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 572 | 573 | [[package]] 574 | name = "winapi-util" 575 | version = "0.1.5" 576 | source = "registry+https://github.com/rust-lang/crates.io-index" 577 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 578 | dependencies = [ 579 | "winapi", 580 | ] 581 | 582 | [[package]] 583 | name = "winapi-x86_64-pc-windows-gnu" 584 | version = "0.4.0" 585 | source = "registry+https://github.com/rust-lang/crates.io-index" 586 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 587 | 588 | [[package]] 589 | name = "windows-sys" 590 | version = "0.36.1" 591 | source = "registry+https://github.com/rust-lang/crates.io-index" 592 | checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" 593 | dependencies = [ 594 | "windows_aarch64_msvc", 595 | "windows_i686_gnu", 596 | "windows_i686_msvc", 597 | "windows_x86_64_gnu", 598 | "windows_x86_64_msvc", 599 | ] 600 | 601 | [[package]] 602 | name = "windows_aarch64_msvc" 603 | version = "0.36.1" 604 | source = "registry+https://github.com/rust-lang/crates.io-index" 605 | checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" 606 | 607 | [[package]] 608 | name = "windows_i686_gnu" 609 | version = "0.36.1" 610 | source = "registry+https://github.com/rust-lang/crates.io-index" 611 | checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" 612 | 613 | [[package]] 614 | name = "windows_i686_msvc" 615 | version = "0.36.1" 616 | source = "registry+https://github.com/rust-lang/crates.io-index" 617 | checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" 618 | 619 | [[package]] 620 | name = "windows_x86_64_gnu" 621 | version = "0.36.1" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" 624 | 625 | [[package]] 626 | name = "windows_x86_64_msvc" 627 | version = "0.36.1" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" 630 | 631 | [[package]] 632 | name = "xattr" 633 | version = "0.2.3" 634 | source = "registry+https://github.com/rust-lang/crates.io-index" 635 | checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" 636 | dependencies = [ 637 | "libc", 638 | ] 639 | -------------------------------------------------------------------------------- /cargo-download/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cargo-download" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | clap = { version = "3.2", features = ["derive"] } 10 | ureq = "2.5" 11 | tar = "0.4" 12 | flate2 = "1" 13 | json = "0.12.4" 14 | -------------------------------------------------------------------------------- /cargo-download/src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use std::path::PathBuf; 3 | 4 | #[derive(Parser, Debug)] 5 | struct Args { 6 | #[clap(value_name = "CRATE")] 7 | krate: String, 8 | 9 | output: Option, 10 | } 11 | 12 | fn main() -> Result<(), Box> { 13 | let args = Args::parse_from(std::env::args().skip(1)); 14 | 15 | let mut it = args.krate.split('@'); 16 | let name = it.next().unwrap(); 17 | let version = it.next(); 18 | let version = match version { 19 | Some(v) => v.to_string(), 20 | None => { 21 | let response = ureq::get(&format!("https://crates.io/api/v1/crates/{name}")).call()?; 22 | let body = response.into_string()?; 23 | let body = json::parse(&body)?; 24 | body["crate"]["max_version"].as_str().unwrap().to_string() 25 | } 26 | }; 27 | 28 | // Fail fast if the destination directory can't be made 29 | let output: PathBuf = args 30 | .output 31 | .unwrap_or_else(|| format!("{name}-{version}")) 32 | .into(); 33 | std::fs::create_dir_all(&output)?; 34 | 35 | assert!(it.next().is_none()); 36 | let download_url = format!( 37 | "https://static.crates.io/crates/{}/{}-{}.crate", 38 | name, name, version 39 | ); 40 | 41 | let response = ureq::get(&download_url).call()?; 42 | let reader = flate2::read::GzDecoder::new(response.into_reader()); 43 | let mut archive = tar::Archive::new(reader); 44 | 45 | for entry in archive.entries()? { 46 | let mut entry = entry?; 47 | let relpath = entry.path()?; 48 | let mut components = relpath.components(); 49 | // Throw away the first path component 50 | components.next(); 51 | let full_path = output.join(&components.as_path()); 52 | // unpack doesn't make directories for us, we need to handle that 53 | if let Some(parent) = full_path.parent() { 54 | std::fs::create_dir_all(parent)?; 55 | } 56 | entry.unpack(&full_path)?; 57 | } 58 | 59 | Ok(()) 60 | } 61 | -------------------------------------------------------------------------------- /ci-crates: -------------------------------------------------------------------------------- 1 | libc 2 | serde 3 | itoa 4 | -------------------------------------------------------------------------------- /ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | exec 2>&1 5 | export TERM=xterm-256color 6 | 7 | function group { 8 | echo "::group::$@" 9 | $@ 10 | echo "::endgroup::" 11 | } 12 | 13 | if [[ "$1" == "style" ]] 14 | then 15 | group cargo fmt --check 16 | group cargo clippy -- -Dclippy::all 17 | elif [[ "$1" == "fuzz" ]] 18 | then 19 | group rustup toolchain install nightly 20 | group cargo install cargo-fuzz 21 | cd ansi-to-html 22 | group cargo +nightly fuzz run render -- -runs=100000 23 | else 24 | group cargo build 25 | group cargo run -- sync --tool="$1" --bucket=miri-bot-dev 26 | group cargo run -- run --tool="$1" --bucket=miri-bot-dev --crate-list=ci-crates --rerun 27 | fi 28 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/saethlin/crates-build-env:latest 2 | 3 | ENV PATH=/root/.cargo/bin:$PATH 4 | 5 | RUN apt-get update && apt-get install -y curl build-essential && \ 6 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ 7 | sh -s -- --default-toolchain=nightly --component=miri --component=rust-src --profile=minimal -y && \ 8 | cargo install --git https://github.com/saethlin/miri-tools cargo-download inapty get-args && \ 9 | cargo install --git https://github.com/RalfJung/cargo-careful cargo-careful && \ 10 | curl -L https://get.nexte.st/latest/linux | tar zxf - && mv cargo-nextest /root/.cargo/bin/ && \ 11 | rm -rf /var/lib/apt/lists/* 12 | 13 | COPY nextest.toml /root/.cargo/nextest.toml 14 | COPY run.sh /root/run.sh 15 | 16 | RUN mkdir /build && \ 17 | rm -rf /root/.cache && \ 18 | for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do \ 19 | echo "cfg-if@1.0.0" | TOOL=miri TARGET=$target TEST_END_DELIMITER="-" bash /root/run.sh; \ 20 | done && \ 21 | echo "cfg-if@1.0.0" | TOOL=asan TARGET=$(rustc -vV | grep host | cut -d' ' -f2) TEST_END_DELIMITER="-" bash /root/run.sh && \ 22 | tar czvf /cache.tar.gz /root/.cache && \ 23 | rm -rf /root/.cache /build 24 | 25 | ENTRYPOINT ["/usr/bin/nice", "-n19", "bash", "/root/run.sh"] 26 | -------------------------------------------------------------------------------- /docker/Dockerfile.base: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | 3 | # Install the packages contained in `packages.txt` 4 | COPY packages.txt /packages.txt 5 | RUN apt-get update && \ 6 | cat /packages.txt | DEBIAN_FRONTEND=noninteractive xargs apt-get install -y && \ 7 | rm -rf /var/lib/apt/lists/* && \ 8 | rm /packages.txt 9 | -------------------------------------------------------------------------------- /docker/Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/saethlin/crater-at-home-ci:latest 2 | 3 | COPY nextest.toml /root/.cargo/nextest.toml 4 | COPY run.sh /root/run.sh 5 | 6 | ENTRYPOINT ["bash", "/root/run.sh"] 7 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | # Start up docker-buildx 6 | docker run --rm --privileged tonistiigi/binfmt:latest --install all 7 | docker buildx create --driver docker-container --use 8 | 9 | # Full base image is just all the packages in packages.txt 10 | docker buildx build --no-cache --file ./Dockerfile.base --platform linux/amd64,linux/arm64 --tag ghcr.io/saethlin/crates-build-env:latest --push . 11 | 12 | # CI base image is the full image, but with no packages. 13 | echo "FROM ubuntu:latest" > Dockerfile.ci-base 14 | tail -n+2 Dockerfile >> Dockerfile.ci-base 15 | docker buildx build --no-cache --file ./Dockerfile.ci-base --platform linux/amd64 --tag ghcr.io/saethlin/crater-at-home-ci:latest --push . 16 | -------------------------------------------------------------------------------- /docker/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.default-miri] 2 | slow-timeout = { period = "60s", terminate-after = 1 } 3 | -------------------------------------------------------------------------------- /docker/packages.txt: -------------------------------------------------------------------------------- 1 | aspell 2 | aspell-en 3 | autoconf 4 | automake 5 | autopoint 6 | autotools-dev 7 | binutils-arm-none-eabi 8 | binutils-mingw-w64-x86-64 9 | bison 10 | bluetooth 11 | bsdmainutils 12 | build-essential 13 | bzip2 14 | capnproto 15 | clang 16 | cmake 17 | comerr-dev 18 | cpp 19 | cpp-10 20 | cron 21 | curl 22 | dmsetup 23 | docker.io 24 | doctest-dev 25 | dselect 26 | emacsen-common 27 | gfortran 28 | git 29 | gnupg 30 | golang 31 | graphicsmagick 32 | gstreamer1.0-plugins-base 33 | gstreamer1.0-plugins-good 34 | gstreamer1.0-pulseaudio 35 | gstreamer1.0-x 36 | krb5-multidev 37 | libaa1 38 | libaacs0 39 | libacl1 40 | libaec0 41 | libalgorithm-diff-perl 42 | libalgorithm-diff-xs-perl 43 | libalgorithm-merge-perl 44 | liballegro-acodec5-dev 45 | liballegro-acodec5.2 46 | liballegro-audio5-dev 47 | liballegro-audio5.2 48 | liballegro-dialog5-dev 49 | liballegro-dialog5.2 50 | liballegro-image5-dev 51 | liballegro-image5.2 52 | liballegro-physfs5-dev 53 | liballegro-physfs5.2 54 | liballegro-ttf5-dev 55 | liballegro-ttf5.2 56 | liballegro-video5-dev 57 | liballegro-video5.2 58 | liballegro5-dev 59 | liballegro5.2 60 | libapparmor1 61 | libapt-pkg-dev 62 | libapt-pkg6.0 63 | libarchive-cpio-perl 64 | libarchive-dev 65 | libarchive-zip-perl 66 | libarchive13 67 | libargon2-0 68 | libarmadillo10 69 | libarpack2 70 | libasan6 71 | libasn1-8-heimdal 72 | libasound2 73 | libasound2-data 74 | libasound2-dev 75 | libaspell15 76 | libass-dev 77 | libass9 78 | libassuan0 79 | libasyncns0 80 | libatk-bridge2.0-0 81 | libatk-bridge2.0-dev 82 | libatk1.0-0 83 | libatk1.0-data 84 | libatk1.0-dev 85 | libatm1 86 | libatomic1 87 | libatspi2.0-0 88 | libatspi2.0-dev 89 | libattr1 90 | libaudit-common 91 | libaudit1 92 | libauthen-sasl-perl 93 | libavahi-client-dev 94 | libavahi-client3 95 | libavahi-common-data 96 | libavahi-common-dev 97 | libavahi-common3 98 | libavahi-compat-libdnssd-dev 99 | libavahi-compat-libdnssd1 100 | libavahi-core7 101 | libavc1394-0 102 | libavcodec-dev 103 | libavcodec58 104 | libavdevice-dev 105 | libavdevice58 106 | libavfilter-dev 107 | libavfilter7 108 | libavformat-dev 109 | libavformat58 110 | libavutil-dev 111 | libavutil56 112 | libayatana-appindicator3-dev 113 | libbdplus0 114 | libbind-dev 115 | libbind9-161 116 | libbinutils 117 | libblas-dev 118 | libblas3 119 | libblkid-dev 120 | libblkid1 121 | libbluetooth-dev 122 | libbluray-dev 123 | libbluray2 124 | libboost-dev 125 | libboost-filesystem1.74.0 126 | libboost-iostreams1.74.0 127 | libboost-system1.74.0 128 | libbrotli1 129 | libbs2b0 130 | libbsd0 131 | libbtrfsutil-dev 132 | libbz2-1.0 133 | libbz2-dev 134 | libc-ares-dev 135 | libc-ares2 136 | libc-bin 137 | libc-dev-bin 138 | libc6 139 | libc6-dev 140 | libcaca0 141 | libcairo-gobject2 142 | libcairo-script-interpreter2 143 | libcairo2 144 | libcairo2-dev 145 | libcap-ng0 146 | libcap2 147 | libcap2-bin 148 | libcapnp-0.8.0 149 | libcapnp-dev 150 | libcapstone-dev 151 | libcapstone4 152 | libcc1-0 153 | libcdio-cdda-dev 154 | libcdio-cdda2 155 | libcdio-dev 156 | libcdio-paranoia-dev 157 | libcdio-paranoia2 158 | libcdio19 159 | libcdparanoia-dev 160 | libcdparanoia0 161 | libcfitsio-dev 162 | libcfitsio-doc 163 | libcfitsio9 164 | libcgi-fast-perl 165 | libcgi-pm-perl 166 | libcharls2 167 | libchewing3-dev 168 | libchromaprint1 169 | libclang-dev 170 | libclang1 171 | libclass-accessor-perl 172 | libcolord-dev 173 | libcolord2 174 | libcolorhug-dev 175 | libcolorhug2 176 | libcom-err2 177 | libcryptsetup-dev 178 | libcryptsetup12 179 | libcsfml-audio2.5 180 | libcsfml-dev 181 | libcsfml-graphics2.5 182 | libcsfml-network2.5 183 | libcsfml-system2.5 184 | libcsfml-window2.5 185 | libcups2 186 | libcurl3-gnutls 187 | libcurl4 188 | libcwidget4 189 | libdaemon0 190 | libdap27 191 | libdapclient6v5 192 | libdata-dump-perl 193 | libdatrie1 194 | libdb5.3 195 | libdbus-1-3 196 | libdbus-1-dev 197 | libdbus-glib-1-2 198 | libdbus-glib-1-dev 199 | libdc1394-25 200 | libdconf1 201 | libdebconfclient0 202 | libdeflate-dev 203 | libdevmapper-dev 204 | libdevmapper-event1.02.1 205 | libdevmapper1.02.1 206 | libdmx-dev 207 | libdmx1 208 | libdns-export1110 209 | libdns1110 210 | libdouble-conversion3 211 | libdpkg-perl 212 | libdrm-amdgpu1 213 | libdrm-common 214 | libdrm-dev 215 | libdrm-nouveau2 216 | libdrm-radeon1 217 | libdrm2 218 | libdumb1 219 | libdumb1-dev 220 | libdv4 221 | libdw-dev 222 | libebur128-1 223 | libedit2 224 | libegl-mesa0 225 | libegl1 226 | libegl1-mesa 227 | libegl1-mesa-dev 228 | libelf-dev 229 | libenca0 230 | libenchant-2-dev 231 | libencode-locale-perl 232 | libepoxy-dev 233 | libepoxy0 234 | liberfa-dev 235 | liberror-perl 236 | libestr0 237 | libev-dev 238 | libev4 239 | libevdev-dev 240 | libevdev2 241 | libevent-2.1-7 242 | libevent-core-2.1-7 243 | libevent-dev 244 | libevent-extra-2.1-7 245 | libevent-openssl-2.1-7 246 | libevent-pthreads-2.1-7 247 | libexif12 248 | libexiv2-27 249 | libexiv2-dev 250 | libexpat1 251 | libexpat1-dev 252 | libext2fs-dev 253 | libext2fs2 254 | libfabric1 255 | libfakeroot 256 | libfastjson4 257 | libfcgi-perl 258 | libfdisk1 259 | libffi-dev 260 | libffi7 261 | libfftw3-dev 262 | libfftw3-double3 263 | libfile-copy-recursive-perl 264 | libfile-fcntllock-perl 265 | libfile-listing-perl 266 | libfile-stripnondeterminism-perl 267 | libflac-dev 268 | libflac8 269 | libflite1 270 | libfont-afm-perl 271 | libfontconfig1 272 | libfontconfig1-dev 273 | libfontenc-dev 274 | libfontenc1 275 | libfprint-2-dev 276 | libfreetype6 277 | libfreetype6-dev 278 | libfreexl1 279 | libfribidi-dev 280 | libfribidi0 281 | libfs-dev 282 | libfs6 283 | libfuse-dev 284 | libfuse2 285 | libfyba0 286 | libgail-common 287 | libgail18 288 | libgbm1 289 | libgc1 290 | libgcc-10-dev 291 | libgcc-9-dev 292 | libgcc1 293 | libgcrypt20 294 | libgcrypt20-dev 295 | libgd3 296 | libgdal30 297 | libgdbm-compat4 298 | libgdbm6 299 | libgdcm3.0 300 | libgdk-pixbuf2.0-0 301 | libgdk-pixbuf2.0-bin 302 | libgdk-pixbuf2.0-common 303 | libgdk-pixbuf2.0-dev 304 | libgdm-dev 305 | libgdm1 306 | libgeoip1 307 | libgeos-c1v5 308 | libgeos-dev 309 | libgeos3.10.2 310 | libgeotiff5 311 | libgettextpo-dev 312 | libgettextpo0 313 | libgexiv2-2 314 | libgexiv2-dev 315 | libgfortran-10-dev 316 | libgfortran5 317 | libgif7 318 | libgirepository-1.0-1 319 | libgl1 320 | libgl1-mesa-dev 321 | libgl1-mesa-dri 322 | libgl1-mesa-glx 323 | libgl2ps1.4 324 | libglapi-mesa 325 | libgles1 326 | libgles2 327 | libgles2-mesa 328 | libgles2-mesa-dev 329 | libglew2.2 330 | libglib2.0-0 331 | libglib2.0-bin 332 | libglib2.0-data 333 | libglib2.0-dev 334 | libglib2.0-dev-bin 335 | libglu1-mesa 336 | libglu1-mesa-dev 337 | libglvnd-core-dev 338 | libglvnd-dev 339 | libglvnd0 340 | libglx-mesa0 341 | libglx0 342 | libgme0 343 | libgmp10 344 | libgnutls30 345 | libgomp1 346 | libgpg-error-dev 347 | libgpg-error0 348 | libgpgme-dev 349 | libgpgme11 350 | libgphoto2-6 351 | libgphoto2-l10n 352 | libgphoto2-port12 353 | libgpm2 354 | libgraphene-1.0-0 355 | libgraphicsmagick1-dev 356 | libgraphite2-3 357 | libgraphite2-dev 358 | libgsm1 359 | libgssapi-krb5-2 360 | libgssapi3-heimdal 361 | libgssrpc4 362 | libgstreamer-gl1.0-0 363 | libgstreamer-plugins-bad1.0-0 364 | libgstreamer-plugins-base1.0-0 365 | libgstreamer-plugins-good1.0-0 366 | libgstreamer1.0-0 367 | libgstreamer1.0-dev 368 | libgtk-3-0 369 | libgtk-3-bin 370 | libgtk-3-common 371 | libgtk-3-dev 372 | libgtk2.0-0 373 | libgtk2.0-bin 374 | libgtk2.0-common 375 | libgtk2.0-dev 376 | libgudev-1.0-0 377 | libgudev-1.0-dev 378 | libgusb-dev 379 | libgusb2 380 | libharfbuzz-dev 381 | libharfbuzz-gobject0 382 | libharfbuzz-icu0 383 | libharfbuzz0b 384 | libhcrypto4-heimdal 385 | libhdf4-0-alt 386 | libhdf5-103 387 | libhdf5-dev 388 | libhdf5-openmpi-103 389 | libheimbase1-heimdal 390 | libheimntlm0-heimdal 391 | libhogweed6 392 | libhtml-form-perl 393 | libhtml-format-perl 394 | libhtml-parser-perl 395 | libhtml-tagset-perl 396 | libhtml-tree-perl 397 | libhttp-cookies-perl 398 | libhttp-daemon-perl 399 | libhttp-date-perl 400 | libhttp-message-perl 401 | libhttp-negotiate-perl 402 | libhttp-parser2.9 403 | libhunspell-1.7-0 404 | libhunspell-dev 405 | libhwloc-plugins 406 | libhwloc15 407 | libhx509-5-heimdal 408 | libhyperscan-dev 409 | libhyperscan5 410 | libhyphen0 411 | libibus-1.0-5 412 | libibus-1.0-dev 413 | libibverbs-dev 414 | libibverbs1 415 | libical-dev 416 | libice-dev 417 | libice6 418 | libicu-dev 419 | libicu-le-hb-dev 420 | libicu-le-hb0 421 | libicu70 422 | libidn12 423 | libidn2-0 424 | libiec61883-0 425 | libieee1284-3 426 | libilmbase25 427 | libinput-bin 428 | libinput10 429 | libintl-perl 430 | libintl-xs-perl 431 | libio-html-perl 432 | libio-socket-ssl-perl 433 | libio-string-perl 434 | libip4tc2 435 | libipset-dev 436 | libirs161 437 | libisc-export1105 438 | libisc1105 439 | libisccc161 440 | libisccfg163 441 | libisl23 442 | libiso9660-11 443 | libiso9660-dev 444 | libitm1 445 | libiw-dev 446 | libjack-jackd2-0 447 | libjansson4 448 | libjavascriptcoregtk-4.0-18 449 | libjavascriptcoregtk-4.0-dev 450 | libjbig0 451 | libjpeg-dev 452 | libjpeg-turbo8 453 | libjpeg-turbo8-dev 454 | libjpeg8 455 | libjpeg8-dev 456 | libjs-jquery 457 | libjs-jquery-metadata 458 | libjs-jquery-tablesorter 459 | libjson-c5 460 | libjson-glib-1.0-0 461 | libjson-glib-1.0-common 462 | libjsoncpp25 463 | libjudy-dev 464 | libk5crypto3 465 | libkadm5clnt-mit12 466 | libkadm5srv-mit12 467 | libkdb5-10 468 | libkeyutils1 469 | libkf5config-dev 470 | libkf5i18n-dev 471 | libkf5kdelibs4support5-bin 472 | libklibc 473 | libkmlbase1 474 | libkmldom1 475 | libkmlengine1 476 | libkmod2 477 | libkrb5-26-heimdal 478 | libkrb5-3 479 | libkrb5support0 480 | libksba8 481 | liblapack-dev 482 | liblapack3 483 | liblapacke-dev 484 | liblcms2-2 485 | liblcms2-dev 486 | libldap-2.5-0 487 | libldap-common 488 | libldap2-dev 489 | libldb2 490 | liblept5 491 | libleptonica-dev 492 | liblmdb-dev 493 | liblocale-gettext-perl 494 | liblsan0 495 | libltdl-dev 496 | libltdl7 497 | liblua5.3-0 498 | liblua5.3-dev 499 | liblwp-mediatypes-perl 500 | liblwp-protocol-https-perl 501 | liblwres161 502 | liblxc1 503 | liblz4-1 504 | liblzma-dev 505 | liblzma5 506 | liblzo2-2 507 | libmagic-dev 508 | libmagic-mgc 509 | libmagic1 510 | libmail-sendmail-perl 511 | libmailtools-perl 512 | libmariadb-dev 513 | libmariadb-dev-compat 514 | libmetis-dev 515 | libmilter-dev 516 | libminizip1 517 | libmircommon-dev 518 | libmircommon8 519 | libmircookie-dev 520 | libmircookie2 521 | libmircore-dev 522 | libmircore1 523 | libmnl0 524 | libmodplug1 525 | libmount1 526 | libmp3lame0 527 | libmpc3 528 | libmpdec3 529 | libmpfr6 530 | libmpg123-0 531 | libmtdev1 532 | libmumps-seq-dev 533 | libmysqlclient21 534 | libncurses5 535 | libncurses5-dev 536 | libncursesw5 537 | libnet-http-perl 538 | libnet-smtp-ssl-perl 539 | libnet-ssleay-perl 540 | libnetcdf-c++4 541 | libnetcdf19 542 | libnettle8 543 | libnewlib-arm-none-eabi 544 | libnewlib-dev 545 | libnewt0.52 546 | libnghttp2-14 547 | libnl-3-200 548 | libnl-3-dev 549 | libnl-genl-3-200 550 | libnl-genl-3-dev 551 | libnl-route-3-200 552 | libnl-route-3-dev 553 | libnorm1 554 | libnotify-dev 555 | libnotify4 556 | libnpth0 557 | libnspr4 558 | libnss-mdns 559 | libnss-systemd 560 | libnss3 561 | libnuma1 562 | libobjc-10-dev 563 | libobjc4 564 | libodbc1 565 | libogdi4.1 566 | libogg-dev 567 | libogg0 568 | libomp-dev 569 | libomp5 570 | libopenal-data 571 | libopenal-dev 572 | libopenal1 573 | libopenblas-dev 574 | libopencv-calib3d4.5 575 | libopencv-contrib4.5 576 | libopencv-core4.5 577 | libopencv-dev 578 | libopencv-features2d4.5 579 | libopencv-flann4.5 580 | libopencv-highgui4.5 581 | libopencv-imgcodecs4.5 582 | libopencv-imgproc4.5 583 | libopencv-ml4.5 584 | libopencv-objdetect4.5 585 | libopencv-photo4.5 586 | libopencv-shape4.5 587 | libopencv-stitching4.5 588 | libopencv-superres4.5 589 | libopencv-video4.5 590 | libopencv-videoio4.5 591 | libopencv-videostab4.5 592 | libopencv-viz4.5 593 | libopencv4.5-java 594 | libopencv4.5d-jni 595 | libopenexr25 596 | libopengl0 597 | libopenjp2-7 598 | libopenmpi3 599 | libopenmpt0 600 | libopenslide-dev 601 | libopus-dev 602 | libopus0 603 | libopusfile0 604 | liborc-0.4-0 605 | libp11-kit0 606 | libpam-cap 607 | libpam-modules 608 | libpam-modules-bin 609 | libpam-runtime 610 | libpam-systemd 611 | libpam0g 612 | libpam0g-dev 613 | libpango-1.0-0 614 | libpango1.0-dev 615 | libpangocairo-1.0-0 616 | libpangoft2-1.0-0 617 | libpangoxft-1.0-0 618 | libparted-dev 619 | libparted-fs-resize0 620 | libparted2 621 | libpcap-dev 622 | libpciaccess-dev 623 | libpciaccess0 624 | libpcre16-3 625 | libpcre3 626 | libpcre3-dev 627 | libpcre32-3 628 | libpcrecpp0v5 629 | libpcsclite-dev 630 | libpcsclite1 631 | libperl4-corelibs-perl 632 | libperl5.34 633 | libpgm-5.3-0 634 | libphysfs-dev 635 | libphysfs1 636 | libpipeline1 637 | libpixman-1-0 638 | libpixman-1-dev 639 | libplist-dev 640 | libplist3 641 | libpng-dev 642 | libpng-tools 643 | libpng16-16 644 | libpocketsphinx-dev 645 | libpocketsphinx3 646 | libpolkit-agent-1-0 647 | libpolkit-gobject-1-0 648 | libpoppler118 649 | libpopt0 650 | libpostproc-dev 651 | libpostproc55 652 | libpq-dev 653 | libpq5 654 | libprocps8 655 | libproj22 656 | libprotobuf-dev 657 | libprotobuf-lite23 658 | libprotobuf23 659 | libprotoc23 660 | libprotozero-dev 661 | libproxy1v5 662 | libpsl5 663 | libpthread-stubs0-dev 664 | libpulse-dev 665 | libpulse-mainloop-glib0 666 | libpulse0 667 | libpython-all-dev 668 | libpython2.7 669 | libpython2.7-dev 670 | libpython2.7-minimal 671 | libpython2.7-stdlib 672 | libpython3.10 673 | libpython3.10-dev 674 | libpython3.10-minimal 675 | libpython3.10-stdlib 676 | libqhull8.0 677 | libqt5charts5-dev 678 | libqt5concurrent5 679 | libqt5core5a 680 | libqt5dbus5 681 | libqt5gui5 682 | libqt5network5 683 | libqt5opengl5 684 | libqt5opengl5-dev 685 | libqt5printsupport5 686 | libqt5qml5 687 | libqt5quick5 688 | libqt5quickparticles5 689 | libqt5quicktest5 690 | libqt5quickwidgets5 691 | libqt5sql5 692 | libqt5sql5-sqlite 693 | libqt5svg5 694 | libqt5test5 695 | libqt5widgets5 696 | libqt5xml5 697 | libraw1394-11 698 | librdmacm-dev 699 | librdmacm1 700 | libreadline-dev 701 | libreadline8 702 | libreofficekit-dev 703 | librest-0.7-0 704 | librhash0 705 | librlottie-dev 706 | libroken18-heimdal 707 | librsvg2-2 708 | librsvg2-common 709 | librsvg2-dev 710 | librtmidi-dev 711 | librtmp1 712 | librubberband2 713 | libsamplerate0 714 | libsane 715 | libsane-common 716 | libsane1 717 | libsasl2-2 718 | libsasl2-modules 719 | libsasl2-modules-db 720 | libsctp1 721 | libsdl2-2.0-0 722 | libsdl2-dev 723 | libseccomp2 724 | libsecret-1-0 725 | libsecret-common 726 | libselinux1 727 | libselinux1-dev 728 | libsemanage-common 729 | libsemanage2 730 | libsensors5 731 | libsepol-dev 732 | libsepol2 733 | libsfml-audio2.5 734 | libsfml-dev 735 | libsfml-graphics2.5 736 | libsfml-network2.5 737 | libsfml-system2.5 738 | libsfml-window2.5 739 | libshine3 740 | libshout3 741 | libsigc++-2.0-0v5 742 | libsigsegv2 743 | libslang2 744 | libslang2-dev 745 | libsm-dev 746 | libsm6 747 | libsmartcols1 748 | libsmbclient 749 | libsmbclient-dev 750 | libsnappy1v5 751 | libsndfile1 752 | libsndio-dev 753 | libsndio7.0 754 | libsocket++1 755 | libsodium-dev 756 | libsodium23 757 | libsoup-gnome2.4-1 758 | libsoup2.4-1 759 | libsoup2.4-dev 760 | libsoxr0 761 | libspatialite7 762 | libspeechd-dev 763 | libspeex1 764 | libsphinxbase-dev 765 | libsphinxbase3 766 | libsqlite3-0 767 | libsqlite3-dev 768 | libss2 769 | libssh-gcrypt-4 770 | libssh2-1 771 | libssl-dev 772 | libssl-doc 773 | libssl3 774 | libstatgrab-dev 775 | libstdc++-10-dev 776 | libstdc++-arm-none-eabi-newlib 777 | libstdc++6 778 | libsub-name-perl 779 | libsuitesparse-dev 780 | libsuperlu5 781 | libsvm-dev 782 | libsvm3 783 | libswresample-dev 784 | libswresample3 785 | libswscale-dev 786 | libswscale5 787 | libsys-hostname-long-perl 788 | libsystemd-dev 789 | libsystemd0 790 | libsz2 791 | libtag1v5 792 | libtag1v5-vanilla 793 | libtalloc2 794 | libtasn1-6 795 | libtbb2 796 | libtcl8.6 797 | libtdb-dev 798 | libtdb1 799 | libtesseract-dev 800 | libtesseract4 801 | libtevent0 802 | libtext-charwidth-perl 803 | libtext-iconv-perl 804 | libtext-unidecode-perl 805 | libtext-wrapi18n-perl 806 | libthai-data 807 | libthai0 808 | libtheora-dev 809 | libtheora0 810 | libtiff5 811 | libtimedate-perl 812 | libtinfo-dev 813 | libtinfo5 814 | libtk8.6 815 | libtool 816 | libtool-bin 817 | libtracecmd-dev 818 | libtraceevent-dev 819 | libtracefs-dev 820 | libtry-tiny-perl 821 | libtsan0 822 | libtwolame0 823 | libubsan1 824 | libudev-dev 825 | libudev1 826 | libudf-dev 827 | libudf0 828 | libunistring-dev 829 | libunistring2 830 | libunwind-dev 831 | libunwind8 832 | liburi-perl 833 | liburiparser1 834 | libusb-0.1-4 835 | libusb-1.0-0 836 | libusb-1.0-0-dev 837 | libusb-1.0-doc 838 | libusb-dev 839 | libutempter0 840 | libuuid1 841 | libuv1 842 | libv4l-0 843 | libv4lconvert0 844 | libva-drm2 845 | libva-x11-2 846 | libva2 847 | libvdpau-va-gl1 848 | libvdpau1 849 | libvisual-0.4-0 850 | libvncserver-dev 851 | libvorbis-dev 852 | libvorbis0a 853 | libvorbisenc2 854 | libvorbisfile3 855 | libvpx-dev 856 | libvpx7 857 | libvtk9.1 858 | libwacom-bin 859 | libwacom-common 860 | libwacom9 861 | libwavpack1 862 | libwayland-bin 863 | libwayland-client0 864 | libwayland-cursor0 865 | libwayland-dev 866 | libwayland-egl1 867 | libwayland-egl1-mesa 868 | libwayland-server0 869 | libwbclient0 870 | libwebkit2gtk-4.0-37 871 | libwebkit2gtk-4.0-dev 872 | libwebp-dev 873 | libwebp7 874 | libwebpdemux2 875 | libwebpmux3 876 | libwind0-heimdal 877 | libwoff1 878 | libwrap0 879 | libwww-perl 880 | libwww-robotrules-perl 881 | libwxbase3.0-0v5 882 | libwxgtk3.0-gtk3-0v5 883 | libwxgtk3.0-gtk3-dev 884 | libx11-6 885 | libx11-data 886 | libx11-dev 887 | libx11-doc 888 | libx11-xcb-dev 889 | libx11-xcb1 890 | libx264-163 891 | libx264-dev 892 | libx265-199 893 | libx265-dev 894 | libxapian30 895 | libxau-dev 896 | libxau6 897 | libxaw7 898 | libxaw7-dev 899 | libxcb-cursor-dev 900 | libxcb-dri2-0 901 | libxcb-dri2-0-dev 902 | libxcb-dri3-0 903 | libxcb-dri3-dev 904 | libxcb-glx0 905 | libxcb-glx0-dev 906 | libxcb-icccm4 907 | libxcb-image0 908 | libxcb-keysyms1 909 | libxcb-present-dev 910 | libxcb-present0 911 | libxcb-randr0 912 | libxcb-randr0-dev 913 | libxcb-render-util0 914 | libxcb-render0 915 | libxcb-render0-dev 916 | libxcb-shape0 917 | libxcb-shape0-dev 918 | libxcb-shm0 919 | libxcb-shm0-dev 920 | libxcb-sync-dev 921 | libxcb-sync1 922 | libxcb-util1 923 | libxcb-xfixes0 924 | libxcb-xfixes0-dev 925 | libxcb-xinerama0 926 | libxcb-xkb1 927 | libxcb1 928 | libxcb1-dev 929 | libxcomposite-dev 930 | libxcomposite1 931 | libxcursor-dev 932 | libxcursor1 933 | libxdamage-dev 934 | libxdamage1 935 | libxdmcp-dev 936 | libxdmcp6 937 | libxerces-c3.2 938 | libxext-dev 939 | libxext6 940 | libxfixes-dev 941 | libxfixes3 942 | libxfont-dev 943 | libxfont2 944 | libxft-dev 945 | libxft2 946 | libxi-dev 947 | libxi6 948 | libxinerama-dev 949 | libxinerama1 950 | libxkbcommon-dev 951 | libxkbcommon-x11-0 952 | libxkbcommon0 953 | libxkbfile-dev 954 | libxkbfile1 955 | libxml-libxml-perl 956 | libxml-namespacesupport-perl 957 | libxml-parser-perl 958 | libxml-sax-base-perl 959 | libxml-sax-expat-perl 960 | libxml-sax-perl 961 | libxml2 962 | libxml2-dev 963 | libxml2-utils 964 | libxmu-dev 965 | libxmu-headers 966 | libxmu6 967 | libxmuu-dev 968 | libxmuu1 969 | libxosd-dev 970 | libxpm-dev 971 | libxpm4 972 | libxrandr-dev 973 | libxrandr2 974 | libxrender-dev 975 | libxrender1 976 | libxres-dev 977 | libxres1 978 | libxshmfence-dev 979 | libxshmfence1 980 | libxslt1.1 981 | libxss-dev 982 | libxss1 983 | libxt-dev 984 | libxt6 985 | libxtables12 986 | libxtst-dev 987 | libxtst6 988 | libxv-dev 989 | libxv1 990 | libxvidcore4 991 | libxvmc-dev 992 | libxvmc1 993 | libxxf86dga-dev 994 | libxxf86dga1 995 | libxxf86vm-dev 996 | libxxf86vm1 997 | libyaml-0-2 998 | libzfslinux-dev 999 | libzmq3-dev 1000 | libzmq5 1001 | libzstd1 1002 | libzvbi-common 1003 | libzvbi0 1004 | linux-headers-generic 1005 | linux-libc-dev 1006 | llvm 1007 | llvm-12 1008 | lxc-dev 1009 | mdbtools-dev 1010 | meson 1011 | mosquitto-dev 1012 | mpich 1013 | musl-tools 1014 | nasm 1015 | nettle-dev 1016 | ngspice 1017 | ninja-build 1018 | nodejs 1019 | openjdk-8-jre-headless 1020 | perl 1021 | php-dev 1022 | pkg-config 1023 | policykit-1 1024 | protobuf-compiler 1025 | python2-dev 1026 | python3-dev 1027 | python3-distutils 1028 | python3-gi 1029 | python3-lib2to3 1030 | python3-minimal 1031 | python3-netifaces 1032 | python3-talloc 1033 | python3-yaml 1034 | qt3d5-dev 1035 | qt5-gtk-platformtheme 1036 | qt5-qmake 1037 | qt5-qmake-bin 1038 | qt5-qmltooling-plugins 1039 | qtbase5-dev 1040 | qtbase5-dev-tools 1041 | qtchooser 1042 | qtdeclarative5-dev 1043 | qttools5-dev 1044 | qttranslations5-l10n 1045 | r-base-core 1046 | rake 1047 | re2c 1048 | readline-common 1049 | rsyslog 1050 | ruby 1051 | ruby-dev 1052 | ruby-did-you-mean 1053 | ruby-minitest 1054 | ruby-net-telnet 1055 | ruby-power-assert 1056 | ruby-test-unit 1057 | rubygems-integration 1058 | samba-libs 1059 | sane-utils 1060 | sgml-base 1061 | sudo 1062 | tcl 1063 | tcl8.6 1064 | tex-common 1065 | texi2html 1066 | texinfo 1067 | tk 1068 | tk8.6 1069 | tzdata 1070 | ucf 1071 | udev 1072 | uuid-dev 1073 | va-driver-all 1074 | vdpau-driver-all 1075 | wget 1076 | xbitmaps 1077 | xorg-dev 1078 | xorg-sgml-doctools 1079 | xserver-xorg-dev 1080 | xtrans-dev 1081 | xxd 1082 | xz-utils 1083 | zlib1g-dev 1084 | -------------------------------------------------------------------------------- /docker/run.sh: -------------------------------------------------------------------------------- 1 | set -u 2 | exec 2>&1 3 | 4 | export TERM=xterm-256color 5 | 6 | # Extract the cache if it exists 7 | # Ideally it's just an error to not have a cache, but this script is executed to build the cache. 8 | if [ -e /cache.tar.gz ] 9 | then 10 | tar xf /cache.tar.gz 11 | fi 12 | 13 | TOOLCHAIN=nightly 14 | 15 | export CARGO_INCREMENTAL=0 16 | export RUST_BACKTRACE=1 17 | export RUSTFLAGS="--cap-lints=warn -Copt-level=0 -Zvalidate-mir -Aunexpected-cfgs" 18 | if [[ $TARGET == "x86_64-unknown-linux-gnu" ]]; then 19 | export RUSTFLAGS="$RUSTFLAGS -Ctarget-cpu=x86-64-v2" 20 | elif [[ $TARGET == "aarch64-unknown-linux-gnu" ]]; then 21 | export RUSTFLAGS="$RUSTFLAGS -Ctarget-cpu=apple-a14" 22 | fi 23 | 24 | if [[ $TOOL == "build" ]]; then 25 | export RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=4 -Zinline-mir -Cdebuginfo=2 -Cdebug-assertions=yes -Copt-level=3 -Zcross-crate-inline-threshold=always -Zthreads=64 -Zinline-mir-hint-threshold=10000 -Zinline-mir-threshold=10000 -Zmir-enable-passes=-DataflowConstProp" 26 | elif [[ $TOOL == "asan" ]]; then 27 | # Use 1 GB for a default stack size. 28 | # We really want to only run out of stack in true infinite recursion. 29 | ulimit -s 1048576 30 | export RUST_MIN_STACK=1073741824 31 | export RUSTFLAGS="$RUSTFLAGS -Cdebuginfo=1 -Zstrict-init-checks=no" 32 | export ASAN_OPTIONS="color=always:detect_leaks=0:detect_stack_use_after_return=true:allocator_may_return_null=1:detect_invalid_pointer_pairs=2" 33 | elif [[ $TOOL == "miri" ]]; then 34 | export RUSTFLAGS="$RUSTFLAGS -Zrandomize-layout -Cdebuginfo=1" 35 | export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-num-cpus=64" 36 | elif [[ $TOOL == "check" ]]; then 37 | export RUSTFLAGS="$RUSTFLAGS -Zthreads=64" 38 | fi 39 | export RUSTDOCFLAGS=$RUSTFLAGS 40 | 41 | function timed { 42 | timeout --kill-after=10s 1h inapty cargo +$TOOLCHAIN "$@" --target=$TARGET 43 | } 44 | 45 | function run_build { 46 | timed test --no-run $ARGS 47 | } 48 | 49 | function run_check { 50 | timed check $ARGS 51 | } 52 | 53 | function run_asan { 54 | timed careful test -Zcareful-sanitizer=address --no-run $ARGS &> /dev/null 55 | timed careful test -Zcareful-sanitizer=address --color=always --no-fail-fast $ARGS 56 | } 57 | 58 | function run_miri { 59 | timed miri test --no-run $ARGS &> /dev/null 60 | # rustdoc is already passed --color=always, so adding it to the global MIRIFLAGS is just an error 61 | MIRIFLAGS="$MIRIFLAGS --color=always" timed miri nextest run --color=always --no-fail-fast --config-file=/root/.cargo/nextest.toml $ARGS 62 | # nextest runs one interpreter per test, so unsupported errors only terminate the test not the whole suite. 63 | # but the doctests implementation is quite different and already creates a new interpreter for every test. 64 | timed miri test --doc --no-fail-fast $ARGS 65 | } 66 | 67 | if [[ $TOOL == "miri" ]]; then 68 | timed miri setup &> /dev/null 69 | fi 70 | 71 | while read crate; 72 | do 73 | cd /build 74 | # Delete everything in our writable mount points 75 | find /build /tmp /root/.cargo/registry -mindepth 1 -delete 76 | if cargo download $crate /build; then 77 | ARGS=$(get-args $crate) 78 | cargo update &> /dev/null 79 | if [[ $TOOL == "build" ]]; then 80 | run_build 81 | elif [[ $TOOL == "check" ]]; then 82 | run_check 83 | elif [[ $TOOL == "asan" ]]; then 84 | run_asan 85 | elif [[ $TOOL == "miri" ]]; then 86 | run_miri 87 | else 88 | exit 1 89 | fi 90 | fi 91 | echo "-${TEST_END_DELIMITER}-" 92 | # Delete everything in our writable mount points 93 | find /build /tmp /root/.cargo/registry -mindepth 1 -delete 94 | done < /dev/stdin 95 | -------------------------------------------------------------------------------- /get-args/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "get-args" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "serde", 10 | "serde_json", 11 | ] 12 | 13 | [[package]] 14 | name = "itoa" 15 | version = "1.0.9" 16 | source = "registry+https://github.com/rust-lang/crates.io-index" 17 | checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" 18 | 19 | [[package]] 20 | name = "proc-macro2" 21 | version = "1.0.66" 22 | source = "registry+https://github.com/rust-lang/crates.io-index" 23 | checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" 24 | dependencies = [ 25 | "unicode-ident", 26 | ] 27 | 28 | [[package]] 29 | name = "quote" 30 | version = "1.0.31" 31 | source = "registry+https://github.com/rust-lang/crates.io-index" 32 | checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" 33 | dependencies = [ 34 | "proc-macro2", 35 | ] 36 | 37 | [[package]] 38 | name = "ryu" 39 | version = "1.0.15" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" 42 | 43 | [[package]] 44 | name = "serde" 45 | version = "1.0.174" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1" 48 | dependencies = [ 49 | "serde_derive", 50 | ] 51 | 52 | [[package]] 53 | name = "serde_derive" 54 | version = "1.0.174" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e" 57 | dependencies = [ 58 | "proc-macro2", 59 | "quote", 60 | "syn", 61 | ] 62 | 63 | [[package]] 64 | name = "serde_json" 65 | version = "1.0.103" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" 68 | dependencies = [ 69 | "itoa", 70 | "ryu", 71 | "serde", 72 | ] 73 | 74 | [[package]] 75 | name = "syn" 76 | version = "2.0.27" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" 79 | dependencies = [ 80 | "proc-macro2", 81 | "quote", 82 | "unicode-ident", 83 | ] 84 | 85 | [[package]] 86 | name = "unicode-ident" 87 | version = "1.0.11" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" 90 | -------------------------------------------------------------------------------- /get-args/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "get-args" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | serde = { version = "1.0.174", features = ["derive"] } 10 | serde_json = "1.0.103" 11 | 12 | [package.metadata.playground] 13 | features = ["feature"] 14 | no-default-features = true 15 | all-features = true 16 | 17 | [package.metadata.docs.rs] 18 | features = ["ouch"] 19 | 20 | [package.metadata."docs.rs"] 21 | features = ["ahhhh"] 22 | -------------------------------------------------------------------------------- /get-args/src/main.rs: -------------------------------------------------------------------------------- 1 | use serde::Deserialize; 2 | use serde_json::Value; 3 | use std::collections::{HashMap, HashSet}; 4 | use std::process::Command; 5 | 6 | #[derive(Debug, Deserialize)] 7 | struct Config { 8 | packages: Vec, 9 | } 10 | 11 | #[derive(Debug, Deserialize)] 12 | struct Package { 13 | name: String, 14 | metadata: Option>, 15 | } 16 | 17 | impl Package { 18 | fn playground(&self) -> Option { 19 | let value = self.metadata.as_ref()?.get("playground")?.clone(); 20 | serde_json::from_value(value).ok() 21 | } 22 | 23 | fn docsrs(&self) -> Option { 24 | let value = self.metadata.as_ref()?.get("docs.rs")?.clone(); 25 | serde_json::from_value(value).ok() 26 | } 27 | 28 | fn docs_rs(&self) -> Option { 29 | let value = self.metadata.as_ref()?.get("docs")?.clone(); 30 | let value = value.as_object()?.get("rs")?.clone(); 31 | serde_json::from_value(value).ok() 32 | } 33 | } 34 | 35 | #[derive(Default, Debug, Deserialize)] 36 | struct Metadata { 37 | #[serde(default)] 38 | features: HashSet, 39 | #[serde(rename = "no-default-features", default)] 40 | no_default_features: bool, 41 | #[serde(rename = "all-features", default)] 42 | all_features: bool, 43 | } 44 | 45 | impl Metadata { 46 | fn merge(&mut self, other: Self) { 47 | self.features.extend(other.features); 48 | self.no_default_features |= other.no_default_features; 49 | self.all_features |= other.all_features; 50 | } 51 | } 52 | 53 | fn run() -> Option { 54 | let krate = std::env::args().nth(1)?; 55 | let krate = krate.split('@').next()?; 56 | 57 | let mut cmd = Command::new("cargo"); 58 | if let Ok(toolchain) = std::env::var("TOOLCHAIN") { 59 | cmd.arg(format!("+{toolchain}")); 60 | } 61 | let config = cmd.arg("metadata").output().ok()?; 62 | if !config.status.success() { 63 | std::process::exit(config.status.code().unwrap_or(1)); 64 | } 65 | 66 | let config: Config = serde_json::from_slice(&config.stdout).ok()?; 67 | 68 | let krate = config 69 | .packages 70 | .iter() 71 | .find(|package| package.name == krate)?; 72 | 73 | let mut metadata = Metadata::default(); 74 | 75 | for section in [krate.playground(), krate.docsrs(), krate.docs_rs()] { 76 | if let Some(section) = section { 77 | metadata.merge(section); 78 | } 79 | } 80 | 81 | let mut args: Vec = Vec::new(); 82 | if metadata.no_default_features { 83 | args.push("--no-default-features".to_string()); 84 | } 85 | if metadata.all_features { 86 | args.push("--all-features".to_string()); 87 | } 88 | 89 | let mut features = metadata.features.into_iter().collect::>(); 90 | features.sort(); 91 | let features = features.join(","); 92 | if !features.is_empty() { 93 | args.push(format!("--features={}", features)); 94 | } 95 | 96 | Some(args.join(" ")) 97 | } 98 | 99 | fn main() { 100 | if let Some(args) = run() { 101 | println!("{args}"); 102 | } else { 103 | std::process::exit(1); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /htmlpty/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ansi-to-html" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "env_logger", 10 | "log", 11 | "vte", 12 | ] 13 | 14 | [[package]] 15 | name = "arrayvec" 16 | version = "0.7.4" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" 19 | 20 | [[package]] 21 | name = "env_logger" 22 | version = "0.10.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" 25 | dependencies = [ 26 | "log", 27 | ] 28 | 29 | [[package]] 30 | name = "htmlpty" 31 | version = "0.1.0" 32 | dependencies = [ 33 | "ansi-to-html", 34 | "libc", 35 | ] 36 | 37 | [[package]] 38 | name = "libc" 39 | version = "0.2.147" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" 42 | 43 | [[package]] 44 | name = "log" 45 | version = "0.4.20" 46 | source = "registry+https://github.com/rust-lang/crates.io-index" 47 | checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" 48 | 49 | [[package]] 50 | name = "proc-macro2" 51 | version = "1.0.66" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" 54 | dependencies = [ 55 | "unicode-ident", 56 | ] 57 | 58 | [[package]] 59 | name = "quote" 60 | version = "1.0.33" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" 63 | dependencies = [ 64 | "proc-macro2", 65 | ] 66 | 67 | [[package]] 68 | name = "unicode-ident" 69 | version = "1.0.11" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" 72 | 73 | [[package]] 74 | name = "utf8parse" 75 | version = "0.2.1" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" 78 | 79 | [[package]] 80 | name = "vte" 81 | version = "0.11.1" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" 84 | dependencies = [ 85 | "arrayvec", 86 | "utf8parse", 87 | "vte_generate_state_changes", 88 | ] 89 | 90 | [[package]] 91 | name = "vte_generate_state_changes" 92 | version = "0.1.1" 93 | source = "registry+https://github.com/rust-lang/crates.io-index" 94 | checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" 95 | dependencies = [ 96 | "proc-macro2", 97 | "quote", 98 | ] 99 | -------------------------------------------------------------------------------- /htmlpty/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "htmlpty" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | libc = "0.2.147" 10 | ansi-to-html = { path = "../ansi-to-html" } 11 | -------------------------------------------------------------------------------- /htmlpty/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io::{stderr, stdout, Read, Write}; 3 | use std::os::fd::FromRawFd; 4 | use std::os::unix::process::CommandExt; 5 | use std::process::Command; 6 | use std::ptr; 7 | 8 | fn main() { 9 | let winsz = libc::winsize { 10 | ws_col: 128, 11 | ws_row: 64, 12 | ws_xpixel: 1280, 13 | ws_ypixel: 1408, 14 | }; 15 | 16 | let mut pty: i32 = 0; 17 | 18 | // SAFETY: Pointer arguments are valid (and thus ignored) or null 19 | let pid = unsafe { libc::forkpty(&mut pty, ptr::null_mut(), ptr::null_mut(), &winsz) }; 20 | 21 | if pid == 0 { 22 | // We are the child. Spawn the subprocess based off our arguments. 23 | let mut args = std::env::args_os().skip(1); 24 | Command::new(args.next().unwrap()).args(args).exec(); 25 | unreachable!("exec should not return"); 26 | } else { 27 | // We are the originating process. Copy from the pty to output. 28 | // SAFETY: master is open and valid, it was just opened by forkpty 29 | let mut pty = unsafe { File::from_raw_fd(pty) }; 30 | 31 | let mut buf = [0u8; 4096]; 32 | let mut stdout = stdout().lock(); 33 | let stderr = stderr().lock(); 34 | let mut renderer = ansi_to_html::Handle::new(stderr); 35 | while let Ok(n) = pty.read(&mut buf) { 36 | let bytes = &buf[..n]; 37 | stdout.write_all(bytes).unwrap(); 38 | renderer.write_all(bytes).unwrap(); 39 | } 40 | 41 | stdout.flush().unwrap(); 42 | renderer.finish().unwrap(); 43 | 44 | // Exit according to our child's status 45 | // SAFETY: No preconditions 46 | let status = unsafe { 47 | let mut status = 0; 48 | libc::waitpid(pid, &mut status, 0); 49 | libc::WEXITSTATUS(status) 50 | }; 51 | std::process::exit(status); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /inapty/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "inapty" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "libc", 10 | ] 11 | 12 | [[package]] 13 | name = "libc" 14 | version = "0.2.147" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" 17 | -------------------------------------------------------------------------------- /inapty/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "inapty" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | libc = "0.2.147" 10 | -------------------------------------------------------------------------------- /inapty/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::os::fd::FromRawFd; 3 | use std::os::unix::process::CommandExt; 4 | use std::process::Command; 5 | use std::ptr; 6 | 7 | fn main() { 8 | let winsz = libc::winsize { 9 | ws_col: 128, 10 | ws_row: 64, 11 | ws_xpixel: 1280, 12 | ws_ypixel: 1408, 13 | }; 14 | 15 | let mut pty: i32 = 0; 16 | 17 | // SAFETY: Pointer arguments are valid (and thus ignored) or null 18 | let pid = unsafe { libc::forkpty(&mut pty, ptr::null_mut(), ptr::null_mut(), &winsz) }; 19 | 20 | if pid == 0 { 21 | // We are the child. Spawn the subprocess based off our arguments. 22 | let mut args = std::env::args_os().skip(1); 23 | Command::new(args.next().unwrap()).args(args).exec(); 24 | unreachable!("exec should not return"); 25 | } else { 26 | // We are the originating process. Copy from the pty to output. 27 | // SAFETY: master is open and valid, it was just opened by forkpty 28 | let mut pty = unsafe { File::from_raw_fd(pty) }; 29 | // Copy all the output from the child pty to our stdout 30 | while std::io::copy(&mut pty, &mut std::io::stdout()).is_ok() {} 31 | // Exit according to our child's status 32 | // SAFETY: No preconditions 33 | let status = unsafe { 34 | let mut status = 0; 35 | libc::waitpid(pid, &mut status, 0); 36 | libc::WEXITSTATUS(status) 37 | }; 38 | std::process::exit(status); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/client.rs: -------------------------------------------------------------------------------- 1 | use crate::{Crate, Status, Tool, Version}; 2 | use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart, Object}; 3 | use aws_smithy_types_convert::date_time::DateTimeExt; 4 | use backoff::Error; 5 | use backoff::ExponentialBackoff; 6 | use color_eyre::Result; 7 | use futures_util::StreamExt; 8 | use futures_util::TryFutureExt; 9 | use std::collections::HashMap; 10 | use std::future::Future; 11 | 12 | #[derive(Clone)] 13 | pub struct Client { 14 | inner: aws_sdk_s3::Client, 15 | bucket: String, 16 | tool: Tool, 17 | } 18 | 19 | const CHUNK_SIZE: usize = 5 * 1024 * 1024; 20 | 21 | impl Client { 22 | pub async fn new(tool: Tool, bucket: &str) -> Result { 23 | let config = aws_config::load_from_env().await; 24 | let inner = aws_sdk_s3::Client::new(&config); 25 | Ok(Self { 26 | inner, 27 | bucket: bucket.to_string(), 28 | tool, 29 | }) 30 | } 31 | 32 | pub fn tool(&self) -> Tool { 33 | self.tool 34 | } 35 | 36 | pub async fn upload(&self, key: &str, data: &[u8], content_type: &str) -> Result<()> { 37 | retry(|| self._upload(key, data, content_type)).await 38 | } 39 | 40 | async fn _upload(&self, key: &str, data: &[u8], content_type: &str) -> Result<()> { 41 | // S3 has a minimum multipart upload size of 5 MB. If we are below that, we need to use 42 | // PutObject. 43 | if data.len() < CHUNK_SIZE { 44 | self.inner 45 | .put_object() 46 | .bucket(&self.bucket) 47 | .key(key) 48 | .body(data.to_vec().into()) 49 | .content_type(content_type) 50 | .send() 51 | .await?; 52 | return Ok(()); 53 | } 54 | 55 | let res = self 56 | .inner 57 | .create_multipart_upload() 58 | .bucket(&self.bucket) 59 | .key(key) 60 | .content_type(content_type) 61 | .send() 62 | .await?; 63 | let upload_id = res.upload_id().unwrap(); 64 | let mut parts = Vec::new(); 65 | for (part_number, chunk) in data.chunks(CHUNK_SIZE).enumerate() { 66 | // part numbers must start at 1 67 | let part_number = part_number as i32 + 1; 68 | let upload_part_res = self 69 | .inner 70 | .upload_part() 71 | .key(key) 72 | .bucket(&self.bucket) 73 | .upload_id(upload_id) 74 | .body(chunk.to_vec().into()) 75 | .part_number(part_number) 76 | .send() 77 | .await?; 78 | parts.push( 79 | CompletedPart::builder() 80 | .e_tag(upload_part_res.e_tag.unwrap_or_default()) 81 | .part_number(part_number) 82 | .build(), 83 | ) 84 | } 85 | let completed_multipart_upload = CompletedMultipartUpload::builder() 86 | .set_parts(Some(parts)) 87 | .build(); 88 | self.inner 89 | .complete_multipart_upload() 90 | .bucket(&self.bucket) 91 | .key(key) 92 | .multipart_upload(completed_multipart_upload) 93 | .upload_id(upload_id) 94 | .send() 95 | .await?; 96 | 97 | Ok(()) 98 | } 99 | 100 | pub async fn upload_raw(&self, krate: &Crate, data: Vec) -> Result<()> { 101 | self.upload(&self.tool.raw_crate_path(krate), &data, "text/plain") 102 | .await 103 | } 104 | 105 | pub async fn upload_html(&self, krate: &Crate, data: Vec) -> Result<()> { 106 | let key = self.tool.rendered_crate_path(krate); 107 | self.upload(&key, &data, "text/html;charset=utf-8").await 108 | } 109 | 110 | pub async fn download_raw(&self, krate: &Crate) -> Result> { 111 | self.download(&self.tool.raw_crate_path(krate)).await 112 | } 113 | 114 | pub async fn download_html(&self, krate: &Crate) -> Result> { 115 | self.download(&self.tool.rendered_crate_path(krate)).await 116 | } 117 | 118 | async fn download(&self, key: &str) -> Result> { 119 | retry(|| self._download(key)).await 120 | } 121 | 122 | async fn _download(&self, key: &str) -> Result> { 123 | let response = self 124 | .inner 125 | .get_object() 126 | .bucket(&self.bucket) 127 | .key(key) 128 | .send() 129 | .await?; 130 | let bytes = response.body.collect().await?; 131 | Ok(bytes.to_vec()) 132 | } 133 | 134 | pub async fn get_crate_downloads(&self) -> Result>> { 135 | let response = self 136 | .inner 137 | .get_object() 138 | .bucket(&self.bucket) 139 | .key("downloads.json") 140 | .send() 141 | .await?; 142 | let bytes = response.body.collect().await?; 143 | let blob = bytes.to_vec(); 144 | let crates: HashMap<_, _> = serde_json::from_slice(&blob)?; 145 | Ok(crates) 146 | } 147 | 148 | pub async fn get_crate_versions(&self) -> Result> { 149 | let response = self 150 | .inner 151 | .get_object() 152 | .bucket(&self.bucket) 153 | .key("crates.json") 154 | .send() 155 | .await?; 156 | let bytes = response.body.collect().await?; 157 | let blob = bytes.to_vec(); 158 | let crates: Vec<(String, String)> = serde_json::from_slice(&blob)?; 159 | let crates = crates 160 | .into_iter() 161 | .map(|krate| Crate { 162 | name: krate.0, 163 | version: Version::parse(&krate.1), 164 | status: Status::Unknown, 165 | recent_downloads: None, 166 | }) 167 | .collect(); 168 | Ok(crates) 169 | } 170 | 171 | pub async fn list_finished_crates(&self, dur: Option) -> Result> { 172 | let now = time::OffsetDateTime::now_utc(); 173 | let prefix = format!("{}/", self.tool.raw_path()); 174 | let mut res = self 175 | .inner 176 | .list_objects_v2() 177 | .bucket(&self.bucket) 178 | .prefix(&prefix) 179 | .into_paginator() 180 | .send(); 181 | let mut files = Vec::new(); 182 | while let Some(res) = res.next().await { 183 | let page = res?; 184 | for obj in page.contents().unwrap_or_default() { 185 | if let Some(dur) = dur { 186 | if let Some(modified) = obj.last_modified() { 187 | let modified = modified.to_time().unwrap(); 188 | // Ignore crates older than dur 189 | if now - modified > dur { 190 | continue; 191 | } 192 | } 193 | } 194 | if let Some(key) = obj.key().and_then(|key| key.strip_prefix(&prefix)) { 195 | let mut it = key.split('/'); 196 | let Some(name) = it.next() else { 197 | continue; 198 | }; 199 | let Some(version) = it.next() else { 200 | continue; 201 | }; 202 | files.push(Crate { 203 | name: name.to_string(), 204 | version: Version::parse(version), 205 | status: Status::Unknown, 206 | recent_downloads: None, 207 | }); 208 | } 209 | } 210 | } 211 | Ok(files) 212 | } 213 | 214 | pub async fn list_rendered_crates(&self) -> Result> { 215 | let prefix = format!("{}/", self.tool.html_path()); 216 | let mut res = self 217 | .inner 218 | .list_objects_v2() 219 | .bucket(&self.bucket) 220 | .prefix(&prefix) 221 | .into_paginator() 222 | .send(); 223 | let mut files = Vec::new(); 224 | while let Some(res) = res.next().await { 225 | let page = res?; 226 | for obj in page.contents().unwrap_or_default() { 227 | if let Some(key) = obj.key().and_then(|key| key.strip_prefix(&prefix)) { 228 | files.push(key.to_string()); 229 | } 230 | } 231 | } 232 | Ok(files) 233 | } 234 | 235 | pub async fn upload_landing_page(&self, data: Vec) -> Result<()> { 236 | self.upload( 237 | self.tool.landing_page_path(), 238 | &data, 239 | "text/html;charset=utf-8", 240 | ) 241 | .await 242 | } 243 | 244 | pub async fn list_db(&self) -> Result> { 245 | let res = retry(move || { 246 | self.inner 247 | .list_objects_v2() 248 | .bucket(&self.bucket) 249 | .prefix("crates.json") 250 | .max_keys(1) 251 | .send() 252 | }) 253 | .await?; 254 | let meta = res.contents.and_then(|c| c.first().cloned()); 255 | Ok(meta) 256 | } 257 | } 258 | 259 | async fn retry(mut f: Func) -> std::result::Result 260 | where 261 | Func: FnMut() -> Fut, 262 | Fut: Future>, 263 | E: std::fmt::Display, 264 | { 265 | backoff::future::retry_notify( 266 | ExponentialBackoff::default(), 267 | || f().map_err(Error::transient), 268 | |e, _| { 269 | log::warn!("{}", e); 270 | }, 271 | ) 272 | .await 273 | } 274 | -------------------------------------------------------------------------------- /src/db_dump.rs: -------------------------------------------------------------------------------- 1 | use crate::{Crate, Status, Version}; 2 | use color_eyre::Result; 3 | use flate2::read::GzDecoder; 4 | use fxhash::FxHashMap; 5 | use serde::Deserialize; 6 | use std::{collections::hash_map::Entry, io::Read}; 7 | use tar::Archive; 8 | 9 | struct PublishedCrate { 10 | crate_id: u64, 11 | recent_downloads: u64, 12 | version: Version, 13 | } 14 | 15 | pub fn download() -> Result> { 16 | log::info!("Downloading crate database"); 17 | 18 | let mut archive = Vec::new(); 19 | GzDecoder::new( 20 | ureq::get("https://static.crates.io/db-dump.tar.gz") 21 | .call()? 22 | .into_reader(), 23 | ) 24 | .read_to_end(&mut archive)?; 25 | 26 | log::info!("Processing crate database"); 27 | 28 | let mut tar = Archive::new(archive.as_slice()); 29 | 30 | let mut version_to_downloads = FxHashMap::default(); 31 | let mut version_to_crate = FxHashMap::default(); 32 | let mut num_to_name = FxHashMap::default(); 33 | 34 | for entry in tar.entries()? { 35 | let entry = entry?; 36 | let path = entry.path()?; 37 | let mut components = path.components(); 38 | components.next(); // The first element of the path is the date 39 | 40 | if components.as_path().to_str() == Some("data/version_downloads.csv") { 41 | version_to_downloads = decode_downloads( 42 | &archive[entry.raw_file_position() as usize..][..entry.size() as usize], 43 | )?; 44 | } 45 | if components.as_path().to_str() == Some("data/versions.csv") { 46 | version_to_crate = decode_versions( 47 | &archive[entry.raw_file_position() as usize..][..entry.size() as usize], 48 | )?; 49 | } 50 | if components.as_path().to_str() == Some("data/crates.csv") { 51 | num_to_name = decode_crates( 52 | &archive[entry.raw_file_position() as usize..][..entry.size() as usize], 53 | )?; 54 | } 55 | } 56 | 57 | // Aggregate download statistics by crate 58 | let mut crate_to_downloads: FxHashMap = FxHashMap::default(); 59 | 60 | for (version_id, mut krate) in version_to_crate { 61 | if let Some(downloads) = version_to_downloads.get(&version_id) { 62 | match crate_to_downloads.entry(krate.crate_id) { 63 | Entry::Vacant(v) => { 64 | krate.recent_downloads += downloads; 65 | v.insert(krate); 66 | } 67 | Entry::Occupied(mut v) => { 68 | let existing = v.get_mut(); 69 | existing.recent_downloads += downloads; 70 | if krate.version > existing.version { 71 | existing.version = krate.version; 72 | } 73 | } 74 | } 75 | } 76 | } 77 | 78 | // Sort by downloads 79 | let mut crates = crate_to_downloads 80 | .into_iter() 81 | .filter_map(|(_id, krate)| { 82 | num_to_name.get(&krate.crate_id).map(|name| Crate { 83 | name: name.clone(), 84 | recent_downloads: Some(krate.recent_downloads), 85 | version: krate.version, 86 | status: Status::Unknown, 87 | }) 88 | }) 89 | .collect::>(); 90 | crates.sort_by(|a, b| b.recent_downloads.cmp(&a.recent_downloads)); 91 | Ok(crates) 92 | } 93 | 94 | #[derive(Deserialize)] 95 | struct DownloadsRecord { 96 | #[serde(skip)] 97 | _date: String, 98 | downloads: u64, 99 | version_id: u64, 100 | } 101 | 102 | fn decode_downloads(csv: &[u8]) -> Result> { 103 | let mut downloads = FxHashMap::default(); 104 | 105 | let mut reader = csv::Reader::from_reader(csv); 106 | for record in reader.deserialize::() { 107 | let record = record?; 108 | *downloads.entry(record.version_id).or_default() += record.downloads; 109 | } 110 | 111 | Ok(downloads) 112 | } 113 | 114 | #[derive(Deserialize)] 115 | struct VersionsRecord { 116 | crate_id: u64, 117 | #[serde(skip)] 118 | _crate_size: u64, 119 | #[serde(skip)] 120 | _created_at: String, 121 | #[serde(skip)] 122 | _downloads: u64, 123 | #[serde(skip)] 124 | _features: String, 125 | id: u64, 126 | #[serde(skip)] 127 | _license: String, 128 | num: String, // crate version 129 | #[serde(skip)] 130 | _published_by: String, 131 | #[serde(skip)] 132 | _updated_at: String, 133 | #[serde(skip)] 134 | _yanked: String, 135 | } 136 | 137 | fn decode_versions(csv: &[u8]) -> Result> { 138 | let mut map = FxHashMap::default(); 139 | 140 | let mut reader = csv::Reader::from_reader(csv); 141 | for record in reader.deserialize::() { 142 | let record = record?; 143 | map.insert( 144 | record.id, 145 | PublishedCrate { 146 | crate_id: record.crate_id, 147 | recent_downloads: 0, 148 | version: Version::parse(&record.num), 149 | }, 150 | ); 151 | } 152 | 153 | Ok(map) 154 | } 155 | 156 | #[derive(Deserialize)] 157 | struct CratesRecord { 158 | #[serde(skip)] 159 | _created_at: String, 160 | #[serde(skip)] 161 | _description: String, 162 | #[serde(skip)] 163 | _documentation: String, 164 | #[serde(skip)] 165 | _downloads: u64, 166 | id: u64, 167 | #[serde(skip)] 168 | _max_upload_size: u64, 169 | name: String, 170 | #[serde(skip)] 171 | _readme: String, 172 | #[serde(skip)] 173 | _repository: String, 174 | #[serde(skip)] 175 | _updated_at: String, 176 | } 177 | 178 | fn decode_crates(csv: &[u8]) -> Result> { 179 | let mut map = FxHashMap::default(); 180 | 181 | let mut reader = csv::Reader::from_reader(csv); 182 | for record in reader.deserialize::() { 183 | let record = record?; 184 | map.insert(record.id, record.name); 185 | } 186 | Ok(map) 187 | } 188 | -------------------------------------------------------------------------------- /src/diagnose.rs: -------------------------------------------------------------------------------- 1 | use crate::{Cause, Crate, Status}; 2 | 3 | use color_eyre::Result; 4 | use once_cell::sync::Lazy; 5 | use regex::Regex; 6 | 7 | static ANSI_REGEX: Lazy = 8 | Lazy::new(|| Regex::new("\x1b(\\[[0-9;?]*[A-HJKSTfhilmnsu]|\\(B)").unwrap()); 9 | 10 | pub fn diagnose(krate: &mut Crate, output: &[u8]) -> Result<()> { 11 | let output = String::from_utf8_lossy(output); 12 | let output = ANSI_REGEX.replace_all(&output, "").to_string(); 13 | // Strip ANSI escape codes from the output; 14 | krate.status = if output.contains("Undefined Behavior: ") { 15 | Status::UB { 16 | cause: diagnose_output(&output), 17 | } 18 | } else if output.contains("ERROR: AddressSanitizer: ") { 19 | if output 20 | .contains("WARNING: ASan is ignoring requested __asan_handle_no_return: stack type") 21 | { 22 | Status::Error("ASan false positive?".to_string()) 23 | } else { 24 | Status::UB { 25 | cause: diagnose_asan(&output), 26 | } 27 | } 28 | } else if output.contains("SIGILL: illegal instruction") { 29 | Status::UB { 30 | cause: vec![Cause { 31 | kind: "SIGILL".to_string(), 32 | source_crate: None, 33 | }], 34 | } 35 | } else if output.contains("misaligned pointer dereference") { 36 | Status::UB { 37 | cause: vec![Cause { 38 | kind: "misaligned pointer dereference".to_string(), 39 | source_crate: None, 40 | }], 41 | } 42 | } else if output.contains("attempted to leave type") { 43 | Status::UB { 44 | cause: vec![Cause { 45 | kind: "uninit type which does not permit uninit".to_string(), 46 | source_crate: None, 47 | }], 48 | } 49 | } else if output.contains("unsafe precondition(s) violated") { 50 | Status::UB { 51 | cause: vec![Cause { 52 | kind: "precondition check failed".to_string(), 53 | source_crate: None, 54 | }], 55 | } 56 | } else if output.contains("Command exited with non-zero status 124") { 57 | Status::Error("Timeout".to_string()) 58 | } else if output.contains("Command exited with non-zero status 255") { 59 | Status::Error("OOM".to_string()) 60 | } else if output.contains("Command exited with non-zero status") { 61 | Status::Error(String::new()) 62 | } else { 63 | Status::Passing 64 | }; 65 | Ok(()) 66 | } 67 | 68 | fn diagnose_asan(output: &str) -> Vec { 69 | let mut causes = Vec::new(); 70 | 71 | let lines = output.lines().collect::>(); 72 | 73 | for line in lines 74 | .iter() 75 | .filter(|line| line.contains("ERROR: AddressSanitizer: ")) 76 | { 77 | if line.contains("requested allocation size") { 78 | causes.push(Cause { 79 | kind: "requested allocation size exceeds maximum supported size".to_string(), 80 | source_crate: None, 81 | }); 82 | } else if let Some(kind) = line.split_whitespace().nth(2) { 83 | causes.push(Cause { 84 | kind: kind.to_string(), 85 | source_crate: None, 86 | }); 87 | } 88 | } 89 | causes.sort(); 90 | causes.dedup(); 91 | causes 92 | } 93 | 94 | fn diagnose_output(output: &str) -> Vec { 95 | let mut causes = Vec::new(); 96 | 97 | let lines = output.lines().collect::>(); 98 | 99 | for (l, line) in lines 100 | .iter() 101 | .enumerate() 102 | .filter(|(_, line)| line.contains("Undefined Behavior: ")) 103 | { 104 | let end = lines 105 | .iter() 106 | .enumerate() 107 | .skip(l) 108 | .find_map(|(l, line)| { 109 | if line.trim().is_empty() { 110 | Some(l) 111 | } else { 112 | None 113 | } 114 | }) 115 | .unwrap_or(l + 1); 116 | 117 | let kind; 118 | if line.contains("Data race detected") { 119 | kind = "data race".to_string() 120 | } else if line.contains("encountered uninitialized") 121 | || line.contains("this operation requires initialized memory") 122 | { 123 | kind = "uninitialized memory".to_string(); 124 | } else if line.contains("out-of-bounds") { 125 | kind = "invalid pointer offset".to_string(); 126 | } else if line.contains("dereferencing pointer failed: null pointer is not a valid pointer") 127 | { 128 | kind = "null pointer dereference".to_string(); 129 | } else if line.contains("encountered 0, but expected something greater or equal to 1") { 130 | kind = "zero-initialized nonzero type".to_string(); 131 | } else if line.contains("encountered a null reference") { 132 | kind = "null reference".to_string(); 133 | } else if line.contains("accessing memory with alignment") { 134 | kind = "misaligned pointer dereference".to_string(); 135 | } else if line.contains("dangling reference") { 136 | kind = "dangling reference".to_string(); 137 | } else if line.contains("unaligned reference") { 138 | kind = "unaligned reference".to_string(); 139 | } else if line.contains("incorrect layout on deallocation") { 140 | kind = "incorrect layout on deallocation".to_string(); 141 | } else if line.contains("deallocating while") && line.contains("is strongly protected") { 142 | kind = "deallocation conflict with dereferenceable".to_string(); 143 | } else if line.contains("which is strongly protected because it is an argument of call") { 144 | kind = "protector invalidation".to_string(); 145 | } else if line.contains("attempting a write access") 146 | && line.contains("only grants SharedReadOnly") 147 | { 148 | kind = "SB-write-via-&".to_string(); 149 | } else if line.contains("borrow stack") 150 | || line.contains("reborrow") 151 | || line.contains("retag") 152 | { 153 | if line.contains("") { 154 | kind = "int-to-ptr cast".to_string(); 155 | } else { 156 | kind = diagnose_sb(&lines[l..end]); 157 | } 158 | } else if line.contains("type validation failed") { 159 | let second = line.split(": encountered").nth(1).unwrap().trim(); 160 | kind = format!("type validation failed: encountered {}", second); 161 | } else { 162 | kind = line 163 | .split("Undefined Behavior: ") 164 | .nth(1) 165 | .unwrap() 166 | .trim() 167 | .to_string(); 168 | } 169 | 170 | let mut source_crate = None; 171 | 172 | for line in &lines[l..] { 173 | if line.contains("inside `") && line.contains(" at ") { 174 | let path = line.split(" at ").nth(1).unwrap(); 175 | if path.starts_with("/build") || !path.starts_with('/') { 176 | break; 177 | } else if path.contains(".cargo/registry/src/") { 178 | let crate_name = path.split('/').nth(6).unwrap(); 179 | source_crate = Some(crate_name.to_string()); 180 | break; 181 | } 182 | } 183 | } 184 | causes.push(Cause { kind, source_crate }) 185 | } 186 | 187 | causes.sort(); 188 | causes.dedup(); 189 | causes 190 | } 191 | 192 | fn diagnose_sb(lines: &[&str]) -> String { 193 | if lines[0].contains("only grants SharedReadOnly") && lines[0].contains("for Unique") { 194 | String::from("&->&mut") 195 | } else if lines.iter().any(|line| { 196 | line.contains("attempting a write access") && line.contains("only grants SharedReadOnly") 197 | }) { 198 | String::from("write through pointer based on &") 199 | } else if lines.iter().any(|line| line.contains("invalidated")) { 200 | String::from("SB-invalidation") 201 | } else if lines 202 | .iter() 203 | .any(|line| line.contains("created due to a retag at offsets [0x0..0x0]")) 204 | { 205 | String::from("SB-null-provenance") 206 | } else if lines[0].contains("does not exist in the borrow stack") { 207 | String::from("SB-use-outside-provenance") 208 | } else if lines[0].contains("no item granting write access for deallocation") { 209 | String::from("SB-invalid-dealloc") 210 | } else { 211 | String::from("SB-uncategorized") 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use color_eyre::Result; 3 | use diagnose::diagnose; 4 | use std::{fmt, str::FromStr}; 5 | 6 | mod client; 7 | mod db_dump; 8 | mod diagnose; 9 | mod render; 10 | mod run; 11 | mod sync; 12 | 13 | #[derive(Parser)] 14 | struct Cli { 15 | #[command(subcommand)] 16 | command: Commands, 17 | } 18 | 19 | #[derive(Parser)] 20 | enum Commands { 21 | Run(run::Args), 22 | Sync(sync::Args), 23 | } 24 | 25 | fn main() -> Result<()> { 26 | if std::env::var("RUST_BACKTRACE").is_err() { 27 | std::env::set_var("RUST_BACKTRACE", "1"); 28 | } 29 | if std::env::var("RUST_LOG").is_err() { 30 | std::env::set_var("RUST_LOG", "info"); 31 | } 32 | env_logger::init(); 33 | color_eyre::install()?; 34 | 35 | let args = Cli::parse(); 36 | match args.command { 37 | Commands::Run(args) => run::run(args), 38 | Commands::Sync(args) => sync::run(args), 39 | } 40 | } 41 | 42 | #[derive(Clone, Copy)] 43 | pub enum Tool { 44 | Miri, 45 | Asan, 46 | Build, 47 | Check, 48 | } 49 | 50 | impl Tool { 51 | pub fn raw_path(self) -> &'static str { 52 | match self { 53 | Tool::Miri => "miri/raw", 54 | Tool::Asan => "asan/raw", 55 | Tool::Build => "build/raw", 56 | Tool::Check => "check/raw", 57 | } 58 | } 59 | 60 | pub fn raw_crate_path(self, krate: &Crate) -> String { 61 | format!("{}/{}/{}", self.raw_path(), krate.name, krate.version) 62 | } 63 | 64 | pub fn html_path(self) -> &'static str { 65 | match self { 66 | Tool::Miri => "miri/logs", 67 | Tool::Asan => "asan/logs", 68 | Tool::Build => "build/logs", 69 | Tool::Check => "check/logs", 70 | } 71 | } 72 | 73 | pub fn rendered_crate_path(self, krate: &Crate) -> String { 74 | format!("{}/{}/{}", self.html_path(), krate.name, krate.version) 75 | } 76 | 77 | pub fn landing_page_path(self) -> &'static str { 78 | match self { 79 | Tool::Miri => "miri/index.html", 80 | Tool::Asan => "asan/index.html", 81 | Tool::Build => "build/index.html", 82 | Tool::Check => "check/index.html", 83 | } 84 | } 85 | } 86 | 87 | impl fmt::Display for Tool { 88 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { 89 | let s = match self { 90 | Tool::Miri => "miri", 91 | Tool::Asan => "asan", 92 | Tool::Build => "build", 93 | Tool::Check => "check", 94 | }; 95 | f.write_str(s) 96 | } 97 | } 98 | 99 | impl FromStr for Tool { 100 | type Err = String; 101 | 102 | fn from_str(s: &str) -> Result { 103 | match s { 104 | "miri" => Ok(Self::Miri), 105 | "asan" => Ok(Self::Asan), 106 | "build" => Ok(Self::Build), 107 | "check" => Ok(Self::Check), 108 | _ => Err(format!("Invalid tool {}", s)), 109 | } 110 | } 111 | } 112 | 113 | #[derive(Clone, Debug)] 114 | pub struct Crate { 115 | pub name: String, 116 | pub version: Version, 117 | pub recent_downloads: Option, 118 | pub status: Status, 119 | } 120 | 121 | #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Debug)] 122 | pub enum Version { 123 | Parsed(semver::Version), 124 | Unparsed(String), 125 | } 126 | 127 | impl serde::Serialize for Version { 128 | fn serialize(&self, serializer: S) -> Result 129 | where 130 | S: serde::Serializer, 131 | { 132 | serializer.serialize_str(&self.to_string()) 133 | } 134 | } 135 | 136 | impl Version { 137 | pub fn parse(s: &str) -> Self { 138 | semver::Version::parse(s) 139 | .map(Version::Parsed) 140 | .unwrap_or_else(|_| Version::Unparsed(s.to_string())) 141 | } 142 | } 143 | 144 | impl fmt::Display for Version { 145 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 146 | match self { 147 | Version::Parsed(v) => write!(f, "{}", v), 148 | Version::Unparsed(v) => write!(f, "{}", v), 149 | } 150 | } 151 | } 152 | 153 | #[derive(Clone, Debug)] 154 | pub enum Status { 155 | Unknown, 156 | Passing, 157 | Error(String), 158 | UB { cause: Vec }, 159 | } 160 | 161 | #[derive(Clone, Debug, Ord, Eq, PartialEq, PartialOrd)] 162 | pub struct Cause { 163 | pub kind: String, 164 | pub source_crate: Option, 165 | } 166 | -------------------------------------------------------------------------------- /src/render.rs: -------------------------------------------------------------------------------- 1 | use crate::{Crate, Status}; 2 | use color_eyre::eyre::Result; 3 | use std::fmt::Write; 4 | 5 | #[rustfmt::skip] 6 | macro_rules! log_format { 7 | () => { 8 | r#"{} {} 25 | 33 | 34 | 
{} {}
35 | 
{}
"# 36 | } 37 | } 38 | 39 | pub fn render_crate(krate: &Crate, output: &[u8]) -> String { 40 | let (css, mut encoded) = 41 | ansi_to_html::render(format!("{}/{}", krate.name, krate.version), output); 42 | 43 | // Remove blank rows from the bottom of the terminal output 44 | let ending = "\n"; 45 | while encoded.ends_with(ending) { 46 | encoded.remove(encoded.len() - ending.len()); 47 | } 48 | 49 | for pat in [ 50 | "Undefined Behavior:", 51 | "ERROR: AddressSanitizer:", 52 | "SIGILL: illegal instruction", 53 | "attempted to leave type", 54 | "misaligned pointer dereference", 55 | "unsafe precondition(s) violated", 56 | ] { 57 | if encoded.contains(pat) { 58 | let replacement = format!("{}", pat); 59 | encoded = encoded.replacen(pat, &replacement, 1); 60 | break; 61 | } 62 | } 63 | 64 | format!( 65 | log_format!(), 66 | css, krate.name, krate.version, krate.name, krate.version, encoded 67 | ) 68 | } 69 | 70 | const OUTPUT_HEADER: &str = r#" 71 | 119 | 174 |
175 |
176 | 177 | 
193 | Click on a crate to the right to display its build log
194 |
195 | 196 |
197 |
198 | "#; 199 | 200 | pub fn render_ub(crates: &[Crate]) -> Result { 201 | let mut output = String::new(); 202 | writeln!(output, "{}", OUTPUT_HEADER)?; 203 | for c in crates { 204 | if let Status::UB { cause: causes, .. } = &c.status { 205 | write!(output, "
{} {}
", c.name, c.version,)?; 206 | for cause in causes { 207 | write!(output, "{}", cause.kind)?; 208 | if let Some(source_crate) = &cause.source_crate { 209 | write!(output, " ({source_crate})")?; 210 | } 211 | write!(output, ", ")?; 212 | } 213 | output.pop(); 214 | output.pop(); 215 | writeln!(output, "
")?; 216 | } 217 | } 218 | 219 | Ok(output) 220 | } 221 | 222 | pub const LANDING_PAGE: &str = r#" 223 | 238 | 278 |

Hello! This website hosts a library of logs, displayed as if you have just run cargo miri test on every published crate on crates.io. 279 |

Try searching for a crate below, if one is found you will be redirected to the build output for its most recently published version. For crates where Miri detects UB, the page will be automatically scrolled to the first UB report. 280 |

281 | 282 |

283 |

284 | "); 238 | 239 | client.upload_landing_page(output.into_bytes()).await?; 240 | 241 | Ok(()) 242 | } 243 | 244 | static ERROR_PAGE: &str = r#" 245 | oops 261 | 
error: No such file or directory (http error 404)
262 | 
263 | error: aborting due to previous error
"#; 264 | --------------------------------------------------------------------------------