├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .pre-commit-hooks.yaml ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── _config.yml ├── ci.sh ├── default.nix ├── flake.lock ├── flake.nix ├── renovate.json ├── samples ├── code.rb └── example.md ├── shell.nix └── src ├── cli.rs ├── lib.rs └── main.rs /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | on: 3 | pull_request: 4 | push: 5 | jobs: 6 | tests: 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest, macos-latest] 10 | runs-on: ${{ matrix.os }} 11 | steps: 12 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 13 | - uses: cachix/install-nix-action@91a071959513ca103b54280ac0bef5b825791d4d # v31 14 | - run: ./ci.sh 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Rust 2 | /target 3 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/doublify/pre-commit-rust 5 | rev: ebc9050d3d3434417feff68e3d847ad4123f5ba8 6 | hooks: 7 | - id: fmt 8 | - id: cargo-check 9 | 10 | - repo: local 11 | hooks: 12 | - id: mdsh 13 | name: mdsh 14 | description: README.md shell pre-processor. 15 | entry: cargo run -- --input 16 | language: rust 17 | files: README.md 18 | always_run: true 19 | -------------------------------------------------------------------------------- /.pre-commit-hooks.yaml: -------------------------------------------------------------------------------- 1 | - id: mdsh 2 | name: mdsh 3 | description: README.md shell pre-processor. 4 | entry: mdsh --inputs 5 | language: rust 6 | files: README.md 7 | minimum_pre_commit_version: 1.18.1 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 0.9.2 / 2025-03-17 3 | ================== 4 | 5 | * chore: update `--help` output to fix test (#81) 6 | * fix: add `--version` back (#80) 7 | * chore: fmt 8 | 9 | 0.9.1 / 2025-03-14 10 | ================== 11 | 12 | * fix(README): update mdsh output 13 | * fix(deps): update rust crate clap to v4.5.32 (#79) 14 | * fix(deps): update rust crate lazy_static to v1.5.0 (#68) 15 | * fix(deps): update rust crate regex to v1.11.1 (#71) 16 | * chore(deps): update actions/checkout digest to 11bd719 (#73) 17 | * chore(deps): update cachix/install-nix-action action to v31 (#77) 18 | * cli: port from structopt to clap/derive (#75) 19 | * chore(deps): update cachix/install-nix-action action to v30 (#72) 20 | * chore(deps): update cachix/install-nix-action action to v29 (#70) 21 | * fix(deps): update rust crate regex to v1.10.6 (#69) 22 | * chore(nix): read deps from Cargo.lock directly 23 | 24 | 0.9.0 / 2024-06-17 25 | ================== 26 | 27 | * chore(deps): update actions/checkout digest to 692973e (#64) 28 | * chore(nix): fix shell invocation 29 | * chore(deps): cargo update 30 | * chore(deps): flake update 31 | * chore(flake): replace flake-utils with systems 32 | * chore(deps): update cachix/install-nix-action action to v27 (#66) 33 | * Merge pull request #63 from deemp/main 34 | * fix: mdsh derivation - don't depend on readme - update hash - bump patch version 35 | * chore: bump version 36 | * fix: improve messages about failing commands 37 | * fix: set RUST_SRC_PATH 38 | * chore(deps): update cachix/install-nix-action action to v26 (#60) 39 | 40 | 0.8.0 / 2024-02-27 41 | ================== 42 | 43 | * FEAT: support multiline commands (#59) 44 | * FEAT: add Nix package and describe usage with flakes (#59) 45 | * FIX: print newline after command 46 | * CHORE: update readme 47 | * CHORE: switch default branch to main 48 | * CHORE: update deps 49 | 50 | 0.7.0 / 2023-02-03 51 | ================== 52 | 53 | * FEAT: add support for multiple inputs (#33) 54 | * FIX: add libiconv as a dev dependency 55 | * FIX: avoid writing if no change 56 | * README: make the run reproducible 57 | * CHORE: fix CI on macOS 58 | * CHORE: fix warning 59 | * CHORE: Bump regex from 1.4.3 to 1.5.5 (#31) 60 | 61 | 0.6.0 / 2021-02-26 62 | ================== 63 | 64 | * CHANGE: handle empty lines between command and result 65 | * bump dependencies 66 | 67 | 0.5.0 / 2020-05-08 68 | ================== 69 | 70 | * NEW: add variables support (#27) 71 | 72 | 0.4.0 / 2020-01-12 73 | ================== 74 | 75 | * NEW: Codefence type (#26) 76 | 77 | 0.3.0 / 2019-10-19 78 | ================== 79 | 80 | * CHANGE: use the RHS of the link as a source. 81 | Eg: `$ [before.rb](after.rb)` now loads `after.rb` instead of `before.rb` 82 | 83 | 0.2.0 / 2019-10-08 84 | ================== 85 | 86 | * FEAT: add support for commented-out commands 87 | * FIX: fix line collapsing 88 | 89 | 0.1.5 / 2019-08-24 90 | ================== 91 | 92 | * FEAT: add pre-commit hooks 93 | * improve diff output for --frozen 94 | 95 | 0.1.4 / 2019-08-01 96 | ================== 97 | 98 | * FEAT: implement --frozen option (#13) 99 | * FEAT: filter out ANSI escape characters (#22) 100 | * FEAT: better error messages on read/write errors (#18) 101 | * DOC: improved documentation overall 102 | 103 | 0.1.3 / 2019-02-18 104 | ================== 105 | 106 | * FEAT: allow switching between outputs 107 | * FEAT: add support for work_dir. Fixes #5 108 | * README: add installation instructions 109 | * README: clarify the syntax 110 | * README: Fix typos (#3) 111 | 112 | 0.1.2 / 2019-02-17 113 | ================== 114 | 115 | * pin nixpkgs 116 | * README: improve the docs 117 | 118 | 0.1.1 / 2019-02-16 119 | ================== 120 | 121 | * README: add badges 122 | * cargo fmt 123 | * Cargo.toml: add metadata 124 | 125 | 0.1.0 / 2019-02-16 126 | ================== 127 | 128 | * add linking support 129 | * support stdin and stdout 130 | * basic implementation 131 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anstream" 16 | version = "0.6.18" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" 19 | dependencies = [ 20 | "anstyle", 21 | "anstyle-parse", 22 | "anstyle-query", 23 | "anstyle-wincon", 24 | "colorchoice", 25 | "is_terminal_polyfill", 26 | "utf8parse", 27 | ] 28 | 29 | [[package]] 30 | name = "anstyle" 31 | version = "1.0.10" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 34 | 35 | [[package]] 36 | name = "anstyle-parse" 37 | version = "0.2.6" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" 40 | dependencies = [ 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle-query" 46 | version = "1.1.2" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" 49 | dependencies = [ 50 | "windows-sys", 51 | ] 52 | 53 | [[package]] 54 | name = "anstyle-wincon" 55 | version = "3.0.7" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" 58 | dependencies = [ 59 | "anstyle", 60 | "once_cell", 61 | "windows-sys", 62 | ] 63 | 64 | [[package]] 65 | name = "clap" 66 | version = "4.5.32" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" 69 | dependencies = [ 70 | "clap_builder", 71 | "clap_derive", 72 | ] 73 | 74 | [[package]] 75 | name = "clap_builder" 76 | version = "4.5.32" 77 | source = "registry+https://github.com/rust-lang/crates.io-index" 78 | checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" 79 | dependencies = [ 80 | "anstream", 81 | "anstyle", 82 | "clap_lex", 83 | "strsim", 84 | ] 85 | 86 | [[package]] 87 | name = "clap_derive" 88 | version = "4.5.32" 89 | source = "registry+https://github.com/rust-lang/crates.io-index" 90 | checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" 91 | dependencies = [ 92 | "heck", 93 | "proc-macro2", 94 | "quote", 95 | "syn", 96 | ] 97 | 98 | [[package]] 99 | name = "clap_lex" 100 | version = "0.7.4" 101 | source = "registry+https://github.com/rust-lang/crates.io-index" 102 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 103 | 104 | [[package]] 105 | name = "colorchoice" 106 | version = "1.0.3" 107 | source = "registry+https://github.com/rust-lang/crates.io-index" 108 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 109 | 110 | [[package]] 111 | name = "difference" 112 | version = "2.0.0" 113 | source = "registry+https://github.com/rust-lang/crates.io-index" 114 | checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" 115 | 116 | [[package]] 117 | name = "heck" 118 | version = "0.5.0" 119 | source = "registry+https://github.com/rust-lang/crates.io-index" 120 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 121 | 122 | [[package]] 123 | name = "is_terminal_polyfill" 124 | version = "1.70.1" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 127 | 128 | [[package]] 129 | name = "lazy_static" 130 | version = "1.5.0" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 133 | 134 | [[package]] 135 | name = "mdsh" 136 | version = "0.9.2" 137 | dependencies = [ 138 | "clap", 139 | "difference", 140 | "lazy_static", 141 | "regex", 142 | ] 143 | 144 | [[package]] 145 | name = "memchr" 146 | version = "2.7.4" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 149 | 150 | [[package]] 151 | name = "once_cell" 152 | version = "1.20.2" 153 | source = "registry+https://github.com/rust-lang/crates.io-index" 154 | checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" 155 | 156 | [[package]] 157 | name = "proc-macro2" 158 | version = "1.0.85" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" 161 | dependencies = [ 162 | "unicode-ident", 163 | ] 164 | 165 | [[package]] 166 | name = "quote" 167 | version = "1.0.36" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" 170 | dependencies = [ 171 | "proc-macro2", 172 | ] 173 | 174 | [[package]] 175 | name = "regex" 176 | version = "1.11.1" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 179 | dependencies = [ 180 | "aho-corasick", 181 | "memchr", 182 | "regex-automata", 183 | "regex-syntax", 184 | ] 185 | 186 | [[package]] 187 | name = "regex-automata" 188 | version = "0.4.8" 189 | source = "registry+https://github.com/rust-lang/crates.io-index" 190 | checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" 191 | dependencies = [ 192 | "aho-corasick", 193 | "memchr", 194 | "regex-syntax", 195 | ] 196 | 197 | [[package]] 198 | name = "regex-syntax" 199 | version = "0.8.5" 200 | source = "registry+https://github.com/rust-lang/crates.io-index" 201 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 202 | 203 | [[package]] 204 | name = "strsim" 205 | version = "0.11.1" 206 | source = "registry+https://github.com/rust-lang/crates.io-index" 207 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 208 | 209 | [[package]] 210 | name = "syn" 211 | version = "2.0.87" 212 | source = "registry+https://github.com/rust-lang/crates.io-index" 213 | checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" 214 | dependencies = [ 215 | "proc-macro2", 216 | "quote", 217 | "unicode-ident", 218 | ] 219 | 220 | [[package]] 221 | name = "unicode-ident" 222 | version = "1.0.12" 223 | source = "registry+https://github.com/rust-lang/crates.io-index" 224 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 225 | 226 | [[package]] 227 | name = "utf8parse" 228 | version = "0.2.2" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 231 | 232 | [[package]] 233 | name = "windows-sys" 234 | version = "0.59.0" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 237 | dependencies = [ 238 | "windows-targets", 239 | ] 240 | 241 | [[package]] 242 | name = "windows-targets" 243 | version = "0.52.6" 244 | source = "registry+https://github.com/rust-lang/crates.io-index" 245 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 246 | dependencies = [ 247 | "windows_aarch64_gnullvm", 248 | "windows_aarch64_msvc", 249 | "windows_i686_gnu", 250 | "windows_i686_gnullvm", 251 | "windows_i686_msvc", 252 | "windows_x86_64_gnu", 253 | "windows_x86_64_gnullvm", 254 | "windows_x86_64_msvc", 255 | ] 256 | 257 | [[package]] 258 | name = "windows_aarch64_gnullvm" 259 | version = "0.52.6" 260 | source = "registry+https://github.com/rust-lang/crates.io-index" 261 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 262 | 263 | [[package]] 264 | name = "windows_aarch64_msvc" 265 | version = "0.52.6" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 268 | 269 | [[package]] 270 | name = "windows_i686_gnu" 271 | version = "0.52.6" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 274 | 275 | [[package]] 276 | name = "windows_i686_gnullvm" 277 | version = "0.52.6" 278 | source = "registry+https://github.com/rust-lang/crates.io-index" 279 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 280 | 281 | [[package]] 282 | name = "windows_i686_msvc" 283 | version = "0.52.6" 284 | source = "registry+https://github.com/rust-lang/crates.io-index" 285 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 286 | 287 | [[package]] 288 | name = "windows_x86_64_gnu" 289 | version = "0.52.6" 290 | source = "registry+https://github.com/rust-lang/crates.io-index" 291 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 292 | 293 | [[package]] 294 | name = "windows_x86_64_gnullvm" 295 | version = "0.52.6" 296 | source = "registry+https://github.com/rust-lang/crates.io-index" 297 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 298 | 299 | [[package]] 300 | name = "windows_x86_64_msvc" 301 | version = "0.52.6" 302 | source = "registry+https://github.com/rust-lang/crates.io-index" 303 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 304 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mdsh" 3 | version = "0.9.2" 4 | authors = ["zimbatm "] 5 | edition = "2021" 6 | description = "Markdown shell pre-processor" 7 | homepage = "https://github.com/zimbatm/mdsh" 8 | repository = "https://github.com/zimbatm/mdsh" 9 | keywords = [ 10 | "markdown", 11 | "shell", 12 | ] 13 | readme = "README.md" 14 | license = "MIT" 15 | 16 | [badges.travis-ci] 17 | repository = "zimbatm/mdsh" 18 | 19 | [dependencies] 20 | clap = { version = "4", features = ["derive"] } 21 | difference = "2.0.0" 22 | lazy_static = "1.4.0" 23 | regex = "1.9.5" 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 zimbatm and contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `$ mdsh` - a markdown shell pre-processor 2 | 3 | [![Build Status](https://github.com/zimbatm/mdsh/actions/workflows/ci.yaml/badge.svg)](https://github.com/zimbatm/mdsh/actions/workflows/ci.yaml?branch=master) [![crates.io](https://img.shields.io/crates/v/mdsh.svg)](https://crates.io/crates/mdsh) 4 | 5 | The mdsh project describes a Markdown language extension that can be used to 6 | automate some common tasks in README.md files. Quite often I find myself 7 | needing to embed a snippet of code or markdown from a different file. Or I 8 | want to show the output of a command. In both cases this can be done manually, 9 | but what all you had to do was run `mdsh` and have the file updated 10 | automatically? 11 | 12 | So the goal of this tool is first to extend the syntax of Markdown in a 13 | natural way. Something that you might type. And if the `mdsh` tool is run, the 14 | related blocks get updated in place. Most other tools would produce a new file 15 | but we really want a sort of idempotent operation here. 16 | 17 | In the end this gives a tool that is a bit akin to literate programming or 18 | jupyer notebooks but for shell commands. It adds a bit of verbosity to the 19 | file and in exchange it allows to automate the refresh of those outputs. 20 | 21 | ## Usage 22 | 23 | `$ mdsh --help` 24 | 25 | ``` 26 | Markdown shell pre-processor. Never let your READMEs and tutorials get out of sync again. 27 | 28 | Exits non-zero if a sub-command failed. 29 | 30 | Usage: mdsh [OPTIONS] 31 | 32 | Options: 33 | -i, --inputs 34 | Path to the markdown files. `-` for stdin 35 | 36 | [default: ./README.md] 37 | 38 | -o, --output 39 | Path to the output file, `-` for stdout [defaults to updating the input file in-place] 40 | 41 | --work_dir 42 | Directory to execute the scripts under [defaults to the input file’s directory] 43 | 44 | --frozen 45 | Fail if the output is different from the input. Useful for CI. 46 | 47 | Using `--frozen`, you can guarantee that developers update documentation when they make a change. Just add `mdsh --frozen` as a check to your continuous integration setup. 48 | 49 | --clean 50 | Remove all generated blocks 51 | 52 | -h, --help 53 | Print help (see a summary with '-h') 54 | 55 | -V, --version 56 | Print version 57 | ``` 58 | 59 | ## Syntax Extensions 60 | 61 | ### Inline Shell Code 62 | 63 | Syntax regexp: 64 | ```regexp 65 | ^`[$>] ([^`]+)`\s*$ 66 | ``` 67 | 68 | Inline Shell Code are normal `inline code` that: 69 | 70 | * start at the beginning of a line 71 | * include either `$` or `>` at the beginning of their content 72 | * contain a shell command 73 | 74 | When those are enountered, the command is executed by `mdsh` and output as 75 | either a fenced code block (`$`) or markdown code (`>`). 76 | 77 | * `$` runs the command and outputs a code block 78 | * `>` runs the command and outputs markdown 79 | 80 | Examples: 81 | 82 | ~~~ 83 | `$ seq 4 | sort -r` 84 | 85 | ``` 86 | 4 87 | 3 88 | 2 89 | 1 90 | ``` 91 | ~~~ 92 | 93 | ~~~ 94 | `> echo 'I *can* include markdown. Hehe.'` 95 | 96 | 97 | I *can* include markdown. Hehe. 98 | 99 | ~~~ 100 | 101 | ### Multiline Shell Code 102 | 103 | Syntax regexp: 104 | ```regexp 105 | ^```[$^]\n.*\n```$ 106 | ``` 107 | 108 | Multiline Shell Code are normal multiline code that: 109 | 110 | * start at the beginning of a line 111 | * include `$` or `^` as "language" 112 | * contain a shell command 113 | 114 | When those are enountered, the command is executed by `mdsh` and output as 115 | either a fenced code block (`$`) or markdown code (`>`). 116 | 117 | * `$` runs the command and outputs a code block 118 | * `>` runs the command and outputs markdown 119 | 120 | Examples: 121 | 122 | ~~~ 123 | ```$ as bash 124 | seq 3 | sort -r 125 | seq 2 | sort -r 126 | ``` 127 | 128 | ```bash 129 | 3 130 | 2 131 | 1 132 | 2 133 | 1 134 | ``` 135 | ~~~ 136 | 137 | ~~~ 138 | ```> 139 | echo 'I *can* include markdown. Hehe.' 140 | ``` 141 | 142 | 143 | I *can* include markdown. Hehe. 144 | 145 | ~~~ 146 | 147 | ### Variables 148 | 149 | Syntax regexp: 150 | ```regexp 151 | ^`! ([\w_]+)=([^`]+)`\s*$ 152 | ``` 153 | 154 | Variables allow you to set new variables in the environment and reachable by 155 | the next blocks that are being executed. 156 | 157 | The value part is being evaluated by bash and can thus spawn sub-shells. 158 | 159 | Examples: 160 | 161 | `! user=bob` 162 | 163 | Now the `$user` environment variable is available: 164 | 165 | `$ echo hello $user` 166 | 167 | ``` 168 | hello bob 169 | ``` 170 | 171 | Now capitalize the user 172 | 173 | `! USER=$(echo $user | tr '[[:lower:]]' '[[:upper:]]')` 174 | 175 | `$ echo hello $USER` 176 | 177 | ``` 178 | hello BOB 179 | ``` 180 | 181 | ### Link Includes 182 | 183 | Syntax regexp: 184 | ```regexp 185 | ^\[[$>] ([^\]]+)]\([^\)]+\)\s*$ 186 | ``` 187 | 188 | Link Includes work similarily to code blocks but with the link syntax. 189 | 190 | * `$` loads the file and embeds it as a code block 191 | * `>` loads the file and embeds it as markdown 192 | 193 | Examples: 194 | 195 | ~~~ 196 | [$ code.rb](samples/code.rb) as ruby 197 | 198 | ```ruby 199 | require "pp" 200 | 201 | pp ({ foo: 3 }) 202 | ``` 203 | ~~~ 204 | 205 | ~~~ 206 | [> example.md](samples/example.md) 207 | 208 | 209 | *this is part of the example.md file* 210 | 211 | ~~~ 212 | 213 | ### ANSI escapes 214 | 215 | ANSI escape sequences are filtered from command outputs: 216 | 217 | `$ echo $'\e[33m'yellow` 218 | 219 | ``` 220 | yellow 221 | ``` 222 | 223 | ### Commented-out commands 224 | 225 | Sometimes it's useful not to render the command that is being shown. All the 226 | commands support being hidden inside of a HTML comment like so: 227 | 228 | ~~~ 229 | 230 | 231 | ``` 232 | example 233 | ``` 234 | ~~~ 235 | 236 | ### Fenced code type 237 | 238 | If you want GitHub to highlight the outputted code fences, it's possible to 239 | postfix the line with `as `. For example: 240 | 241 | ~~~ 242 | `$ echo '{ key: "value" }'` as json 243 | 244 | ```json 245 | { key: "value" } 246 | ``` 247 | ~~~ 248 | 249 | ## Installation 250 | 251 | The best way to install `mdsh` is with the rust tool cargo. 252 | 253 | ```bash 254 | cargo install mdsh 255 | ``` 256 | 257 | If you are lucky enough to be a nix user: 258 | 259 | ```bash 260 | nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -iA mdsh 261 | ``` 262 | 263 | If you are a nix + flakes user: 264 | 265 | ```bash 266 | nix profile install github:zimbatm/mdsh 267 | ``` 268 | 269 | ## Running without installation 270 | 271 | If you are a nix + flakes user: 272 | 273 | ```bash 274 | nix run github:zimbatm/mdsh -- --help 275 | ``` 276 | 277 | ### Pre-commit hook 278 | 279 | This project can also be installed as a [pre-commit](https://pre-commit.com/) 280 | hook. 281 | 282 | Add to your project's `.pre-commit-config.yaml`: 283 | 284 | ```yaml 285 | - repo: https://github.com/zimbatm/mdsh.git 286 | rev: main 287 | hooks: 288 | - id: mdsh 289 | ``` 290 | 291 | Make sure to have rust available in your environment. 292 | 293 | Then run `pre-commit install-hooks` 294 | 295 | ## Known issues 296 | 297 | The tool currently lacks in precision as it doesn't parse the Markdown file, 298 | it just looks for the desired blocks by regexp. It means that in some cases it 299 | might misintepret some of the commands. Most existing Markdown parsers are 300 | used to generate HTML in the end and are thus not position-preserving. Eg: 301 | pulldown-cmark 302 | 303 | The block removal algorithm doesn't support output that contains triple 304 | backtick or ``. 305 | 306 | ## Related projects 307 | 308 | * is the closest to this project. It 309 | has some interesting Pandoc filters that capture code blocks into outputs. 310 | The transformation is not in-place like `mdsh`. 311 | * [Enola.dev's ExecMD](https://docs.enola.dev/use/execmd) is another similar tool. 312 | * [Literate Programming](https://en.wikipedia.org/wiki/Literate_programming) 313 | is the practice of interspesing executable code into documents. There are 314 | many language-specific implementations out there. `mdsh` is a bit like a 315 | bash literate programming language. 316 | * [Jupyter Notebooks](https://jupyter.org/) is a whole other universe of 317 | documentation and code. It's great but stores the notebooks as JSON files. A 318 | special viewer program is required to render them to HTML or text. 319 | 320 | ## User Feedback 321 | 322 | ### Issues 323 | 324 | If you have any problems with or questions about this project, please contact 325 | use through a [GitHub issue](https://github.com/zimbatm/mdsh/issues). 326 | 327 | ### Contributing 328 | 329 | You are invited to contribute new features, fixes or updates, large or small; 330 | we are always thrilled to receive pull requests, and do our best to process 331 | them as fast as we can. 332 | 333 | ## License 334 | 335 | [> LICENSE](LICENSE) 336 | 337 | 338 | MIT License 339 | 340 | Copyright (c) 2019 zimbatm and contributors 341 | 342 | Permission is hereby granted, free of charge, to any person obtaining a copy 343 | of this software and associated documentation files (the "Software"), to deal 344 | in the Software without restriction, including without limitation the rights 345 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 346 | copies of the Software, and to permit persons to whom the Software is 347 | furnished to do so, subject to the following conditions: 348 | 349 | The above copyright notice and this permission notice shall be included in all 350 | copies or substantial portions of the Software. 351 | 352 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 353 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 354 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 355 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 356 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 357 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 358 | SOFTWARE. 359 | 360 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-primer 2 | title: "`$ mdsh`" 3 | description: "a markdown shell pre-processor" 4 | url: "https://zimbatm.github.io/mdsh" 5 | author: 6 | twitter: zimbatm 7 | github: zimbatm 8 | 9 | # see https://github.com/github/pages-gem/blob/754a725e4766d4329bb1dd0e07c638a045ad2c04/lib/github-pages/plugins.rb#L6-L42 10 | plugins: 11 | - jemoji 12 | - jekyll-avatar 13 | - jekyll-default-layout 14 | - jekyll-feed 15 | - jekyll-mentions 16 | - jekyll-readme-index 17 | - jekyll-sitemap 18 | 19 | markdown: CommonMarkGhPages 20 | # see https://github.com/gjtorikian/commonmarker#parse-options 21 | commonmark: 22 | options: 23 | - FOOTNOTES 24 | - SMART 25 | extensions: 26 | - autolink 27 | - strikethrough 28 | - table 29 | -------------------------------------------------------------------------------- /ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nix-shell 2 | #!nix-shell -i bash -I nixpkgs=channel:nixos-22.05 3 | # shellcheck shell=bash 4 | # 5 | # Travis CI specific build script 6 | set -euo pipefail 7 | 8 | ## Functions ## 9 | 10 | run() { 11 | echo >&2 12 | echo "$ $*" >&2 13 | "$@" 14 | } 15 | 16 | ## Main ## 17 | 18 | mkdir -p "${TMPDIR}" 19 | 20 | # build mdsh 21 | run cargo build --verbose 22 | 23 | # run after build, pre-commit needs mdsh 24 | run pre-commit run --all-files 25 | 26 | # run the tests 27 | run cargo test --verbose 28 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | # Compatibility function to allow flakes to be used by 2 | # non-flake-enabled Nix versions. Given a source tree containing a 3 | # 'flake.nix' and 'flake.lock' file, it fetches the flake inputs and 4 | # calls the flake's 'outputs' function. It then returns an attrset 5 | # containing 'defaultNix' (to be used in 'default.nix'), 'shellNix' 6 | # (to be used in 'shell.nix'). 7 | 8 | { src, system ? builtins.currentSystem or "unknown-system" }: 9 | 10 | let 11 | 12 | lockFilePath = src + "/flake.lock"; 13 | 14 | lockFile = builtins.fromJSON (builtins.readFile lockFilePath); 15 | 16 | fetchTree = 17 | info: 18 | if info.type == "github" then 19 | { 20 | outPath = 21 | fetchTarball 22 | ({ url = "https://api.${info.host or "github.com"}/repos/${info.owner}/${info.repo}/tarball/${info.rev}"; } 23 | // (if info ? narHash then { sha256 = info.narHash; } else { }) 24 | ); 25 | rev = info.rev; 26 | shortRev = builtins.substring 0 7 info.rev; 27 | lastModified = info.lastModified; 28 | lastModifiedDate = formatSecondsSinceEpoch info.lastModified; 29 | narHash = info.narHash; 30 | } 31 | else if info.type == "git" then 32 | { 33 | outPath = 34 | builtins.fetchGit 35 | ({ url = info.url; } 36 | // (if info ? rev then { inherit (info) rev; } else { }) 37 | // (if info ? ref then { inherit (info) ref; } else { }) 38 | // (if info ? submodules then { inherit (info) submodules; } else { }) 39 | ); 40 | lastModified = info.lastModified; 41 | lastModifiedDate = formatSecondsSinceEpoch info.lastModified; 42 | narHash = info.narHash; 43 | } // (if info ? rev then { 44 | rev = info.rev; 45 | shortRev = builtins.substring 0 7 info.rev; 46 | } else { }) 47 | else if info.type == "path" then 48 | { 49 | outPath = builtins.path { 50 | path = 51 | if builtins.substring 0 1 info.path != "/" 52 | then src + ("/" + info.path) 53 | else info.path; 54 | }; 55 | narHash = info.narHash; 56 | } 57 | else if info.type == "tarball" then 58 | { 59 | outPath = 60 | fetchTarball 61 | ({ inherit (info) url; } 62 | // (if info ? narHash then { sha256 = info.narHash; } else { }) 63 | ); 64 | } 65 | else if info.type == "gitlab" then 66 | { 67 | inherit (info) rev narHash lastModified; 68 | outPath = 69 | fetchTarball 70 | ({ url = "https://${info.host or "gitlab.com"}/api/v4/projects/${info.owner}%2F${info.repo}/repository/archive.tar.gz?sha=${info.rev}"; } 71 | // (if info ? narHash then { sha256 = info.narHash; } else { }) 72 | ); 73 | shortRev = builtins.substring 0 7 info.rev; 74 | } 75 | else 76 | # FIXME: add Mercurial, tarball inputs. 77 | throw "flake input has unsupported input type '${info.type}'"; 78 | 79 | callLocklessFlake = flakeSrc: 80 | let 81 | flake = import (flakeSrc + "/flake.nix"); 82 | outputs = flakeSrc // (flake.outputs ({ self = outputs; })); 83 | in 84 | outputs; 85 | 86 | rootSrc = 87 | let 88 | # Try to clean the source tree by using fetchGit, if this source 89 | # tree is a valid git repository. 90 | tryFetchGit = src: 91 | if isGit && !isShallow 92 | then 93 | let res = builtins.fetchGit src; 94 | in if res.rev == "0000000000000000000000000000000000000000" then removeAttrs res [ "rev" "shortRev" ] else res 95 | else { outPath = src; }; 96 | # NB git worktrees have a file for .git, so we don't check the type of .git 97 | isGit = builtins.pathExists (src + "/.git"); 98 | isShallow = builtins.pathExists (src + "/.git/shallow"); 99 | 100 | in 101 | { lastModified = 0; lastModifiedDate = formatSecondsSinceEpoch 0; } 102 | // (if src ? outPath then src else tryFetchGit src); 103 | 104 | # Format number of seconds in the Unix epoch as %Y%m%d%H%M%S. 105 | formatSecondsSinceEpoch = t: 106 | let 107 | rem = x: y: x - x / y * y; 108 | days = t / 86400; 109 | secondsInDay = rem t 86400; 110 | hours = secondsInDay / 3600; 111 | minutes = (rem secondsInDay 3600) / 60; 112 | seconds = rem t 60; 113 | 114 | # Courtesy of https://stackoverflow.com/a/32158604. 115 | z = days + 719468; 116 | era = (if z >= 0 then z else z - 146096) / 146097; 117 | doe = z - era * 146097; 118 | yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; 119 | y = yoe + era * 400; 120 | doy = doe - (365 * yoe + yoe / 4 - yoe / 100); 121 | mp = (5 * doy + 2) / 153; 122 | d = doy - (153 * mp + 2) / 5 + 1; 123 | m = mp + (if mp < 10 then 3 else -9); 124 | y' = y + (if m <= 2 then 1 else 0); 125 | 126 | pad = s: if builtins.stringLength s < 2 then "0" + s else s; 127 | in 128 | "${toString y'}${pad (toString m)}${pad (toString d)}${pad (toString hours)}${pad (toString minutes)}${pad (toString seconds)}"; 129 | 130 | allNodes = 131 | builtins.mapAttrs 132 | (key: node: 133 | let 134 | sourceInfo = 135 | if key == lockFile.root 136 | then rootSrc 137 | else fetchTree (node.info or { } // removeAttrs node.locked [ "dir" ]); 138 | 139 | subdir = if key == lockFile.root then "" else node.locked.dir or ""; 140 | 141 | flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); 142 | 143 | inputs = builtins.mapAttrs 144 | (_inputName: inputSpec: allNodes.${resolveInput inputSpec}) 145 | (node.inputs or { }); 146 | 147 | # Resolve a input spec into a node name. An input spec is 148 | # either a node name, or a 'follows' path from the root 149 | # node. 150 | resolveInput = inputSpec: 151 | if builtins.isList inputSpec 152 | then getInputByPath lockFile.root inputSpec 153 | else inputSpec; 154 | 155 | # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the 156 | # root node, returning the final node. 157 | getInputByPath = nodeName: path: 158 | if path == [ ] 159 | then nodeName 160 | else 161 | getInputByPath 162 | # Since this could be a 'follows' input, call resolveInput. 163 | (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) 164 | (builtins.tail path); 165 | 166 | outputs = flake.outputs (inputs // { self = result; }); 167 | 168 | result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; }; 169 | 170 | in 171 | if node.flake or true then 172 | assert builtins.isFunction flake.outputs; 173 | result 174 | else 175 | sourceInfo 176 | ) 177 | lockFile.nodes; 178 | 179 | result = 180 | if !(builtins.pathExists lockFilePath) 181 | then callLocklessFlake rootSrc 182 | else if lockFile.version >= 5 && lockFile.version <= 7 183 | then allNodes.${lockFile.root} 184 | else throw "lock file '${lockFilePath}' has unsupported version ${toString lockFile.version}"; 185 | 186 | in 187 | rec { 188 | defaultNix = 189 | (builtins.removeAttrs result [ "__functor" ]) 190 | // (if result ? defaultPackage.${system} then { default = result.defaultPackage.${system}; } else { }) 191 | // (if result ? packages.${system}.default then { default = result.packages.${system}.default; } else { }); 192 | 193 | shellNix = 194 | defaultNix 195 | // (if result ? devShell.${system} then { default = result.devShell.${system}; } else { }) 196 | // (if result ? devShells.${system}.default then { default = result.devShells.${system}.default; } else { }); 197 | } 198 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1718396522, 6 | "narHash": "sha256-C0re6ZtCqC1ndL7ib7vOqmgwvZDhOhJ1W0wQgX1tTIo=", 7 | "owner": "nixos", 8 | "repo": "nixpkgs", 9 | "rev": "3e6b9369165397184774a4b7c5e8e5e46531b53f", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nixos", 14 | "ref": "nixpkgs-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "root": { 20 | "inputs": { 21 | "nixpkgs": "nixpkgs", 22 | "systems": "systems" 23 | } 24 | }, 25 | "systems": { 26 | "locked": { 27 | "lastModified": 1681028828, 28 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 29 | "owner": "nix-systems", 30 | "repo": "default", 31 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 32 | "type": "github" 33 | }, 34 | "original": { 35 | "owner": "nix-systems", 36 | "repo": "default", 37 | "type": "github" 38 | } 39 | } 40 | }, 41 | "root": "root", 42 | "version": 7 43 | } 44 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "mdsh - a markdown shell pre-processor"; 3 | inputs = { 4 | nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; 5 | systems.url = "github:nix-systems/default"; 6 | }; 7 | outputs = 8 | { 9 | self, 10 | nixpkgs, 11 | systems, 12 | }: 13 | let 14 | version = (lib.importTOML "${self}/Cargo.toml").package.version; 15 | 16 | inherit (nixpkgs) lib; 17 | fs = lib.fileset; 18 | eachSystem = f: lib.genAttrs (import systems) (system: f nixpkgs.legacyPackages.${system}); 19 | in 20 | { 21 | devShells = eachSystem (pkgs: { 22 | default = pkgs.mkShell { 23 | buildInputs = [ 24 | pkgs.cargo 25 | pkgs.gitAndTools.git-extras 26 | pkgs.gitAndTools.pre-commit 27 | pkgs.libiconv 28 | pkgs.rust-analyzer 29 | pkgs.rustc 30 | pkgs.rustfmt 31 | ]; 32 | 33 | shellHook = '' 34 | export PATH=$PWD/target/debug:$PATH 35 | export RUST_SRC_PATH="${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; 36 | ''; 37 | }; 38 | }); 39 | 40 | packages = eachSystem (pkgs: { 41 | default = pkgs.rustPlatform.buildRustPackage { 42 | pname = "mdsh"; 43 | inherit version; 44 | 45 | src = fs.toSource { 46 | root = ./.; 47 | fileset = fs.unions [ 48 | ./Cargo.toml 49 | ./Cargo.lock 50 | ./src 51 | ]; 52 | }; 53 | 54 | cargoLock.lockFile = ./Cargo.lock; 55 | 56 | meta = with lib; { 57 | description = "Markdown shell pre-processor"; 58 | homepage = "https://github.com/zimbatm/mdsh"; 59 | license = with licenses; [ mit ]; 60 | maintainers = with maintainers; [ zimbatm ]; 61 | mainProgram = "mdsh"; 62 | }; 63 | }; 64 | }); 65 | }; 66 | } 67 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /samples/code.rb: -------------------------------------------------------------------------------- 1 | require "pp" 2 | 3 | pp ({ foo: 3 }) 4 | -------------------------------------------------------------------------------- /samples/example.md: -------------------------------------------------------------------------------- 1 | *this is part of the example.md file* 2 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | { system ? builtins.currentSystem }: 2 | (import ./. { src = ./.; inherit system; }).shellNix.default 3 | -------------------------------------------------------------------------------- /src/cli.rs: -------------------------------------------------------------------------------- 1 | //! Command line interface 2 | use clap::Parser; 3 | use std::path::{Path, PathBuf}; 4 | use std::str::FromStr; 5 | 6 | /// Markdown shell pre-processor. 7 | /// Never let your READMEs and tutorials get out of sync again. 8 | /// 9 | /// Exits non-zero if a sub-command failed. 10 | #[derive(Debug, Parser)] 11 | #[clap(name = "mdsh", version = env!("CARGO_PKG_VERSION"))] 12 | pub struct Opt { 13 | /// Path to the markdown files. `-` for stdin. 14 | #[clap( 15 | short = 'i', 16 | long = "inputs", 17 | alias = "input", 18 | default_value = "./README.md" 19 | )] 20 | pub inputs: Vec, 21 | 22 | /// Path to the output file, `-` for stdout [defaults to updating the input file in-place]. 23 | #[clap(short = 'o', long = "output")] 24 | pub output: Option, 25 | 26 | /// Directory to execute the scripts under [defaults to the input file’s directory]. 27 | #[clap(long = "work_dir")] 28 | pub work_dir: Option, 29 | 30 | /// Fail if the output is different from the input. Useful for CI. 31 | /// 32 | /// Using `--frozen`, you can guarantee that developers update 33 | /// documentation when they make a change. Just add `mdsh --frozen` 34 | /// as a check to your continuous integration setup. 35 | #[clap(long = "frozen", conflicts_with = "clean")] 36 | pub frozen: bool, 37 | 38 | /// Remove all generated blocks. 39 | #[clap(long = "clean")] 40 | pub clean: bool, 41 | } 42 | 43 | /// Possible file input (either a file name or `-`) 44 | #[derive(Debug, Clone)] 45 | pub enum FileArg { 46 | /// equal to - (so stdin or stdout) 47 | StdHandle, 48 | File(PathBuf), 49 | } 50 | 51 | impl FileArg { 52 | /// Return the parent, if it is a `StdHandle` use the current directory. 53 | /// Returns `None` if there is no parent (that is we are `/`). 54 | pub fn parent(&self) -> Option { 55 | match self { 56 | FileArg::StdHandle => Some(Parent::current_dir()), 57 | FileArg::File(buf) => Parent::of(buf), 58 | } 59 | } 60 | 61 | /// return a `FileArg::File`, don’t parse 62 | pub fn from_str_unsafe(s: &str) -> Self { 63 | FileArg::File(PathBuf::from(s)) 64 | } 65 | } 66 | 67 | impl FromStr for FileArg { 68 | type Err = std::string::ParseError; 69 | fn from_str(s: &str) -> Result { 70 | match s { 71 | "-" => Ok(FileArg::StdHandle), 72 | p => Ok(FileArg::File(PathBuf::from(p))), 73 | } 74 | } 75 | } 76 | 77 | /// Parent path, gracefully handling relative path inputs 78 | #[derive(Debug, Clone)] 79 | pub struct Parent(PathBuf); 80 | 81 | impl Parent { 82 | /// Create from a `Path`, falling back to the 83 | /// `current_dir()` if necessary. 84 | /// Returns `None` if there is no parent (that is we are `/`). 85 | pub fn of(p: &Path) -> Option { 86 | let prnt = p.parent()?; 87 | if prnt.as_os_str().is_empty() { 88 | Some(Self::current_dir()) 89 | } else { 90 | Some(Parent(prnt.to_path_buf())) 91 | } 92 | } 93 | 94 | /// Creates a `Parent` that is the current directory. 95 | /// Asks the operating system for the path. 96 | pub fn current_dir() -> Self { 97 | Parent( 98 | std::env::current_dir().expect( 99 | "fatal: current working directory not accessible and `--work_dir` not given", 100 | ), 101 | ) 102 | } 103 | 104 | /// Convert from a `PathBuf` that is already a parent. 105 | pub fn from_parent_path_buf(buf: PathBuf) -> Self { 106 | Parent(buf) 107 | } 108 | 109 | pub fn as_path_buf(&self) -> &PathBuf { 110 | &self.0 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod cli; 2 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate lazy_static; 3 | 4 | use std::fs::File; 5 | use std::io::prelude::*; 6 | use std::io::{self, ErrorKind, Write}; 7 | use std::process::{Command, Output, Stdio}; 8 | 9 | use clap::Parser; 10 | use difference::Changeset; 11 | use mdsh::cli::{FileArg, Opt, Parent}; 12 | use regex::{Captures, Regex}; 13 | 14 | fn run_command(command: &str, work_dir: &Parent) -> Output { 15 | let mut cli = Command::new("bash"); 16 | cli.arg("-c") 17 | .arg(format!("set -euo pipefail && {command}")) 18 | .stdin(Stdio::null()) // don't read from stdin 19 | .current_dir(work_dir.as_path_buf()) 20 | .output() 21 | .expect( 22 | format!( 23 | "fatal: failed to execute command `{:?}` in {}", 24 | cli, 25 | work_dir.as_path_buf().display() 26 | ) 27 | .as_str(), 28 | ) 29 | } 30 | 31 | fn die(msg: String) -> A { 32 | std::io::stderr() 33 | .write_all(format!("fatal: {}\n", msg).as_bytes()) 34 | .unwrap(); 35 | std::process::exit(1) 36 | } 37 | 38 | fn read_file(f: &FileArg) -> String { 39 | let mut buffer = String::new(); 40 | 41 | match f { 42 | FileArg::StdHandle => { 43 | let stdin = io::stdin(); 44 | let mut handle = stdin.lock(); 45 | handle 46 | .read_to_string(&mut buffer) 47 | .unwrap_or_else(|err| die(format!("failed to read from stdin: {}", err))); 48 | } 49 | FileArg::File(path_buf) => { 50 | File::open(path_buf) 51 | .and_then(|mut file| file.read_to_string(&mut buffer)) 52 | .unwrap_or_else(|err| { 53 | die(format!( 54 | "failed to read from {}: {}", 55 | path_buf.display(), 56 | err 57 | )) 58 | }); 59 | } 60 | } 61 | 62 | buffer 63 | } 64 | 65 | fn write_file(f: &FileArg, contents: String) { 66 | match f { 67 | FileArg::StdHandle => { 68 | let stdout = io::stdout(); 69 | let mut handle = stdout.lock(); 70 | write!(handle, "{}", contents) 71 | .unwrap_or_else(|err| die(format!("failed to write to stdout: {}", err))); 72 | } 73 | FileArg::File(path_buf) => { 74 | File::create(path_buf) 75 | .and_then(|mut file| { 76 | write!(file, "{}", contents)?; 77 | file.sync_all() 78 | }) 79 | .unwrap_or_else(|err| { 80 | die(format!( 81 | "failed to write to {}: {}", 82 | path_buf.display(), 83 | err 84 | )) 85 | }); 86 | } 87 | } 88 | } 89 | 90 | fn trail_nl>(s: T) -> String { 91 | let r = s.as_ref(); 92 | if r.ends_with('\n') { 93 | r.to_string() 94 | } else { 95 | format!("{}\n", r) 96 | } 97 | } 98 | 99 | // make sure that the string starts and ends with new lines 100 | fn wrap_nl(s: String) -> String { 101 | if s.starts_with('\n') { 102 | trail_nl(s) 103 | } else if s.ends_with('\n') { 104 | format!("\n{}", s) 105 | } else { 106 | format!("\n{}\n", s) 107 | } 108 | } 109 | 110 | // remove all ANSI escape characters 111 | fn filter_ansi(s: String) -> String { 112 | RE_ANSI_FILTER.replace_all(&s, "").to_string() 113 | } 114 | 115 | /// Link text block include of form `[$ description](./filename)` 116 | static RE_FENCE_LINK_STR: &str = r"\[\$ [^\]]+\]\((?P[^\)]+)\)"; 117 | /// Link markdown block include of form `[> description](./filename)` 118 | static RE_MD_LINK_STR: &str = r"\[> [^\]]+\]\((?P[^\)]+)\)"; 119 | /// Command text block include of form `\`$ command\`` 120 | static RE_FENCE_COMMAND_STR: &str = r"`\$ (?P[^`]+)`"; 121 | /// Command text block include of form 122 | /// ~~~ 123 | /// ```$ 124 | /// command 125 | /// ``` 126 | /// ~~~ 127 | static RE_MULTILINE_FENCE_COMMAND_STR: &str = 128 | r"```\$( as (?P\w+))?\n(?P[^`]+)\n```"; 129 | /// Command markdown block include of form `\`> command\`` 130 | static RE_MD_COMMAND_STR: &str = r"`> (?P[^`]+)`"; 131 | /// Command markdown block include of form 132 | /// ~~~ 133 | /// ```> 134 | /// command 135 | /// ``` 136 | /// ~~~ 137 | static RE_MULTILINE_MD_COMMAND_STR: &str = r"```>\n(?P[^`]+)\n```"; 138 | /// Command to set a variable 139 | static RE_VAR_COMMAND_STR: &str = r"`! (?P[\w_]+)=(?P[^`]+)`"; 140 | /// Delimiter block for marking automatically inserted text 141 | static RE_FENCE_BLOCK_STR: &str = r"^```.+?^```"; 142 | /// Delimiter block for marking automatically inserted markdown 143 | static RE_MD_BLOCK_STR: &str = r"^.+?^"; 144 | 145 | /// HTML comment wrappers 146 | static RE_COMMENT_BEGIN_STR: &str = r"(?:)?"; 148 | 149 | /// Fenced code type specifier 150 | static RE_FENCE_TYPE_STR: &str = r"(?: as (?P\w+))?"; 151 | 152 | lazy_static! { 153 | /// Match a whole text block (`$` command or link and then delimiter block) 154 | static ref RE_MATCH_FENCE_BLOCK_STR: String = format!( 155 | r"(?sm)(^{}(?:({}|{}){}|{}){} *$)\n+({}|{})", 156 | RE_COMMENT_BEGIN_STR, RE_FENCE_COMMAND_STR, RE_FENCE_LINK_STR, RE_FENCE_TYPE_STR, RE_MULTILINE_FENCE_COMMAND_STR, RE_COMMENT_END_STR, 157 | RE_FENCE_BLOCK_STR, RE_MD_BLOCK_STR, 158 | ); 159 | /// Match a whole markdown block (`>` command or link and then delimiter block) 160 | static ref RE_MATCH_MD_BLOCK_STR: String = format!( 161 | r"(?sm)(^{}(?:{}|{}|{}){} *$)\n+({}|{})", 162 | RE_COMMENT_BEGIN_STR, RE_MD_COMMAND_STR, RE_MD_LINK_STR, RE_MULTILINE_MD_COMMAND_STR, RE_COMMENT_END_STR, 163 | RE_MD_BLOCK_STR, RE_FENCE_BLOCK_STR, 164 | ); 165 | 166 | static ref RE_MATCH_ANY_COMMAND_STR: String = format!(r"(?sm)^{}(`[^`\n]+`{}|```(\$( as (?P\w+))?|>)\n[^`]+\n```){} *$", RE_COMMENT_BEGIN_STR, RE_FENCE_TYPE_STR, RE_COMMENT_END_STR); 167 | /// Match `RE_FENCE_COMMAND_STR` 168 | static ref RE_MATCH_FENCE_COMMAND_STR: String = format!(r"(?sm)^({}{}|{})$", RE_FENCE_COMMAND_STR, RE_FENCE_TYPE_STR, RE_MULTILINE_FENCE_COMMAND_STR); 169 | /// Match `RE_MD_COMMAND_STR` 170 | static ref RE_MATCH_MD_COMMAND_STR: String = format!(r"(?sm)^({}|{})$", RE_MD_COMMAND_STR, RE_MULTILINE_MD_COMMAND_STR); 171 | /// Match `RE_VAR_COMMAND_STR` 172 | static ref RE_MATCH_VAR_COMMAND_STR: String = format!(r"(?sm)^{}$", RE_VAR_COMMAND_STR); 173 | 174 | /// Match `RE_FENCE_LINK_STR` 175 | static ref RE_MATCH_FENCE_LINK_STR: String = format!(r"(?sm)^{}{}{}{} *$", RE_COMMENT_BEGIN_STR, RE_FENCE_LINK_STR, RE_FENCE_TYPE_STR, RE_COMMENT_END_STR); 176 | /// Match `RE_MD_LINK_STR` 177 | static ref RE_MATCH_MD_LINK_STR: String = format!(r"(?sm)^{}{}{} *$", RE_COMMENT_BEGIN_STR, RE_MD_LINK_STR, RE_COMMENT_END_STR); 178 | 179 | 180 | static ref RE_MATCH_ANY_COMMAND: Regex = Regex::new(&RE_MATCH_ANY_COMMAND_STR).unwrap(); 181 | static ref RE_MATCH_CODE_BLOCK: Regex = Regex::new(&RE_MATCH_FENCE_BLOCK_STR).unwrap(); 182 | static ref RE_MATCH_MD_BLOCK: Regex = Regex::new(&RE_MATCH_MD_BLOCK_STR).unwrap(); 183 | static ref RE_MATCH_FENCE_COMMAND: Regex = Regex::new(&RE_MATCH_FENCE_COMMAND_STR).unwrap(); 184 | static ref RE_MATCH_MD_COMMAND: Regex = Regex::new(&RE_MATCH_MD_COMMAND_STR).unwrap(); 185 | static ref RE_MATCH_VAR_COMMAND: Regex = Regex::new(&RE_MATCH_VAR_COMMAND_STR).unwrap(); 186 | static ref RE_MATCH_FENCE_LINK: Regex = Regex::new(&RE_MATCH_FENCE_LINK_STR).unwrap(); 187 | static ref RE_MATCH_MD_LINK: Regex = Regex::new(&RE_MATCH_MD_LINK_STR).unwrap(); 188 | 189 | /// ANSI characters filter 190 | /// https://superuser.com/questions/380772/removing-ansi-color-codes-from-text-stream 191 | static ref RE_ANSI_FILTER: Regex = Regex::new(r"\x1b\[[0-9;]*[mGKH]").unwrap(); 192 | } 193 | 194 | struct FailingCommand { 195 | output: Output, 196 | command: String, 197 | command_char: char, 198 | is_multiline: bool, 199 | } 200 | 201 | fn main() -> std::io::Result<()> { 202 | let opt = Opt::parse(); 203 | let clean = opt.clean; 204 | let frozen = opt.frozen; 205 | let inputs = opt.inputs; 206 | 207 | if inputs.len() == 0 { 208 | // Nothing to do 209 | return Ok(()); 210 | } else if inputs.len() == 1 { 211 | let input = inputs.first().unwrap(); 212 | let output = opt.output.unwrap_or_else(|| input.clone()); 213 | let work_dir: Parent = opt.work_dir.map_or_else( 214 | || { 215 | input 216 | .clone() 217 | .parent() 218 | .expect("fatal: your input file has no parent directory.") 219 | }, 220 | |buf| Parent::from_parent_path_buf(buf), 221 | ); 222 | process_file(&input, &output, &work_dir, clean, frozen)?; 223 | } else { 224 | if opt.output.is_some() { 225 | return Err(std::io::Error::new( 226 | ErrorKind::Other, 227 | "--output is not compatible with multiple inputs", 228 | )); 229 | } 230 | if opt.work_dir.is_some() { 231 | return Err(std::io::Error::new( 232 | ErrorKind::Other, 233 | "--work-dir is not compatible with multiple inputs", 234 | )); 235 | } 236 | for input in inputs { 237 | let work_dir = input 238 | .clone() 239 | .parent() 240 | .expect("fatal: your input file has no parent directory."); 241 | let output = input.clone(); 242 | process_file(&input, &output, &work_dir, clean, frozen)?; 243 | } 244 | } 245 | 246 | Ok(()) 247 | } 248 | 249 | fn process_file( 250 | input: &FileArg, 251 | output: &FileArg, 252 | work_dir: &Parent, 253 | clean: bool, 254 | frozen: bool, 255 | ) -> std::io::Result<()> { 256 | let original_contents = read_file(&input); 257 | let mut contents = original_contents.clone(); 258 | 259 | eprintln!( 260 | "Using input={:?} output={:?} work_dir={:?} clean={:?} frozen={:?}", 261 | &input, output, work_dir, clean, frozen 262 | ); 263 | 264 | /// Remove all outputs of blocks 265 | fn clean_blocks(file: &mut String, block_regex: &Regex) { 266 | *file = block_regex 267 | .replace_all(file, |caps: &Captures| { 268 | // the 1 group is our command, 269 | // the 2nd is the block, which we ignore and thus erase 270 | caps[1].to_string() 271 | }) 272 | .into_owned() 273 | } 274 | 275 | clean_blocks(&mut contents, &RE_MATCH_CODE_BLOCK); 276 | clean_blocks(&mut contents, &RE_MATCH_MD_BLOCK); 277 | 278 | // Write the contents and return if --clean is passed 279 | if clean { 280 | write_file(&output, contents.to_string()); 281 | return Ok(()); 282 | } 283 | 284 | // Return either the captures fence type with a whitespace in front, 285 | // or an empty string. 286 | // That way if the fence type doesn't apply, nothing is being added. 287 | fn get_fence_type(caps: &Captures) -> String { 288 | if let Some(name) = &caps.name("fence_type1") { 289 | format!("{}", name.as_str()) 290 | } else if let Some(name) = &caps.name("fence_type2") { 291 | format!("{}", name.as_str()) 292 | } else { 293 | format!("") 294 | } 295 | } 296 | 297 | let mut failures = Vec::new(); 298 | 299 | // Run all commands and fill their blocks. 300 | let fill_commands = 301 | |data: &mut String, command_regex: &Regex| -> Result<(), Vec> { 302 | *data = command_regex 303 | .replace_all(data, |caps: &Captures| { 304 | let original_line = &caps[0]; 305 | let command_line = &caps[1]; 306 | let fence_type = get_fence_type(caps); 307 | eprintln!("{}", command_line); 308 | // eprintln!("command_line: {}", command_line); 309 | // eprintln!("fence_type: {}", fence_type); 310 | 311 | if let Some(caps) = RE_MATCH_FENCE_COMMAND.captures(command_line) { 312 | let command1 = caps.name("command1").map_or("", |m| m.as_str()); 313 | let command: &str = if command1 != "" { 314 | command1 315 | } else { 316 | &caps["command2"] 317 | }; 318 | // eprintln!("command: {}", command); 319 | let start_delimiter = "```"; 320 | let end_delimiter = "```"; 321 | let command_char = '$'; 322 | 323 | // TODO: now match on any of the known commands 324 | 325 | let is_multiline = command.lines().count() > 1; 326 | let result = run_command(command, &work_dir); 327 | if result.status.success() { 328 | let stdout = String::from_utf8_lossy(&result.stdout); 329 | // remove ANSI escape sequences 330 | let stdout = filter_ansi(stdout.to_string()); 331 | // we can leave the output block if stdout was empty 332 | if stdout.trim().is_empty() { 333 | format!("{}", trail_nl(&original_line)) 334 | } else { 335 | format!( 336 | "{}\n{}{}{}{}", 337 | trail_nl(&original_line), 338 | start_delimiter, 339 | fence_type, 340 | wrap_nl(stdout.to_string()), 341 | end_delimiter 342 | ) 343 | } 344 | } else { 345 | failures.push(FailingCommand { 346 | output: result, 347 | command: command.to_string(), 348 | command_char: command_char, 349 | is_multiline: is_multiline, 350 | }); 351 | // re-insert what was there before 352 | original_line.to_string() 353 | } 354 | } else if let Some(caps) = RE_MATCH_MD_COMMAND.captures(command_line) { 355 | let command1 = caps.name("command1").map_or("", |m| m.as_str()); 356 | let command = if command1 != "" { 357 | command1 358 | } else { 359 | &caps["command2"] 360 | }; 361 | // eprintln!("command: {}", command); 362 | let start_delimiter = ""; 363 | let end_delimiter = ""; 364 | let command_char = '>'; 365 | 366 | let result = run_command(command, &work_dir); 367 | if result.status.success() { 368 | let stdout = String::from_utf8_lossy(&result.stdout); 369 | // remove ANSI escape sequences 370 | let stdout = filter_ansi(stdout.to_string()); 371 | // we can leave the output block if STDOUT was empty 372 | if stdout.trim().is_empty() { 373 | format!("{}", trail_nl(&original_line)) 374 | } else { 375 | format!( 376 | "{}\n{}{}{}{}", 377 | trail_nl(&original_line), 378 | start_delimiter, 379 | fence_type, 380 | wrap_nl(stdout.to_string()), 381 | end_delimiter 382 | ) 383 | } 384 | } else { 385 | failures.push(FailingCommand { 386 | output: result, 387 | command: command.to_string(), 388 | command_char: command_char, 389 | is_multiline: false, 390 | }); 391 | // re-insert what was there before 392 | original_line.to_string() 393 | } 394 | } else if let Some(caps) = RE_MATCH_VAR_COMMAND.captures(command_line) { 395 | let key = &caps["key"]; 396 | let raw_value = &caps["raw_value"]; 397 | // eprintln!("key: {}", key); 398 | // eprintln!("raw_value: {}", raw_value); 399 | let command = format!("echo {}", raw_value.trim()); 400 | let result = run_command(&command, &work_dir); 401 | if result.status.success() { 402 | let stdout = String::from_utf8_lossy(&result.stdout); 403 | // remove ANSI escape sequences 404 | let stdout = filter_ansi(stdout.to_string()); 405 | // set the environment variable 406 | std::env::set_var(key, stdout.trim()); 407 | } else { 408 | failures.push(FailingCommand { 409 | output: result, 410 | command: command.to_string(), 411 | command_char: '!', 412 | is_multiline: false, 413 | }); 414 | }; 415 | 416 | // re-insert what was there before 417 | original_line.to_string() 418 | } else { 419 | panic!("WTF, not supported") 420 | } 421 | }) 422 | .into_owned(); 423 | if failures.is_empty() { 424 | Ok(()) 425 | } else { 426 | Err(failures) 427 | } 428 | }; 429 | 430 | fn print_failures(fs: Vec) { 431 | eprintln!("\nERROR: some commands failed:\n"); 432 | for f in fs { 433 | let stderr = match String::from_utf8_lossy(&f.output.stderr) 434 | .into_owned() 435 | .as_str() 436 | { 437 | "" => String::from(""), 438 | s => String::from("\nIts stderr was:\n") + s.trim_end(), 439 | }; 440 | let command_string_ = format!("{} ", f.command_char); 441 | let (delimiter, newline, command_string) = if f.is_multiline { 442 | ("```", "\n", "") 443 | } else { 444 | ("`", "", command_string_.as_str()) 445 | }; 446 | eprintln!( 447 | "{}{}{}{}{}{}\nfailed with {}.{}\n", 448 | delimiter, 449 | newline, 450 | command_string, 451 | f.command, 452 | newline, 453 | delimiter, 454 | f.output.status, 455 | stderr 456 | ); 457 | } 458 | } 459 | 460 | fill_commands(&mut contents, &RE_MATCH_ANY_COMMAND).or_else( 461 | |failures| -> std::io::Result<()> { 462 | print_failures(failures); 463 | std::process::exit(1); 464 | }, 465 | )?; 466 | 467 | /// Run all link includes and fill their blocks 468 | fn fill_includes( 469 | file: &mut String, 470 | link_regex: &Regex, 471 | link_char: char, 472 | start_delimiter: &str, 473 | end_delimiter: &str, 474 | ) { 475 | *file = link_regex 476 | .replace_all(file, |caps: &Captures| { 477 | let link = &caps["link"]; 478 | let fence_type = get_fence_type(caps); 479 | 480 | eprintln!("[{} {}]", link_char, link); 481 | 482 | let result = read_file(&FileArg::from_str_unsafe(link)); 483 | 484 | format!( 485 | "{}\n{}{}{}{}", 486 | trail_nl(&caps[0]), 487 | start_delimiter, 488 | fence_type, 489 | wrap_nl(result.to_owned()), 490 | end_delimiter 491 | ) 492 | }) 493 | .into_owned() 494 | } 495 | 496 | fill_includes(&mut contents, &RE_MATCH_FENCE_LINK, '$', "```", "```"); 497 | fill_includes( 498 | &mut contents, 499 | &RE_MATCH_MD_LINK, 500 | '>', 501 | "", 502 | "", 503 | ); 504 | 505 | // If there is no change, these is nothing left to do. 506 | if original_contents == contents { 507 | return Ok(()); 508 | } 509 | 510 | // Let the user know where things have changed 511 | let changeset = Changeset::new(&original_contents, &contents, "\n"); 512 | eprintln!("{}", changeset); 513 | 514 | // If there are changes and the file is frozen, abort 515 | if frozen { 516 | return Err(std::io::Error::new( 517 | ErrorKind::Other, 518 | "--frozen: output is not the same", 519 | )); 520 | } 521 | 522 | // Write the file 523 | write_file(&output, contents.to_string()); 524 | 525 | return Ok(()); 526 | } 527 | --------------------------------------------------------------------------------