├── .appveyor.yml ├── .circleci ├── cache-key ├── cargo-lint └── config.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── benches ├── circuit_breaker.rs ├── futures.rs ├── state_machine.rs └── windowed_adder.rs └── src ├── backoff.rs ├── circuit_breaker.rs ├── clock.rs ├── config.rs ├── ema.rs ├── error.rs ├── failure_policy.rs ├── failure_predicate.rs ├── futures ├── mod.rs └── stream.rs ├── instrument.rs ├── lib.rs ├── state_machine.rs └── windowed_adder.rs /.appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | global: 3 | PROJECT_NAME: failsafe 4 | RUST_BACKTRACE: 1 5 | matrix: 6 | # Stable channel 7 | - TARGET: x86_64-pc-windows-gnu 8 | CHANNEL: stable 9 | - TARGET: x86_64-pc-windows-msvc 10 | CHANNEL: stable 11 | 12 | # Install Rust and Cargo 13 | # (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml) 14 | install: 15 | - curl -sSf -o rustup-init.exe https://win.rustup.rs 16 | - rustup-init.exe --default-host %TARGET% --default-toolchain %CHANNEL% -y 17 | - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin 18 | - rustc -vV 19 | - cargo -vV 20 | 21 | # 'cargo test' takes care of building for us, so disable Appveyor's build stage. This prevents 22 | # the "directory does not contain a project or solution file" error. 23 | # source: https://github.com/starkat99/appveyor-rust/blob/master/appveyor.yml#L113 24 | build: false 25 | 26 | # Equivalent to Travis' `script` phase 27 | # TODO modify this phase as you see fit 28 | test_script: 29 | - cargo build 30 | - cargo test 31 | 32 | branches: 33 | only: 34 | - master 35 | -------------------------------------------------------------------------------- /.circleci/cache-key: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | CACHE_KEY=${1:-.cache-key} 6 | CARGO=${CARGO:-cargo} 7 | RUSTC=${RUSTC:-rustc} 8 | 9 | GREEN='\033[0;32m' 10 | NC='\033[0m' # No Color 11 | 12 | echo -e "${GREEN}${RUSTC} --version --verbose${NC}" 13 | rustc --version --verbose | tee -a ${CACHE_KEY} 14 | echo 15 | 16 | echo -e "${GREEN}${CARGO} --version --verbose${NC}" 17 | ${CARGO} --version --verbose | tee -a ${CACHE_KEY} 18 | -------------------------------------------------------------------------------- /.circleci/cargo-lint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | GREEN='\033[0;32m' 6 | YELLOW='\033[1;33m' 7 | NC='\033[0m' # No Color 8 | 9 | CARGO=${CARGO:-cargo} 10 | 11 | function rust_add_component() { 12 | local name=$1 13 | 14 | echo -e "${GREEN}rustup component add ${name}${NC}" 15 | rustup component add ${name} 16 | } 17 | 18 | # cargo fmt 19 | rust_add_component rustfmt-preview 20 | ${CARGO} fmt -- --version 21 | ${CARGO} fmt -- --check 22 | 23 | echo 24 | 25 | rust_add_component clippy-preview 26 | ${CARGO} clippy --version 27 | ${CARGO} clippy -- -Dwarnings 28 | 29 | echo -e "${YELLOW}OK${NC}" 30 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | default_env: &default_env 4 | environment: 5 | CARGO_HOME: "/home/circleci/.cargo" 6 | CARGO: "cargo --color always" 7 | CARGO_ARGS: "" 8 | 9 | default_steps: &default_steps 10 | steps: 11 | - checkout 12 | - run: 13 | name: "Print versions" 14 | command: | 15 | mkdir -p /home/circleci 16 | .circleci/cache-key /home/circleci/.cache-key 17 | - restore_cache: 18 | key: cache-v4-{{ checksum "/home/circleci/.cache-key" }}-{{ checksum "./Cargo.toml" }} 19 | - run: 20 | name: "Lint" 21 | command: | 22 | if [ "$CIRCLE_JOB" == "rust_stable" ]; then 23 | .circleci/cargo-lint 24 | fi 25 | - run: 26 | name: "Fetch dependencies" 27 | command: ${CARGO} fetch 28 | - run: 29 | name: "Build" 30 | command: | 31 | if [ "$CIRCLE_JOB" == "rust_1_60" ]; then 32 | ${CARGO} build ${CARGO_ARGS} 33 | else 34 | ${CARGO} build ${CARGO_ARGS} --tests 35 | fi 36 | - run: 37 | name: "Test" 38 | command: | 39 | if [ "$CIRCLE_JOB" != "rust_1_60" ]; then 40 | ${CARGO} build ${CARGO_ARGS} --tests 41 | fi 42 | - run: 43 | name: "Bench" 44 | command: | 45 | if [ "$CIRCLE_JOB" == "rust_stable" ]; then 46 | ${CARGO} bench ${CARGO_ARGS} 47 | fi 48 | - save_cache: 49 | key: cache-v4-{{ checksum "/home/circleci/.cache-key" }}-{{ checksum "./Cargo.toml" }} 50 | paths: 51 | - "/home/circleci/.cargo" 52 | jobs: 53 | rust_1_60: 54 | <<: *default_env 55 | <<: *default_steps 56 | docker: 57 | - image: "cimg/rust:1.60" 58 | 59 | rust_stable: 60 | <<: *default_env 61 | <<: *default_steps 62 | docker: 63 | - image: "cimg/rust:1.79" 64 | 65 | rust_nightly: 66 | <<: *default_env 67 | <<: *default_steps 68 | docker: 69 | - image: "rustlang/rust:nightly" 70 | 71 | workflows: 72 | version: 2 73 | commit: 74 | jobs: 75 | - rust_1_60 76 | - rust_stable 77 | - rust_nightly 78 | nightly: 79 | jobs: 80 | - rust_nightly 81 | triggers: 82 | - schedule: 83 | cron: "0 0 * * 0" 84 | filters: { branches: { only: [master] } } 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /.idea 3 | /*.iml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### [Unreleased] 2 | 3 | Breaking changes: 4 | * minimum rust version is 1.60 5 | 6 | Improvements: 7 | * drop `pin-project` dependency, use `pin-project-lite` instead 8 | 9 | ### [1.3.0] - 2024-05-05 10 | 11 | Added: 12 | * Implements circuit breaker for `futures::Stream` (thanks to https://github.com/leshow) 13 | 14 | Breaking changes: 15 | * minimum rust version is 1.59 16 | 17 | ### [1.2.0] - 2022-08-22 18 | 19 | Added: 20 | * the `reset` method to the `StateMachine`, (thanks to https://github.com/eg-fxia) 21 | 22 | Breaking changes: 23 | * minimum rust version is 1.49 24 | 25 | Updates: 26 | * `pin_project` has been updated to `0.12` 27 | * `criterion` has been updated to `0.3.6` 28 | 29 | ### [1.1.0] - 2021-09-18 30 | 31 | Fixes: 32 | * fix `FullJittered` implementation: the exponential behavior was not being applied 33 | 34 | Updates: 35 | * `pin_project` and `rand` has been updated to the latest versions 36 | 37 | Breaking changes: 38 | * minimum rust version is 1.45.0 39 | 40 | ### [1.0.0] - 2020-08-17 41 | 42 | Breaking changes: 43 | * use rust 2018 edition 44 | * use `std::future::Future` and `futures==0.3`, supporting `async`/`await` 45 | * minimum rust version is 1.39.0 46 | 47 | Improvements: 48 | * drop `spin` dependency, use `parking_lot` 49 | * add `futures-support` feature, to allow opt-out for `futures` support 50 | 51 | ### [0.3.1] - 2019-06-10 52 | 53 | Fixes: 54 | * add explicitly `dyn` definition to trait objects 55 | 56 | ### [0.3.0] - 2018-10-26 57 | 58 | Breaking changes: 59 | * remove `instrument::NoopInstrument`, use `()` instead. 60 | * added optional feature `parking_lot_mutex` when it exists the crate `parking_lot` 61 | would be using for `Mutex` instead of the default `spin`. 62 | 63 | ### [0.2.0] - 2018-09-10 64 | 65 | Breaking changes: 66 | * `success_rate` policy now accepts `min_request_threshold`. 67 | * the `CircuitBreaker` turned into a trait which implements `call` and `call_with` methods. 68 | * the trait `Callable` was removed 69 | 70 | Improvements: 71 | * remove `tokio-timer` dependency. 72 | * use spin lock instead `std::sync::Mutex` 73 | 74 | [Unreleased]: https://github.com/dmexe/failsafe-rs/compare/v1.3.0...master 75 | [1.2.0]: https://github.com/dmexe/failsafe-rs/compare/v1.2.0...v1.3.0 76 | [1.2.0]: https://github.com/dmexe/failsafe-rs/compare/v1.1.0...v1.2.0 77 | [1.1.0]: https://github.com/dmexe/failsafe-rs/compare/v1.0.0...v1.1.0 78 | [1.0.0]: https://github.com/dmexe/failsafe-rs/compare/v0.3.1...v1.0.0 79 | [0.3.1]: https://github.com/dmexe/failsafe-rs/compare/v0.3.0...v0.3.1 80 | [0.3.0]: https://github.com/dmexe/failsafe-rs/compare/v0.2.0...v0.3.0 81 | [0.2.0]: https://github.com/dmexe/failsafe-rs/releases/tag/v0.2.0 82 | 83 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.22.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "aho-corasick" 22 | version = "1.1.3" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 25 | dependencies = [ 26 | "memchr", 27 | ] 28 | 29 | [[package]] 30 | name = "anes" 31 | version = "0.1.6" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" 34 | 35 | [[package]] 36 | name = "atty" 37 | version = "0.2.14" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 40 | dependencies = [ 41 | "hermit-abi 0.1.19", 42 | "libc", 43 | "winapi", 44 | ] 45 | 46 | [[package]] 47 | name = "autocfg" 48 | version = "1.3.0" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" 51 | 52 | [[package]] 53 | name = "backtrace" 54 | version = "0.3.73" 55 | source = "registry+https://github.com/rust-lang/crates.io-index" 56 | checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" 57 | dependencies = [ 58 | "addr2line", 59 | "cc", 60 | "cfg-if", 61 | "libc", 62 | "miniz_oxide", 63 | "object", 64 | "rustc-demangle", 65 | ] 66 | 67 | [[package]] 68 | name = "bitflags" 69 | version = "1.3.2" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 72 | 73 | [[package]] 74 | name = "bitflags" 75 | version = "2.6.0" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" 78 | 79 | [[package]] 80 | name = "bumpalo" 81 | version = "3.16.0" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" 84 | 85 | [[package]] 86 | name = "cast" 87 | version = "0.3.0" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" 90 | 91 | [[package]] 92 | name = "cc" 93 | version = "1.0.104" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" 96 | 97 | [[package]] 98 | name = "cfg-if" 99 | version = "1.0.0" 100 | source = "registry+https://github.com/rust-lang/crates.io-index" 101 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 102 | 103 | [[package]] 104 | name = "ciborium" 105 | version = "0.2.2" 106 | source = "registry+https://github.com/rust-lang/crates.io-index" 107 | checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" 108 | dependencies = [ 109 | "ciborium-io", 110 | "ciborium-ll", 111 | "serde", 112 | ] 113 | 114 | [[package]] 115 | name = "ciborium-io" 116 | version = "0.2.2" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" 119 | 120 | [[package]] 121 | name = "ciborium-ll" 122 | version = "0.2.2" 123 | source = "registry+https://github.com/rust-lang/crates.io-index" 124 | checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" 125 | dependencies = [ 126 | "ciborium-io", 127 | "half", 128 | ] 129 | 130 | [[package]] 131 | name = "clap" 132 | version = "3.2.25" 133 | source = "registry+https://github.com/rust-lang/crates.io-index" 134 | checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" 135 | dependencies = [ 136 | "bitflags 1.3.2", 137 | "clap_lex", 138 | "indexmap", 139 | "textwrap", 140 | ] 141 | 142 | [[package]] 143 | name = "clap_lex" 144 | version = "0.2.4" 145 | source = "registry+https://github.com/rust-lang/crates.io-index" 146 | checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" 147 | dependencies = [ 148 | "os_str_bytes", 149 | ] 150 | 151 | [[package]] 152 | name = "criterion" 153 | version = "0.4.0" 154 | source = "registry+https://github.com/rust-lang/crates.io-index" 155 | checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" 156 | dependencies = [ 157 | "anes", 158 | "atty", 159 | "cast", 160 | "ciborium", 161 | "clap", 162 | "criterion-plot", 163 | "itertools", 164 | "lazy_static", 165 | "num-traits", 166 | "oorandom", 167 | "plotters", 168 | "rayon", 169 | "regex", 170 | "serde", 171 | "serde_derive", 172 | "serde_json", 173 | "tinytemplate", 174 | "walkdir", 175 | ] 176 | 177 | [[package]] 178 | name = "criterion-plot" 179 | version = "0.5.0" 180 | source = "registry+https://github.com/rust-lang/crates.io-index" 181 | checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" 182 | dependencies = [ 183 | "cast", 184 | "itertools", 185 | ] 186 | 187 | [[package]] 188 | name = "crossbeam-deque" 189 | version = "0.8.5" 190 | source = "registry+https://github.com/rust-lang/crates.io-index" 191 | checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" 192 | dependencies = [ 193 | "crossbeam-epoch", 194 | "crossbeam-utils", 195 | ] 196 | 197 | [[package]] 198 | name = "crossbeam-epoch" 199 | version = "0.9.18" 200 | source = "registry+https://github.com/rust-lang/crates.io-index" 201 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 202 | dependencies = [ 203 | "crossbeam-utils", 204 | ] 205 | 206 | [[package]] 207 | name = "crossbeam-utils" 208 | version = "0.8.20" 209 | source = "registry+https://github.com/rust-lang/crates.io-index" 210 | checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" 211 | 212 | [[package]] 213 | name = "crunchy" 214 | version = "0.2.2" 215 | source = "registry+https://github.com/rust-lang/crates.io-index" 216 | checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" 217 | 218 | [[package]] 219 | name = "either" 220 | version = "1.13.0" 221 | source = "registry+https://github.com/rust-lang/crates.io-index" 222 | checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" 223 | 224 | [[package]] 225 | name = "failsafe" 226 | version = "1.3.0" 227 | dependencies = [ 228 | "criterion", 229 | "futures", 230 | "futures-core", 231 | "parking_lot", 232 | "pin-project-lite", 233 | "rand", 234 | "rand_xorshift", 235 | "tokio", 236 | ] 237 | 238 | [[package]] 239 | name = "futures" 240 | version = "0.3.30" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" 243 | dependencies = [ 244 | "futures-channel", 245 | "futures-core", 246 | "futures-executor", 247 | "futures-io", 248 | "futures-sink", 249 | "futures-task", 250 | "futures-util", 251 | ] 252 | 253 | [[package]] 254 | name = "futures-channel" 255 | version = "0.3.30" 256 | source = "registry+https://github.com/rust-lang/crates.io-index" 257 | checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" 258 | dependencies = [ 259 | "futures-core", 260 | "futures-sink", 261 | ] 262 | 263 | [[package]] 264 | name = "futures-core" 265 | version = "0.3.30" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" 268 | 269 | [[package]] 270 | name = "futures-executor" 271 | version = "0.3.30" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" 274 | dependencies = [ 275 | "futures-core", 276 | "futures-task", 277 | "futures-util", 278 | ] 279 | 280 | [[package]] 281 | name = "futures-io" 282 | version = "0.3.30" 283 | source = "registry+https://github.com/rust-lang/crates.io-index" 284 | checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" 285 | 286 | [[package]] 287 | name = "futures-macro" 288 | version = "0.3.30" 289 | source = "registry+https://github.com/rust-lang/crates.io-index" 290 | checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" 291 | dependencies = [ 292 | "proc-macro2", 293 | "quote", 294 | "syn", 295 | ] 296 | 297 | [[package]] 298 | name = "futures-sink" 299 | version = "0.3.30" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" 302 | 303 | [[package]] 304 | name = "futures-task" 305 | version = "0.3.30" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" 308 | 309 | [[package]] 310 | name = "futures-util" 311 | version = "0.3.30" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" 314 | dependencies = [ 315 | "futures-channel", 316 | "futures-core", 317 | "futures-io", 318 | "futures-macro", 319 | "futures-sink", 320 | "futures-task", 321 | "memchr", 322 | "pin-project-lite", 323 | "pin-utils", 324 | "slab", 325 | ] 326 | 327 | [[package]] 328 | name = "getrandom" 329 | version = "0.2.15" 330 | source = "registry+https://github.com/rust-lang/crates.io-index" 331 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 332 | dependencies = [ 333 | "cfg-if", 334 | "libc", 335 | "wasi", 336 | ] 337 | 338 | [[package]] 339 | name = "gimli" 340 | version = "0.29.0" 341 | source = "registry+https://github.com/rust-lang/crates.io-index" 342 | checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" 343 | 344 | [[package]] 345 | name = "half" 346 | version = "2.4.1" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" 349 | dependencies = [ 350 | "cfg-if", 351 | "crunchy", 352 | ] 353 | 354 | [[package]] 355 | name = "hashbrown" 356 | version = "0.12.3" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 359 | 360 | [[package]] 361 | name = "hermit-abi" 362 | version = "0.1.19" 363 | source = "registry+https://github.com/rust-lang/crates.io-index" 364 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 365 | dependencies = [ 366 | "libc", 367 | ] 368 | 369 | [[package]] 370 | name = "hermit-abi" 371 | version = "0.3.9" 372 | source = "registry+https://github.com/rust-lang/crates.io-index" 373 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" 374 | 375 | [[package]] 376 | name = "indexmap" 377 | version = "1.9.3" 378 | source = "registry+https://github.com/rust-lang/crates.io-index" 379 | checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" 380 | dependencies = [ 381 | "autocfg", 382 | "hashbrown", 383 | ] 384 | 385 | [[package]] 386 | name = "itertools" 387 | version = "0.10.5" 388 | source = "registry+https://github.com/rust-lang/crates.io-index" 389 | checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" 390 | dependencies = [ 391 | "either", 392 | ] 393 | 394 | [[package]] 395 | name = "itoa" 396 | version = "1.0.11" 397 | source = "registry+https://github.com/rust-lang/crates.io-index" 398 | checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" 399 | 400 | [[package]] 401 | name = "js-sys" 402 | version = "0.3.69" 403 | source = "registry+https://github.com/rust-lang/crates.io-index" 404 | checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" 405 | dependencies = [ 406 | "wasm-bindgen", 407 | ] 408 | 409 | [[package]] 410 | name = "lazy_static" 411 | version = "1.5.0" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 414 | 415 | [[package]] 416 | name = "libc" 417 | version = "0.2.155" 418 | source = "registry+https://github.com/rust-lang/crates.io-index" 419 | checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" 420 | 421 | [[package]] 422 | name = "lock_api" 423 | version = "0.4.12" 424 | source = "registry+https://github.com/rust-lang/crates.io-index" 425 | checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" 426 | dependencies = [ 427 | "autocfg", 428 | "scopeguard", 429 | ] 430 | 431 | [[package]] 432 | name = "log" 433 | version = "0.4.22" 434 | source = "registry+https://github.com/rust-lang/crates.io-index" 435 | checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" 436 | 437 | [[package]] 438 | name = "memchr" 439 | version = "2.7.4" 440 | source = "registry+https://github.com/rust-lang/crates.io-index" 441 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 442 | 443 | [[package]] 444 | name = "miniz_oxide" 445 | version = "0.7.4" 446 | source = "registry+https://github.com/rust-lang/crates.io-index" 447 | checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" 448 | dependencies = [ 449 | "adler", 450 | ] 451 | 452 | [[package]] 453 | name = "num-traits" 454 | version = "0.2.19" 455 | source = "registry+https://github.com/rust-lang/crates.io-index" 456 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 457 | dependencies = [ 458 | "autocfg", 459 | ] 460 | 461 | [[package]] 462 | name = "num_cpus" 463 | version = "1.16.0" 464 | source = "registry+https://github.com/rust-lang/crates.io-index" 465 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 466 | dependencies = [ 467 | "hermit-abi 0.3.9", 468 | "libc", 469 | ] 470 | 471 | [[package]] 472 | name = "object" 473 | version = "0.36.1" 474 | source = "registry+https://github.com/rust-lang/crates.io-index" 475 | checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" 476 | dependencies = [ 477 | "memchr", 478 | ] 479 | 480 | [[package]] 481 | name = "once_cell" 482 | version = "1.19.0" 483 | source = "registry+https://github.com/rust-lang/crates.io-index" 484 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 485 | 486 | [[package]] 487 | name = "oorandom" 488 | version = "11.1.3" 489 | source = "registry+https://github.com/rust-lang/crates.io-index" 490 | checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" 491 | 492 | [[package]] 493 | name = "os_str_bytes" 494 | version = "6.6.1" 495 | source = "registry+https://github.com/rust-lang/crates.io-index" 496 | checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" 497 | 498 | [[package]] 499 | name = "parking_lot" 500 | version = "0.12.3" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" 503 | dependencies = [ 504 | "lock_api", 505 | "parking_lot_core", 506 | ] 507 | 508 | [[package]] 509 | name = "parking_lot_core" 510 | version = "0.9.10" 511 | source = "registry+https://github.com/rust-lang/crates.io-index" 512 | checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" 513 | dependencies = [ 514 | "cfg-if", 515 | "libc", 516 | "redox_syscall", 517 | "smallvec", 518 | "windows-targets", 519 | ] 520 | 521 | [[package]] 522 | name = "pin-project-lite" 523 | version = "0.2.14" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" 526 | 527 | [[package]] 528 | name = "pin-utils" 529 | version = "0.1.0" 530 | source = "registry+https://github.com/rust-lang/crates.io-index" 531 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 532 | 533 | [[package]] 534 | name = "plotters" 535 | version = "0.3.6" 536 | source = "registry+https://github.com/rust-lang/crates.io-index" 537 | checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" 538 | dependencies = [ 539 | "num-traits", 540 | "plotters-backend", 541 | "plotters-svg", 542 | "wasm-bindgen", 543 | "web-sys", 544 | ] 545 | 546 | [[package]] 547 | name = "plotters-backend" 548 | version = "0.3.6" 549 | source = "registry+https://github.com/rust-lang/crates.io-index" 550 | checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" 551 | 552 | [[package]] 553 | name = "plotters-svg" 554 | version = "0.3.6" 555 | source = "registry+https://github.com/rust-lang/crates.io-index" 556 | checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" 557 | dependencies = [ 558 | "plotters-backend", 559 | ] 560 | 561 | [[package]] 562 | name = "ppv-lite86" 563 | version = "0.2.17" 564 | source = "registry+https://github.com/rust-lang/crates.io-index" 565 | checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" 566 | 567 | [[package]] 568 | name = "proc-macro2" 569 | version = "1.0.86" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" 572 | dependencies = [ 573 | "unicode-ident", 574 | ] 575 | 576 | [[package]] 577 | name = "quote" 578 | version = "1.0.36" 579 | source = "registry+https://github.com/rust-lang/crates.io-index" 580 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" 581 | dependencies = [ 582 | "proc-macro2", 583 | ] 584 | 585 | [[package]] 586 | name = "rand" 587 | version = "0.8.5" 588 | source = "registry+https://github.com/rust-lang/crates.io-index" 589 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 590 | dependencies = [ 591 | "libc", 592 | "rand_chacha", 593 | "rand_core", 594 | ] 595 | 596 | [[package]] 597 | name = "rand_chacha" 598 | version = "0.3.1" 599 | source = "registry+https://github.com/rust-lang/crates.io-index" 600 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 601 | dependencies = [ 602 | "ppv-lite86", 603 | "rand_core", 604 | ] 605 | 606 | [[package]] 607 | name = "rand_core" 608 | version = "0.6.4" 609 | source = "registry+https://github.com/rust-lang/crates.io-index" 610 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 611 | dependencies = [ 612 | "getrandom", 613 | ] 614 | 615 | [[package]] 616 | name = "rand_xorshift" 617 | version = "0.3.0" 618 | source = "registry+https://github.com/rust-lang/crates.io-index" 619 | checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" 620 | dependencies = [ 621 | "rand_core", 622 | ] 623 | 624 | [[package]] 625 | name = "rayon" 626 | version = "1.10.0" 627 | source = "registry+https://github.com/rust-lang/crates.io-index" 628 | checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" 629 | dependencies = [ 630 | "either", 631 | "rayon-core", 632 | ] 633 | 634 | [[package]] 635 | name = "rayon-core" 636 | version = "1.12.1" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" 639 | dependencies = [ 640 | "crossbeam-deque", 641 | "crossbeam-utils", 642 | ] 643 | 644 | [[package]] 645 | name = "redox_syscall" 646 | version = "0.5.2" 647 | source = "registry+https://github.com/rust-lang/crates.io-index" 648 | checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" 649 | dependencies = [ 650 | "bitflags 2.6.0", 651 | ] 652 | 653 | [[package]] 654 | name = "regex" 655 | version = "1.10.5" 656 | source = "registry+https://github.com/rust-lang/crates.io-index" 657 | checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" 658 | dependencies = [ 659 | "aho-corasick", 660 | "memchr", 661 | "regex-automata", 662 | "regex-syntax", 663 | ] 664 | 665 | [[package]] 666 | name = "regex-automata" 667 | version = "0.4.7" 668 | source = "registry+https://github.com/rust-lang/crates.io-index" 669 | checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" 670 | dependencies = [ 671 | "aho-corasick", 672 | "memchr", 673 | "regex-syntax", 674 | ] 675 | 676 | [[package]] 677 | name = "regex-syntax" 678 | version = "0.8.4" 679 | source = "registry+https://github.com/rust-lang/crates.io-index" 680 | checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" 681 | 682 | [[package]] 683 | name = "rustc-demangle" 684 | version = "0.1.24" 685 | source = "registry+https://github.com/rust-lang/crates.io-index" 686 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" 687 | 688 | [[package]] 689 | name = "ryu" 690 | version = "1.0.18" 691 | source = "registry+https://github.com/rust-lang/crates.io-index" 692 | checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" 693 | 694 | [[package]] 695 | name = "same-file" 696 | version = "1.0.6" 697 | source = "registry+https://github.com/rust-lang/crates.io-index" 698 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 699 | dependencies = [ 700 | "winapi-util", 701 | ] 702 | 703 | [[package]] 704 | name = "scopeguard" 705 | version = "1.2.0" 706 | source = "registry+https://github.com/rust-lang/crates.io-index" 707 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 708 | 709 | [[package]] 710 | name = "serde" 711 | version = "1.0.203" 712 | source = "registry+https://github.com/rust-lang/crates.io-index" 713 | checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" 714 | dependencies = [ 715 | "serde_derive", 716 | ] 717 | 718 | [[package]] 719 | name = "serde_derive" 720 | version = "1.0.203" 721 | source = "registry+https://github.com/rust-lang/crates.io-index" 722 | checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" 723 | dependencies = [ 724 | "proc-macro2", 725 | "quote", 726 | "syn", 727 | ] 728 | 729 | [[package]] 730 | name = "serde_json" 731 | version = "1.0.120" 732 | source = "registry+https://github.com/rust-lang/crates.io-index" 733 | checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" 734 | dependencies = [ 735 | "itoa", 736 | "ryu", 737 | "serde", 738 | ] 739 | 740 | [[package]] 741 | name = "slab" 742 | version = "0.4.9" 743 | source = "registry+https://github.com/rust-lang/crates.io-index" 744 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 745 | dependencies = [ 746 | "autocfg", 747 | ] 748 | 749 | [[package]] 750 | name = "smallvec" 751 | version = "1.13.2" 752 | source = "registry+https://github.com/rust-lang/crates.io-index" 753 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" 754 | 755 | [[package]] 756 | name = "syn" 757 | version = "2.0.68" 758 | source = "registry+https://github.com/rust-lang/crates.io-index" 759 | checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" 760 | dependencies = [ 761 | "proc-macro2", 762 | "quote", 763 | "unicode-ident", 764 | ] 765 | 766 | [[package]] 767 | name = "textwrap" 768 | version = "0.16.1" 769 | source = "registry+https://github.com/rust-lang/crates.io-index" 770 | checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" 771 | 772 | [[package]] 773 | name = "tinytemplate" 774 | version = "1.2.1" 775 | source = "registry+https://github.com/rust-lang/crates.io-index" 776 | checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" 777 | dependencies = [ 778 | "serde", 779 | "serde_json", 780 | ] 781 | 782 | [[package]] 783 | name = "tokio" 784 | version = "1.38.0" 785 | source = "registry+https://github.com/rust-lang/crates.io-index" 786 | checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" 787 | dependencies = [ 788 | "backtrace", 789 | "num_cpus", 790 | "pin-project-lite", 791 | "tokio-macros", 792 | ] 793 | 794 | [[package]] 795 | name = "tokio-macros" 796 | version = "2.3.0" 797 | source = "registry+https://github.com/rust-lang/crates.io-index" 798 | checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" 799 | dependencies = [ 800 | "proc-macro2", 801 | "quote", 802 | "syn", 803 | ] 804 | 805 | [[package]] 806 | name = "unicode-ident" 807 | version = "1.0.12" 808 | source = "registry+https://github.com/rust-lang/crates.io-index" 809 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 810 | 811 | [[package]] 812 | name = "walkdir" 813 | version = "2.5.0" 814 | source = "registry+https://github.com/rust-lang/crates.io-index" 815 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 816 | dependencies = [ 817 | "same-file", 818 | "winapi-util", 819 | ] 820 | 821 | [[package]] 822 | name = "wasi" 823 | version = "0.11.0+wasi-snapshot-preview1" 824 | source = "registry+https://github.com/rust-lang/crates.io-index" 825 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 826 | 827 | [[package]] 828 | name = "wasm-bindgen" 829 | version = "0.2.92" 830 | source = "registry+https://github.com/rust-lang/crates.io-index" 831 | checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" 832 | dependencies = [ 833 | "cfg-if", 834 | "wasm-bindgen-macro", 835 | ] 836 | 837 | [[package]] 838 | name = "wasm-bindgen-backend" 839 | version = "0.2.92" 840 | source = "registry+https://github.com/rust-lang/crates.io-index" 841 | checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" 842 | dependencies = [ 843 | "bumpalo", 844 | "log", 845 | "once_cell", 846 | "proc-macro2", 847 | "quote", 848 | "syn", 849 | "wasm-bindgen-shared", 850 | ] 851 | 852 | [[package]] 853 | name = "wasm-bindgen-macro" 854 | version = "0.2.92" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" 857 | dependencies = [ 858 | "quote", 859 | "wasm-bindgen-macro-support", 860 | ] 861 | 862 | [[package]] 863 | name = "wasm-bindgen-macro-support" 864 | version = "0.2.92" 865 | source = "registry+https://github.com/rust-lang/crates.io-index" 866 | checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" 867 | dependencies = [ 868 | "proc-macro2", 869 | "quote", 870 | "syn", 871 | "wasm-bindgen-backend", 872 | "wasm-bindgen-shared", 873 | ] 874 | 875 | [[package]] 876 | name = "wasm-bindgen-shared" 877 | version = "0.2.92" 878 | source = "registry+https://github.com/rust-lang/crates.io-index" 879 | checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" 880 | 881 | [[package]] 882 | name = "web-sys" 883 | version = "0.3.69" 884 | source = "registry+https://github.com/rust-lang/crates.io-index" 885 | checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" 886 | dependencies = [ 887 | "js-sys", 888 | "wasm-bindgen", 889 | ] 890 | 891 | [[package]] 892 | name = "winapi" 893 | version = "0.3.9" 894 | source = "registry+https://github.com/rust-lang/crates.io-index" 895 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 896 | dependencies = [ 897 | "winapi-i686-pc-windows-gnu", 898 | "winapi-x86_64-pc-windows-gnu", 899 | ] 900 | 901 | [[package]] 902 | name = "winapi-i686-pc-windows-gnu" 903 | version = "0.4.0" 904 | source = "registry+https://github.com/rust-lang/crates.io-index" 905 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 906 | 907 | [[package]] 908 | name = "winapi-util" 909 | version = "0.1.8" 910 | source = "registry+https://github.com/rust-lang/crates.io-index" 911 | checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" 912 | dependencies = [ 913 | "windows-sys", 914 | ] 915 | 916 | [[package]] 917 | name = "winapi-x86_64-pc-windows-gnu" 918 | version = "0.4.0" 919 | source = "registry+https://github.com/rust-lang/crates.io-index" 920 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 921 | 922 | [[package]] 923 | name = "windows-sys" 924 | version = "0.52.0" 925 | source = "registry+https://github.com/rust-lang/crates.io-index" 926 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 927 | dependencies = [ 928 | "windows-targets", 929 | ] 930 | 931 | [[package]] 932 | name = "windows-targets" 933 | version = "0.52.6" 934 | source = "registry+https://github.com/rust-lang/crates.io-index" 935 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 936 | dependencies = [ 937 | "windows_aarch64_gnullvm", 938 | "windows_aarch64_msvc", 939 | "windows_i686_gnu", 940 | "windows_i686_gnullvm", 941 | "windows_i686_msvc", 942 | "windows_x86_64_gnu", 943 | "windows_x86_64_gnullvm", 944 | "windows_x86_64_msvc", 945 | ] 946 | 947 | [[package]] 948 | name = "windows_aarch64_gnullvm" 949 | version = "0.52.6" 950 | source = "registry+https://github.com/rust-lang/crates.io-index" 951 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 952 | 953 | [[package]] 954 | name = "windows_aarch64_msvc" 955 | version = "0.52.6" 956 | source = "registry+https://github.com/rust-lang/crates.io-index" 957 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 958 | 959 | [[package]] 960 | name = "windows_i686_gnu" 961 | version = "0.52.6" 962 | source = "registry+https://github.com/rust-lang/crates.io-index" 963 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 964 | 965 | [[package]] 966 | name = "windows_i686_gnullvm" 967 | version = "0.52.6" 968 | source = "registry+https://github.com/rust-lang/crates.io-index" 969 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 970 | 971 | [[package]] 972 | name = "windows_i686_msvc" 973 | version = "0.52.6" 974 | source = "registry+https://github.com/rust-lang/crates.io-index" 975 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 976 | 977 | [[package]] 978 | name = "windows_x86_64_gnu" 979 | version = "0.52.6" 980 | source = "registry+https://github.com/rust-lang/crates.io-index" 981 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 982 | 983 | [[package]] 984 | name = "windows_x86_64_gnullvm" 985 | version = "0.52.6" 986 | source = "registry+https://github.com/rust-lang/crates.io-index" 987 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 988 | 989 | [[package]] 990 | name = "windows_x86_64_msvc" 991 | version = "0.52.6" 992 | source = "registry+https://github.com/rust-lang/crates.io-index" 993 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 994 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "failsafe" 3 | version = "1.3.0" 4 | authors = ["Dmitry Galinsky "] 5 | description = "A circuit breaker implementation" 6 | license = "MIT" 7 | repository = "https://github.com/dmexe/failsafe-rs" 8 | edition = "2018" 9 | rust-version = "1.60" 10 | 11 | [dependencies] 12 | futures-core = { version = "0.3", optional = true } 13 | pin-project-lite = { version = "0.2", optional = true } 14 | rand = "0.8" 15 | parking_lot = "0.12" 16 | 17 | [dev-dependencies] 18 | futures = { version = "0.3", features = ["std"] } 19 | tokio = { version = "1.20", features = ["rt", "rt-multi-thread", "macros", "time"] } 20 | criterion = { version = "0.4", features = ["html_reports"] } 21 | rand_xorshift = "0.3" 22 | 23 | [features] 24 | default = ["futures-support"] 25 | futures-support = ["futures-core", "pin-project-lite"] 26 | 27 | [[bench]] 28 | name = "windowed_adder" 29 | harness = false 30 | 31 | [[bench]] 32 | name = "state_machine" 33 | harness = false 34 | 35 | [[bench]] 36 | name = "futures" 37 | harness = false 38 | 39 | [[bench]] 40 | name = "circuit_breaker" 41 | harness = false 42 | 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Dmitry Galinsky 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Failsafe 2 | 3 | [![Сrate](https://img.shields.io/crates/v/failsafe.svg)](https://crates.io/crates/failsafe) 4 | [![Вocumentation](https://docs.rs/failsafe/badge.svg)](https://docs.rs/failsafe) 5 | [![CircleCI](https://circleci.com/gh/dmexe/failsafe-rs.svg?style=svg)](https://circleci.com/gh/dmexe/failsafe-rs) 6 | [![Appveyor](https://ci.appveyor.com/api/projects/status/c0qrj9dbskneunjg/branch/master?svg=true)](https://ci.appveyor.com/project/dmexe/failsafe-rs/branch/master) 7 | 8 | A circuit breaker implementation which used to detect failures and encapsulates the logic of preventing a 9 | failure from constantly recurring, during maintenance, temporary external system failure or unexpected 10 | system difficulties. 11 | 12 | * [https://martinfowler.com/bliki/CircuitBreaker.html](https://martinfowler.com/bliki/CircuitBreaker.html) 13 | * [Read documentation](https://docs.rs/failsafe/1.3.0/failsafe) 14 | 15 | # Features 16 | 17 | * Working with both `Fn() -> Result` and `Future` (optional via default 18 | `futures-support` feature). 19 | * Backoff strategies: `constant`, `exponential`, `equal_jittered`, `full_jittered` 20 | * Failure detection policies: `consecutive_failures`, `success_rate_over_time_window` 21 | * Minimum rust version: 1.63 22 | 23 | # Usage 24 | 25 | Add this to your Cargo.toml: 26 | 27 | ```toml 28 | failsafe = "1.3.0" 29 | ``` 30 | 31 | # Example 32 | 33 | Using default backoff strategy and failure accrual policy. 34 | 35 | ```rust 36 | use failsafe::{Config, CircuitBreaker, Error}; 37 | 38 | // A function that sometimes failed. 39 | fn dangerous_call() -> Result<(), ()> { 40 | if thread_rng().gen_range(0, 2) == 0 { 41 | return Err(()) 42 | } 43 | Ok(()) 44 | } 45 | 46 | // Create a circuit breaker which configured by reasonable default backoff and 47 | // failure accrual policy. 48 | let circuit_breaker = Config::new().build(); 49 | 50 | // Call the function in a loop, after some iterations the circuit breaker will 51 | // be in a open state and reject next calls. 52 | for n in 0..100 { 53 | match circuit_breaker.call(|| dangerous_call()) { 54 | Err(Error::Inner(_)) => { 55 | eprintln!("{}: fail", n); 56 | }, 57 | Err(Error::Rejected) => { 58 | eprintln!("{}: rejected", n); 59 | break; 60 | }, 61 | _ => {} 62 | } 63 | } 64 | ``` 65 | 66 | Or configure custom backoff and policy: 67 | 68 | ```rust 69 | use std::time::Duration; 70 | use failsafe::{backoff, failure_policy, CircuitBreaker}; 71 | 72 | // Create an exponential growth backoff which starts from 10s and ends with 60s. 73 | let backoff = backoff::exponential(Duration::from_secs(10), Duration::from_secs(60)); 74 | 75 | // Create a policy which failed when three consecutive failures were made. 76 | let policy = failure_policy::consecutive_failures(3, backoff); 77 | 78 | // Creates a circuit breaker with given policy. 79 | let circuit_breaker = Config::new() 80 | .failure_policy(policy) 81 | .build(); 82 | ``` 83 | 84 | -------------------------------------------------------------------------------- /benches/circuit_breaker.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | 3 | use std::thread; 4 | 5 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 6 | 7 | use failsafe::{CircuitBreaker, Config, Error}; 8 | 9 | fn single_threaded(c: &mut Criterion) { 10 | let circuit_breaker = Config::new().build(); 11 | let mut n = 0; 12 | 13 | c.bench_function("single_threaded", |b| { 14 | b.iter(|| { 15 | match circuit_breaker.call(|| dangerous_call(n)) { 16 | Ok(_) => {} 17 | Err(Error::Inner(_)) => {} 18 | Err(err) => unreachable!("{:?}", err), 19 | } 20 | n += 1; 21 | }) 22 | }); 23 | } 24 | 25 | fn multi_threaded_in_batch(c: &mut Criterion) { 26 | let circuit_breaker = Config::new().build(); 27 | let batch_size = 10; 28 | 29 | c.bench_function("multi_threaded_in_batch", |b| { 30 | b.iter(|| { 31 | let mut threads = Vec::new(); 32 | 33 | for n in 0..batch_size { 34 | let circuit_breaker = circuit_breaker.clone(); 35 | let thr = thread::spawn(move || { 36 | let res = match circuit_breaker.call(|| dangerous_call(n)) { 37 | Ok(_) => true, 38 | Err(Error::Inner(_)) => false, 39 | Err(err) => unreachable!("{:?}", err), 40 | }; 41 | black_box(res); 42 | }); 43 | 44 | threads.push(thr); 45 | } 46 | 47 | threads.into_iter().for_each(|it| it.join().unwrap()); 48 | }) 49 | }); 50 | } 51 | 52 | fn dangerous_call(n: usize) -> Result { 53 | if n % 5 == 0 { 54 | black_box(Err(n)) 55 | } else { 56 | black_box(Ok(n)) 57 | } 58 | } 59 | 60 | criterion_group!(benches, single_threaded, multi_threaded_in_batch); 61 | criterion_main!(benches); 62 | -------------------------------------------------------------------------------- /benches/futures.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | 3 | use std::cell::RefCell; 4 | 5 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 6 | use futures::{ 7 | stream::{self, StreamExt, TryStreamExt}, 8 | FutureExt, 9 | }; 10 | use tokio::runtime::Runtime; 11 | 12 | use failsafe::{futures::CircuitBreaker, Config, Error}; 13 | 14 | fn multi_threaded_in_batch(c: &mut Criterion) { 15 | let circuit_breaker = Config::new().build(); 16 | let runtime = RefCell::new(Runtime::new().unwrap()); 17 | let batch_size = 10; 18 | 19 | c.bench_function("multi_threaded_in_batch", |b| { 20 | b.iter(|| { 21 | let circuit_breaker = circuit_breaker.clone(); 22 | 23 | let batch = (0..batch_size).map(move |n| { 24 | circuit_breaker 25 | .call(dangerous_call(n)) 26 | .map(|res| match res { 27 | Ok(n) => Ok(n), 28 | Err(Error::Inner(n)) => Ok(n), 29 | Err(Error::Rejected) => Err(0), 30 | }) 31 | }); 32 | 33 | let batch = stream::iter(batch) 34 | .buffer_unordered(batch_size) 35 | .try_collect(); 36 | 37 | let runtime = runtime.borrow_mut(); 38 | let res: Vec<_> = runtime.block_on(batch).unwrap(); 39 | assert_eq!(45usize, res.iter().sum::()); 40 | }) 41 | }); 42 | } 43 | 44 | async fn dangerous_call(n: usize) -> Result { 45 | if n % 5 == 0 { 46 | black_box(Err(n)) 47 | } else { 48 | black_box(Ok(n)) 49 | } 50 | } 51 | 52 | criterion_group!(benches, multi_threaded_in_batch); 53 | criterion_main!(benches); 54 | -------------------------------------------------------------------------------- /benches/state_machine.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | 3 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 4 | use std::time::Duration; 5 | 6 | use failsafe::{backoff, clock, failure_policy, StateMachine}; 7 | 8 | #[allow(clippy::unit_arg)] 9 | fn consecutive_failures_policy(c: &mut Criterion) { 10 | let backoff = backoff::constant(Duration::from_secs(5)); 11 | let policy = failure_policy::consecutive_failures(3, backoff); 12 | let state_machine = StateMachine::new(policy, ()); 13 | 14 | c.bench_function("consecutive_failures_policy", |b| { 15 | b.iter(|| { 16 | black_box(state_machine.is_call_permitted()); 17 | black_box(state_machine.on_success()); 18 | black_box(state_machine.on_error()); 19 | }) 20 | }); 21 | } 22 | 23 | #[allow(clippy::unit_arg)] 24 | fn success_rate_over_time_window_policy(c: &mut Criterion) { 25 | let backoff = backoff::constant(Duration::from_secs(5)); 26 | let policy = 27 | failure_policy::success_rate_over_time_window(0.5, 0, Duration::from_secs(10), backoff); 28 | let state_machine = StateMachine::new(policy, ()); 29 | 30 | clock::freeze(|time| { 31 | c.bench_function("success_rate_over_time_window_policy", |b| { 32 | b.iter(|| { 33 | time.advance(Duration::from_secs(1)); 34 | black_box(state_machine.is_call_permitted()); 35 | black_box(state_machine.on_success()); 36 | black_box(state_machine.on_error()); 37 | }) 38 | }) 39 | }); 40 | } 41 | 42 | criterion_group!( 43 | benches, 44 | consecutive_failures_policy, 45 | success_rate_over_time_window_policy 46 | ); 47 | criterion_main!(benches); 48 | -------------------------------------------------------------------------------- /benches/windowed_adder.rs: -------------------------------------------------------------------------------- 1 | #![deny(warnings)] 2 | 3 | use criterion::{black_box, criterion_group, criterion_main, Criterion}; 4 | use std::thread; 5 | use std::time::Duration; 6 | 7 | use failsafe::WindowedAdder; 8 | 9 | fn add_and_sum(c: &mut Criterion) { 10 | let mut adder = WindowedAdder::new(Duration::from_millis(1000), 10); 11 | 12 | for _ in 0..10 { 13 | adder.add(42); 14 | thread::sleep(Duration::from_millis(100)); 15 | } 16 | 17 | c.bench_function("add", |b| { 18 | b.iter(|| { 19 | adder.add(42); 20 | black_box(adder.sum()); 21 | }) 22 | }); 23 | } 24 | 25 | criterion_group!(benches, add_and_sum); 26 | criterion_main!(benches); 27 | -------------------------------------------------------------------------------- /src/backoff.rs: -------------------------------------------------------------------------------- 1 | //! Contains various backoff strategies. 2 | //! 3 | //! Strategies are defined as `Iterator`. 4 | 5 | use std::iter::{self, Iterator}; 6 | use std::time::Duration; 7 | 8 | use rand::prelude::thread_rng; 9 | pub use rand::prelude::ThreadRng; 10 | 11 | const MAX_RETRIES: u32 = 30; 12 | 13 | /// A type alias for backoff strategy. 14 | pub type Backoff = dyn Iterator; 15 | 16 | /// Creates a infinite stream of given `duration` 17 | pub fn constant(duration: Duration) -> Constant { 18 | iter::repeat(duration) 19 | } 20 | 21 | /// Creates infinite stream of backoffs that keep the exponential growth from `start` until it 22 | /// reaches `max`. 23 | pub fn exponential(start: Duration, max: Duration) -> Exponential { 24 | assert!( 25 | start.as_secs() > 0, 26 | "start must be > 1s: {}", 27 | start.as_secs() 28 | ); 29 | assert!(max.as_secs() > 0, "max must be > 1s: {}", max.as_secs()); 30 | assert!( 31 | max >= start, 32 | "max must be greater then start: {} < {}", 33 | max.as_secs(), 34 | start.as_secs() 35 | ); 36 | 37 | Exponential { 38 | start, 39 | max, 40 | attempt: 0, 41 | } 42 | } 43 | 44 | /// Creates infinite stream of backoffs that keep half of the exponential growth, and jitter 45 | /// between 0 and that amount. 46 | /// 47 | /// See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. 48 | pub fn equal_jittered(start: Duration, max: Duration) -> EqualJittered { 49 | assert!( 50 | start.as_secs() > 0, 51 | "start must be > 1s: {}", 52 | start.as_secs() 53 | ); 54 | assert!(max.as_secs() > 0, "max must be > 1s: {}", max.as_secs()); 55 | assert!( 56 | max >= start, 57 | "max must be greater then start: {} < {}", 58 | max.as_secs(), 59 | start.as_secs() 60 | ); 61 | 62 | EqualJittered { 63 | start, 64 | max, 65 | attempt: 0, 66 | rng: ThreadLocalGenRange, 67 | } 68 | } 69 | 70 | /// Creates infinite stream of backoffs that keep the exponential growth, and jitter 71 | /// between 0 and that amount. 72 | /// 73 | /// See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. 74 | pub fn full_jittered(start: Duration, max: Duration) -> FullJittered { 75 | assert!( 76 | start.as_secs() > 0, 77 | "start must be > 1s: {}", 78 | start.as_secs() 79 | ); 80 | assert!(max.as_secs() > 0, "max must be > 1s: {}", max.as_secs()); 81 | assert!( 82 | max >= start, 83 | "max must be greater then start: {} < {}", 84 | max.as_secs(), 85 | start.as_secs() 86 | ); 87 | 88 | FullJittered { 89 | start, 90 | max, 91 | attempt: 0, 92 | rng: ThreadLocalGenRange, 93 | } 94 | } 95 | 96 | /// Random generator. 97 | pub trait GenRange { 98 | /// Generates a random value within range low and high. 99 | fn gen_range(&mut self, low: u64, high: u64) -> u64; 100 | } 101 | 102 | /// Thread local random generator, invokes `rand::thread_rng`. 103 | #[derive(Debug, Clone)] 104 | pub struct ThreadLocalGenRange; 105 | 106 | impl GenRange for ThreadLocalGenRange { 107 | #[inline] 108 | fn gen_range(&mut self, low: u64, high: u64) -> u64 { 109 | use rand::Rng; 110 | thread_rng().gen_range(low..high) 111 | } 112 | } 113 | 114 | /// A type alias for constant backoff strategy, which is just iterator. 115 | pub type Constant = iter::Repeat; 116 | 117 | /// An infinite stream of backoffs that keep the exponential growth from `start` until it 118 | /// reaches `max`. 119 | #[derive(Clone, Debug)] 120 | pub struct Exponential { 121 | start: Duration, 122 | max: Duration, 123 | attempt: u32, 124 | } 125 | 126 | impl Iterator for Exponential { 127 | type Item = Duration; 128 | 129 | fn next(&mut self) -> Option { 130 | let exp = exponential_backoff_seconds(self.attempt, self.start, self.max); 131 | 132 | if self.attempt < MAX_RETRIES { 133 | self.attempt += 1; 134 | } 135 | 136 | Some(Duration::from_secs(exp)) 137 | } 138 | } 139 | 140 | /// An infinite stream of backoffs that keep half of the exponential growth, and jitter 141 | /// between 0 and that amount. 142 | /// 143 | /// See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. 144 | #[derive(Clone, Debug)] 145 | pub struct FullJittered { 146 | start: Duration, 147 | max: Duration, 148 | attempt: u32, 149 | rng: R, 150 | } 151 | 152 | #[cfg(test)] 153 | impl FullJittered { 154 | fn with_rng(self, rng: T) -> FullJittered { 155 | FullJittered { 156 | rng, 157 | start: self.start, 158 | max: self.max, 159 | attempt: self.attempt, 160 | } 161 | } 162 | } 163 | 164 | impl Iterator for FullJittered { 165 | type Item = Duration; 166 | 167 | fn next(&mut self) -> Option { 168 | let exp = exponential_backoff_seconds(self.attempt, self.start, self.max); 169 | let seconds = self.rng.gen_range(0, exp + 1); 170 | 171 | if self.attempt < MAX_RETRIES { 172 | self.attempt += 1; 173 | } 174 | 175 | Some(Duration::from_secs(seconds)) 176 | } 177 | } 178 | 179 | /// Creates infinite stream of backoffs that keep the exponential growth, and jitter 180 | /// between 0 and that amount. 181 | /// 182 | /// See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. 183 | #[derive(Clone, Debug)] 184 | pub struct EqualJittered { 185 | start: Duration, 186 | max: Duration, 187 | attempt: u32, 188 | rng: R, 189 | } 190 | 191 | #[cfg(test)] 192 | impl EqualJittered { 193 | fn with_rng(self, rng: T) -> EqualJittered { 194 | EqualJittered { 195 | rng, 196 | start: self.start, 197 | max: self.max, 198 | attempt: self.attempt, 199 | } 200 | } 201 | } 202 | 203 | impl Iterator for EqualJittered { 204 | type Item = Duration; 205 | 206 | fn next(&mut self) -> Option { 207 | let exp = exponential_backoff_seconds(self.attempt, self.start, self.max); 208 | let seconds = (exp / 2) + self.rng.gen_range(0, (exp / 2) + 1); 209 | 210 | if self.attempt < MAX_RETRIES { 211 | self.attempt += 1; 212 | } 213 | 214 | Some(Duration::from_secs(seconds)) 215 | } 216 | } 217 | 218 | fn exponential_backoff_seconds(attempt: u32, base: Duration, max: Duration) -> u64 { 219 | ((1_u64 << attempt) * base.as_secs()).min(max.as_secs()) 220 | } 221 | 222 | #[cfg(test)] 223 | mod tests { 224 | use super::*; 225 | use rand::{RngCore, SeedableRng}; 226 | use rand_xorshift::XorShiftRng; 227 | 228 | const SEED: &[u8; 16] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 3, 2]; 229 | struct TestGenRage(T); 230 | 231 | impl Default for TestGenRage { 232 | fn default() -> Self { 233 | TestGenRage(XorShiftRng::from_seed(*SEED)) 234 | } 235 | } 236 | 237 | impl GenRange for TestGenRage { 238 | fn gen_range(&mut self, low: u64, high: u64) -> u64 { 239 | use rand::Rng; 240 | self.0.gen_range(low..high) 241 | } 242 | } 243 | 244 | #[test] 245 | fn exponential_growth() { 246 | let backoff = exponential(Duration::from_secs(10), Duration::from_secs(100)); 247 | 248 | let actual = backoff.take(6).map(|it| it.as_secs()).collect::>(); 249 | let expected = vec![10, 20, 40, 80, 100, 100]; 250 | assert_eq!(expected, actual); 251 | } 252 | 253 | #[test] 254 | fn full_jittered_growth() { 255 | let backoff = full_jittered(Duration::from_secs(10), Duration::from_secs(300)) 256 | .with_rng(TestGenRage::default()); 257 | 258 | let actual = backoff.take(10).map(|it| it.as_secs()).collect::>(); 259 | let expected = vec![0, 0, 33, 53, 80, 6, 132, 121, 234, 79]; 260 | assert_eq!(expected, actual); 261 | } 262 | 263 | #[test] 264 | fn equal_jittered_growth() { 265 | let backoff = equal_jittered(Duration::from_secs(5), Duration::from_secs(300)) 266 | .with_rng(TestGenRage::default()); 267 | 268 | let actual = backoff.take(10).map(|it| it.as_secs()).collect::>(); 269 | let expected = vec![2, 5, 10, 37, 63, 133, 225, 153, 216, 170]; 270 | assert_eq!(expected, actual) 271 | } 272 | 273 | #[test] 274 | fn constant_growth() { 275 | let backoff = constant(Duration::from_secs(3)); 276 | 277 | let actual = backoff.take(3).map(|it| it.as_secs()).collect::>(); 278 | let expected = vec![3, 3, 3]; 279 | assert_eq!(expected, actual); 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /src/circuit_breaker.rs: -------------------------------------------------------------------------------- 1 | use super::error::Error; 2 | use super::failure_policy::FailurePolicy; 3 | use super::failure_predicate::{self, FailurePredicate}; 4 | use super::instrument::Instrument; 5 | use super::state_machine::StateMachine; 6 | 7 | /// A circuit breaker's public interface. 8 | pub trait CircuitBreaker { 9 | /// Requests permission to call. 10 | /// 11 | /// It returns `true` if a call is allowed, or `false` if prohibited. 12 | fn is_call_permitted(&self) -> bool; 13 | 14 | /// Executes a given function within circuit breaker. 15 | /// 16 | /// Depending on function result value, the call will be recorded as success or failure. 17 | #[inline] 18 | fn call(&self, f: F) -> Result> 19 | where 20 | F: FnOnce() -> Result, 21 | { 22 | self.call_with(failure_predicate::Any, f) 23 | } 24 | 25 | /// Executes a given function within circuit breaker. 26 | /// 27 | /// Depending on function result value, the call will be recorded as success or failure. 28 | /// It checks error by the provided predicate. If the predicate returns `true` for the 29 | /// error, the call is recorded as failure otherwise considered this error as a success. 30 | fn call_with(&self, predicate: P, f: F) -> Result> 31 | where 32 | P: FailurePredicate, 33 | F: FnOnce() -> Result; 34 | } 35 | 36 | impl CircuitBreaker for StateMachine 37 | where 38 | POLICY: FailurePolicy, 39 | INSTRUMENT: Instrument, 40 | { 41 | #[inline] 42 | fn is_call_permitted(&self) -> bool { 43 | self.is_call_permitted() 44 | } 45 | 46 | fn call_with(&self, predicate: P, f: F) -> Result> 47 | where 48 | P: FailurePredicate, 49 | F: FnOnce() -> Result, 50 | { 51 | if !self.is_call_permitted() { 52 | return Err(Error::Rejected); 53 | } 54 | 55 | match f() { 56 | Ok(ok) => { 57 | self.on_success(); 58 | Ok(ok) 59 | } 60 | Err(err) => { 61 | if predicate.is_err(&err) { 62 | self.on_error(); 63 | } else { 64 | self.on_success(); 65 | } 66 | Err(Error::Inner(err)) 67 | } 68 | } 69 | } 70 | } 71 | 72 | #[cfg(test)] 73 | mod tests { 74 | use std::time::Duration; 75 | 76 | use super::super::backoff; 77 | use super::super::config::Config; 78 | use super::super::failure_policy::consecutive_failures; 79 | use super::*; 80 | 81 | #[test] 82 | fn call_with() { 83 | let circuit_breaker = new_circuit_breaker(); 84 | let is_err = |err: &bool| !(*err); 85 | 86 | for _ in 0..2 { 87 | match circuit_breaker.call_with(is_err, || Err::<(), _>(true)) { 88 | Err(Error::Inner(true)) => {} 89 | x => unreachable!("{:?}", x), 90 | } 91 | assert!(circuit_breaker.is_call_permitted()); 92 | } 93 | 94 | match circuit_breaker.call_with(is_err, || Err::<(), _>(false)) { 95 | Err(Error::Inner(false)) => {} 96 | x => unreachable!("{:?}", x), 97 | } 98 | assert!(!circuit_breaker.is_call_permitted()); 99 | } 100 | 101 | #[test] 102 | fn call_ok() { 103 | let circuit_breaker = new_circuit_breaker(); 104 | 105 | circuit_breaker.call(|| Ok::<_, ()>(())).unwrap(); 106 | assert!(circuit_breaker.is_call_permitted()); 107 | } 108 | 109 | #[test] 110 | fn call_err() { 111 | let circuit_breaker = new_circuit_breaker(); 112 | 113 | match circuit_breaker.call(|| Err::<(), _>(())) { 114 | Err(Error::Inner(())) => {} 115 | x => unreachable!("{:?}", x), 116 | } 117 | assert!(!circuit_breaker.is_call_permitted()); 118 | 119 | match circuit_breaker.call(|| Err::<(), _>(())) { 120 | Err(Error::Rejected) => {} 121 | x => unreachable!("{:?}", x), 122 | } 123 | assert!(!circuit_breaker.is_call_permitted()); 124 | } 125 | 126 | fn new_circuit_breaker() -> impl CircuitBreaker { 127 | let backoff = backoff::constant(Duration::from_secs(5)); 128 | let policy = consecutive_failures(1, backoff); 129 | Config::new().failure_policy(policy).build() 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/clock.rs: -------------------------------------------------------------------------------- 1 | use std::cell::Cell; 2 | use std::time::{Duration, Instant}; 3 | 4 | thread_local!(static CLOCK: Cell> = const { Cell::new(None) }); 5 | 6 | #[derive(Debug)] 7 | pub struct MockClock(Instant); 8 | 9 | impl MockClock { 10 | fn new() -> MockClock { 11 | MockClock(Instant::now()) 12 | } 13 | 14 | #[inline] 15 | pub fn now(&self) -> Instant { 16 | self.0 17 | } 18 | 19 | #[inline] 20 | pub fn advance(&mut self, diff: Duration) { 21 | self.0 += diff 22 | } 23 | } 24 | 25 | pub fn freeze(f: F) -> R 26 | where 27 | F: FnOnce(&mut MockClock) -> R, 28 | { 29 | CLOCK.with(|cell| { 30 | let mut clock = MockClock::new(); 31 | 32 | assert!( 33 | cell.get().is_none(), 34 | "default clock already set for execution context" 35 | ); 36 | 37 | // Ensure that the clock is removed from the thread-local context 38 | // when leaving the scope. This handles cases that involve panicking. 39 | struct Reset<'a>(&'a Cell>); 40 | 41 | impl<'a> Drop for Reset<'a> { 42 | fn drop(&mut self) { 43 | self.0.set(None); 44 | } 45 | } 46 | 47 | let _reset = Reset(cell); 48 | 49 | cell.set(Some(&clock as *const MockClock)); 50 | 51 | f(&mut clock) 52 | }) 53 | } 54 | 55 | #[inline] 56 | pub fn now() -> Instant { 57 | CLOCK.with(|current| match current.get() { 58 | Some(ptr) => unsafe { (*ptr).now() }, 59 | None => Instant::now(), 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | use super::backoff; 2 | use super::failure_policy::{self, ConsecutiveFailures, FailurePolicy, SuccessRateOverTimeWindow}; 3 | use super::instrument::Instrument; 4 | use super::state_machine::StateMachine; 5 | 6 | /// A `CircuitBreaker`'s configuration. 7 | #[derive(Debug)] 8 | pub struct Config { 9 | pub(crate) failure_policy: POLICY, 10 | pub(crate) instrument: INSTRUMENT, 11 | } 12 | 13 | impl Config<(), ()> { 14 | /// Creates a new circuit breaker's default configuration. 15 | #[allow(clippy::new_ret_no_self)] 16 | pub fn new() -> Config< 17 | failure_policy::OrElse< 18 | SuccessRateOverTimeWindow, 19 | ConsecutiveFailures, 20 | >, 21 | (), 22 | > { 23 | let failure_policy = 24 | SuccessRateOverTimeWindow::default().or_else(ConsecutiveFailures::default()); 25 | 26 | Config { 27 | failure_policy, 28 | instrument: (), 29 | } 30 | } 31 | } 32 | 33 | impl Config { 34 | /// Configures `FailurePolicy` for a circuit breaker. 35 | pub fn failure_policy(self, failure_policy: T) -> Config 36 | where 37 | T: FailurePolicy, 38 | { 39 | Config { 40 | failure_policy, 41 | instrument: self.instrument, 42 | } 43 | } 44 | 45 | /// Configures `Instrument` for a circuit breaker. 46 | pub fn instrument(self, instrument: T) -> Config 47 | where 48 | T: Instrument, 49 | { 50 | Config { 51 | failure_policy: self.failure_policy, 52 | instrument, 53 | } 54 | } 55 | 56 | /// Builds a new circuit breaker instance. 57 | pub fn build(self) -> StateMachine 58 | where 59 | POLICY: FailurePolicy, 60 | INSTRUMENT: Instrument, 61 | { 62 | StateMachine::new(self.failure_policy, self.instrument) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ema.rs: -------------------------------------------------------------------------------- 1 | /// Maintain an exponential moving average of Long-typed values over a 2 | /// given window on a user-defined clock. 3 | /// 4 | /// Ema requires monotonic timestamps 5 | #[derive(Debug)] 6 | pub struct Ema { 7 | window: u64, 8 | timestamp: u64, 9 | ema: f64, 10 | } 11 | 12 | impl Ema { 13 | /// Constructs a new `Ema` instance. 14 | /// 15 | /// * `window` - The mean lifetime of observations. 16 | pub fn new(window: u64) -> Self { 17 | Ema { 18 | window, 19 | timestamp: 0, 20 | ema: 0.0, 21 | } 22 | } 23 | 24 | /// `true` if `Ema` contains no values. 25 | #[allow(dead_code)] 26 | pub fn is_empty(&self) -> bool { 27 | self.timestamp == 0 28 | } 29 | 30 | /// Updates the average with observed value. and return the new average. 31 | /// 32 | /// Since `update` requires monotonic timestamps. it is up to the caller to 33 | /// ensure that calls to update do not race. 34 | /// 35 | /// # Panics 36 | /// 37 | /// When timestamp isn't monotonic. 38 | pub fn update(&mut self, timestamp: u64, value: f64) -> f64 { 39 | if self.timestamp == 0 { 40 | self.timestamp = timestamp; 41 | self.ema = value; 42 | } else { 43 | assert!( 44 | timestamp >= self.timestamp, 45 | "non monotonic timestamp detected" 46 | ); 47 | let time_diff = timestamp - self.timestamp; 48 | self.timestamp = timestamp; 49 | 50 | let w = if self.window == 0 { 51 | 0_f64 52 | } else { 53 | (-(time_diff as f64) / self.window as f64).exp() 54 | }; 55 | 56 | self.ema = value * (1_f64 - w) + self.ema * w; 57 | } 58 | 59 | self.ema 60 | } 61 | 62 | /// Returns the last observation. 63 | #[allow(dead_code)] 64 | pub fn last(&self) -> f64 { 65 | self.ema 66 | } 67 | 68 | /// Resets the average to 0 and erase all observations. 69 | pub fn reset(&mut self) { 70 | self.timestamp = 0; 71 | self.ema = 0_f64; 72 | } 73 | } 74 | 75 | #[cfg(test)] 76 | mod tests { 77 | use super::*; 78 | 79 | #[test] 80 | fn compute() { 81 | // http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:moving_averages 82 | const PER_DAY_DATA: [f64; 30] = [ 83 | 22.27, 22.19, 22.08, 22.17, 22.18, 22.13, 22.23, 22.43, 22.24, 22.29, 22.15, 22.39, 84 | 22.38, 22.61, 23.36, 24.05, 23.75, 23.83, 23.95, 23.63, 23.82, 23.87, 23.65, 23.19, 85 | 23.10, 23.33, 22.68, 23.10, 22.40, 22.17, 86 | ]; 87 | 88 | const EMA_10_OVER_DAYS: [f64; 30] = [ 89 | 22.27, 22.26, 22.25, 22.24, 22.23, 22.22, 22.22, 22.24, 22.24, 22.25, 22.24, 22.25, 90 | 22.26, 22.3, 22.4, 22.56, 22.67, 22.78, 22.89, 22.96, 23.04, 23.12, 23.17, 23.17, 91 | 23.17, 23.18, 23.13, 23.13, 23.06, 22.98, 92 | ]; 93 | 94 | let mut ema = Ema::new(10); 95 | 96 | let result = PER_DAY_DATA 97 | .iter() 98 | .enumerate() 99 | .map(|(i, x)| ((i + 1) as u64, x)) 100 | .map(|(i, x)| round_to(ema.update(i, *x), 2)) 101 | .collect::>(); 102 | 103 | assert_eq!(EMA_10_OVER_DAYS.to_vec(), result); 104 | } 105 | 106 | #[test] 107 | #[should_panic] 108 | fn non_monotonic_timestamp() { 109 | let mut ema = Ema::new(10); 110 | ema.update(10, 10.0); 111 | ema.update(1, 10.0); 112 | } 113 | 114 | #[test] 115 | fn updates_are_time_invariant() { 116 | let mut a = Ema::new(1000); 117 | let mut b = Ema::new(1000); 118 | 119 | assert_eq!(10.0, a.update(10, 10.0)); 120 | assert_eq!(10.0, a.update(20, 10.0)); 121 | assert_eq!(10.0, a.update(30, 10.0)); 122 | 123 | assert_eq!(10.0, b.update(10, 10.0)); 124 | assert_eq!(10.0, b.update(30, 10.0)); 125 | 126 | assert_eq!(a.update(40, 5.0), b.update(40, 5.0)); 127 | assert!(a.update(50, 5.0) > b.update(60, 5.0)); 128 | 129 | assert_eq!( 130 | round_to(a.update(60, 5.0), 4), 131 | round_to(b.update(60, 5.0), 4) 132 | ); 133 | } 134 | 135 | #[test] 136 | fn reset() { 137 | let mut ema = Ema::new(5); 138 | 139 | assert_eq!(3.0, ema.update(1, 3.0)); 140 | 141 | ema.reset(); 142 | 143 | assert!(ema.is_empty()); 144 | assert_eq!(0.0, ema.last()); 145 | assert_eq!(5.0, ema.update(2, 5.0)); 146 | } 147 | 148 | fn round_to(x: f64, power: i32) -> f64 { 149 | let power = f64::powi(10.0, power); 150 | (x * power).round() / power 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error as StdError; 2 | use std::fmt::{self, Display}; 3 | 4 | /// A `CircuitBreaker`'s error. 5 | #[derive(Debug)] 6 | pub enum Error { 7 | /// An error from inner call. 8 | Inner(E), 9 | /// An error when call was rejected. 10 | Rejected, 11 | } 12 | 13 | impl Display for Error 14 | where 15 | E: Display, 16 | { 17 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 18 | match self { 19 | Error::Rejected => write!(f, "call was rejected"), 20 | Error::Inner(err) => write!(f, "{}", err), 21 | } 22 | } 23 | } 24 | 25 | impl StdError for Error 26 | where 27 | E: StdError + 'static, 28 | { 29 | fn source(&self) -> Option<&(dyn StdError + 'static)> { 30 | match self { 31 | Error::Inner(ref err) => Some(err), 32 | _ => None, 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/failure_policy.rs: -------------------------------------------------------------------------------- 1 | //! Contains various failure accrual policies, which are used for the failure rate detection. 2 | 3 | use std::iter::Iterator; 4 | use std::time::{Duration, Instant}; 5 | 6 | use super::backoff; 7 | use super::clock; 8 | use super::ema::Ema; 9 | use super::windowed_adder::WindowedAdder; 10 | 11 | static DEFAULT_BACKOFF: Duration = Duration::from_secs(300); 12 | 13 | const SUCCESS: f64 = 1.0; 14 | const FAILURE: f64 = 0.0; 15 | const MILLIS_PER_SECOND: u64 = 1_000; 16 | const DEFAULT_SUCCESS_RATE_THRESHOLD: f64 = 0.8; 17 | const DEFAULT_SUCCESS_RATE_WINDOW_SECONDS: u64 = 30; 18 | const DEFAULT_CONSECUTIVE_FAILURES: u32 = 5; 19 | const DEFAULT_MINIMUM_REQUEST_THRESHOLD: u32 = 5; 20 | 21 | /// A `FailurePolicy` is used to determine whether or not the backend died. 22 | pub trait FailurePolicy { 23 | /// Invoked when a request is successful. 24 | fn record_success(&mut self); 25 | 26 | /// Invoked when a non-probing request fails. If it returns `Some(Duration)`, 27 | /// the backend will mark as the dead for the specified `Duration`. 28 | fn mark_dead_on_failure(&mut self) -> Option; 29 | 30 | /// Invoked when a backend is revived after probing. Used to reset any history. 31 | fn revived(&mut self); 32 | 33 | /// Creates a `FailurePolicy` which uses both `self` and `rhs`. 34 | fn or_else(self, rhs: R) -> OrElse 35 | where 36 | Self: Sized, 37 | { 38 | OrElse { 39 | left: self, 40 | right: rhs, 41 | } 42 | } 43 | } 44 | 45 | /// Returns a policy based on an exponentially-weighted moving average success 46 | /// rate over a time window. A moving average is used so the success rate 47 | /// calculation is biased towards more recent requests. 48 | /// 49 | /// If the computed weighted success rate is less than the required success rate, 50 | /// `mark_dead_on_failure` will return `Some(Duration)`. 51 | /// 52 | /// See `ema::Ema` for how the success rate is computed. 53 | /// 54 | /// * `required_success_rate` - a success rate that must be met. 55 | /// * `min_request_threshold` - minimum number of requests in the past `window` 56 | /// for `mark_dead_on_failure` to return a duration. 57 | /// * `window` - window over which the success rate is tracked. `mark_dead_on_failure` 58 | /// will return None, until we get requests for a duration of at least `window`. 59 | /// * `backoff` - stream of durations to use for the next duration 60 | /// returned from `mark_dead_on_failure` 61 | /// 62 | /// # Panics 63 | /// 64 | /// When `required_success_rate` isn't in `[0.0, 1.0]` interval. 65 | pub fn success_rate_over_time_window( 66 | required_success_rate: f64, 67 | min_request_threshold: u32, 68 | window: Duration, 69 | backoff: BACKOFF, 70 | ) -> SuccessRateOverTimeWindow 71 | where 72 | BACKOFF: Iterator + Clone, 73 | { 74 | assert!( 75 | (0.0..=1.0).contains(&required_success_rate), 76 | "required_success_rate must be [0, 1]: {}", 77 | required_success_rate 78 | ); 79 | 80 | let window_millis = window.as_secs() * MILLIS_PER_SECOND; 81 | let request_counter = WindowedAdder::new(window, 5); 82 | 83 | SuccessRateOverTimeWindow { 84 | required_success_rate, 85 | min_request_threshold, 86 | ema: Ema::new(window_millis), 87 | now: clock::now(), 88 | window_millis, 89 | backoff: backoff.clone(), 90 | fresh_backoff: backoff, 91 | request_counter, 92 | } 93 | } 94 | 95 | /// A policy based on a maximum number of consecutive failures. If `num_failures` 96 | /// occur consecutively, `mark_dead_on_failure` will return a Some(Duration) to 97 | /// mark an endpoint dead for. 98 | /// 99 | /// * `num_failures` - number of consecutive failures. 100 | /// * `backoff` - stream of durations to use for the next duration 101 | /// returned from `mark_dead_on_failure` 102 | pub fn consecutive_failures( 103 | num_failures: u32, 104 | backoff: BACKOFF, 105 | ) -> ConsecutiveFailures 106 | where 107 | BACKOFF: Iterator + Clone, 108 | { 109 | ConsecutiveFailures { 110 | num_failures, 111 | consecutive_failures: 0, 112 | backoff: backoff.clone(), 113 | fresh_backoff: backoff, 114 | } 115 | } 116 | 117 | impl Default for SuccessRateOverTimeWindow { 118 | fn default() -> Self { 119 | let backoff = backoff::equal_jittered(Duration::from_secs(10), Duration::from_secs(300)); 120 | let window = Duration::from_secs(DEFAULT_SUCCESS_RATE_WINDOW_SECONDS); 121 | success_rate_over_time_window( 122 | DEFAULT_SUCCESS_RATE_THRESHOLD, 123 | DEFAULT_MINIMUM_REQUEST_THRESHOLD, 124 | window, 125 | backoff, 126 | ) 127 | } 128 | } 129 | 130 | impl Default for ConsecutiveFailures { 131 | fn default() -> Self { 132 | let backoff = backoff::equal_jittered(Duration::from_secs(10), Duration::from_secs(300)); 133 | consecutive_failures(DEFAULT_CONSECUTIVE_FAILURES, backoff) 134 | } 135 | } 136 | 137 | /// A policy based on an exponentially-weighted moving average success 138 | /// rate over a time window. A moving average is used so the success rate 139 | /// calculation is biased towards more recent requests. 140 | #[derive(Debug)] 141 | pub struct SuccessRateOverTimeWindow { 142 | required_success_rate: f64, 143 | min_request_threshold: u32, 144 | ema: Ema, 145 | now: Instant, 146 | window_millis: u64, 147 | backoff: BACKOFF, 148 | fresh_backoff: BACKOFF, 149 | request_counter: WindowedAdder, 150 | } 151 | 152 | impl SuccessRateOverTimeWindow 153 | where 154 | BACKOFF: Clone, 155 | { 156 | /// Returns seconds since instance was created. 157 | fn elapsed_millis(&self) -> u64 { 158 | let diff = clock::now() - self.now; 159 | (diff.as_secs() * MILLIS_PER_SECOND) + u64::from(diff.subsec_millis()) 160 | } 161 | 162 | /// We can trigger failure accrual if the `window` has passed, success rate is below 163 | /// `required_success_rate`. 164 | fn can_remove(&mut self, success_rate: f64) -> bool { 165 | self.elapsed_millis() >= self.window_millis 166 | && success_rate < self.required_success_rate 167 | && self.request_counter.sum() >= i64::from(self.min_request_threshold) 168 | } 169 | } 170 | 171 | impl FailurePolicy for SuccessRateOverTimeWindow 172 | where 173 | BACKOFF: Iterator + Clone, 174 | { 175 | #[inline] 176 | fn record_success(&mut self) { 177 | let timestamp = self.elapsed_millis(); 178 | self.ema.update(timestamp, SUCCESS); 179 | self.request_counter.add(1); 180 | } 181 | 182 | #[inline] 183 | fn mark_dead_on_failure(&mut self) -> Option { 184 | self.request_counter.add(1); 185 | 186 | let timestamp = self.elapsed_millis(); 187 | let success_rate = self.ema.update(timestamp, FAILURE); 188 | 189 | if self.can_remove(success_rate) { 190 | let duration = self.backoff.next().unwrap_or(DEFAULT_BACKOFF); 191 | Some(duration) 192 | } else { 193 | None 194 | } 195 | } 196 | 197 | #[inline] 198 | fn revived(&mut self) { 199 | self.now = clock::now(); 200 | self.ema.reset(); 201 | self.request_counter.reset(); 202 | self.backoff = self.fresh_backoff.clone(); 203 | } 204 | } 205 | 206 | /// A policy based on a maximum number of consecutive failure 207 | #[derive(Debug)] 208 | pub struct ConsecutiveFailures { 209 | num_failures: u32, 210 | consecutive_failures: u32, 211 | backoff: BACKOFF, 212 | fresh_backoff: BACKOFF, 213 | } 214 | 215 | impl FailurePolicy for ConsecutiveFailures 216 | where 217 | BACKOFF: Iterator + Clone, 218 | { 219 | #[inline] 220 | fn record_success(&mut self) { 221 | self.consecutive_failures = 0; 222 | } 223 | 224 | #[inline] 225 | fn mark_dead_on_failure(&mut self) -> Option { 226 | self.consecutive_failures += 1; 227 | 228 | if self.consecutive_failures >= self.num_failures { 229 | let duration = self.backoff.next().unwrap_or(DEFAULT_BACKOFF); 230 | Some(duration) 231 | } else { 232 | None 233 | } 234 | } 235 | 236 | #[inline] 237 | fn revived(&mut self) { 238 | self.consecutive_failures = 0; 239 | self.backoff = self.fresh_backoff.clone(); 240 | } 241 | } 242 | 243 | /// A combinator used for join two policies into new one. 244 | #[derive(Debug)] 245 | pub struct OrElse { 246 | left: LEFT, 247 | right: RIGHT, 248 | } 249 | 250 | impl FailurePolicy for OrElse 251 | where 252 | LEFT: FailurePolicy, 253 | RIGHT: FailurePolicy, 254 | { 255 | #[inline] 256 | fn record_success(&mut self) { 257 | self.left.record_success(); 258 | self.right.record_success(); 259 | } 260 | 261 | #[inline] 262 | fn mark_dead_on_failure(&mut self) -> Option { 263 | let left = self.left.mark_dead_on_failure(); 264 | let right = self.right.mark_dead_on_failure(); 265 | 266 | match (left, right) { 267 | (Some(_), None) => left, 268 | (None, Some(_)) => right, 269 | (Some(l), Some(r)) => Some(l.max(r)), 270 | _ => None, 271 | } 272 | } 273 | 274 | #[inline] 275 | fn revived(&mut self) { 276 | self.left.revived(); 277 | self.right.revived(); 278 | } 279 | } 280 | 281 | #[cfg(test)] 282 | mod tests { 283 | use super::*; 284 | 285 | use super::super::backoff; 286 | use super::super::clock; 287 | 288 | mod consecutive_failures { 289 | use super::*; 290 | 291 | #[test] 292 | fn fail_on_nth_attempt() { 293 | let mut policy = consecutive_failures(3, constant_backoff()); 294 | 295 | assert_eq!(None, policy.mark_dead_on_failure()); 296 | assert_eq!(None, policy.mark_dead_on_failure()); 297 | assert_eq!(Some(5.seconds()), policy.mark_dead_on_failure()); 298 | } 299 | 300 | #[test] 301 | fn reset_to_zero_on_revived() { 302 | let mut policy = consecutive_failures(3, constant_backoff()); 303 | 304 | assert_eq!(None, policy.mark_dead_on_failure()); 305 | 306 | policy.revived(); 307 | 308 | assert_eq!(None, policy.mark_dead_on_failure()); 309 | assert_eq!(None, policy.mark_dead_on_failure()); 310 | assert_eq!(Some(5.seconds()), policy.mark_dead_on_failure()); 311 | } 312 | 313 | #[test] 314 | fn reset_to_zero_on_success() { 315 | let mut policy = consecutive_failures(3, constant_backoff()); 316 | 317 | assert_eq!(None, policy.mark_dead_on_failure()); 318 | 319 | policy.record_success(); 320 | 321 | assert_eq!(None, policy.mark_dead_on_failure()); 322 | assert_eq!(None, policy.mark_dead_on_failure()); 323 | assert_eq!(Some(5.seconds()), policy.mark_dead_on_failure()); 324 | } 325 | 326 | #[test] 327 | fn iterates_over_backoff() { 328 | let exp_backoff = exp_backoff(); 329 | let mut policy = consecutive_failures(1, exp_backoff.clone()); 330 | 331 | for i in exp_backoff.take(6) { 332 | assert_eq!(Some(i), policy.mark_dead_on_failure()); 333 | } 334 | } 335 | } 336 | 337 | mod success_rate_over_time_window { 338 | use super::*; 339 | 340 | #[test] 341 | fn fail_when_success_rate_not_met() { 342 | clock::freeze(|time| { 343 | let exp_backoff = exp_backoff(); 344 | let success_rate_duration = 30.seconds(); 345 | let mut policy = success_rate_over_time_window( 346 | 0.5, 347 | 1, 348 | success_rate_duration, 349 | exp_backoff.clone(), 350 | ); 351 | 352 | assert_eq!(None, policy.mark_dead_on_failure()); 353 | 354 | // Advance the time with 'success_rate_duration'. 355 | // All mark_dead_on_failure calls should now return Some(Duration), 356 | // and should iterate over expBackoffList. 357 | time.advance(success_rate_duration); 358 | 359 | for i in exp_backoff.take(6) { 360 | assert_eq!(Some(i), policy.mark_dead_on_failure()); 361 | } 362 | }) 363 | } 364 | 365 | #[test] 366 | fn respects_rps_threshold() { 367 | clock::freeze(|time| { 368 | let exp_backoff = exp_backoff(); 369 | let mut policy = success_rate_over_time_window(1.0, 5, 30.seconds(), exp_backoff); 370 | 371 | time.advance(30.seconds()); 372 | 373 | assert_eq!(None, policy.mark_dead_on_failure()); 374 | assert_eq!(None, policy.mark_dead_on_failure()); 375 | assert_eq!(None, policy.mark_dead_on_failure()); 376 | assert_eq!(None, policy.mark_dead_on_failure()); 377 | assert_eq!(Some(5.seconds()), policy.mark_dead_on_failure()); 378 | }); 379 | } 380 | 381 | #[test] 382 | fn revived_resets_failures() { 383 | clock::freeze(|time| { 384 | let exp_backoff = constant_backoff(); 385 | let success_rate_duration = 30.seconds(); 386 | let mut policy = success_rate_over_time_window( 387 | 0.5, 388 | 1, 389 | success_rate_duration, 390 | exp_backoff.clone(), 391 | ); 392 | 393 | time.advance(success_rate_duration); 394 | for i in exp_backoff.take(6) { 395 | assert_eq!(Some(i), policy.mark_dead_on_failure()); 396 | } 397 | 398 | policy.revived(); 399 | 400 | // Make sure the failure status has been reset. 401 | // This will also be registered as the timestamp of the first request. 402 | assert_eq!(None, policy.mark_dead_on_failure()); 403 | 404 | // One failure after 'success_rate_duration' should mark the node dead again. 405 | time.advance(success_rate_duration); 406 | assert!(policy.mark_dead_on_failure().is_some()) 407 | }) 408 | } 409 | 410 | #[test] 411 | fn fractional_success_rate() { 412 | clock::freeze(|time| { 413 | let exp_backoff = exp_backoff(); 414 | let success_rate_duration = 100.seconds(); 415 | let mut policy = success_rate_over_time_window( 416 | 0.5, 417 | 1, 418 | success_rate_duration, 419 | exp_backoff.clone(), 420 | ); 421 | 422 | for _i in 0..100 { 423 | time.advance(1.seconds()); 424 | policy.record_success(); 425 | } 426 | 427 | // With a window of 100 seconds, it will take 100 * ln(2) + 1 = 70 seconds of failures 428 | // for the success rate to drop below 0.5 (half-life). 429 | for _i in 0..69 { 430 | time.advance(1.seconds()); 431 | assert_eq!(None, policy.mark_dead_on_failure(), "n={}", _i); 432 | } 433 | 434 | // 70th failure should make markDeadOnFailure() return Some(_) 435 | time.advance(1.seconds()); 436 | assert_eq!(Some(5.seconds()), policy.mark_dead_on_failure()); 437 | }) 438 | } 439 | } 440 | 441 | mod or_else { 442 | use super::*; 443 | 444 | #[test] 445 | fn compose_policies() { 446 | let mut policy = consecutive_failures(3, constant_backoff()).or_else( 447 | success_rate_over_time_window(0.5, 100, 10.seconds(), constant_backoff()), 448 | ); 449 | 450 | policy.record_success(); 451 | assert_eq!(None, policy.mark_dead_on_failure()); 452 | } 453 | } 454 | 455 | fn constant_backoff() -> backoff::Constant { 456 | backoff::constant(5.seconds()) 457 | } 458 | 459 | fn exp_backoff() -> backoff::Exponential { 460 | backoff::exponential(5.seconds(), 60.seconds()) 461 | } 462 | 463 | trait IntoDuration { 464 | fn seconds(self) -> Duration; 465 | } 466 | 467 | impl IntoDuration for u64 { 468 | fn seconds(self) -> Duration { 469 | Duration::from_secs(self) 470 | } 471 | } 472 | } 473 | -------------------------------------------------------------------------------- /src/failure_predicate.rs: -------------------------------------------------------------------------------- 1 | /// Evaluates if an error should be recorded as a failure and thus increase the failure rate. 2 | pub trait FailurePredicate { 3 | /// Must return `true` if the error should count as a failure, otherwise it must return `false`. 4 | fn is_err(&self, err: &ERROR) -> bool; 5 | } 6 | 7 | impl FailurePredicate for F 8 | where 9 | F: Fn(&ERROR) -> bool, 10 | { 11 | #[inline] 12 | fn is_err(&self, err: &ERROR) -> bool { 13 | self(err) 14 | } 15 | } 16 | 17 | /// the Any predicate always returns true 18 | #[derive(Debug, Copy, Clone)] 19 | pub struct Any; 20 | 21 | impl FailurePredicate for Any { 22 | #[inline] 23 | fn is_err(&self, _err: &ERROR) -> bool { 24 | true 25 | } 26 | } 27 | 28 | #[cfg(test)] 29 | mod tests { 30 | use super::*; 31 | 32 | #[test] 33 | fn use_func_as_failure_predicate() { 34 | fn is_err(err: &bool) -> bool { 35 | *err 36 | } 37 | assert!(FailurePredicate::is_err(&is_err, &true)); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/futures/mod.rs: -------------------------------------------------------------------------------- 1 | //! Futures aware circuit breaker. 2 | //! 3 | //! # Example 4 | //! 5 | //! Using default backoff strategy and failure accrual policy. 6 | //! 7 | //! ``` 8 | //! # extern crate rand; 9 | //! # use rand::{thread_rng, Rng}; 10 | //! # async { 11 | //! 12 | //! use failsafe::Config; 13 | //! use failsafe::futures::CircuitBreaker; 14 | //! 15 | //! // A function that sometimes fails. 16 | //! async fn dangerous_call() -> Result<(), ()> { 17 | //! if thread_rng().gen_range(0..2) == 0 { 18 | //! return Err(()) 19 | //! } 20 | //! Ok(()) 21 | //! } 22 | //! 23 | //! // Create a circuit breaker which configured by reasonable default backoff and 24 | //! // failure accrual policy. 25 | //! let circuit_breaker = Config::new().build(); 26 | //! 27 | //! // Wraps `dangerous_call` result future within circuit breaker. 28 | //! let future = circuit_breaker.call(dangerous_call()); 29 | //! let result = future.await; 30 | //! 31 | //! # }; // async 32 | 33 | use std::future::Future; 34 | use std::pin::Pin; 35 | use std::task::{Context, Poll}; 36 | 37 | use futures_core::future::TryFuture; 38 | 39 | use super::error::Error; 40 | use super::failure_policy::FailurePolicy; 41 | use super::failure_predicate::{self, FailurePredicate}; 42 | use super::instrument::Instrument; 43 | use super::state_machine::StateMachine; 44 | 45 | pub mod stream; 46 | 47 | /// A futures aware circuit breaker's public interface. 48 | pub trait CircuitBreaker { 49 | #[doc(hidden)] 50 | type FailurePolicy: FailurePolicy + Send + Sync; 51 | #[doc(hidden)] 52 | type Instrument: Instrument + Send + Sync; 53 | 54 | /// Requests permission to call. 55 | /// 56 | /// It returns `true` if a call is allowed, or `false` if prohibited. 57 | fn is_call_permitted(&self) -> bool; 58 | 59 | /// Executes a given future within circuit breaker. 60 | /// 61 | /// Depending on future result value, the call will be recorded as success or failure. 62 | #[inline] 63 | fn call( 64 | &self, 65 | f: F, 66 | ) -> ResponseFuture 67 | where 68 | F: TryFuture, 69 | { 70 | self.call_with(failure_predicate::Any, f) 71 | } 72 | 73 | /// Executes a given future within circuit breaker. 74 | /// 75 | /// Depending on future result value, the call will be recorded as success or failure. 76 | /// It checks error by the provided predicate. If the predicate returns `true` for the 77 | /// error, the call is recorded as failure otherwise considered this error as a success. 78 | fn call_with( 79 | &self, 80 | predicate: P, 81 | f: F, 82 | ) -> ResponseFuture 83 | where 84 | F: TryFuture, 85 | P: FailurePredicate; 86 | } 87 | 88 | impl CircuitBreaker for StateMachine 89 | where 90 | POLICY: FailurePolicy + Send + Sync, 91 | INSTRUMENT: Instrument + Send + Sync, 92 | { 93 | type FailurePolicy = POLICY; 94 | type Instrument = INSTRUMENT; 95 | 96 | #[inline] 97 | fn is_call_permitted(&self) -> bool { 98 | self.is_call_permitted() 99 | } 100 | 101 | #[inline] 102 | fn call_with( 103 | &self, 104 | predicate: P, 105 | f: F, 106 | ) -> ResponseFuture 107 | where 108 | F: TryFuture, 109 | P: FailurePredicate, 110 | { 111 | ResponseFuture { 112 | future: f, 113 | state_machine: self.clone(), 114 | predicate, 115 | ask: false, 116 | } 117 | } 118 | } 119 | 120 | pin_project_lite::pin_project! { 121 | /// A circuit breaker's future. 122 | #[allow(missing_debug_implementations)] 123 | pub struct ResponseFuture { 124 | #[pin] 125 | future: FUTURE, 126 | state_machine: StateMachine, 127 | predicate: PREDICATE, 128 | ask: bool, 129 | } 130 | } 131 | 132 | impl Future 133 | for ResponseFuture 134 | where 135 | FUTURE: TryFuture, 136 | POLICY: FailurePolicy, 137 | INSTRUMENT: Instrument, 138 | PREDICATE: FailurePredicate, 139 | { 140 | type Output = Result>; 141 | 142 | fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { 143 | let this = self.project(); 144 | 145 | if !*this.ask { 146 | *this.ask = true; 147 | if !this.state_machine.is_call_permitted() { 148 | return Poll::Ready(Err(Error::Rejected)); 149 | } 150 | } 151 | 152 | match this.future.try_poll(cx) { 153 | Poll::Ready(Ok(ok)) => { 154 | this.state_machine.on_success(); 155 | Poll::Ready(Ok(ok)) 156 | } 157 | Poll::Ready(Err(err)) => { 158 | if this.predicate.is_err(&err) { 159 | this.state_machine.on_error(); 160 | } else { 161 | this.state_machine.on_success(); 162 | } 163 | Poll::Ready(Err(Error::Inner(err))) 164 | } 165 | Poll::Pending => Poll::Pending, 166 | } 167 | } 168 | } 169 | 170 | #[cfg(test)] 171 | mod tests { 172 | use std::time::Duration; 173 | 174 | use futures::future; 175 | 176 | use super::super::backoff; 177 | use super::super::config::Config; 178 | use super::super::failure_policy; 179 | use super::*; 180 | 181 | #[tokio::test] 182 | async fn call_ok() { 183 | let circuit_breaker = new_circuit_breaker(); 184 | let future = delay_for(Duration::from_millis(100)); 185 | let future = circuit_breaker.call(future); 186 | 187 | future.await.unwrap(); 188 | assert!(circuit_breaker.is_call_permitted()); 189 | } 190 | 191 | #[tokio::test] 192 | async fn call_err() { 193 | let circuit_breaker = new_circuit_breaker(); 194 | 195 | let future = future::err::<(), ()>(()); 196 | let future = circuit_breaker.call(future); 197 | match future.await { 198 | Err(Error::Inner(_)) => {} 199 | err => unreachable!("{:?}", err), 200 | } 201 | assert!(!circuit_breaker.is_call_permitted()); 202 | 203 | let future = delay_for(Duration::from_secs(1)); 204 | let future = circuit_breaker.call(future); 205 | match future.await { 206 | Err(Error::Rejected) => {} 207 | err => unreachable!("{:?}", err), 208 | } 209 | assert!(!circuit_breaker.is_call_permitted()); 210 | } 211 | 212 | #[tokio::test] 213 | async fn call_with() { 214 | let circuit_breaker = new_circuit_breaker(); 215 | let is_err = |err: &bool| !(*err); 216 | 217 | for _ in 0..2 { 218 | let future = future::err::<(), _>(true); 219 | let future = circuit_breaker.call_with(is_err, future); 220 | match future.await { 221 | Err(Error::Inner(true)) => {} 222 | err => unreachable!("{:?}", err), 223 | } 224 | assert!(circuit_breaker.is_call_permitted()); 225 | } 226 | 227 | let future = future::err::<(), _>(false); 228 | let future = circuit_breaker.call_with(is_err, future); 229 | match future.await { 230 | Err(Error::Inner(false)) => {} 231 | err => unreachable!("{:?}", err), 232 | } 233 | assert!(!circuit_breaker.is_call_permitted()); 234 | } 235 | 236 | fn new_circuit_breaker() -> impl CircuitBreaker { 237 | let backoff = backoff::constant(Duration::from_secs(5)); 238 | let policy = failure_policy::consecutive_failures(1, backoff); 239 | Config::new().failure_policy(policy).build() 240 | } 241 | 242 | async fn delay_for(duration: Duration) -> Result<(), ()> { 243 | tokio::time::sleep(duration).await; 244 | Ok(()) 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/futures/stream.rs: -------------------------------------------------------------------------------- 1 | //! calls CircuitBreaker in a Stream that can be polled with `next()` 2 | use std::task; 3 | 4 | use futures_core::Stream; 5 | 6 | use crate::{failure_predicate, FailurePolicy, FailurePredicate, StateMachine}; 7 | 8 | pin_project_lite::pin_project! { 9 | /// Stream that holds `StateMachine` and calls stream future 10 | #[derive(Debug, Clone)] 11 | pub struct BreakerStream { 12 | breaker: StateMachine, 13 | #[pin] 14 | stream: S, 15 | predicate: P, 16 | } 17 | } 18 | 19 | impl BreakerStream 20 | where 21 | S: Stream>, 22 | { 23 | /// create new circuit breaker stream 24 | pub fn new(breaker: StateMachine, stream: S) -> Self { 25 | Self { 26 | breaker, 27 | stream, 28 | predicate: crate::failure_predicate::Any, 29 | } 30 | } 31 | } 32 | 33 | impl BreakerStream 34 | where 35 | S: Stream>, 36 | P: FailurePredicate, 37 | { 38 | /// create new circuit breaker with predicate 39 | pub fn new_with(breaker: StateMachine, stream: S, predicate: P) -> Self { 40 | Self { 41 | breaker, 42 | stream, 43 | predicate, 44 | } 45 | } 46 | /// return a reference to the underlying state machine 47 | pub fn state_machine(&self) -> &StateMachine { 48 | &self.breaker 49 | } 50 | } 51 | 52 | impl Stream for BreakerStream 53 | where 54 | S: Stream>, 55 | P: FailurePredicate, 56 | Pol: FailurePolicy, 57 | Ins: crate::Instrument, 58 | { 59 | type Item = Result>; 60 | 61 | fn poll_next( 62 | self: std::pin::Pin<&mut Self>, 63 | cx: &mut task::Context<'_>, 64 | ) -> task::Poll> { 65 | use task::Poll; 66 | let this = self.project(); 67 | if !this.breaker.is_call_permitted() { 68 | return Poll::Ready(Some(Err(crate::Error::Rejected))); 69 | } 70 | 71 | match this.stream.poll_next(cx) { 72 | Poll::Ready(Some(Ok(ok))) => { 73 | this.breaker.on_success(); 74 | Poll::Ready(Some(Ok(ok))) 75 | } 76 | Poll::Ready(Some(Err(err))) => { 77 | if this.predicate.is_err(&err) { 78 | this.breaker.on_error(); 79 | } else { 80 | this.breaker.on_success(); 81 | } 82 | Poll::Ready(Some(Err(crate::Error::Inner(err)))) 83 | } 84 | Poll::Ready(None) => Poll::Ready(None), 85 | Poll::Pending => Poll::Pending, 86 | } 87 | } 88 | } 89 | 90 | #[cfg(test)] 91 | mod tests { 92 | use std::time::Duration; 93 | 94 | use futures::StreamExt; 95 | 96 | use crate::{backoff, failure_policy, Config}; 97 | 98 | use super::*; 99 | 100 | #[tokio::test] 101 | async fn call_ok() { 102 | let circuit_breaker = new_circuit_breaker(Duration::from_secs(5)); 103 | let stream = BreakerStream::new( 104 | circuit_breaker, 105 | futures::stream::once(async { delay_for(Duration::from_millis(100)).await }), 106 | ); 107 | tokio::pin!(stream); 108 | while let Some(_x) = stream.next().await {} 109 | 110 | assert!(stream.state_machine().is_call_permitted()); 111 | } 112 | 113 | #[tokio::test] 114 | async fn call_err() { 115 | let stream = BreakerStream::new( 116 | new_circuit_breaker(Duration::from_millis(100)), 117 | futures::stream::iter(vec![Err::<(), ()>(()), Ok(())]), 118 | ); 119 | tokio::pin!(stream); 120 | match stream.next().await { 121 | Some(Err(crate::Error::Inner(_))) => {} 122 | err => unreachable!("{:?}", err), 123 | } 124 | assert!(!stream.state_machine().is_call_permitted()); 125 | 126 | match stream.next().await { 127 | Some(Err(crate::Error::Rejected)) => {} 128 | err => unreachable!("{:?}", err), 129 | } 130 | assert!(!stream.state_machine().is_call_permitted()); 131 | tokio::time::sleep(Duration::from_millis(200)).await; 132 | // permitted now 133 | assert!(stream.state_machine().is_call_permitted()); 134 | match stream.next().await { 135 | Some(Ok(())) => {} 136 | err => unreachable!("{:?}", err), 137 | } 138 | } 139 | 140 | fn new_circuit_breaker( 141 | duration: Duration, 142 | ) -> StateMachine>, ()> { 143 | let backoff = backoff::constant(duration); 144 | let policy = failure_policy::consecutive_failures(1, backoff); 145 | Config::new().failure_policy(policy).build() 146 | } 147 | 148 | async fn delay_for(duration: Duration) -> Result<(), ()> { 149 | tokio::time::sleep(duration).await; 150 | Ok(()) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/instrument.rs: -------------------------------------------------------------------------------- 1 | //! State machine instrumentation. 2 | 3 | /// Consumes the state machine events. May used for metrics and/or logs. 4 | pub trait Instrument { 5 | /// Calls when state machine reject a call. 6 | fn on_call_rejected(&self); 7 | 8 | /// Calls when the circuit breaker become to open state. 9 | fn on_open(&self); 10 | 11 | /// Calls when the circuit breaker become to half open state. 12 | fn on_half_open(&self); 13 | 14 | /// Calls when the circuit breaker become to closed state. 15 | fn on_closed(&self); 16 | } 17 | 18 | /// An instrumentation which does noting. 19 | impl Instrument for () { 20 | #[inline] 21 | fn on_call_rejected(&self) {} 22 | 23 | #[inline] 24 | fn on_open(&self) {} 25 | 26 | #[inline] 27 | fn on_half_open(&self) {} 28 | 29 | #[inline] 30 | fn on_closed(&self) {} 31 | } 32 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! CircuitBreaker is used to detect failures and encapsulates the logic of preventing a failure 2 | //! from constantly recurring, during maintenance, temporary external system failure or unexpected 3 | //! system difficulties. 4 | //! 5 | //! # Links 6 | //! 7 | //! * Future aware circuit breaker's interface [futures::CircuitBreaker](futures/index.html). 8 | //! * The state machine which is used [StateMachine](state_machine/StateMachine.t.html). 9 | //! * More about circuit breakers [https://martinfowler.com/bliki/CircuitBreaker.html](https://martinfowler.com/bliki/CircuitBreaker.html) 10 | //! 11 | //! # Example 12 | //! 13 | //! Using default backoff strategy and failure accrual policy. 14 | //! 15 | //! ``` 16 | //! use rand::{thread_rng, Rng}; 17 | //! use failsafe::{Config, CircuitBreaker, Error}; 18 | //! 19 | //! // A function that sometimes failed. 20 | //! fn dangerous_call() -> Result<(), ()> { 21 | //! if thread_rng().gen_range(0..2) == 0 { 22 | //! return Err(()) 23 | //! } 24 | //! Ok(()) 25 | //! } 26 | //! 27 | //! // Create a circuit breaker which configured by reasonable default backoff and 28 | //! // failure accrual policy. 29 | //! let circuit_breaker = Config::new().build(); 30 | //! 31 | //! // Call the function in a loop, after some iterations the circuit breaker will 32 | //! // be in a open state and reject next calls. 33 | //! for n in 0..100 { 34 | //! match circuit_breaker.call(|| dangerous_call()) { 35 | //! Err(Error::Inner(_)) => { 36 | //! eprintln!("{}: fail", n); 37 | //! }, 38 | //! Err(Error::Rejected) => { 39 | //! eprintln!("{}: rejected", n); 40 | //! break; 41 | //! }, 42 | //! _ => {} 43 | //! } 44 | //! } 45 | //! ``` 46 | //! 47 | //! Or configure custom backoff and policy: 48 | //! 49 | //! ``` 50 | //! use std::time::Duration; 51 | //! use failsafe::{backoff, failure_policy, Config, CircuitBreaker}; 52 | //! 53 | //! fn circuit_breaker() -> impl CircuitBreaker { 54 | //! // Create an exponential growth backoff which starts from 10s and ends with 60s. 55 | //! let backoff = backoff::exponential(Duration::from_secs(10), Duration::from_secs(60)); 56 | //! 57 | //! // Create a policy which failed when three consecutive failures were made. 58 | //! let policy = failure_policy::consecutive_failures(3, backoff); 59 | //! 60 | //! // Creates a circuit breaker with given policy. 61 | //! Config::new() 62 | //! .failure_policy(policy) 63 | //! .build() 64 | //! } 65 | //! ``` 66 | 67 | #![deny(missing_debug_implementations)] 68 | #![deny(missing_docs)] 69 | #![cfg_attr(test, deny(warnings))] 70 | 71 | mod circuit_breaker; 72 | mod config; 73 | mod ema; 74 | mod error; 75 | mod failure_predicate; 76 | mod instrument; 77 | mod state_machine; 78 | mod windowed_adder; 79 | 80 | pub mod backoff; 81 | pub mod failure_policy; 82 | #[cfg(feature = "futures-support")] 83 | pub mod futures; 84 | 85 | #[doc(hidden)] 86 | pub mod clock; 87 | 88 | pub use self::circuit_breaker::CircuitBreaker; 89 | pub use self::config::Config; 90 | pub use self::error::Error; 91 | pub use self::failure_policy::FailurePolicy; 92 | pub use self::failure_predicate::{Any, FailurePredicate}; 93 | pub use self::instrument::Instrument; 94 | pub use self::state_machine::StateMachine; 95 | pub use self::windowed_adder::WindowedAdder; 96 | -------------------------------------------------------------------------------- /src/state_machine.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::{self, Debug}; 2 | use std::sync::Arc; 3 | use std::time::{Duration, Instant}; 4 | 5 | use parking_lot::Mutex; 6 | 7 | use super::clock; 8 | use super::failure_policy::FailurePolicy; 9 | use super::instrument::Instrument; 10 | 11 | const ON_CLOSED: u8 = 0b0000_0001; 12 | const ON_HALF_OPEN: u8 = 0b0000_0010; 13 | const ON_REJECTED: u8 = 0b0000_0100; 14 | const ON_OPEN: u8 = 0b0000_1000; 15 | 16 | /// States of the state machine. 17 | #[derive(Debug)] 18 | enum State { 19 | /// A closed breaker is operating normally and allowing. 20 | Closed, 21 | /// An open breaker has tripped and will not allow requests through until an interval expired. 22 | Open(Instant, Duration), 23 | /// A half open breaker has completed its wait interval and will allow requests. The state keeps 24 | /// the previous duration in an open state. 25 | HalfOpen(Duration), 26 | } 27 | 28 | struct Shared { 29 | state: State, 30 | failure_policy: POLICY, 31 | } 32 | 33 | struct Inner { 34 | shared: Mutex>, 35 | instrument: INSTRUMENT, 36 | } 37 | 38 | /// A circuit breaker implementation backed by state machine. 39 | /// 40 | /// It is implemented via a finite state machine with three states: `Closed`, `Open` and `HalfOpen`. 41 | /// The state machine does not know anything about the backend's state by itself, but uses the 42 | /// information provided by the method via `on_success` and `on_error` events. Before communicating 43 | /// with the backend, the the permission to do so must be obtained via the method `is_call_permitted`. 44 | /// 45 | /// The state of the state machine changes from `Closed` to `Open` when the `FailurePolicy` 46 | /// reports that the failure rate is above a (configurable) threshold. Then, all access to the backend 47 | /// is blocked for a time duration provided by `FailurePolicy`. 48 | /// 49 | /// After the time duration has elapsed, the state changes from `Open` to `HalfOpen` and allows 50 | /// calls to see if the backend is still unavailable or has become available again. If the circuit 51 | /// breaker receives a failure on the next call, the state will change back to `Open`. Otherwise 52 | /// it changes to `Closed`. 53 | pub struct StateMachine { 54 | inner: Arc>, 55 | } 56 | 57 | impl State { 58 | /// Returns a string value for the state identifier. 59 | #[inline] 60 | pub fn as_str(&self) -> &'static str { 61 | match self { 62 | State::Open(_, _) => "open", 63 | State::Closed => "closed", 64 | State::HalfOpen(_) => "half_open", 65 | } 66 | } 67 | } 68 | 69 | impl Debug for StateMachine { 70 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 71 | let shared = self.inner.shared.lock(); 72 | f.debug_struct("StateMachine") 73 | .field("state", &(shared.state.as_str())) 74 | .finish() 75 | } 76 | } 77 | 78 | impl Clone for StateMachine { 79 | fn clone(&self) -> Self { 80 | StateMachine { 81 | inner: self.inner.clone(), 82 | } 83 | } 84 | } 85 | 86 | impl Shared 87 | where 88 | POLICY: FailurePolicy, 89 | { 90 | #[inline] 91 | fn transit_to_closed(&mut self) { 92 | self.state = State::Closed; 93 | self.failure_policy.revived(); 94 | } 95 | 96 | #[inline] 97 | fn transit_to_half_open(&mut self, delay: Duration) { 98 | self.state = State::HalfOpen(delay); 99 | } 100 | 101 | #[inline] 102 | fn transit_to_open(&mut self, delay: Duration) { 103 | let until = clock::now() + delay; 104 | self.state = State::Open(until, delay); 105 | } 106 | } 107 | 108 | impl StateMachine 109 | where 110 | POLICY: FailurePolicy, 111 | INSTRUMENT: Instrument, 112 | { 113 | /// Creates a new state machine with given failure policy and instrument. 114 | pub fn new(failure_policy: POLICY, instrument: INSTRUMENT) -> Self { 115 | instrument.on_closed(); 116 | 117 | StateMachine { 118 | inner: Arc::new(Inner { 119 | shared: Mutex::new(Shared { 120 | state: State::Closed, 121 | failure_policy, 122 | }), 123 | instrument, 124 | }), 125 | } 126 | } 127 | 128 | /// Requests permission to call. 129 | /// 130 | /// It returns `true` if a call is allowed, or `false` if prohibited. 131 | pub fn is_call_permitted(&self) -> bool { 132 | let mut instrument: u8 = 0; 133 | 134 | let res = { 135 | let mut shared = self.inner.shared.lock(); 136 | 137 | match shared.state { 138 | State::Closed => true, 139 | State::HalfOpen(_) => true, 140 | State::Open(until, delay) => { 141 | if clock::now() > until { 142 | shared.transit_to_half_open(delay); 143 | instrument |= ON_HALF_OPEN; 144 | true 145 | } else { 146 | instrument |= ON_REJECTED; 147 | false 148 | } 149 | } 150 | } 151 | }; 152 | 153 | if instrument & ON_HALF_OPEN != 0 { 154 | self.inner.instrument.on_half_open(); 155 | } 156 | 157 | if instrument & ON_REJECTED != 0 { 158 | self.inner.instrument.on_call_rejected(); 159 | } 160 | 161 | res 162 | } 163 | 164 | /// Reset state machine to Closed 165 | /// 166 | pub fn reset(&self) { 167 | let mut shared = self.inner.shared.lock(); 168 | match shared.state { 169 | State::HalfOpen(_) => { 170 | shared.transit_to_closed(); 171 | self.inner.instrument.on_closed(); 172 | } 173 | State::Open(_, _) => { 174 | shared.transit_to_closed(); 175 | self.inner.instrument.on_closed(); 176 | } 177 | _ => {} 178 | } 179 | } 180 | 181 | /// Records a successful call. 182 | /// 183 | /// This method must be invoked when a call was success. 184 | pub fn on_success(&self) { 185 | let mut instrument: u8 = 0; 186 | { 187 | let mut shared = self.inner.shared.lock(); 188 | if let State::HalfOpen(_) = shared.state { 189 | shared.transit_to_closed(); 190 | instrument |= ON_CLOSED; 191 | } 192 | shared.failure_policy.record_success() 193 | } 194 | 195 | if instrument & ON_CLOSED != 0 { 196 | self.inner.instrument.on_closed(); 197 | } 198 | } 199 | 200 | /// Records a failed call. 201 | /// 202 | /// This method must be invoked when a call failed. 203 | pub fn on_error(&self) { 204 | let mut instrument: u8 = 0; 205 | { 206 | let mut shared = self.inner.shared.lock(); 207 | match shared.state { 208 | State::Closed => { 209 | if let Some(delay) = shared.failure_policy.mark_dead_on_failure() { 210 | shared.transit_to_open(delay); 211 | instrument |= ON_OPEN; 212 | } 213 | } 214 | State::HalfOpen(delay_in_half_open) => { 215 | // Pick up the next open state's delay from the policy, if policy returns Some(_) 216 | // use it, otherwise reuse the delay from the current state. 217 | let delay = shared 218 | .failure_policy 219 | .mark_dead_on_failure() 220 | .unwrap_or(delay_in_half_open); 221 | shared.transit_to_open(delay); 222 | instrument |= ON_OPEN; 223 | } 224 | _ => {} 225 | } 226 | } 227 | 228 | if instrument & ON_OPEN != 0 { 229 | self.inner.instrument.on_open(); 230 | } 231 | } 232 | } 233 | 234 | #[cfg(test)] 235 | mod tests { 236 | use std::sync::atomic::{AtomicUsize, Ordering}; 237 | use std::sync::{Arc, Mutex}; 238 | 239 | use super::super::backoff; 240 | use super::super::failure_policy::consecutive_failures; 241 | use super::*; 242 | 243 | /// Perform `Closed` -> `Open` -> `HalfOpen` -> `Open` -> `HalfOpen` -> `Closed` transitions. 244 | #[test] 245 | fn state_machine() { 246 | clock::freeze(move |time| { 247 | let observe = Observer::new(); 248 | let backoff = backoff::exponential(5.seconds(), 300.seconds()); 249 | let policy = consecutive_failures(3, backoff); 250 | let state_machine = StateMachine::new(policy, observe.clone()); 251 | 252 | assert!(state_machine.is_call_permitted()); 253 | 254 | // Perform success requests. the circuit breaker must be closed. 255 | for _i in 0..10 { 256 | assert!(state_machine.is_call_permitted()); 257 | state_machine.on_success(); 258 | assert!(observe.is_closed()); 259 | } 260 | 261 | // Perform failed requests, the circuit breaker still closed. 262 | for _i in 0..2 { 263 | assert!(state_machine.is_call_permitted()); 264 | state_machine.on_error(); 265 | assert!(observe.is_closed()); 266 | } 267 | 268 | // Perform a failed request and transit to the open state for 5s. 269 | assert!(state_machine.is_call_permitted()); 270 | state_machine.on_error(); 271 | assert!(observe.is_open()); 272 | 273 | // Reject call attempts, the circuit breaker in open state. 274 | for i in 0..10 { 275 | assert!(!state_machine.is_call_permitted()); 276 | assert_eq!(i + 1, observe.rejected_calls()); 277 | } 278 | 279 | // Wait 2s, the circuit breaker still open. 280 | time.advance(2.seconds()); 281 | assert!(!state_machine.is_call_permitted()); 282 | assert!(observe.is_open()); 283 | 284 | clock::now(); 285 | 286 | // Wait 4s (6s total), the circuit breaker now in the half open state. 287 | time.advance(4.seconds()); 288 | assert!(state_machine.is_call_permitted()); 289 | assert!(observe.is_half_open()); 290 | 291 | // Perform a failed request and transit back to the open state for 10s. 292 | state_machine.on_error(); 293 | assert!(!state_machine.is_call_permitted()); 294 | assert!(observe.is_open()); 295 | 296 | // Wait 5s, the circuit breaker still open. 297 | time.advance(5.seconds()); 298 | assert!(!state_machine.is_call_permitted()); 299 | assert!(observe.is_open()); 300 | 301 | // Wait 6s (11s total), the circuit breaker now in the half open state. 302 | time.advance(6.seconds()); 303 | assert!(state_machine.is_call_permitted()); 304 | assert!(observe.is_half_open()); 305 | 306 | // Perform a success request and transit to the closed state. 307 | state_machine.on_success(); 308 | assert!(state_machine.is_call_permitted()); 309 | assert!(observe.is_closed()); 310 | 311 | // Perform success requests. 312 | for _i in 0..10 { 313 | assert!(state_machine.is_call_permitted()); 314 | state_machine.on_success(); 315 | } 316 | 317 | // Perform failed request and transit back to open state 318 | for _i in 0..3 { 319 | state_machine.on_error(); 320 | } 321 | assert!(observe.is_open()); 322 | 323 | // Reset state machine 324 | state_machine.reset(); 325 | assert!(observe.is_closed()); 326 | }); 327 | } 328 | 329 | #[derive(Debug)] 330 | enum State { 331 | Open, 332 | HalfOpen, 333 | Closed, 334 | } 335 | 336 | #[derive(Clone, Debug)] 337 | struct Observer { 338 | state: Arc>, 339 | rejected_calls: Arc, 340 | } 341 | 342 | impl Observer { 343 | fn new() -> Self { 344 | Observer { 345 | state: Arc::new(Mutex::new(State::Closed)), 346 | rejected_calls: Arc::new(AtomicUsize::new(0)), 347 | } 348 | } 349 | 350 | fn is_closed(&self) -> bool { 351 | matches!(*self.state.lock().unwrap(), State::Closed) 352 | } 353 | 354 | fn is_open(&self) -> bool { 355 | matches!(*self.state.lock().unwrap(), State::Open) 356 | } 357 | 358 | fn is_half_open(&self) -> bool { 359 | matches!(*self.state.lock().unwrap(), State::HalfOpen) 360 | } 361 | 362 | fn rejected_calls(&self) -> usize { 363 | self.rejected_calls.load(Ordering::SeqCst) 364 | } 365 | } 366 | 367 | impl Instrument for Observer { 368 | fn on_call_rejected(&self) { 369 | self.rejected_calls.fetch_add(1, Ordering::SeqCst); 370 | } 371 | 372 | fn on_open(&self) { 373 | println!("state=open"); 374 | let mut own_state = self.state.lock().unwrap(); 375 | *own_state = State::Open 376 | } 377 | 378 | fn on_half_open(&self) { 379 | println!("state=half_open"); 380 | let mut own_state = self.state.lock().unwrap(); 381 | *own_state = State::HalfOpen 382 | } 383 | 384 | fn on_closed(&self) { 385 | println!("state=closed"); 386 | let mut own_state = self.state.lock().unwrap(); 387 | *own_state = State::Closed 388 | } 389 | } 390 | 391 | trait IntoDuration { 392 | fn seconds(self) -> Duration; 393 | } 394 | 395 | impl IntoDuration for u64 { 396 | fn seconds(self) -> Duration { 397 | Duration::from_secs(self) 398 | } 399 | } 400 | } 401 | -------------------------------------------------------------------------------- /src/windowed_adder.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | use super::clock; 4 | 5 | /// Time windowed counter. 6 | #[derive(Debug)] 7 | pub struct WindowedAdder { 8 | window: u64, 9 | slices: Vec, 10 | index: usize, 11 | elapsed: Instant, 12 | } 13 | 14 | impl WindowedAdder { 15 | /// Creates a new counter. 16 | /// 17 | /// * `window` - The range of time to be kept in the counter. 18 | /// * `slices` - The number of slices that are maintained; a higher number of slices 19 | /// means finer granularity but also more memory consumption. Must be more than 1 and 20 | /// less then 10. 21 | /// 22 | /// # Panics 23 | /// 24 | /// * When `slices` isn't in range [1;10]. 25 | pub fn new(window: Duration, slices: u8) -> Self { 26 | assert!(slices <= 10); 27 | assert!(slices > 1); 28 | 29 | let window = window.millis() / u64::from(slices); 30 | 31 | Self { 32 | window, 33 | slices: vec![0; slices as usize], 34 | index: 0, 35 | elapsed: clock::now(), 36 | } 37 | } 38 | 39 | /// Purge outdated slices. 40 | pub fn expire(&mut self) { 41 | let now = clock::now(); 42 | let time_diff = (now - self.elapsed).millis(); 43 | 44 | if time_diff < self.window { 45 | return; 46 | } 47 | 48 | let len = self.slices.len(); 49 | let mut idx = (self.index + 1) % len; 50 | 51 | let n_skip = ((time_diff / self.window) - 1).min(len as u64); 52 | if n_skip > 0 { 53 | let r = n_skip.min((len - idx) as u64); 54 | self.zero_slices(idx, idx + r as usize); 55 | self.zero_slices(0usize, (n_skip - r) as usize); 56 | //println!("zero {}-{} {}-{}", idx, idx + r as usize, 0, n_skip - r); 57 | idx = (idx + n_skip as usize) % len; 58 | } 59 | 60 | self.slices[idx] = 0; 61 | self.index = idx; 62 | self.elapsed = now; 63 | 64 | //println!("inc {} vec={:?}", idx, self.slices); 65 | } 66 | 67 | /// Resets state of the counter. 68 | pub fn reset(&mut self) { 69 | self.slices.iter_mut().for_each(|it| *it = 0); 70 | self.elapsed = clock::now(); 71 | } 72 | 73 | /// Increments counter by `value`. 74 | pub fn add(&mut self, value: i64) { 75 | self.expire(); 76 | self.slices[self.index] += value; 77 | //println!("add {} {:?}", value, self.slices); 78 | } 79 | 80 | /// Returns the current sum of the counter. 81 | pub fn sum(&mut self) -> i64 { 82 | self.expire(); 83 | self.slices.iter().sum() 84 | } 85 | 86 | /// Writes zero into slices starting `from` and ending `to`. 87 | fn zero_slices(&mut self, from: usize, to: usize) { 88 | self.slices 89 | .iter_mut() 90 | .take(to) 91 | .skip(from) 92 | .for_each(|it| *it = 0); 93 | } 94 | } 95 | 96 | /// `Duration::as_millis` is unstable at the current(1.28) rust version, so it returns milliseconds 97 | /// in given duration. 98 | trait Millis { 99 | fn millis(&self) -> u64; 100 | } 101 | 102 | impl Millis for Duration { 103 | fn millis(&self) -> u64 { 104 | const MILLIS_PER_SEC: u64 = 1_000; 105 | (self.as_secs() * MILLIS_PER_SEC) + u64::from(self.subsec_millis()) 106 | } 107 | } 108 | 109 | #[cfg(test)] 110 | mod tests { 111 | use super::*; 112 | 113 | #[test] 114 | fn sum_when_time_stands_still() { 115 | clock::freeze(|_| { 116 | let mut adder = new_windowed_adder(); 117 | 118 | adder.add(1); 119 | assert_eq!(1, adder.sum()); 120 | adder.add(1); 121 | assert_eq!(2, adder.sum()); 122 | adder.add(3); 123 | assert_eq!(5, adder.sum()); 124 | }); 125 | } 126 | 127 | #[test] 128 | fn sliding_over_small_window() { 129 | clock::freeze(|time| { 130 | let mut adder = new_windowed_adder(); 131 | 132 | adder.add(1); 133 | assert_eq!(1, adder.sum()); 134 | 135 | time.advance(1.seconds()); 136 | assert_eq!(1, adder.sum()); 137 | 138 | adder.add(2); 139 | assert_eq!(3, adder.sum()); 140 | 141 | time.advance(1.seconds()); 142 | assert_eq!(3, adder.sum()); 143 | 144 | time.advance(1.seconds()); 145 | assert_eq!(2, adder.sum()); 146 | 147 | time.advance(1.seconds()); 148 | assert_eq!(0, adder.sum()); 149 | }) 150 | } 151 | 152 | #[test] 153 | fn sliding_over_large_window() { 154 | clock::freeze(|time| { 155 | let mut adder = WindowedAdder::new(20.seconds(), 10); 156 | 157 | for i in 0..21 { 158 | adder.add(i % 3); 159 | time.advance(1.seconds()); 160 | } 161 | 162 | assert_eq!(20, adder.sum()); 163 | 164 | time.advance(1.seconds()); 165 | assert_eq!(18, adder.sum()); 166 | 167 | time.advance(1.seconds()); 168 | assert_eq!(18, adder.sum()); 169 | 170 | time.advance(5.seconds()); 171 | assert_eq!(12, adder.sum()); 172 | adder.add(1); 173 | 174 | time.advance(10.seconds()); 175 | assert_eq!(3, adder.sum()); 176 | }) 177 | } 178 | 179 | #[test] 180 | fn sliding_window_when_slices_are_skipped() { 181 | clock::freeze(|time| { 182 | let mut adder = new_windowed_adder(); 183 | 184 | adder.add(1); 185 | assert_eq!(1, adder.sum()); 186 | 187 | time.advance(1.seconds()); 188 | adder.add(2); 189 | assert_eq!(3, adder.sum()); 190 | 191 | time.advance(1.seconds()); 192 | adder.add(1); 193 | assert_eq!(4, adder.sum()); 194 | 195 | time.advance(2.seconds()); 196 | assert_eq!(1, adder.sum()); 197 | 198 | time.advance(100.seconds()); 199 | assert_eq!(0, adder.sum()); 200 | 201 | adder.add(100); 202 | time.advance(1.seconds()); 203 | assert_eq!(100, adder.sum()); 204 | 205 | adder.add(100); 206 | time.advance(1.seconds()); 207 | 208 | adder.add(100); 209 | assert_eq!(300, adder.sum()); 210 | 211 | time.advance(100.seconds()); 212 | assert_eq!(0, adder.sum()); 213 | }) 214 | } 215 | 216 | #[test] 217 | fn negative_sums() { 218 | clock::freeze(|time| { 219 | let mut adder = new_windowed_adder(); 220 | 221 | // net: 2 222 | adder.add(-2); 223 | assert_eq!(-2, adder.sum()); 224 | 225 | adder.add(4); 226 | assert_eq!(2, adder.sum()); 227 | 228 | // net: -4 229 | time.advance(1.seconds()); 230 | adder.add(-2); 231 | assert_eq!(0, adder.sum()); 232 | 233 | adder.add(-2); 234 | assert_eq!(-2, adder.sum()); 235 | 236 | // net: -2 237 | time.advance(1.seconds()); 238 | adder.add(-2); 239 | assert_eq!(-4, adder.sum()); 240 | 241 | time.advance(1.seconds()); 242 | assert_eq!(-6, adder.sum()); 243 | 244 | time.advance(1.seconds()); 245 | assert_eq!(-2, adder.sum()); 246 | 247 | time.advance(1.seconds()); 248 | assert_eq!(0, adder.sum()); 249 | 250 | time.advance(100.seconds()); 251 | assert_eq!(0, adder.sum()); 252 | }); 253 | } 254 | 255 | fn new_windowed_adder() -> WindowedAdder { 256 | WindowedAdder::new(3.seconds(), 3) 257 | } 258 | 259 | trait IntoDuration { 260 | fn seconds(self) -> Duration; 261 | } 262 | 263 | impl IntoDuration for u64 { 264 | fn seconds(self) -> Duration { 265 | Duration::from_secs(self) 266 | } 267 | } 268 | } 269 | --------------------------------------------------------------------------------