├── .gitignore ├── release.toml ├── examples ├── deterministic.rs ├── random_pair.rs └── simple.rs ├── test.sh ├── benches └── medium.rs ├── CHANGELOG.md ├── add-git-hook.sh ├── tests ├── feistel_network.rs ├── permutor.rs └── randomness.rs ├── Cargo.toml ├── .travis.yml ├── README.md ├── LICENSE └── src └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | .idea 5 | -------------------------------------------------------------------------------- /release.toml: -------------------------------------------------------------------------------- 1 | pre-release-replacements = [ 2 | {file="README.md", search="permutation_iterator = .*", replace="{{crate_name}} = \"{{version}}\""} 3 | ] -------------------------------------------------------------------------------- /examples/deterministic.rs: -------------------------------------------------------------------------------- 1 | use permutation_iterator::Permutor; 2 | 3 | fn main() -> anyhow::Result<()> { 4 | let max = 10; 5 | let key: [u8; 32] = [0xBA; 32]; 6 | let permutor = Permutor::new_with_slice_key(max, key)?; 7 | for permuted in permutor { 8 | println!("{}", permuted); 9 | } 10 | Ok(()) 11 | } 12 | -------------------------------------------------------------------------------- /examples/random_pair.rs: -------------------------------------------------------------------------------- 1 | use permutation_iterator::RandomPairPermutor; 2 | 3 | fn main() -> anyhow::Result<()> { 4 | let xs = [1, 2, 3]; 5 | let ys = [4, 5, 6, 7, 8]; 6 | 7 | let permutor = RandomPairPermutor::new(xs.len() as u64, ys.len() as u64)?; 8 | for (i, j) in permutor { 9 | println!("({}, {})", xs[i as usize], ys[j as usize]); 10 | } 11 | 12 | Ok(()) 13 | } 14 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | cargo check 6 | cargo check --target wasm32-unknown-unknown 7 | cargo check --target wasm32-unknown-emscripten 8 | cargo check --target wasm32-wasi 9 | 10 | rustup run nightly cargo clippy -- -D warnings 11 | cargo test --release 12 | rustup run nightly cargo bench 13 | # cargo test --test randomness --release -- --nocapture --ignored 14 | 15 | -------------------------------------------------------------------------------- /examples/simple.rs: -------------------------------------------------------------------------------- 1 | use permutation_iterator::Permutor; 2 | 3 | fn main() -> anyhow::Result<()> { 4 | let max = 10; 5 | 6 | // Since we don't pass in a key we will get a random permutation every time we run this. 7 | // Try it out! 8 | let permutor = Permutor::new(max)?; 9 | 10 | for (index, permuted) in permutor.enumerate() { 11 | println!("{} -> {}", index, permuted); 12 | } 13 | 14 | Ok(()) 15 | } 16 | -------------------------------------------------------------------------------- /benches/medium.rs: -------------------------------------------------------------------------------- 1 | #![feature(test)] 2 | 3 | extern crate test; 4 | 5 | #[cfg(test)] 6 | mod benches { 7 | use permutation_iterator::FeistelNetwork; 8 | use test::Bencher; 9 | 10 | const ZERO_KEY: [u8; 32] = [0; 32]; 11 | 12 | #[bench] 13 | fn bench_medium(b: &mut Bencher) { 14 | let feistel = FeistelNetwork::new_with_slice_key(100_000, ZERO_KEY); 15 | b.iter(|| feistel.permute(50_000)); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.2 2 | 3 | - Changed Feistel network rounds from 4 to 12 (and later will make this configurable). 4 | - Randomness on small max_values was very poor at 4 rounds, so just increased it to 12 for now. 5 | - Have the first automated randomness test, that compares chi squared values against a "real randomness" shuffler. 6 | - Not the greatest test of randomness but it's a start. 7 | - Better Travis continuous integration config. 8 | 9 | # 0.1.1 10 | 11 | - Documentation updates. 12 | - Update `Cargo.toml` to better describe project. 13 | 14 | # 0.1.0 15 | 16 | - Initial release. 17 | - Unstable, will almost certainly change API. 18 | -------------------------------------------------------------------------------- /add-git-hook.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Adds the git-hook described below. Appends to the hook file 3 | # if it already exists or creates the file if it does not. 4 | # Note: CWD must be inside target repository 5 | 6 | set -euxo pipefail 7 | 8 | HOOK_DIR=$(git rev-parse --show-toplevel)/.git/hooks 9 | HOOK_FILE="$HOOK_DIR"/pre-commit 10 | 11 | # Create script file if doesn't exist 12 | if [ ! -e "$HOOK_FILE" ] ; then 13 | echo '#!/usr/bin/env bash' >> "$HOOK_FILE" 14 | echo 'set -euxo pipefail' >> "$HOOK_FILE" 15 | chmod a+x "$HOOK_FILE" 16 | fi 17 | chmod a+x "$HOOK_FILE" 18 | 19 | # Append hook code into script 20 | cat >> "$HOOK_FILE" <"] 6 | 7 | description = """ 8 | A Rust library for iterating over random permutations using O(1) (i.e. constant) space. 9 | """ 10 | keywords = ["permutation", "permute", "random"] 11 | categories = ["algorithms", "no-std"] 12 | documentation = "https://docs.rs/permutation-iterator-rs/" 13 | repository = "https://github.com/asimihsan/permutation-iterator-rs" 14 | homepage = "https://github.com/asimihsan/permutation-iterator-rs" 15 | readme = "README.md" 16 | license = "Apache-2.0" 17 | 18 | [badges] 19 | travis-ci = { repository = "asimihsan/permutation-iterator-rs", branch = "master" } 20 | 21 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 22 | 23 | [profile.release] 24 | lto = true 25 | 26 | [[bench]] 27 | name = "medium" 28 | 29 | # To use these `cargo install cargo-template-ci` then run `cargo template-ci`. This will generate Travis config for you. 30 | [package.metadata.template_ci] 31 | dist = "bionic" 32 | versions = ["stable", "beta", "nightly"] 33 | clippy = { allow_failure = false } 34 | rustfmt = { allow_failure = false } 35 | bench = { run = true } 36 | 37 | [dependencies.getrandom] 38 | version = "0.1.14" 39 | features = ["wasm-bindgen"] 40 | 41 | [dependencies.anyhow] 42 | version = "1.0.13" 43 | default-features = false 44 | 45 | [dependencies] 46 | wyhash = "0.3.0" 47 | 48 | [dev-dependencies] 49 | rand = "0.7.3" -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # TemplateCIConfig { bench: BenchEntry(MatrixEntry { run: true, run_cron: false, version: "nightly", install_commandline: None, commandline: "cargo bench" }), clippy: ClippyEntry(MatrixEntry { run: true, run_cron: false, version: "stable", install_commandline: Some("rustup component add clippy"), commandline: "cargo clippy -- -D warnings" }), rustfmt: RustfmtEntry(MatrixEntry { run: true, run_cron: false, version: "stable", install_commandline: Some("rustup component add rustfmt"), commandline: "cargo fmt -v -- --check" }), additional_matrix_entries: {}, cache: "cargo", os: "linux", dist: "bionic", versions: ["stable", "beta", "nightly"], test_commandline: "cargo test --verbose --all", scheduled_test_branches: ["master"], test_schedule: "0 0 * * 0" } 2 | os: 3 | - "linux" 4 | dist: "bionic" 5 | 6 | language: rust 7 | sudo: required 8 | cache: cargo 9 | 10 | rust: 11 | - stable 12 | - beta 13 | - nightly 14 | 15 | env: 16 | global: 17 | - RUN_TEST=true 18 | - RUN_CLIPPY=false 19 | - RUN_BENCH=false 20 | 21 | matrix: 22 | fast_finish: true 23 | include: 24 | - &rustfmt_build 25 | rust: "stable" 26 | env: 27 | - RUN_RUSTFMT=true 28 | - RUN_TEST=false 29 | - &bench_build 30 | rust: "nightly" 31 | env: 32 | - RUN_BENCH=true 33 | - RUN_TEST=false 34 | - &clippy_build 35 | rust: "stable" 36 | env: 37 | - RUN_CLIPPY=true 38 | - RUN_TEST=false 39 | 40 | before_script: 41 | - bash -c 'if [[ "$RUN_RUSTFMT" == "true" ]]; then 42 | rustup component add rustfmt 43 | ; 44 | fi' 45 | - bash -c 'if [[ "$RUN_CLIPPY" == "true" ]]; then 46 | rustup component add clippy 47 | ; 48 | fi' 49 | 50 | script: 51 | - bash -c 'if [[ "$RUN_TEST" == "true" ]]; then 52 | cargo test --release && cargo test --release -- --nocapture --ignored 53 | ; 54 | fi' 55 | - bash -c 'if [[ "$RUN_RUSTFMT" == "true" ]]; then 56 | cargo fmt -v -- --check 57 | ; 58 | fi' 59 | - bash -c 'if [[ "$RUN_BENCH" == "true" ]]; then 60 | cargo bench 61 | ; 62 | fi' 63 | - bash -c 'if [[ "$RUN_CLIPPY" == "true" ]]; then 64 | cargo clippy -- -D warnings 65 | ; 66 | fi' 67 | 68 | branches: 69 | only: 70 | # release tags 71 | - /^v\d+\.\d+\.\d+.*$/ 72 | - master 73 | - trying 74 | - staging 75 | 76 | notifications: 77 | email: 78 | on_success: never 79 | -------------------------------------------------------------------------------- /tests/permutor.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use permutation_iterator::Permutor; 4 | use std::collections::HashSet; 5 | use std::iter::FromIterator; 6 | 7 | #[test] 8 | fn test_different_key_means_different_result() { 9 | // === given === 10 | let max = 10; 11 | let key1 = 0; 12 | let key2 = 1; 13 | 14 | // === when === 15 | let values1 = get_permutation_values(max, key1); 16 | let values2 = get_permutation_values(max, key2); 17 | 18 | // === then === 19 | assert_ne!( 20 | values1, values2, 21 | "expected different permutations given max {}, first key {}, second key {}", 22 | max, key1, key2 23 | ); 24 | } 25 | 26 | #[test] 27 | fn test_same_key_means_same_result() { 28 | // === given === 29 | let max = 10; 30 | let key = 2; 31 | 32 | // === when === 33 | let values1 = get_permutation_values(max, key); 34 | let values2 = get_permutation_values(max, key); 35 | 36 | // === then === 37 | assert_eq!( 38 | values1, values2, 39 | "expected same permutations given max {}, key {} used twice", 40 | max, key 41 | ); 42 | } 43 | 44 | #[test] 45 | fn test_small_battery_returns_correct_permutations() { 46 | for max_value in 2..50 { 47 | for key in 0..10 { 48 | println!("testing max_value {} key {}", max_value, key); 49 | get_permutation_values(max_value, key); 50 | } 51 | } 52 | } 53 | 54 | #[test] 55 | fn test_large_max_returns_correct_permutation() { 56 | get_permutation_values(25_000, 0); 57 | } 58 | 59 | fn get_permutation_values(max_value: u128, key: u64) -> Vec { 60 | // === given === 61 | let permutor = Permutor::new_with_u64_key(max_value, key) 62 | .expect("expected new permutator with u64 key"); 63 | 64 | // === when === 65 | let result: Vec = permutor.collect(); 66 | 67 | // === then === 68 | assert_eq!( 69 | max_value as usize, 70 | result.len(), 71 | "incorrect size result given max {} key {}", 72 | max_value, 73 | key 74 | ); 75 | let result_unique: HashSet = HashSet::from_iter(result.clone()); 76 | assert_eq!( 77 | max_value as usize, 78 | result_unique.len(), 79 | "values aren't unique i.e. not a permutation given max {} key {}", 80 | max_value, 81 | key 82 | ); 83 | 84 | result 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /tests/randomness.rs: -------------------------------------------------------------------------------- 1 | #[cfg(test)] 2 | mod tests { 3 | use permutation_iterator::Permutor; 4 | use rand::prelude::StdRng; 5 | use rand::seq::SliceRandom; 6 | use rand::Rng; 7 | use std::collections::HashMap; 8 | 9 | /// Given a Permutor instance, for a given maximum value over which we're looking for permutations 10 | /// [0, max), see if each given returned value is evenly distributed. 11 | /// 12 | /// Only run this test in release mode, or else it will take too long. 13 | /// 14 | /// Test isn't very reliable especially for small max_value's but better than nothing. 15 | #[test] 16 | #[ignore] 17 | fn test_randomness_g_test() { 18 | for (max_value, max_key, ratio_diff_threshold) in vec![ 19 | // The true random chi-squared value is very variable, [1, 15] almost, so testing very 20 | // small max_value permutations reliably is difficult. 21 | // (4, 100_000), 22 | 23 | // Small-value tests are still pretty unreliable...can't really leave it in! 24 | (10, 200_000, 0.3), 25 | (17, 100_000, 0.2), 26 | (50, 50_000, 0.05), 27 | (100, 50_000, 0.05), 28 | (1000, 50_000, 0.05), 29 | ] { 30 | let g_test_permutor = randomness_g_test(max_value, max_key, false); 31 | let g_test_true_random = randomness_g_test(max_value, max_key, true); 32 | let ratio_diff = (g_test_permutor - g_test_true_random) / g_test_permutor; 33 | println!( 34 | "max_value: {}, max_key: {}, g_test_permutor: {:.2}, g_test_true_random: {:.2}, ratio_diff: {:.2}", 35 | max_value, max_key, g_test_permutor, g_test_true_random, ratio_diff 36 | ); 37 | 38 | // If ratio_diff is negative, permutor is "more random" than true randomness (which is 39 | // absurd, just a test artifact). 40 | assert!( 41 | ratio_diff < ratio_diff_threshold, 42 | "Expected permutor to be as random or worse by {:.2} than true randomness!", 43 | ratio_diff_threshold 44 | ); 45 | } 46 | } 47 | 48 | /// Reference: https://en.wikipedia.org/wiki/G-test 49 | fn randomness_g_test(max_value: u128, max_key: u64, true_random: bool) -> f64 { 50 | let mut rng: StdRng = rand::SeedableRng::seed_from_u64(42); 51 | let min_key = 0; 52 | let mut cell_counts: HashMap> = 53 | HashMap::with_capacity(max_value as usize); 54 | for key in min_key..max_key { 55 | let values: Vec; 56 | if true_random { 57 | values = get_random_permutation_true_random(max_value, &mut rng); 58 | } else { 59 | values = get_random_permutation_permutor(max_value, key); 60 | } 61 | for (i, value) in values.into_iter().enumerate() { 62 | if !cell_counts.contains_key(&i) { 63 | cell_counts.insert(i, HashMap::with_capacity(max_value as usize)); 64 | } 65 | let cell_count = cell_counts.get_mut(&i).unwrap(); 66 | let current_value = cell_count.get(&value).cloned().unwrap_or(0); 67 | cell_count.insert(value, current_value + 1); 68 | } 69 | } 70 | 71 | let mut g_test_sum: f64 = 0.0; 72 | let expected_count: f64 = (max_key - min_key) as f64 / max_value as f64; 73 | for (_cell_index, cell_count) in cell_counts { 74 | for (_cell_value, observed_count) in cell_count { 75 | let g_test_subvalue = 76 | (observed_count as f64 / expected_count).ln() * observed_count as f64; 77 | g_test_sum += g_test_subvalue; 78 | } 79 | } 80 | 2.0 * g_test_sum 81 | } 82 | 83 | fn get_random_permutation_permutor(max_value: u128, key: u64) -> Vec { 84 | let permutor = Permutor::new_with_u64_key(max_value, key).expect("expected new Permutor"); 85 | permutor.collect() 86 | } 87 | 88 | fn get_random_permutation_true_random(max_value: u128, rng: &mut impl Rng) -> Vec { 89 | let mut values: Vec = (0..max_value).collect(); 90 | values.shuffle(rng); 91 | values 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # permutation-iterator 2 | 3 | [![Build Status](https://travis-ci.com/asimihsan/permutation-iterator-rs.svg?branch=master)](https://travis-ci.com/asimihsan/permutation-iterator-rs) 4 | [![Crate](https://img.shields.io/crates/v/permutation_iterator.svg)](https://crates.io/crates/permutation_iterator) 5 | [![API](https://docs.rs/permutation_iterator/badge.svg)](https://docs.rs/permutation_iterator) 6 | ![License](https://img.shields.io/crates/l/permutation_iterator.svg) 7 | 8 | 9 | A Rust library for iterating over random permutations without fully materializing them into memory. 10 | 11 | `permutation-iterator` lets you iterate over a random permutation, for example the values `[0, 1, 2, 3, 4, 5]` in a 12 | random order. It does so in constant space; it does not fully instantiate the values in memory then shuffle them. 13 | 14 | ## Usage 15 | 16 | Add this to your `Cargo.toml`: 17 | 18 | ```toml 19 | [dependencies] 20 | permutation_iterator = "0.1.2" 21 | ``` 22 | 23 | ## Example 24 | 25 | ### Random, single integer range 26 | 27 | Here is how to iterate over a random permutation of integers in the range `[0, max)`, i.e. `0` inclusive to `max` 28 | exclusive. Every time you run this you will get a different permutation. 29 | 30 | ```rust 31 | use permutation_iterator::Permutor; 32 | 33 | fn main() { 34 | let max = 10; 35 | let permutor = Permutor::new(max); 36 | for permuted in permutor { 37 | println!("{}", permuted); 38 | } 39 | } 40 | ``` 41 | 42 | ### Deterministic, single integer range 43 | 44 | You can also pass in a `key` in order to iterate over a deterministically random permutation. Every time you run this 45 | you will get the same permutation: 46 | 47 | ```rust 48 | use permutation_iterator::Permutor; 49 | 50 | fn main() { 51 | let max = 10; 52 | let key: [u8; 32] = [0xBA; 32]; 53 | let permutor = Permutor::new_with_slice_key(max, key); 54 | for permuted in permutor { 55 | println!("{}", permuted); 56 | } 57 | } 58 | ``` 59 | 60 | ### Random, pair of integers 61 | 62 | If you have e.g. two vectors of integers and you want to iterate over a random permutation of pairs from these lists 63 | you can use: 64 | 65 | ```rust 66 | use permutation_iterator::RandomPairPermutor; 67 | 68 | fn main() { 69 | let xs = [1, 2, 3]; 70 | let ys = [4, 5, 6, 7, 8]; 71 | 72 | let permutor = RandomPairPermutor::new(xs.len() as u32, ys.len() as u32); 73 | for (i, j) in permutor { 74 | println!("({}, {})", xs[i as usize], ys[j as usize]); 75 | } 76 | } 77 | ``` 78 | 79 | ## Implementation details 80 | 81 | One way of generating a random permutation is to shuffle a list. For example, given input integers `[0, 1, 2, 3, 4, 5]`, 82 | one can shuffle it to e.g. `[5, 3, 2, 0, 1, 4]`. Each input element maps to one and only one output element, and 83 | vice versa (each output element maps to one and only one input element). As you consume the shuffled list from e.g. 84 | left to right you're consuming this random permutation. 85 | 86 | Shuffling is `O(n)` time using the Fisher-Yates algorithm, however it is also `O(n)` space. We need a copy of the 87 | elements in-memory in order to shuffle them. This is inconvenient if the input range is large, or if the environment 88 | you're running on is memory-constrained. 89 | 90 | Cryptography offers an alternative. Symmetric encryption boils down to mapping a given input to one and only one output, 91 | where the mapping is varied by a single secret key, and vice-versa (each output element maps to one and only one input 92 | element). If this **bijective** mapping did not exist we wouldn't be reliably able to retrieve the original input. One 93 | specific kind of symmetric encryption uses a **block** cipher (operating on n-bits at a time) implemented using a 94 | **Feistel network**. 95 | 96 | A Feistel network is an extraordinary construct that allows you to use a simple, relatively weak **non-invertible** 97 | function over and over again and become a complicated, relatively strong **invertible permutation**. Hence in 98 | constant time we can _encrypt_ inputs as a way of iterating over random permutations. We can similarly _decrypt_ 99 | the output as a way of _resuming_ permutations. 100 | 101 | Consider the example of a bank that is trying to generate unique credit card numbers. Actual credit card numbers need 102 | to be stored very securely and we would rather not have to look them in order to find the next available number. By 103 | storing just a key and the last credit card number generated we can securely and efficiently continue iterating over 104 | the random permutation of all credit card numbers, without risking repeats. 105 | 106 | ## Maintainer notes 107 | 108 | ### How to release a new version 109 | 110 | 111 | 112 | ## License 113 | 114 | `permutation-iterator` is distributed under the terms of the Apache License (Version 2.0). See [LICENSE](LICENSE) for 115 | details. 116 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Asim Ihsan 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 4 | // in compliance with the License. You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software distributed under the License 9 | // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 10 | // or implied. See the License for the specific language governing permissions and limitations under 11 | // the License. 12 | 13 | //! Utilities for iterating over random permutations. 14 | //! 15 | //! `permutation-iterator` provides utilities for iterating over random permutations in constant 16 | //! space. 17 | //! 18 | //! # Quick Start 19 | //! 20 | //! Please check the GitHub repository's `README.md` and `examples` folder for how to get started 21 | //! with this library. 22 | #![no_std] 23 | 24 | use core::hash::Hasher; 25 | use wyhash::WyHash; 26 | 27 | /// Permutor gives you back a permutation iterator that returns a random permutation over 28 | /// [0, max) (0 inclusive to max exclusive). 29 | /// 30 | /// # Examples 31 | /// 32 | /// Permutor can be used to iterate over a random permutation of integers [0..max) (0 inclusive to 33 | /// max exclusive): 34 | /// 35 | /// ``` 36 | /// use crate::permutation_iterator::Permutor; 37 | /// use std::collections::HashSet; 38 | /// 39 | /// let max: u128 = 10; 40 | /// let permutor = Permutor::new(max).expect("Expected new Permutor"); 41 | /// for value in permutor { 42 | /// println!("{}", value); 43 | /// } 44 | /// ``` 45 | pub struct Permutor { 46 | feistel: FeistelNetwork, 47 | max: u128, 48 | current: u128, 49 | values_returned: u128, 50 | } 51 | 52 | impl Permutor { 53 | pub fn new_with_u64_key(max: u128, key: u64) -> anyhow::Result { 54 | let key = u64_to_32slice(key); 55 | Ok(Permutor { 56 | feistel: FeistelNetwork::new_with_slice_key(max, key), 57 | max, 58 | current: 0, 59 | values_returned: 0, 60 | }) 61 | } 62 | 63 | pub fn new_with_slice_key(max: u128, key: [u8; 32]) -> anyhow::Result { 64 | Ok(Permutor { 65 | feistel: FeistelNetwork::new_with_slice_key(max, key), 66 | max, 67 | current: 0, 68 | values_returned: 0, 69 | }) 70 | } 71 | 72 | pub fn new(max: u128) -> anyhow::Result { 73 | Ok(Permutor { 74 | feistel: FeistelNetwork::new(max)?, 75 | max, 76 | current: 0, 77 | values_returned: 0, 78 | }) 79 | } 80 | } 81 | 82 | impl Iterator for Permutor { 83 | type Item = u128; 84 | 85 | fn next(&mut self) -> Option { 86 | while self.values_returned < self.max { 87 | let next = self.feistel.permute(self.current); 88 | self.current += 1; 89 | if next >= self.max { 90 | continue; 91 | } 92 | self.values_returned += 1; 93 | return Some(next); 94 | } 95 | None 96 | } 97 | } 98 | 99 | /// Iterate over a random permutation of a pair of integer sequences. 100 | /// 101 | /// # Examples 102 | /// 103 | /// Suppose you have two lists, first with 3. elements and the second with 7 elements, 104 | /// and you want to iterate over a random permutation of pairs: 105 | /// 106 | /// ``` 107 | /// use permutation_iterator::RandomPairPermutor; 108 | /// 109 | /// let pair_permutor = RandomPairPermutor::new(3, 7).expect("expected new RandomPairPermutor"); 110 | /// for (i, j) in pair_permutor { 111 | /// println!("({}, {})", i, j); 112 | /// } 113 | /// ``` 114 | /// 115 | pub struct RandomPairPermutor { 116 | permutor: Permutor, 117 | max2: u64, 118 | } 119 | 120 | impl RandomPairPermutor { 121 | pub fn new(max1: u64, max2: u64) -> anyhow::Result { 122 | let max: u128 = (max1 as u128) * (max2 as u128); 123 | Ok(RandomPairPermutor { 124 | permutor: Permutor::new(max)?, 125 | max2, 126 | }) 127 | } 128 | } 129 | 130 | impl Iterator for RandomPairPermutor { 131 | type Item = (u64, u64); 132 | 133 | fn next(&mut self) -> Option { 134 | match self.permutor.next() { 135 | Some(value) => { 136 | let first = value as u64 / self.max2; 137 | let second = value as u64 % self.max2; 138 | Some((first, second)) 139 | } 140 | _ => None, 141 | } 142 | } 143 | } 144 | 145 | /// Implements a Feistel network, which can take a non-invertible pseudo-random function (PRF) 146 | /// and turn it into an invertible pseudo-random permutation (PRP). 147 | /// 148 | /// If you use this struct directly note that its intended purpose is to be a PRP and map from 149 | /// an n-bit input to an n-bit output, where n is an even positive integer. For example, if 150 | /// constructed with a `max` of `10`, internally it creates a 4-bit Feistel network, and for all 151 | /// integers in the 4-bit domain `[0, 16)` (`0` inclusive to `16` exclusive) it will map an input 152 | /// to one and only one output, and vice-versa (a given output maps to one and only one input). 153 | /// Even though you specified a max value of `10`, the output range may be larger than expected. 154 | /// Clients like `RandomPermutor` handle this by excluding output values outside of the desired 155 | /// range. 156 | /// 157 | /// This is useful in fields like cryptography, where a block cipher is a PRP. 158 | /// 159 | /// Another great use of a Feistel network is when you want some input to always map to one and only 160 | /// one output (and vice versa). For example, given a 32-bit IP address, we could use some secret 161 | /// key and map each IP address to some other 32-bit IP address. We could log this new 32-bit 162 | /// IP address and people who do not know what the secret key is would find it difficult 163 | /// to determine what the input IP address was. This is Format Preserving Encryption (FPE). 164 | pub struct FeistelNetwork { 165 | /// TODO visible just for testing, fix 166 | pub half_width: u128, 167 | 168 | /// Mask used to keep within the width for the right. 169 | /// TODO visible just for testing, fix 170 | pub right_mask: u128, 171 | 172 | /// Mask used to keep within the width for the left. 173 | /// TODO visible just for testing, fix 174 | pub left_mask: u128, 175 | 176 | /// Private key, some random seed. 256 bits as 32 bytes. 177 | key: [u8; 32], 178 | 179 | rounds: u8, 180 | } 181 | 182 | impl FeistelNetwork { 183 | /// Create a new FeistelNetwork instance that can give you a random permutation of 184 | /// integers. 185 | /// 186 | /// Note that the value of max is rounded up to the nearest even power of 2. If clients are 187 | /// trying to get a permutation of [0, max) they need to iterate over the input range and 188 | /// discard values from FeistelNetwork >= max. 189 | /// 190 | /// The key used for the permutation is made up of securely gathered 32 bytes. 191 | pub fn new(max: u128) -> anyhow::Result { 192 | let mut key: [u8; 32] = [0; 32]; 193 | getrandom::getrandom(&mut key).map_err(anyhow::Error::msg)?; 194 | Ok(FeistelNetwork::new_with_slice_key(max, key)) 195 | } 196 | 197 | /// Create a new FeistelNetwork instance that can give you a random permutation of 198 | /// integers. 199 | /// 200 | /// Note that the value of max is rounded up to the nearest even power of 2. If clients are 201 | /// trying to get a permutation of [0, max) they need to iterate over the input range and 202 | /// discard values from FeistelNetwork >= max. 203 | pub fn new_with_slice_key(max_value: u128, key: [u8; 32]) -> FeistelNetwork { 204 | let mut width = integer_log2(max_value).unwrap(); 205 | if width % 2 != 0 { 206 | width += 1; 207 | } 208 | let half_width = width / 2; 209 | let mut right_mask = 0; 210 | for i in 0..half_width { 211 | right_mask |= 1 << i; 212 | } 213 | let left_mask = right_mask << half_width; 214 | FeistelNetwork { 215 | half_width, 216 | right_mask, 217 | left_mask, 218 | key, 219 | rounds: 32, 220 | } 221 | } 222 | 223 | pub fn permute(&self, input: u128) -> u128 { 224 | let mut left = (input & self.left_mask) >> self.half_width; 225 | let mut right = input & self.right_mask; 226 | 227 | for i in 0..self.rounds as u8 { 228 | let new_left = right; 229 | let f = self.round_function(right, i, self.key, self.right_mask); 230 | right = left ^ f; 231 | left = new_left; 232 | } 233 | 234 | let result = (left << self.half_width) | right; 235 | result & (self.left_mask | self.right_mask) 236 | } 237 | 238 | fn round_function(&self, right: u128, round: u8, key: [u8; 32], mask: u128) -> u128 { 239 | let right_bytes = u128_to_16slice(right); 240 | let round_bytes = u8_to_1slice(round); 241 | 242 | let mut hasher = WyHash::default(); 243 | hasher.write(&key[..]); 244 | hasher.write(&right_bytes[..]); 245 | hasher.write(&round_bytes[..]); 246 | hasher.write(&key[..]); 247 | (hasher.finish() as u128) & mask 248 | } 249 | } 250 | 251 | fn u8_to_1slice(input: u8) -> [u8; 1] { 252 | let mut result: [u8; 1] = [0; 1]; 253 | result[0] = input; 254 | result 255 | } 256 | 257 | /// Convert an unsigned 128 bit number so a slice of 16 bytes in big-endian format (most significant 258 | /// bit first). 259 | /// 260 | /// # Examples 261 | /// 262 | /// ``` 263 | /// use crate::permutation_iterator::u128_to_16slice; 264 | /// let output = u128_to_16slice(42); 265 | /// assert_eq!(output, [0, 0, 0, 0, 0, 0, 0, 0, 266 | /// 0, 0, 0, 0, 0, 0, 0, 0x2A]); 267 | /// ``` 268 | pub fn u128_to_16slice(input: u128) -> [u8; 16] { 269 | let mut result: [u8; 16] = [0; 16]; 270 | result[15] = (input & 0xFF) as u8; 271 | result[14] = ((input >> 8) & 0xFF) as u8; 272 | result[13] = ((input >> 16) & 0xFF) as u8; 273 | result[12] = ((input >> 24) & 0xFF) as u8; 274 | result[11] = ((input >> 32) & 0xFF) as u8; 275 | result[10] = ((input >> 40) & 0xFF) as u8; 276 | result[9] = ((input >> 48) & 0xFF) as u8; 277 | result[8] = ((input >> 56) & 0xFF) as u8; 278 | result[7] = ((input >> 64) & 0xFF) as u8; 279 | result[6] = ((input >> 72) & 0xFF) as u8; 280 | result[5] = ((input >> 80) & 0xFF) as u8; 281 | result[4] = ((input >> 88) & 0xFF) as u8; 282 | result[3] = ((input >> 96) & 0xFF) as u8; 283 | result[2] = ((input >> 104) & 0xFF) as u8; 284 | result[1] = ((input >> 112) & 0xFF) as u8; 285 | result[0] = ((input >> 120) & 0xFF) as u8; 286 | result 287 | } 288 | /// Convert an unsigned 64 bit number so a slice of 8 bytes in big-endian format (most significant 289 | /// bit first). 290 | /// 291 | /// # Examples 292 | /// 293 | /// ``` 294 | /// use crate::permutation_iterator::u64_to_8slice; 295 | /// let output = u64_to_8slice(42); 296 | /// assert_eq!(output, [0, 0, 0, 0, 0, 0, 0, 0x2A]); 297 | /// ``` 298 | pub fn u64_to_8slice(input: u64) -> [u8; 8] { 299 | let mut result: [u8; 8] = [0; 8]; 300 | result[7] = (input & 0xFF) as u8; 301 | result[6] = ((input >> 8) & 0xFF) as u8; 302 | result[5] = ((input >> 16) & 0xFF) as u8; 303 | result[4] = ((input >> 24) & 0xFF) as u8; 304 | result[3] = ((input >> 32) & 0xFF) as u8; 305 | result[2] = ((input >> 40) & 0xFF) as u8; 306 | result[1] = ((input >> 48) & 0xFF) as u8; 307 | result[0] = ((input >> 56) & 0xFF) as u8; 308 | result 309 | } 310 | 311 | /// Convert an unsigned 64 bit number so a slice of 32 bytes in big-endian format (most significant 312 | /// bit first). 313 | /// 314 | /// # Examples 315 | /// 316 | /// ``` 317 | /// use crate::permutation_iterator::u64_to_32slice; 318 | /// let output = u64_to_32slice(42); 319 | /// assert_eq!(output, [0, 0, 0, 0, 0, 0, 0, 0x2A, 320 | /// 0, 0, 0, 0, 0, 0, 0, 0, 321 | /// 0, 0, 0, 0, 0, 0, 0, 0, 322 | /// 0, 0, 0, 0, 0, 0, 0, 0]); 323 | /// ``` 324 | pub fn u64_to_32slice(input: u64) -> [u8; 32] { 325 | let result8 = u64_to_8slice(input); 326 | let mut result: [u8; 32] = [0; 32]; 327 | result[..8].clone_from_slice(&result8[..8]); 328 | result 329 | } 330 | 331 | /// Calculate log2 of an integer input. This tells you how many bits are required to represent the 332 | /// input. 333 | /// 334 | /// # Examples 335 | /// 336 | /// ``` 337 | /// use permutation_iterator::integer_log2; 338 | /// assert_eq!(None, integer_log2(0), "failed for {}", 0); 339 | /// assert_eq!(Some(1), integer_log2(1), "failed for {}", 1); 340 | /// assert_eq!(Some(2), integer_log2(2), "failed for {}", 2); 341 | /// assert_eq!(Some(2), integer_log2(3), "failed for {}", 3); 342 | /// assert_eq!(Some(3), integer_log2(4), "failed for {}", 4); 343 | /// assert_eq!(Some(3), integer_log2(5), "failed for {}", 5); 344 | /// assert_eq!(Some(3), integer_log2(6), "failed for {}", 6); 345 | /// assert_eq!(Some(3), integer_log2(7), "failed for {}", 7); 346 | /// assert_eq!(Some(4), integer_log2(8), "failed for {}", 8); 347 | /// assert_eq!(Some(4), integer_log2(9), "failed for {}", 9); 348 | /// assert_eq!(Some(4), integer_log2(10), "failed for {}", 9); 349 | /// ``` 350 | pub fn integer_log2(input: u128) -> Option { 351 | if input == 0 { 352 | return None; 353 | } 354 | let mut result = 0; 355 | let mut input_copy = input; 356 | while input_copy > 0 { 357 | input_copy >>= 1; 358 | result += 1; 359 | } 360 | Some(result) 361 | } 362 | --------------------------------------------------------------------------------