├── test_cases └── grammar_regex_root.py ├── Cargo.toml ├── regex_mutator ├── Cargo.toml └── src │ └── lib.rs ├── grammartec ├── Cargo.toml └── src │ ├── lib.rs │ ├── newtypes.rs │ ├── recursion_info.rs │ ├── chunkstore.rs │ ├── rule.rs │ ├── context.rs │ ├── tree.rs │ └── mutator.rs ├── forksrv ├── Cargo.toml └── src │ ├── exitreason.rs │ ├── newtypes.rs │ ├── error.rs │ └── lib.rs ├── fuzzer ├── src │ ├── config.rs │ ├── python_grammar_loader.rs │ ├── shared_state.rs │ ├── generator.rs │ ├── mutation_tester.rs │ ├── queue.rs │ ├── state.rs │ ├── fuzzer.rs │ └── main.rs └── Cargo.toml ├── config.ron ├── grammars ├── grammar_py_example.py └── lua.py ├── LICENSE.md ├── test.c ├── README.md └── Cargo.lock /test_cases/grammar_regex_root.py: -------------------------------------------------------------------------------- 1 | ctx.regex("START","[a-z]+") 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "forksrv", 5 | "grammartec", 6 | "fuzzer", 7 | ] 8 | 9 | default-members = ["fuzzer"] 10 | -------------------------------------------------------------------------------- /regex_mutator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "regex_mutator" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | regex-syntax = "0.6.17" -------------------------------------------------------------------------------- /grammartec/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "grammartec" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | 6 | [dependencies] 7 | regex = "1.3.7" 8 | lazy_static= "1.4.0" 9 | rand = "0.7.3" 10 | hash_by_ref = "0.1.0" 11 | forksrv = {path = "../forksrv"} 12 | regex_mutator = {path = "../regex_mutator"} 13 | regex-syntax = "0.6.17" 14 | serde_derive = "1.0" 15 | serde = "1.0" 16 | loaded_dice = "0.2" 17 | num = "0.2.1" 18 | pyo3="0.21.1" 19 | -------------------------------------------------------------------------------- /forksrv/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "forksrv" 3 | version = "0.1.0" 4 | authors = ["coco ", "daniel daniel.teuchert@rub.de"] 5 | 6 | [dependencies] 7 | nix = "0.15.0" 8 | time = "0.1" 9 | subprocess = "0.2.4" 10 | regex = "1.3.7" 11 | lazy_static = "1.4.0" 12 | libc = "0.2.70" 13 | quick-error = "1.2.3" 14 | tempfile = "3.1.0" 15 | rand = "0.7.3" 16 | serde_derive = "1.0" 17 | serde = "1.0" 18 | byteorder = "1.3.4" 19 | snafu = "0.6.8" 20 | timeout-readwrite = "0.3.1" 21 | -------------------------------------------------------------------------------- /grammartec/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | #[macro_use] 5 | extern crate lazy_static; 6 | #[macro_use] 7 | extern crate serde_derive; 8 | extern crate forksrv; 9 | extern crate loaded_dice; 10 | extern crate num; 11 | extern crate pyo3; 12 | extern crate rand; 13 | extern crate regex; 14 | extern crate regex_mutator; 15 | extern crate regex_syntax; 16 | 17 | pub mod chunkstore; 18 | pub mod context; 19 | pub mod mutator; 20 | pub mod newtypes; 21 | pub mod recursion_info; 22 | pub mod rule; 23 | pub mod tree; 24 | -------------------------------------------------------------------------------- /fuzzer/src/config.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | #[derive(Deserialize, Clone)] 5 | pub struct Config { 6 | pub number_of_threads: u8, 7 | pub thread_size: usize, 8 | pub number_of_generate_inputs: u16, 9 | pub number_of_deterministic_mutations: usize, 10 | pub max_tree_size: usize, 11 | pub bitmap_size: usize, 12 | pub timeout_in_millis: u64, 13 | pub path_to_bin_target: String, 14 | pub path_to_grammar: String, 15 | pub path_to_workdir: String, 16 | pub arguments: Vec, 17 | } 18 | -------------------------------------------------------------------------------- /fuzzer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fuzzer" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | default-run = "fuzzer" 6 | 7 | [dependencies] 8 | nix = "0.17.0" 9 | time = "0.1" 10 | subprocess = "0.2.4" 11 | grammartec = {path = "../grammartec"} 12 | forksrv = {path = "../forksrv"} 13 | libc = "*" 14 | serde = "1.0" 15 | serde_json = "1.0" 16 | serde_derive = "1.0" 17 | argparse = "0.2.2" 18 | ron = "*" 19 | clap = "2.33.1" 20 | pyo3 = "0.21.2" 21 | 22 | [[bin]] 23 | name = "fuzzer" 24 | path = "src/main.rs" 25 | 26 | [[bin]] 27 | name = "generator" 28 | path = "src/generator.rs" 29 | 30 | [[bin]] 31 | name = "mutator" 32 | path = "src/mutation_tester.rs" 33 | -------------------------------------------------------------------------------- /forksrv/src/exitreason.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use nix::sys::wait::WaitStatus; 5 | 6 | #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] 7 | pub enum ExitReason { 8 | Normal(i32), 9 | Timeouted, 10 | Signaled(i32), 11 | Stopped(i32), 12 | } 13 | 14 | impl ExitReason { 15 | pub fn from_wait_status(status: WaitStatus) -> ExitReason { 16 | return match status { 17 | WaitStatus::Exited(_, return_value) => ExitReason::Normal(return_value), 18 | WaitStatus::Signaled(_, signal, _) => ExitReason::Signaled(signal as i32), 19 | WaitStatus::Stopped(_, signal) => ExitReason::Stopped(signal as i32), 20 | _ => panic!("Unknown WaitStatus: {:?}", status), 21 | }; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /config.ron: -------------------------------------------------------------------------------- 1 | Config( 2 | //You probably want to change the follwoing options 3 | //File Paths 4 | path_to_bin_target: "./test", 5 | arguments: [ "@@"], //"@@" will be exchanged with the path of a file containing the current input 6 | 7 | path_to_grammar: "grammars/grammar_py_example.py", 8 | path_to_workdir: "/tmp/workdir", 9 | 10 | number_of_threads: 1, 11 | timeout_in_millis: 200, 12 | 13 | 14 | //The rest of the options are probably not something you want to change... 15 | //Forkserver parameter 16 | bitmap_size: 65536, //1<<16 17 | 18 | //Thread Settings: 19 | thread_size: 4194304, 20 | 21 | //Mutation Settings 22 | number_of_generate_inputs: 100, //see main.rs fuzzing_thread 23 | max_tree_size: 1000, //see state.rs generate random 24 | number_of_deterministic_mutations: 1, //see main.rs process_input 25 | 26 | 27 | ) 28 | -------------------------------------------------------------------------------- /grammars/grammar_py_example.py: -------------------------------------------------------------------------------- 1 | 2 | # these 2 rules actually are used to solve test.c 3 | ctx.rule("START", "{CHAR}{START}") 4 | ctx.regex("CHAR","[a-z]") 5 | 6 | #ctx.rule(NONTERM: string, RHS: string|bytes) adds a rule NONTERM->RHS. We can use {NONTERM} in the RHS to request a recursion. 7 | ctx.rule("START","{XML_CONTENT}") 8 | 9 | ctx.rule("XML_CONTENT","{XML}{XML_CONTENT}") 10 | ctx.rule("XML_CONTENT","") 11 | 12 | #ctx.script(NONTERM:string, RHS: [string]], func) adds a rule NONTERM->func(*RHS). 13 | # In contrast to normal `rule`, RHS is an array of nonterminals. 14 | # It's up to the function to combine the values returned for the NONTERMINALS with any fixed content used. 15 | ctx.script("XML",["TAG","ATTR","XML_CONTENT"], lambda tag,attr,body: b"<%s %s>%s"%(tag,attr,body,tag) ) 16 | ctx.rule("ATTR","foo=bar") 17 | ctx.rule("TAG","some_tag") 18 | ctx.rule("TAG","other_tag") 19 | ctx.regex("TAG","[a-z]+") 20 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Sergej Schumilo, Daniel Teuchert, Corenlius Aschermann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /forksrv/src/newtypes.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use snafu::{Backtrace, Snafu}; 5 | 6 | use std::path::PathBuf; 7 | 8 | #[derive(Debug, Snafu)] 9 | #[snafu(visibility = "pub")] 10 | pub enum SubprocessError { 11 | #[snafu(display("Could not handle qemu trace file to {} {}", path.display(), source))] 12 | ReadQemuTrace { 13 | path: PathBuf, 14 | source: std::io::Error, 15 | }, 16 | 17 | #[snafu(display("Could not parse integer in {} {}", line, source))] 18 | ParseIntQemuTrace { 19 | line: String, 20 | source: std::num::ParseIntError, 21 | }, 22 | 23 | #[snafu(display("Could not parse line {}", line))] 24 | ParseLineQemuTrace { line: String, backtrace: Backtrace }, 25 | 26 | #[snafu(display("Qemu did not produce any output"))] 27 | NoQemuOutput { backtrace: Backtrace }, 28 | 29 | #[snafu(display("Could not communicate with QemuForkServer {} {} ", task, source))] 30 | QemuRunNix { task: String, source: nix::Error }, 31 | 32 | #[snafu(display("Could not communicate with QemuForkServer {} {} ", task, source))] 33 | QemuRunIO { 34 | task: String, 35 | source: std::io::Error, 36 | }, 37 | 38 | #[snafu(display("Could not disassemble {}", task))] 39 | DisassemblyError { task: String, backtrace: Backtrace }, 40 | } 41 | -------------------------------------------------------------------------------- /test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int main(int argc, char **argv) { 5 | 6 | char ptr[20]; 7 | if(argc>1){ 8 | FILE *fp = fopen(argv[1], "r"); 9 | fgets(ptr, sizeof(ptr), fp); 10 | } 11 | else{ 12 | fgets(ptr, sizeof(ptr), stdin); 13 | } 14 | printf("%s", ptr); 15 | if(ptr[0] == 'd') { 16 | if(ptr[1] == 'e') { 17 | if(ptr[2] == 'a') { 18 | if(ptr[3] == 'd') { 19 | if(ptr[4] == 'b') { 20 | if(ptr[5] == 'e') { 21 | if(ptr[6] == 'e') { 22 | if(ptr[7] == 'f') { 23 | abort(); 24 | } 25 | else printf("%c",ptr[7]); 26 | } 27 | else printf("%c",ptr[6]); 28 | } 29 | else printf("%c",ptr[5]); 30 | } 31 | else printf("%c",ptr[4]); 32 | } 33 | else printf("%c",ptr[3]); 34 | } 35 | else printf("%c",ptr[2]); 36 | } 37 | else printf("%c",ptr[1]); 38 | } 39 | else printf("%c",ptr[0]); 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /fuzzer/src/python_grammar_loader.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use pyo3::prelude::*; 5 | use pyo3::types::{IntoPyDict}; 6 | 7 | use crate::Context; 8 | 9 | #[pyclass] 10 | struct PyContext { 11 | ctx: Context, 12 | } 13 | impl PyContext { 14 | fn get_context(&self) -> Context { 15 | self.ctx.clone() 16 | } 17 | } 18 | 19 | #[pymethods] 20 | impl PyContext { 21 | #[new] 22 | fn new() -> Self { 23 | PyContext { 24 | ctx: Context::new(), 25 | } 26 | } 27 | 28 | fn rule(&mut self, py: Python, nt: &str, format: &PyAny) -> PyResult<()> { 29 | if let Ok(s) = format.extract::<&str>() { 30 | self.ctx.add_rule(nt, s.as_bytes()); 31 | } else if let Ok(s) = format.extract::<&[u8]>() { 32 | self.ctx.add_rule(nt, s); 33 | } else { 34 | return Err(pyo3::exceptions::PyValueError::new_err( 35 | "format argument should be string or bytes", 36 | )); 37 | } 38 | return Ok(()); 39 | } 40 | 41 | fn script(&mut self, nt: &str, nts: Vec, script: PyObject) { 42 | self.ctx.add_script(nt, nts, script); 43 | } 44 | 45 | fn regex(&mut self, nt: &str, regex: &str) { 46 | self.ctx.add_regex(nt, regex); 47 | } 48 | } 49 | 50 | fn main_(py: Python, grammar_path: &str) -> PyResult { 51 | let py_ctx = PyCell::new(py, PyContext::new()).unwrap(); 52 | let locals = [("ctx", py_ctx)].into_py_dict(py); 53 | py.run( 54 | &std::fs::read_to_string(grammar_path).expect("couldn't read grammar file"), 55 | None, 56 | Some(&locals), 57 | )?; 58 | return Ok(py_ctx.borrow().get_context()); 59 | } 60 | 61 | pub fn load_python_grammar(grammar_path: &str) -> Context { 62 | return Python::with_gil(|py| { 63 | main_(py, grammar_path) 64 | .map_err(|e| e.print_and_set_sys_last_vars(py)) 65 | .unwrap() 66 | }); 67 | } 68 | -------------------------------------------------------------------------------- /forksrv/src/error.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use nix; 5 | use std; 6 | 7 | quick_error! { 8 | #[derive(Debug, Clone)] 9 | pub enum SpawnError { 10 | Fork(err: nix::Error) { 11 | from() 12 | description("execve failed") 13 | display("execve error: {}", err) 14 | cause(err) 15 | } 16 | Path(desc: String){ 17 | description("Invalid Path") 18 | display("Problem with binary path: {}", desc) 19 | } 20 | 21 | Exec(desc: String){ 22 | description("Execution Failed") 23 | display("Execution failed: {}", desc) 24 | } 25 | 26 | FFINull(err: std::ffi::NulError) { 27 | from() 28 | description("argument/path contained Null byte") 29 | display("Null error: {}", err) 30 | cause(err) 31 | } 32 | DevNull(desc: String){ 33 | description("failed to open /dev/null") 34 | display("failed to open /dev/null: {}", desc) 35 | } 36 | } 37 | } 38 | 39 | pub fn path_err(desc: &str) -> Result { 40 | return Err(SpawnError::Path(desc.into())); 41 | } 42 | 43 | quick_error! { 44 | #[derive(Debug)] 45 | pub enum SubprocessError { 46 | Spawn(err: SpawnError) { 47 | from() 48 | description("spawning failed") 49 | display("spawning failed: {}", err) 50 | cause(err) 51 | } 52 | Unspecific(desc: String){ 53 | description("Subprocess Failed") 54 | display("Subprocess failed: {}", desc) 55 | } 56 | Io(err: std::io::Error){ 57 | from() 58 | cause(err) 59 | } 60 | Unix(err: nix::Error){ 61 | from() 62 | cause(err) 63 | } 64 | } 65 | } 66 | 67 | pub fn descr_err(desc: &str) -> Result { 68 | return Err(SubprocessError::Unspecific(desc.into())); 69 | } 70 | -------------------------------------------------------------------------------- /fuzzer/src/shared_state.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use queue::Queue; 5 | use std::collections::HashMap; 6 | 7 | pub struct GlobalSharedState { 8 | pub queue: Queue, 9 | //false for not crashing input. True for crashing inputs 10 | pub bitmaps: HashMap>, 11 | pub execution_count: u64, 12 | pub average_executions_per_sec: u32, 13 | pub bits_found_by_havoc: u64, 14 | pub bits_found_by_havoc_rec: u64, 15 | pub bits_found_by_min: u64, 16 | pub bits_found_by_min_rec: u64, 17 | pub bits_found_by_splice: u64, 18 | pub bits_found_by_det: u64, 19 | pub bits_found_by_gen: u64, 20 | pub asan_found_by_havoc: u64, 21 | pub asan_found_by_havoc_rec: u64, 22 | pub asan_found_by_min: u64, 23 | pub asan_found_by_min_rec: u64, 24 | pub asan_found_by_splice: u64, 25 | pub asan_found_by_det: u64, 26 | pub asan_found_by_gen: u64, 27 | pub last_found_asan: String, 28 | pub last_found_sig: String, 29 | pub last_timeout: String, 30 | pub state_saved: String, 31 | pub total_found_asan: u64, 32 | pub total_found_sig: u64, 33 | } 34 | 35 | impl GlobalSharedState { 36 | pub fn new(work_dir: String, bitmap_size: usize) -> Self { 37 | let queue = Queue::new(work_dir); 38 | //Initialize Empty bitmaps for crashes and normal executions 39 | let mut bitmaps = HashMap::new(); 40 | bitmaps.insert(false, vec![0; bitmap_size]); 41 | bitmaps.insert(true, vec![0; bitmap_size]); 42 | return GlobalSharedState { 43 | queue, 44 | bitmaps, 45 | execution_count: 0, 46 | average_executions_per_sec: 0, 47 | bits_found_by_havoc: 0, 48 | bits_found_by_havoc_rec: 0, 49 | bits_found_by_min: 0, 50 | bits_found_by_min_rec: 0, 51 | bits_found_by_splice: 0, 52 | bits_found_by_det: 0, 53 | bits_found_by_gen: 0, 54 | asan_found_by_havoc: 0, 55 | asan_found_by_havoc_rec: 0, 56 | asan_found_by_min: 0, 57 | asan_found_by_min_rec: 0, 58 | asan_found_by_splice: 0, 59 | asan_found_by_det: 0, 60 | asan_found_by_gen: 0, 61 | last_found_asan: String::from("Not found yet."), 62 | last_found_sig: String::from("Not found yet."), 63 | last_timeout: String::from("No Timeout yet."), 64 | state_saved: String::from("State not saved yet."), 65 | total_found_asan: 0, 66 | total_found_sig: 0, 67 | }; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /grammartec/src/newtypes.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use std::ops::Add; 5 | 6 | #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, Serialize, Deserialize)] 7 | pub struct RuleID(usize); 8 | 9 | #[derive(PartialEq, PartialOrd, Eq, Clone, Copy, Debug, Hash, Serialize, Deserialize)] 10 | pub struct NodeID(usize); 11 | 12 | #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, Serialize, Deserialize)] 13 | pub struct NTermID(usize); 14 | 15 | impl RuleID { 16 | pub fn to_i(&self) -> usize { 17 | self.0 18 | } 19 | } 20 | 21 | impl From for RuleID { 22 | fn from(i: usize) -> Self { 23 | return RuleID(i); 24 | } 25 | } 26 | 27 | impl Into for RuleID { 28 | fn into(self) -> usize { 29 | return self.0; 30 | } 31 | } 32 | 33 | impl Add for RuleID { 34 | type Output = RuleID; 35 | fn add(self, rhs: usize) -> RuleID { 36 | return RuleID(self.0 + rhs); 37 | } 38 | } 39 | 40 | impl NodeID { 41 | pub fn to_i(&self) -> usize { 42 | self.0 43 | } 44 | } 45 | 46 | impl From for NodeID { 47 | fn from(i: usize) -> Self { 48 | return NodeID(i); 49 | } 50 | } 51 | 52 | impl Into for NodeID { 53 | fn into(self) -> usize { 54 | return self.0; 55 | } 56 | } 57 | 58 | impl Add for NodeID { 59 | type Output = NodeID; 60 | fn add(self, rhs: usize) -> NodeID { 61 | return NodeID(self.0 + rhs); 62 | } 63 | } 64 | 65 | //impl Step for NodeID { 66 | // fn steps_between(start: &Self, end: &Self) -> Option { 67 | // let start_i = start.to_i(); 68 | // let end_i = end.to_i(); 69 | // if start > end { 70 | // return None; 71 | // } 72 | // return Some(end_i - start_i); 73 | // } 74 | // fn forward_checked(start: Self, n: usize) -> Option { 75 | // return start.0.checked_add(n).map(NodeID) 76 | // } 77 | // fn backward_checked(start: Self, n: usize) -> Option { 78 | // return start.0.checked_sub(n).map(NodeID); 79 | // } 80 | //} 81 | 82 | impl NTermID { 83 | pub fn to_i(&self) -> usize { 84 | self.0 85 | } 86 | } 87 | 88 | impl From for NTermID { 89 | fn from(i: usize) -> Self { 90 | return NTermID(i); 91 | } 92 | } 93 | 94 | impl Into for NTermID { 95 | fn into(self) -> usize { 96 | return self.0; 97 | } 98 | } 99 | 100 | impl Add for NTermID { 101 | type Output = NTermID; 102 | fn add(self, rhs: usize) -> NTermID { 103 | return NTermID(self.0 + rhs); 104 | } 105 | } 106 | 107 | #[cfg(test)] 108 | mod tests { 109 | use newtypes::NTermID; 110 | use newtypes::NodeID; 111 | use newtypes::RuleID; 112 | 113 | #[test] 114 | fn rule_id() { 115 | let r1: RuleID = 1337.into(); 116 | let r2 = RuleID::from(1338); 117 | let i1: usize = r1.into(); 118 | assert_eq!(i1, 1337); 119 | let i2: usize = 1338; 120 | assert_eq!(i2, r2.into()); 121 | let r3 = r2 + 3; 122 | assert_eq!(r3, 1341.into()); 123 | } 124 | 125 | #[test] 126 | fn node_id() { 127 | let r1: NodeID = 1337.into(); 128 | let r2 = NodeID::from(1338); 129 | let i1: usize = r1.into(); 130 | assert_eq!(i1, 1337); 131 | let i2: usize = 1338; 132 | assert_eq!(i2, r2.into()); 133 | let r3 = r2 + 3; 134 | assert_eq!(r3, 1341.into()); 135 | } 136 | 137 | #[test] 138 | fn nterm_id() { 139 | let r1: NTermID = 1337.into(); 140 | let r2 = NTermID::from(1338); 141 | let i1: usize = r1.into(); 142 | assert_eq!(i1, 1337); 143 | let i2: usize = 1338; 144 | assert_eq!(i2, r2.into()); 145 | let r3 = r2 + 3; 146 | assert_eq!(r3, 1341.into()); 147 | } 148 | #[test] 149 | fn test_node_id_trait_step_impl() { 150 | let x = 1337; 151 | let y = 1360; 152 | let r1: NodeID = x.into(); 153 | let r2 = NodeID::from(y); 154 | let mut sum_from_nodes = 0; 155 | for node in r1..r2 { 156 | sum_from_nodes += node.to_i(); 157 | } 158 | let mut sum_from_ints = 0; 159 | for i in x..y { 160 | sum_from_ints += i; 161 | } 162 | assert_eq!(sum_from_ints, sum_from_nodes); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /fuzzer/src/generator.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | #[macro_use] 5 | extern crate clap; 6 | extern crate grammartec; 7 | extern crate pyo3; 8 | extern crate ron; 9 | extern crate serde_json; 10 | 11 | mod python_grammar_loader; 12 | use grammartec::context::Context; 13 | use grammartec::tree::TreeLike; 14 | 15 | use clap::{App, Arg}; 16 | use std::fs; 17 | use std::fs::File; 18 | use std::io::{self, Write}; 19 | use std::path::Path; 20 | 21 | fn main() { 22 | 23 | pyo3::prepare_freethreaded_python(); 24 | //Parse parameters 25 | let matches = App::new("generator") 26 | .about("Generate strings using a grammar. This can also be used to generate a corpus") 27 | .arg(Arg::with_name("grammar_path") 28 | .short("g") 29 | .value_name("GRAMMAR") 30 | .takes_value(true) 31 | .required(true) 32 | .help("Path to grammar")) 33 | .arg(Arg::with_name("tree_depth") 34 | .short("t") 35 | .value_name("DEPTH") 36 | .takes_value(true) 37 | .required(true) 38 | .help("Size of trees that are generated")) 39 | .arg(Arg::with_name("number_of_trees") 40 | .short("n") 41 | .value_name("NUMBER") 42 | .takes_value(true) 43 | .help("Number of trees to generate [default: 1]")) 44 | .arg(Arg::with_name("store") 45 | .short("s") 46 | .help("Store output to files. This will create a folder called corpus containing one file for each generated tree.")) 47 | .arg(Arg::with_name("verbose") 48 | .short("v") 49 | .help("Be verbose")) 50 | .get_matches(); 51 | 52 | let grammar_path = matches 53 | .value_of("grammar_path") 54 | .expect("grammar_path is a required parameter") 55 | .to_string(); 56 | let tree_depth = 57 | value_t!(matches, "tree_depth", usize).expect("tree_depth is a requried parameter"); 58 | let number_of_trees = value_t!(matches, "number_of_trees", usize).unwrap_or(1); 59 | let store = matches.is_present("store"); 60 | let verbose = matches.is_present("verbose"); 61 | 62 | let mut ctx = Context::new(); 63 | //Create new Context and saved it 64 | if grammar_path.ends_with(".json") { 65 | let gf = File::open(grammar_path).expect("cannot read grammar file"); 66 | let rules: Vec> = 67 | serde_json::from_reader(&gf).expect("cannot parse grammar file"); 68 | assert!(rules.len() > 0, "rule file didn_t include any rules"); 69 | let root = "{".to_string() + &rules[0][0] + "}"; 70 | ctx.add_rule("START", root.as_bytes()); 71 | for rule in rules { 72 | ctx.add_rule(&rule[0], rule[1].as_bytes()); 73 | } 74 | } else if grammar_path.ends_with(".py") { 75 | ctx = python_grammar_loader::load_python_grammar(&grammar_path); 76 | } else { 77 | panic!("Unknown grammar type"); 78 | } 79 | 80 | ctx.initialize(tree_depth); 81 | 82 | //Generate Tree 83 | if store { 84 | if Path::new("corpus").exists() { 85 | } else { 86 | fs::create_dir("corpus").expect("Could not create corpus directory"); 87 | } 88 | } 89 | for i in 0..number_of_trees { 90 | let nonterm = ctx.nt_id("START"); 91 | let len = ctx.get_random_len_for_nt(&nonterm); 92 | let generated_tree = ctx.generate_tree_from_nt(nonterm, len); //1 is the index of the "START" Node 93 | if verbose { 94 | println!("Generating tree {} from {}", i + 1, number_of_trees); 95 | } 96 | if store { 97 | let mut output = 98 | File::create(&format!("corpus/{}", i + 1)).expect("cannot create output file"); 99 | generated_tree.unparse_to(&ctx, &mut output); 100 | } else { 101 | let stdout = io::stdout(); 102 | let mut stdout_handle = stdout.lock(); 103 | generated_tree.unparse_to(&ctx, &mut stdout_handle); 104 | } 105 | 106 | let mut of_tree = File::create(&"/tmp/test_tree.ron").expect("cannot create output file"); 107 | of_tree 108 | .write_all( 109 | ron::ser::to_string(&generated_tree) 110 | .expect("Serialization of Tree failed!") 111 | .as_bytes(), 112 | ) 113 | .expect("Writing to tree file failed"); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /grammartec/src/recursion_info.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use rand::Rng; 5 | use rand::thread_rng; 6 | use rand::SeedableRng; 7 | use rand::rngs::StdRng; 8 | use std::collections::HashMap; 9 | use std::fmt; 10 | 11 | use context::Context; 12 | use loaded_dice::LoadedDiceSampler; 13 | use newtypes::{NTermID, NodeID}; 14 | use tree::Tree; 15 | 16 | pub struct RecursionInfo { 17 | recursive_parents: HashMap, 18 | sampler: LoadedDiceSampler, 19 | depth_by_offset: Vec, 20 | node_by_offset: Vec, 21 | } 22 | 23 | impl fmt::Debug for RecursionInfo { 24 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 25 | f.debug_struct("RecursionInfo") 26 | .field("recursive_parents", &self.recursive_parents) 27 | .field("depth_by_offset", &self.depth_by_offset) 28 | .field("node_by_offset", &self.node_by_offset) 29 | .finish() 30 | } 31 | } 32 | 33 | impl RecursionInfo { 34 | pub fn new(t: &Tree, n: NTermID, ctx: &Context) -> Option { 35 | let (recursive_parents, node_by_offset, depth_by_offset) = 36 | RecursionInfo::find_parents(&t, n, ctx)?; 37 | let sampler = RecursionInfo::build_sampler(&depth_by_offset); 38 | return Some(Self { 39 | recursive_parents, 40 | sampler, 41 | node_by_offset, 42 | depth_by_offset, 43 | }); 44 | } 45 | 46 | // constructs a tree where each node points to the first ancestor with the same nonterminal (e.g. each node points the next node above it, were the pair forms a recursive occurance of a nonterminal). 47 | // This structure is an ''inverted tree''. We use it later to sample efficiently from the set 48 | // of all possible recursive pairs without occuring n^2 overhead. Additionally, we return a 49 | // ordered vec of all nodes with nonterminal n and the depth of this node in the freshly 50 | // constructed 'recursion tree' (weight). Each node is the end point of exactly `weigth` many 51 | // differnt recursions. Therefore we use the weight of the node to sample the endpoint of a path trough the 52 | // recursion tree. Then we just sample the length of this path uniformly as (1.. weight). This 53 | // yields a uniform sample from the whole set of recursions inside the tree. If you read this, Good luck you are on your own. 54 | fn find_parents( 55 | t: &Tree, 56 | nt: NTermID, 57 | ctx: &Context, 58 | ) -> Option<(HashMap, Vec, Vec)> { 59 | let mut stack = vec![(None, 0)]; 60 | let mut res = None; 61 | for (i, rule) in t.rules.iter().enumerate() { 62 | let node = NodeID::from(i); 63 | let (mut maybe_parent, depth) = stack.pop().expect("RAND_3404900492"); 64 | if ctx.get_nt(rule) == nt { 65 | if let Some(parent) = maybe_parent { 66 | let (mut parents, mut ids, mut weights) = 67 | res.unwrap_or_else(|| (HashMap::new(), vec![], vec![])); 68 | parents.insert(node, parent); 69 | ids.push(node); 70 | weights.push(depth); 71 | res = Some((parents, ids, weights)); 72 | } 73 | maybe_parent = Some(node) 74 | } 75 | for _ in 0..ctx.get_num_children(rule) { 76 | stack.push((maybe_parent, depth + 1)); 77 | } 78 | } 79 | return res; 80 | } 81 | 82 | fn build_sampler(depths: &Vec) -> LoadedDiceSampler { 83 | let mut weights = depths.iter().map(|x| *x as f64).collect::>(); 84 | let norm: f64 = weights.iter().sum(); 85 | assert!(norm > 0.0); 86 | for v in weights.iter_mut() { 87 | *v /= norm; 88 | } 89 | return LoadedDiceSampler::new(weights, StdRng::from_rng(thread_rng()).expect("RAND_1769941938")); 90 | } 91 | 92 | pub fn get_random_recursion_pair(&mut self) -> (NodeID, NodeID) { 93 | let offset = self.sampler.sample(); 94 | return self.get_recursion_pair_by_offset(offset); 95 | } 96 | 97 | pub fn get_recursion_pair_by_offset(&self, offset: usize) -> (NodeID, NodeID) { 98 | let node1 = self.node_by_offset[offset]; 99 | let mut node2 = node1; 100 | for _ in 0..(self.depth_by_offset[offset]) { 101 | node2 = self.recursive_parents[&node1]; 102 | } 103 | return (node2, node1); 104 | } 105 | 106 | pub fn get_number_of_recursions(&self) -> usize { 107 | return self.node_by_offset.len(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /fuzzer/src/mutation_tester.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | extern crate grammartec; 5 | extern crate ron; 6 | extern crate serde_json; 7 | 8 | use grammartec::chunkstore::ChunkStore; 9 | use grammartec::context::Context; 10 | use grammartec::mutator::Mutator; 11 | use grammartec::tree::{Tree, TreeLike, TreeMutation}; 12 | 13 | use std::env; 14 | use std::fs::File; 15 | use std::io::{self, Read}; 16 | 17 | enum MutationMethods { 18 | Havoc, 19 | HavocRec, 20 | Splice, 21 | } 22 | 23 | fn main() { 24 | //Parse parameters 25 | if env::args().len() != 5 { 26 | println!("Usage: generator tree_size path_to_serialized_tree path_to_grammar mutation_method(havoc, rec, splice)"); 27 | } else { 28 | let tree_depth = env::args() 29 | .nth(1) 30 | .expect("RAND_1541841394") 31 | .parse::() 32 | .expect("RAND_1541841394"); 33 | let tree_path = env::args().nth(2).expect("RAND_2127457155"); 34 | let grammar_path = env::args().nth(3).expect("RAND_418548630"); 35 | let method = match env::args().nth(4).expect("RAND_1161906828").as_ref() { 36 | "havoc" => MutationMethods::Havoc, 37 | "rec" => MutationMethods::HavocRec, 38 | "splice" => MutationMethods::Splice, 39 | _ => { 40 | panic!("Please use havoc, rec, or splice"); 41 | } 42 | }; 43 | let mut ctx = Context::new(); 44 | 45 | //Generate rules using an antlr grammar: 46 | if grammar_path.ends_with(".json") { 47 | let gf = File::open(grammar_path).expect("cannot read grammar file"); 48 | let rules: Vec<(String, String)> = 49 | serde_json::from_reader(&gf).expect("cannot parse grammar file"); 50 | let root = "{".to_string() + &rules[0].0 + "}"; 51 | ctx.add_rule("START", root.as_bytes()); 52 | for rule in rules { 53 | ctx.add_rule(&rule.0, &rule.1.as_bytes()); 54 | } 55 | } else { 56 | panic!("Unknown grammar type"); 57 | } 58 | 59 | //Deserialize tree 60 | let mut sf = File::open(&tree_path).expect("cannot read tree file"); 61 | let mut tree_as_string = String::new(); 62 | sf.read_to_string(&mut tree_as_string) 63 | .expect("RAND_421233044"); 64 | let mut tree: Tree = 65 | ron::de::from_str(&tree_as_string).expect("Failed to deserialize tree"); 66 | 67 | //Initialize Context 68 | ctx.initialize(tree_depth); 69 | 70 | println!( 71 | "Original tree:\nRules: {:?}\nSizes: {:?}\nParents: {:?}\nUnparsed original tree: ", 72 | tree.rules, tree.sizes, tree.paren 73 | ); 74 | { 75 | let stdout = io::stdout(); 76 | let mut stdout_handle = stdout.lock(); 77 | tree.unparse_to(&ctx, &mut stdout_handle); 78 | } 79 | print!("\n"); 80 | let mut mutator = Mutator::new(&ctx); 81 | let mut tester = |tree_mut: &TreeMutation, ctx: &Context| { 82 | println!("prefix: {:?}", tree_mut.prefix); 83 | println!("repl: {:?}", tree_mut.repl); 84 | println!("postfix: {:?}", tree_mut.postfix); 85 | let mutated_tree = tree_mut.to_tree(ctx); 86 | println!( 87 | "Mutated tree:\nRules: {:?}\nSizes: {:?}\nParents: {:?}\nUnparsed original tree: ", 88 | mutated_tree.rules, mutated_tree.sizes, mutated_tree.paren 89 | ); 90 | let stdout = io::stdout(); 91 | let mut stdout_handle = stdout.lock(); 92 | mutated_tree.unparse_to(&ctx, &mut stdout_handle); 93 | return Ok(()); 94 | }; 95 | match method { 96 | MutationMethods::Havoc => mutator 97 | .mut_random(&mut tree, &ctx, &mut tester) 98 | .expect("RAND_1926416364"), 99 | MutationMethods::HavocRec => { 100 | if let Some(ref mut recursions) = tree.calc_recursions(&ctx) { 101 | mutator 102 | .mut_random_recursion(&mut tree, recursions, &ctx, &mut tester) 103 | .expect("RAND_1905760160"); 104 | } 105 | } 106 | MutationMethods::Splice => { 107 | let mut cks = ChunkStore::new("/tmp/".to_string()); 108 | cks.add_tree(tree.clone(), &ctx); 109 | mutator 110 | .mut_splice(&mut tree, &ctx, &mut cks, &mut tester) 111 | .expect("RAND_842617595"); 112 | } 113 | } 114 | print!("\n"); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /grammartec/src/chunkstore.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use rand::{thread_rng}; 5 | use rand::seq::IteratorRandom; 6 | use std::collections::HashMap; 7 | use std::collections::HashSet; 8 | use std::fs::File; 9 | use std::io::Write; 10 | use std::sync::atomic::AtomicBool; 11 | use std::sync::RwLock; 12 | 13 | use context::Context; 14 | use newtypes::{NTermID, NodeID, RuleID}; 15 | use rule::RuleIDOrCustom; 16 | use tree::{Tree, TreeLike}; 17 | 18 | pub struct ChunkStoreWrapper { 19 | pub chunkstore: RwLock, 20 | pub is_locked: AtomicBool, 21 | } 22 | impl ChunkStoreWrapper { 23 | pub fn new(work_dir: String) -> Self { 24 | return ChunkStoreWrapper { 25 | chunkstore: RwLock::new(ChunkStore::new(work_dir)), 26 | is_locked: AtomicBool::new(false), 27 | }; 28 | } 29 | } 30 | 31 | #[derive(Serialize, Deserialize)] 32 | pub struct ChunkStore { 33 | nts_to_chunks: HashMap>, 34 | seen_outputs: HashSet>, 35 | trees: Vec, 36 | work_dir: String, 37 | number_of_chunks: usize, 38 | } 39 | 40 | impl ChunkStore { 41 | pub fn new(work_dir: String) -> Self { 42 | return ChunkStore { 43 | nts_to_chunks: HashMap::new(), 44 | seen_outputs: HashSet::new(), 45 | trees: vec![], 46 | work_dir: work_dir, 47 | number_of_chunks: 0, 48 | }; 49 | } 50 | 51 | pub fn add_tree(&mut self, tree: Tree, ctx: &Context) { 52 | let mut buffer = vec![]; 53 | let id = self.trees.len(); 54 | let mut contains_new_chunk = false; 55 | for i in 0..tree.size() { 56 | buffer.truncate(0); 57 | if tree.sizes[i] > 30 { 58 | continue; 59 | } 60 | let n = NodeID::from(i); 61 | tree.unparse(n, ctx, &mut buffer); 62 | if !self.seen_outputs.contains(&buffer) { 63 | self.seen_outputs.insert(buffer.clone()); 64 | self.nts_to_chunks 65 | .entry(tree.get_rule(n, ctx).nonterm()) 66 | .or_insert_with(|| vec![]) 67 | .push((id, n)); 68 | let mut file = File::create(format!( 69 | "{}/outputs/chunks/chunk_{:09}", 70 | self.work_dir, self.number_of_chunks 71 | )) 72 | .expect("RAND_596689790"); 73 | self.number_of_chunks += 1; 74 | file.write(&buffer).expect("RAND_606896756"); 75 | contains_new_chunk = true; 76 | } 77 | } 78 | if contains_new_chunk { 79 | self.trees.push(tree); 80 | } 81 | } 82 | 83 | pub fn get_alternative_to<'a>(&'a self, r: RuleID, ctx: &Context) -> Option<(&Tree, NodeID)> { 84 | let chunks = self 85 | .nts_to_chunks 86 | .get(&ctx.get_nt(&RuleIDOrCustom::Rule(r))); 87 | let relevant = chunks.map(|vec| { 88 | vec.iter() 89 | .filter(move |&&(tid, nid)| self.trees[tid].get_rule_id(nid) != r) 90 | }); 91 | //The unwrap_or is just a quick and dirty fix to catch Errors from the sampler 92 | let selected = relevant.and_then(|iter| { 93 | iter.choose(&mut thread_rng()) 94 | }); 95 | return selected.map(|&(tid, nid)| (&self.trees[tid], nid)); 96 | } 97 | 98 | pub fn trees(&self) -> usize { 99 | return self.trees.len(); 100 | } 101 | } 102 | 103 | #[cfg(test)] 104 | mod tests { 105 | use chunkstore::ChunkStore; 106 | use context::Context; 107 | use std::fs; 108 | use tree::TreeLike; 109 | 110 | #[test] 111 | fn chunk_store() { 112 | let mut ctx = Context::new(); 113 | let r1 = ctx.add_rule("A", b"a {B:a}"); 114 | let r2 = ctx.add_rule("B", b"b {C:a}"); 115 | let _ = ctx.add_rule("C", b"c"); 116 | ctx.initialize(101); 117 | let random_size = ctx.get_random_len_for_ruleid(&r1); 118 | println!("random_size: {}", random_size); 119 | let tree = ctx.generate_tree_from_rule(r1, random_size); 120 | fs::create_dir_all("/tmp/outputs/chunks").expect("40234068"); 121 | let mut cks = ChunkStore::new("/tmp/".to_string()); 122 | cks.add_tree(tree, &ctx); 123 | // assert!(cks.seen_outputs.contains("a b c".as_bytes())); 124 | // assert!(cks.seen_outputs.contains("b c".as_bytes())); 125 | // assert!(cks.seen_outputs.contains("c".as_bytes())); 126 | assert_eq!(cks.nts_to_chunks[&ctx.nt_id("A")].len(), 1); 127 | let (tree_id, _) = cks.nts_to_chunks[&ctx.nt_id("A")][0]; 128 | assert_eq!(cks.trees[tree_id].unparse_to_vec(&ctx), "a b c".as_bytes()); 129 | 130 | let random_size = ctx.get_random_len_for_ruleid(&r2); 131 | let tree = ctx.generate_tree_from_rule(r2, random_size); 132 | cks.add_tree(tree, &ctx); 133 | // assert_eq!(cks.seen_outputs.len(), 3); 134 | // assert_eq!(cks.nts_to_chunks[&ctx.nt_id("B")].len(), 1); 135 | let (tree_id, node_id) = cks.nts_to_chunks[&ctx.nt_id("B")][0]; 136 | assert_eq!( 137 | cks.trees[tree_id].unparse_node_to_vec(node_id, &ctx), 138 | "b c".as_bytes() 139 | ); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /regex_mutator/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | extern crate regex_syntax; 5 | 6 | use regex_syntax::hir::{ 7 | Class, ClassBytesRange, ClassUnicodeRange, Hir, Literal, RepetitionKind, RepetitionRange, 8 | }; 9 | 10 | pub struct RomuPrng { 11 | xstate: u64, 12 | ystate: u64, 13 | } 14 | 15 | impl RomuPrng { 16 | pub fn new(xstate: u64, ystate: u64) -> Self { 17 | return Self { xstate, ystate }; 18 | } 19 | 20 | pub fn range(&mut self, min: usize, max: usize) -> usize { 21 | return ((self.next_u64() as usize) % (max - min)) + min; 22 | } 23 | 24 | pub fn new_from_u64(seed: u64) -> Self { 25 | let mut res = Self::new(seed, seed ^ 0xec77152282650854); 26 | for _ in 0..4 { 27 | res.next_u64(); 28 | } 29 | return res; 30 | } 31 | 32 | pub fn next_u32(&mut self) -> u32 { 33 | self.next_u64() as u32 34 | } 35 | 36 | pub fn next_u64(&mut self) -> u64 { 37 | let xp = self.xstate; 38 | self.xstate = 15241094284759029579u64.wrapping_mul(self.ystate); 39 | self.ystate = self.ystate.wrapping_sub(xp); 40 | self.ystate = self.ystate.rotate_left(27); 41 | return xp; 42 | } 43 | } 44 | 45 | pub struct RegexScript { 46 | rng: RomuPrng, 47 | remaining: usize, 48 | } 49 | 50 | impl RegexScript { 51 | pub fn new(seed: u64) -> Self { 52 | let mut rng = RomuPrng::new_from_u64(seed); 53 | 54 | let len = if rng.next_u64() % 256 == 0 { 55 | rng.next_u64() % 0xffff 56 | } else { 57 | let len = 1 << (rng.next_u64() % 8); 58 | rng.next_u64() % len 59 | }; 60 | RegexScript { 61 | rng, 62 | remaining: len as usize, 63 | } 64 | } 65 | 66 | pub fn get_mod(&mut self, val: usize) -> usize { 67 | if self.remaining == 0 { 68 | return 0; 69 | } 70 | return (self.rng.next_u32() as usize) % val; 71 | } 72 | 73 | pub fn get_range(&mut self, min: usize, max: usize) -> usize { 74 | return self.get_mod(max - min) + min; 75 | } 76 | } 77 | 78 | fn append_char(res: &mut Vec, chr: char) { 79 | let mut buf = [0; 4]; 80 | res.extend_from_slice(chr.encode_utf8(&mut buf).as_bytes()) 81 | } 82 | 83 | fn append_lit(res: &mut Vec, lit: &Literal) { 84 | use regex_syntax::hir::Literal::*; 85 | 86 | match lit { 87 | Unicode(chr) => append_char(res, *chr), 88 | Byte(b) => res.push(*b), 89 | } 90 | } 91 | 92 | fn append_unicode_range(res: &mut Vec, scr: &mut RegexScript, cls: &ClassUnicodeRange) { 93 | let mut chr_a_buf = [0; 4]; 94 | let mut chr_b_buf = [0; 4]; 95 | cls.start().encode_utf8(&mut chr_a_buf); 96 | cls.end().encode_utf8(&mut chr_b_buf); 97 | let a = u32::from_le_bytes(chr_a_buf); 98 | let b = u32::from_le_bytes(chr_b_buf); 99 | let c = scr.get_range(a as usize, (b + 1) as usize) as u32; 100 | append_char(res, std::char::from_u32(c).unwrap()); 101 | } 102 | 103 | fn append_byte_range(res: &mut Vec, scr: &mut RegexScript, cls: &ClassBytesRange) { 104 | res.push(scr.get_range(cls.start() as usize, (cls.end() + 1) as usize) as u8); 105 | } 106 | 107 | fn append_class(res: &mut Vec, scr: &mut RegexScript, cls: &Class) { 108 | use regex_syntax::hir::Class::*; 109 | match cls { 110 | Unicode(cls) => { 111 | let rngs = cls.ranges(); 112 | let rng = rngs[scr.get_mod(rngs.len())]; 113 | append_unicode_range(res, scr, &rng); 114 | } 115 | Bytes(cls) => { 116 | let rngs = cls.ranges(); 117 | let rng = rngs[scr.get_mod(rngs.len())]; 118 | append_byte_range(res, scr, &rng); 119 | } 120 | } 121 | } 122 | 123 | fn get_length(scr: &mut RegexScript) -> usize { 124 | let bits = scr.get_mod(8); 125 | return scr.get_mod(2 << bits); 126 | } 127 | 128 | fn get_repetition_range(rep: &RepetitionRange, scr: &mut RegexScript) -> usize { 129 | use regex_syntax::hir::RepetitionRange::*; 130 | match rep { 131 | Exactly(a) => return *a as usize, 132 | AtLeast(a) => return get_length(scr) + (*a as usize), 133 | Bounded(a, b) => return scr.get_range(*a as usize, *b as usize), 134 | } 135 | } 136 | 137 | fn get_repetitions(rep: &RepetitionKind, scr: &mut RegexScript) -> usize { 138 | use regex_syntax::hir::RepetitionKind::*; 139 | match rep { 140 | ZeroOrOne => return scr.get_mod(2), 141 | ZeroOrMore => return get_length(scr), 142 | OneOrMore => return 1 + get_length(scr), 143 | Range(rng) => get_repetition_range(rng, scr), 144 | } 145 | } 146 | 147 | pub fn generate(hir: &Hir, seed: u64) -> Vec { 148 | use regex_syntax::hir::HirKind::*; 149 | let mut scr = RegexScript::new(seed); 150 | let mut stack = vec![hir]; 151 | let mut res = vec![]; 152 | while stack.len() > 0 { 153 | match stack.pop().unwrap().kind() { 154 | Empty => {} 155 | Literal(lit) => append_lit(&mut res, lit), 156 | Class(cls) => append_class(&mut res, &mut scr, cls), 157 | Anchor(_) => {} 158 | WordBoundary(_) => {} 159 | Repetition(rep) => { 160 | let num = get_repetitions(&rep.kind, &mut scr); 161 | for _ in 0..num { 162 | stack.push(&rep.hir); 163 | } 164 | } 165 | Group(grp) => stack.push(&grp.hir), 166 | Concat(hirs) => hirs.iter().rev().for_each(|h| stack.push(h)), 167 | Alternation(hirs) => stack.push(&hirs[scr.get_mod(hirs.len())]), 168 | } 169 | } 170 | return res; 171 | } 172 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nautilus 2.0 2 | 3 | Nautilus is a coverage guided, grammar based fuzzer. You can use it to improve your test coverage and find more bugs. By specifying the grammar of semi valid inputs, Nautilus is able to perform complex mutation and to uncover more interesting test cases. Many of the ideas behind this fuzzer are documented in a Paper published at NDSS 2019. 4 | 5 |

6 | 7 |

8 | 9 | 10 | Version 2.0 has added many improvements to this early prototype and is now 100% compatible with AFL++. Besides general usability improvements, Version 2.0 includes lots of shiny new features: 11 | 12 | * Support for AFL-Qemu mode 13 | * Support for grammars specified in python 14 | * Support for non-context free grammars using python scripts to generate inputs from the structure 15 | * Support for specifying binary protocols/formats 16 | * Support for specifying regex based terminals that aren't part of the directed mutations 17 | * Better ability to avoid generating the same very short inputs over and over 18 | * Massive cleanup of the code base 19 | * Helpful error output on invalid grammars 20 | * Fixed a bug in the the timeout code that occasionally deadlocked the fuzzer 21 | 22 | 23 | ## How Does Nautilus Work? 24 | 25 | You specify a grammar using rules such as `EXPR -> EXPR + EXPR` or `EXPR -> NUM` and `NUM -> 1`. From these rules, the fuzzer constructs a tree. This internal representation allows to apply much more complex mutations than raw bytes. This tree is then turned into a real input for the target application. In normal Context Free Grammars, this process is straightforward: all leaves are concatenated. The left tree in the example below would unparse to the input `a=1+2` and the right one to `a=1+1+1+2`. To increase the expressiveness of your grammars, using Nautilus you are able to provide python functions for the unparsing process to allow much more complex specifications. 26 | 27 |

28 | 29 |

30 | 31 | ## Setup 32 | ```bash 33 | # checkout the git 34 | git clone 'git@github.com:nautilus-fuzz/nautilus.git' 35 | cd nautilus 36 | /path/to/AFLplusplus/afl-clang-fast test.c -o test #afl-clang-fast as provided by AFL 37 | 38 | # all arguments can also be set using the config.ron file 39 | cargo run --release -- -g grammars/grammar_py_example.py -o /tmp/workdir -- ./test @@ 40 | 41 | # or if you want to use QEMU mode: 42 | cargo run /path/to/AFLplusplus/afl-qemu-trace -- ./test_bin @@ 43 | 44 | ``` 45 | 46 | ## Examples 47 | 48 | Here, we use python to generate a grammar for valid xml-like inputs. Notice the use of a script rule to ensure the the opening 49 | and closing tags match. 50 | 51 | ```python 52 | #ctx.rule(NONTERM: string, RHS: string|bytes) adds a rule NONTERM->RHS. We can use {NONTERM} in the RHS to request a recursion. 53 | ctx.rule("START","{XML_CONTENT}") 54 | ctx.rule("XML_CONTENT","{XML}{XML_CONTENT}") 55 | ctx.rule("XML_CONTENT","") 56 | 57 | #ctx.script(NONTERM:string, RHS: [string]], func) adds a rule NONTERM->func(*RHS). 58 | # In contrast to normal `rule`, RHS is an array of nonterminals. 59 | # It's up to the function to combine the values returned for the NONTERMINALS with any fixed content used. 60 | ctx.script("XML",["TAG","ATTR","XML_CONTENT"], lambda tag,attr,body: b"<%s %s>%s"%(tag,attr,body,tag) ) 61 | ctx.rule("ATTR","foo=bar") 62 | ctx.rule("TAG","some_tag") 63 | ctx.rule("TAG","other_tag") 64 | 65 | #sometimes we don't want to explore the set of possible inputs in more detail. For example, if we fuzz a script 66 | #interpreter, we don't want to spend time on fuzzing all different variable names. In such cases we can use Regex 67 | #terminals. Regex terminals are only mutated during generation, but not during normal mutation stages, saving a lot of time. 68 | #The fuzzer still explores different values for the regex, but it won't be able to learn interesting values incrementally. 69 | #Use this when incremantal exploration would most likely waste time. 70 | 71 | ctx.regex("TAG","[a-z]+") 72 | ``` 73 | 74 | To test your grammars you can use the generator: 75 | 76 | ``` 77 | $ cargo run --bin generator -- -g grammars/grammar_py_exmaple.py -t 100 78 | 79 | ``` 80 | 81 | You can also use Nautilus in combination with AFL. Simply point AFL `-o` to the same workdir, and AFL will synchronize 82 | with Nautilus. Note that this is one way. AFL imports Nautilus inputs, but not the other way around. 83 | 84 | ``` 85 | #Terminal/Screen 1 86 | ./afl-fuzz -Safl -i /tmp/seeds -o /tmp/workdir/ ./test @@ 87 | 88 | #Terminal/Screen 2 89 | cargo run --release -- -o /tmp/workdir -- ./test @@ 90 | ``` 91 | 92 | ## Trophies 93 | 94 | * https://github.com/Microsoft/ChakraCore/issues/5503 95 | * https://github.com/mruby/mruby/issues/3995 (**CVE-2018-10191**) 96 | * https://github.com/mruby/mruby/issues/4001 (**CVE-2018-10199**) 97 | * https://github.com/mruby/mruby/issues/4038 (**CVE-2018-12248**) 98 | * https://github.com/mruby/mruby/issues/4027 (**CVE-2018-11743**) 99 | * https://github.com/mruby/mruby/issues/4036 (**CVE-2018-12247**) 100 | * https://github.com/mruby/mruby/issues/4037 (**CVE-2018-12249**) 101 | * https://bugs.php.net/bug.php?id=76410 102 | * https://bugs.php.net/bug.php?id=76244 103 | -------------------------------------------------------------------------------- /fuzzer/src/queue.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use std::collections::HashMap; 5 | use std::collections::HashSet; 6 | use std::fs; 7 | use std::fs::File; 8 | use std::io::ErrorKind; 9 | 10 | use forksrv::exitreason::ExitReason; 11 | use grammartec::context::Context; 12 | use grammartec::recursion_info::RecursionInfo; 13 | use grammartec::tree::Tree; 14 | use grammartec::tree::TreeLike; 15 | 16 | #[derive(Serialize, Clone, Deserialize)] 17 | pub enum InputState { 18 | Init(usize), 19 | Det((usize, usize)), 20 | Random, 21 | } 22 | 23 | pub struct QueueItem { 24 | pub id: usize, 25 | pub tree: Tree, 26 | pub fresh_bits: HashSet, 27 | pub all_bits: Vec, 28 | pub exitreason: ExitReason, 29 | pub state: InputState, 30 | pub recursions: Option>, 31 | pub execution_time: u32, 32 | } 33 | 34 | impl QueueItem { 35 | pub fn new( 36 | id: usize, 37 | tree: Tree, 38 | fresh_bits: HashSet, 39 | all_bits: Vec, 40 | exitreason: ExitReason, 41 | execution_time: u32, 42 | ) -> Self { 43 | return QueueItem { 44 | id, 45 | tree, 46 | fresh_bits, 47 | all_bits, 48 | exitreason, 49 | state: InputState::Init(0), 50 | recursions: None, 51 | execution_time, 52 | }; 53 | } 54 | } 55 | 56 | pub struct Queue { 57 | pub inputs: Vec, 58 | pub processed: Vec, 59 | pub bit_to_inputs: HashMap>, 60 | pub current_id: usize, 61 | pub work_dir: String, 62 | } 63 | 64 | impl Queue { 65 | pub fn add( 66 | &mut self, 67 | tree: Tree, 68 | all_bits: Vec, 69 | exitreason: ExitReason, 70 | ctx: &Context, 71 | execution_time: u32, 72 | ) { 73 | if all_bits 74 | .iter() 75 | .enumerate() 76 | .all(|(i, elem)| (*elem == 0) || self.bit_to_inputs.contains_key(&i)) 77 | { 78 | return; 79 | } 80 | let mut fresh_bits = HashSet::new(); 81 | //Check which bits are new and insert them into fresh_bits 82 | for (i, elem) in all_bits.iter().enumerate() { 83 | if *elem != 0 { 84 | if !self.bit_to_inputs.contains_key(&i) { 85 | fresh_bits.insert(i.clone()); 86 | } 87 | self.bit_to_inputs 88 | .entry(i) 89 | .or_insert(vec![]) 90 | .push(self.current_id); 91 | } 92 | } 93 | 94 | //Create File for entry 95 | let mut file = File::create(format!( 96 | "{}/outputs/queue/id:{:09},er:{:?}", 97 | self.work_dir, self.current_id, exitreason 98 | )) 99 | .expect("RAND_259979732"); 100 | tree.unparse_to(&ctx, &mut file); 101 | 102 | //Add entry to queue 103 | self.inputs.push(QueueItem::new( 104 | self.current_id, 105 | tree, 106 | fresh_bits, 107 | all_bits, 108 | exitreason, 109 | execution_time, 110 | )); 111 | 112 | //Increase current_id 113 | if self.current_id == usize::max_value() { 114 | self.current_id = 0; 115 | } else { 116 | self.current_id += 1; 117 | } 118 | } 119 | 120 | pub fn new(work_dir: String) -> Self { 121 | return Queue { 122 | inputs: vec![], 123 | processed: vec![], 124 | bit_to_inputs: HashMap::new(), 125 | current_id: 0, 126 | work_dir: work_dir, 127 | }; 128 | } 129 | 130 | pub fn pop(&mut self) -> Option { 131 | let option = self.inputs.pop(); 132 | if option.is_some() { 133 | let item = option.expect("RAND_607640468"); 134 | let id = item.id; 135 | let mut keys = Vec::with_capacity(self.bit_to_inputs.keys().len()); //TODO: Find a better solution for this 136 | { 137 | for k in self.bit_to_inputs.keys() { 138 | keys.push(*k); 139 | } 140 | } 141 | for k in keys { 142 | let mut v = self.bit_to_inputs.remove(&k).expect("RAND_2593710501"); 143 | if let Some(pos) = v.iter().position(|x| *x == id) { 144 | v.remove(pos); 145 | } 146 | if !v.is_empty() { 147 | self.bit_to_inputs.insert(k, v); 148 | } 149 | } 150 | return Some(item); 151 | } 152 | return None; 153 | } 154 | 155 | pub fn finished(&mut self, item: QueueItem) { 156 | if item 157 | .all_bits 158 | .iter() 159 | .enumerate() 160 | .all(|(i, elem)| (*elem == 0) || self.bit_to_inputs.contains_key(&i)) 161 | { 162 | //If file was created for this entry, delete it. 163 | match fs::remove_file(format!( 164 | "{}/outputs/queue/id:{:09},er:{:?}", 165 | self.work_dir, item.id, item.exitreason 166 | )) { 167 | Err(ref err) if err.kind() != ErrorKind::NotFound => { 168 | println!("Error while deleting file: {}", err); 169 | } 170 | _ => {} 171 | } 172 | return; 173 | } 174 | 175 | //Check which bits are new and insert them into fresh_bits 176 | let mut fresh_bits = HashSet::new(); 177 | for (i, elem) in item.all_bits.iter().enumerate() { 178 | if *elem != 0 { 179 | if !self.bit_to_inputs.contains_key(&i) { 180 | fresh_bits.insert(i.clone()); 181 | } 182 | self.bit_to_inputs.entry(i).or_insert(vec![]).push(item.id); 183 | } 184 | } 185 | self.processed.push(item); 186 | } 187 | 188 | pub fn len(&self) -> usize { 189 | return self.inputs.len(); 190 | } 191 | 192 | pub fn new_round(&mut self) { 193 | self.inputs.append(&mut self.processed); 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /fuzzer/src/state.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use std::collections::HashSet; 5 | use std::fs::File; 6 | use std::sync::atomic::Ordering; 7 | use std::sync::Arc; 8 | use std::time::Instant; 9 | 10 | use grammartec::chunkstore::ChunkStoreWrapper; 11 | use grammartec::context::Context; 12 | use grammartec::mutator::Mutator; 13 | use grammartec::tree::{TreeLike, TreeMutation}; 14 | 15 | use config::Config; 16 | use forksrv::newtypes::SubprocessError; 17 | use fuzzer::{ExecutionReason, Fuzzer}; 18 | use queue::QueueItem; 19 | 20 | pub struct FuzzingState { 21 | pub cks: Arc, 22 | pub ctx: Context, 23 | pub config: Config, 24 | pub fuzzer: Fuzzer, 25 | pub mutator: Mutator, 26 | } 27 | 28 | impl FuzzingState { 29 | pub fn new(fuzzer: Fuzzer, config: Config, cks: Arc) -> Self { 30 | let ctx = Context::new(); 31 | let mutator = Mutator::new(&ctx); 32 | return FuzzingState { 33 | cks, 34 | ctx, 35 | config, 36 | fuzzer, 37 | mutator, 38 | }; 39 | } 40 | 41 | //Return value indicates if minimization is complete: true: complete, false: not complete 42 | pub fn minimize( 43 | &mut self, 44 | input: &mut QueueItem, 45 | start_index: usize, 46 | end_index: usize, 47 | ) -> Result { 48 | let ctx = &mut self.ctx; 49 | let fuzzer = &mut self.fuzzer; 50 | 51 | let min_simple = self.mutator.minimize_tree( 52 | &mut input.tree, 53 | &input.fresh_bits, 54 | ctx, 55 | start_index, 56 | end_index, 57 | &mut |t: &TreeMutation, fresh_bits: &HashSet, ctx: &Context| { 58 | let res = fuzzer.has_bits(t, fresh_bits, ExecutionReason::Min, ctx)?; 59 | Ok(res) 60 | }, 61 | )?; 62 | 63 | let min_rec = self.mutator.minimize_rec( 64 | &mut input.tree, 65 | &input.fresh_bits, 66 | ctx, 67 | start_index, 68 | end_index, 69 | &mut |t: &TreeMutation, fresh_bits: &HashSet, ctx: &Context| { 70 | let res = fuzzer.has_bits(t, fresh_bits, ExecutionReason::MinRec, ctx)?; 71 | Ok(res) 72 | }, 73 | )?; 74 | 75 | if min_simple && min_rec { 76 | //Only do this when minimization is completely done 77 | let now = Instant::now(); 78 | while self 79 | .cks 80 | .is_locked 81 | .compare_and_swap(false, true, Ordering::Acquire) 82 | { 83 | if now.elapsed().as_secs() > 30 { 84 | panic!("minimize starved!"); 85 | } 86 | } 87 | self.cks 88 | .chunkstore 89 | .write() 90 | .expect("RAND_1217841466") 91 | .add_tree(input.tree.clone(), &ctx); 92 | self.cks.is_locked.store(false, Ordering::Release); 93 | 94 | input.recursions = input.tree.calc_recursions(ctx); 95 | 96 | //Update file corresponding to this entry 97 | let mut file = File::create(format!( 98 | "{}/outputs/queue/id:{:09},er:{:?}.min", //TODO FIX PATH TO WORKDIR 99 | &self.config.path_to_workdir, input.id, input.exitreason 100 | )) 101 | .expect("Could not create queue entry, are you sure $workdir/outputs exists?"); 102 | input.tree.unparse_to(&ctx, &mut file); 103 | return Ok(true); 104 | } 105 | 106 | return Ok(false); 107 | } 108 | 109 | pub fn deterministic_tree_mutation( 110 | &mut self, 111 | input: &mut QueueItem, 112 | start_index: usize, 113 | end_index: usize, 114 | ) -> Result { 115 | let ctx = &mut self.ctx; 116 | let fuzzer = &mut self.fuzzer; 117 | let done = self.mutator.mut_rules( 118 | &input.tree, 119 | ctx, 120 | start_index, 121 | end_index, 122 | &mut |t: &TreeMutation, ctx: &Context| { 123 | fuzzer 124 | .run_on_with_dedup(t, ExecutionReason::Det, ctx) 125 | .map(|_| ()) 126 | }, 127 | )?; 128 | return Ok(done); 129 | } 130 | 131 | pub fn havoc(&mut self, input: &mut QueueItem) -> Result<(), SubprocessError> { 132 | let ctx = &mut self.ctx; 133 | let fuzzer = &mut self.fuzzer; 134 | for _i in 0..100 { 135 | self.mutator 136 | .mut_random(&input.tree, ctx, &mut |t: &TreeMutation, ctx: &Context| { 137 | fuzzer 138 | .run_on_with_dedup(t, ExecutionReason::Havoc, ctx) 139 | .map(|_| ()) 140 | })?; 141 | } 142 | return Ok(()); 143 | } 144 | 145 | pub fn havoc_recursion(&mut self, input: &mut QueueItem) -> Result<(), SubprocessError> { 146 | if let Some(ref mut recursions) = input.recursions 147 | /* input.tree.calc_recursions() */ 148 | { 149 | for _i in 0..20 { 150 | let ctx = &mut self.ctx; 151 | let fuzzer = &mut self.fuzzer; 152 | self.mutator.mut_random_recursion( 153 | &input.tree, 154 | recursions, 155 | ctx, 156 | &mut |t: &TreeMutation, ctx: &Context| { 157 | fuzzer 158 | .run_on_with_dedup(t, ExecutionReason::HavocRec, ctx) 159 | .map(|_| ()) 160 | }, 161 | )?; 162 | } 163 | } 164 | return Ok(()); 165 | } 166 | 167 | pub fn splice(&mut self, input: &mut QueueItem) -> Result<(), SubprocessError> { 168 | let ctx = &mut self.ctx; 169 | let fuzzer = &mut self.fuzzer; 170 | for _i in 0..100 { 171 | let now = Instant::now(); 172 | while self.cks.is_locked.load(Ordering::SeqCst) { 173 | if now.elapsed().as_secs() > 30 { 174 | panic!("splice starved!"); 175 | } 176 | } 177 | self.mutator.mut_splice( 178 | &input.tree, 179 | ctx, 180 | &*self.cks.chunkstore.read().expect("RAND_1290117799"), 181 | &mut |t: &TreeMutation, ctx: &Context| { 182 | fuzzer 183 | .run_on_with_dedup(t, ExecutionReason::Splice, ctx) 184 | .map(|_| ()) 185 | }, 186 | )?; 187 | } 188 | return Ok(()); 189 | } 190 | 191 | pub fn generate_random(&mut self, nt: &str) -> Result<(), SubprocessError> { 192 | let nonterm = self.ctx.nt_id(nt); 193 | let len = self.ctx.get_random_len_for_nt(&nonterm); 194 | let tree = self.ctx.generate_tree_from_nt(nonterm, len); 195 | self.fuzzer 196 | .run_on_with_dedup(&tree, ExecutionReason::Gen, &mut self.ctx)?; 197 | return Ok(()); 198 | } 199 | #[allow(dead_code)] 200 | pub fn inspect(&self, input: &QueueItem) -> String { 201 | return String::from_utf8_lossy(&input.tree.unparse_to_vec(&self.ctx)).into_owned(); 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /forksrv/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | 5 | extern crate byteorder; 6 | extern crate nix; 7 | extern crate snafu; 8 | extern crate tempfile; 9 | extern crate timeout_readwrite; 10 | #[macro_use] 11 | extern crate serde_derive; 12 | 13 | pub mod exitreason; 14 | pub mod newtypes; 15 | 16 | use nix::fcntl; 17 | use nix::libc::{ 18 | __errno_location, shmat, shmctl, shmget, strerror, IPC_CREAT, IPC_EXCL, IPC_PRIVATE, IPC_RMID, 19 | }; 20 | use nix::sys::signal::{self, Signal}; 21 | use nix::sys::stat; 22 | use nix::sys::wait::WaitStatus; 23 | use nix::unistd; 24 | use nix::unistd::Pid; 25 | use nix::unistd::{fork, ForkResult}; 26 | use std::ffi::CString; 27 | use std::os::unix::io::AsRawFd; 28 | use std::os::unix::io::RawFd; 29 | 30 | use std::io::BufReader; 31 | use std::ptr; 32 | use std::time::Duration; 33 | use timeout_readwrite::TimeoutReader; 34 | 35 | use byteorder::{LittleEndian, ReadBytesExt}; 36 | use std::fs::File; 37 | use std::os::unix::io::FromRawFd; 38 | 39 | use exitreason::ExitReason; 40 | use newtypes::*; 41 | use snafu::ResultExt; 42 | 43 | pub struct ForkServer { 44 | inp_file: File, 45 | ctl_in: File, 46 | shared_data: *mut [u8], 47 | st_out: std::io::BufReader>, 48 | } 49 | 50 | impl ForkServer { 51 | pub fn new( 52 | path: String, 53 | args: Vec, 54 | hide_output: bool, 55 | timeout_in_millis: u64, 56 | bitmap_size: usize, 57 | ) -> Self { 58 | let inp_file = tempfile::NamedTempFile::new().expect("couldn't create temp file"); 59 | let (inp_file, in_path) = inp_file 60 | .keep() 61 | .expect("couldn't persists temp file for input"); 62 | let inp_file_path = in_path 63 | .to_str() 64 | .expect("temp path should be unicode!") 65 | .to_string(); 66 | let args = args 67 | .into_iter() 68 | .map(|s| if s == "@@" { inp_file_path.clone() } else { s }) 69 | .collect::>(); 70 | let (ctl_out, ctl_in) = nix::unistd::pipe().expect("failed to create ctl_pipe"); 71 | let (st_out, st_in) = nix::unistd::pipe().expect("failed to create st_pipe"); 72 | let (shm_file, shared_data) = ForkServer::create_shm(bitmap_size); 73 | 74 | match fork().expect("couldn't fork") { 75 | // Parent returns 76 | ForkResult::Parent { child: _, .. } => { 77 | unistd::close(ctl_out).expect("coulnd't close ctl_out"); 78 | unistd::close(st_in).expect("coulnd't close st_out"); 79 | let mut st_out = BufReader::new(TimeoutReader::new( 80 | unsafe { File::from_raw_fd(st_out) }, 81 | Duration::from_millis(timeout_in_millis), 82 | )); 83 | st_out 84 | .read_u32::() 85 | .expect("couldn't read child hello"); 86 | return Self { 87 | inp_file: inp_file, 88 | ctl_in: unsafe { File::from_raw_fd(ctl_in) }, 89 | shared_data: shared_data, 90 | st_out, 91 | }; 92 | } 93 | //Child does complex stuff 94 | ForkResult::Child => { 95 | let forkserver_fd = 198; // from AFL config.h 96 | unistd::dup2(ctl_out, forkserver_fd as RawFd) 97 | .expect("couldn't dup2 ctl_our to FROKSRV_FD"); 98 | unistd::dup2(st_in, (forkserver_fd + 1) as RawFd) 99 | .expect("couldn't dup2 ctl_our to FROKSRV_FD+1"); 100 | 101 | unistd::dup2(inp_file.as_raw_fd(), 0).expect("couldn't dup2 input file to stdin"); 102 | unistd::close(inp_file.as_raw_fd()).expect("couldn't close input file"); 103 | 104 | unistd::close(ctl_in).expect("couldn't close ctl_in"); 105 | unistd::close(ctl_out).expect("couldn't close ctl_out"); 106 | unistd::close(st_in).expect("couldn't close ctl_out"); 107 | unistd::close(st_out).expect("couldn't close ctl_out"); 108 | 109 | let path = CString::new(path).expect("binary path must not contain zero"); 110 | let args = args 111 | .into_iter() 112 | .map(|s| CString::new(s).expect("args must not contain zero")) 113 | .collect::>(); 114 | 115 | let shm_id = CString::new(format!("__AFL_SHM_ID={}", shm_file)).unwrap(); 116 | 117 | //Asan options: set asan SIG to 223 and disable leak detection 118 | let asan_settings = 119 | CString::new("ASAN_OPTIONS=exitcode=223,abort_on_erro=true,detect_leaks=0") 120 | .expect("RAND_2089158993"); 121 | 122 | let env = vec![shm_id, asan_settings]; 123 | 124 | if hide_output { 125 | let null = fcntl::open("/dev/null", fcntl::OFlag::O_RDWR, stat::Mode::empty()) 126 | .expect("couldn't open /dev/null"); 127 | unistd::dup2(null, 1 as RawFd).expect("couldn't dup2 /dev/null to stdout"); 128 | unistd::dup2(null, 2 as RawFd).expect("couldn't dup2 /dev/null to stderr"); 129 | unistd::close(null).expect("couldn't close /dev/null"); 130 | } 131 | println!("EXECVE {:?} {:?} {:?}", path, args, env); 132 | unistd::execve(&path, &args, &env).expect("couldn't execve afl-qemu-tarce"); 133 | unreachable!(); 134 | } 135 | } 136 | } 137 | 138 | pub fn run(&mut self, data: &[u8]) -> Result { 139 | for i in self.get_shared_mut().iter_mut() { 140 | *i = 0; 141 | } 142 | unistd::ftruncate(self.inp_file.as_raw_fd(), 0).context(QemuRunNix { 143 | task: "Couldn't truncate inp_file", 144 | })?; 145 | unistd::lseek(self.inp_file.as_raw_fd(), 0, unistd::Whence::SeekSet).context( 146 | QemuRunNix { 147 | task: "Couldn't seek inp_file", 148 | }, 149 | )?; 150 | unistd::write(self.inp_file.as_raw_fd(), data).context(QemuRunNix { 151 | task: "Couldn't write data to inp_file", 152 | })?; 153 | unistd::lseek(self.inp_file.as_raw_fd(), 0, unistd::Whence::SeekSet).context( 154 | QemuRunNix { 155 | task: "Couldn't seek inp_file", 156 | }, 157 | )?; 158 | 159 | unistd::write(self.ctl_in.as_raw_fd(), &[0, 0, 0, 0]).context(QemuRunNix { 160 | task: "Couldn't send start command", 161 | })?; 162 | 163 | let pid = Pid::from_raw(self.st_out.read_i32::().context(QemuRunIO { 164 | task: "Couldn't read target pid", 165 | })?); 166 | 167 | if let Ok(status) = self.st_out.read_i32::() { 168 | return Ok(ExitReason::from_wait_status( 169 | WaitStatus::from_raw(pid, status).expect("402104968"), 170 | )); 171 | } 172 | signal::kill(pid, Signal::SIGKILL).context(QemuRunNix { 173 | task: "Couldn't kill timed out process", 174 | })?; 175 | self.st_out.read_u32::().context(QemuRunIO { 176 | task: "couldn't read timeout exitcode", 177 | })?; 178 | return Ok(ExitReason::Timeouted); 179 | } 180 | 181 | pub fn get_shared_mut(&mut self) -> &mut [u8] { 182 | unsafe { return &mut *self.shared_data } 183 | } 184 | pub fn get_shared(&self) -> &[u8] { 185 | unsafe { return &*self.shared_data } 186 | } 187 | 188 | fn create_shm(bitmap_size: usize) -> (i32, *mut [u8]) { 189 | unsafe { 190 | let shm_id = shmget(IPC_PRIVATE, bitmap_size, IPC_CREAT | IPC_EXCL | 0o600); 191 | if shm_id < 0 { 192 | panic!( 193 | "shm_id {:?}", 194 | CString::from_raw(strerror(*__errno_location())) 195 | ); 196 | } 197 | 198 | let trace_bits = shmat(shm_id, ptr::null(), 0); 199 | if (trace_bits as isize) < 0 { 200 | panic!( 201 | "shmat {:?}", 202 | CString::from_raw(strerror(*__errno_location())) 203 | ); 204 | } 205 | 206 | let res = shmctl(shm_id, IPC_RMID, 0 as *mut nix::libc::shmid_ds); 207 | if res < 0 { 208 | panic!( 209 | "shmclt {:?}", 210 | CString::from_raw(strerror(*__errno_location())) 211 | ); 212 | } 213 | return (shm_id, trace_bits as *mut [u8; 1 << 16]); 214 | } 215 | } 216 | } 217 | 218 | #[cfg(test)] 219 | mod tests { 220 | use crate::*; 221 | #[test] 222 | fn run_forkserver() { 223 | let hide_output = false; 224 | let timeout_in_millis = 200; 225 | let bitmap_size = 1 << 16; 226 | let target = "../test".to_string(); 227 | let args = vec![]; 228 | let mut fork = ForkServer::new(target, args, hide_output, timeout_in_millis, bitmap_size); 229 | assert!(fork.get_shared()[1..].iter().all(|v| *v == 0)); 230 | assert_eq!( 231 | fork.run(b"deadbeeg").unwrap(), 232 | exitreason::ExitReason::Normal(0) 233 | ); 234 | assert_eq!( 235 | fork.run(b"deadbeef").unwrap(), 236 | exitreason::ExitReason::Signaled(6) 237 | ); 238 | assert!(fork.get_shared()[1..].iter().any(|v| *v != 0)); 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /grammartec/src/rule.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use context::Context; 5 | use newtypes::{NTermID, NodeID, RuleID}; 6 | use pyo3::prelude::{PyObject, Python}; 7 | use rand::thread_rng; 8 | use rand::Rng; 9 | use regex; 10 | use regex_syntax::hir::Hir; 11 | use tree::Tree; 12 | 13 | #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] 14 | pub enum RuleChild { 15 | Term(Vec), 16 | NTerm(NTermID), 17 | } 18 | 19 | fn show_bytes(bs: &[u8]) -> String { 20 | use std::ascii::escape_default; 21 | use std::str; 22 | 23 | let mut visible = String::new(); 24 | for &b in bs { 25 | let part: Vec = escape_default(b).collect(); 26 | visible.push_str(str::from_utf8(&part).unwrap()); 27 | } 28 | return format!("\"{}\"", visible); 29 | } 30 | 31 | impl RuleChild { 32 | pub fn from_lit(lit: &[u8]) -> Self { 33 | return RuleChild::Term(lit.into()); 34 | } 35 | 36 | pub fn from_nt(nt: &str, ctx: &mut Context) -> Self { 37 | let (nonterm, _) = RuleChild::split_nt_description(nt); 38 | return RuleChild::NTerm(ctx.aquire_nt_id(&nonterm)); 39 | } 40 | 41 | fn split_nt_description(nonterm: &str) -> (String, String) { 42 | lazy_static! { 43 | static ref SPLITTER: regex::Regex = 44 | regex::Regex::new(r"^\{([A-Z][a-zA-Z_\-0-9]*)(?::([a-zA-Z_\-0-9]*))?\}$") 45 | .expect("RAND_1363289094"); 46 | } 47 | 48 | //splits {A:a} or {A} into A and maybe a 49 | let descr = SPLITTER.captures(nonterm).expect(&format!("could not interpret Nonterminal {:?}. Nonterminal Descriptions need to match start with a capital letter and con only contain [a-zA-Z_-0-9]",nonterm)); 50 | //let name = descr.get(2).map(|m| m.as_str().into()).unwrap_or(default.to_string())); 51 | return (descr[1].into(), "".into()); 52 | } 53 | 54 | fn debug_show(&self, ctx: &Context) -> String { 55 | match self { 56 | RuleChild::Term(d) => show_bytes(&d), 57 | RuleChild::NTerm(nt) => ctx.nt_id_to_s(*nt), 58 | } 59 | } 60 | } 61 | 62 | #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] 63 | pub enum RuleIDOrCustom { 64 | Rule(RuleID), 65 | Custom(RuleID, Vec), 66 | } 67 | impl RuleIDOrCustom { 68 | pub fn id(&self) -> RuleID { 69 | match self { 70 | RuleIDOrCustom::Rule(id) => return *id, 71 | RuleIDOrCustom::Custom(id, _) => return *id, 72 | } 73 | } 74 | 75 | pub fn data(&self) -> &[u8] { 76 | match self { 77 | RuleIDOrCustom::Custom(_, data) => return data, 78 | RuleIDOrCustom::Rule(_) => panic!("cannot get data on a normal rule"), 79 | } 80 | } 81 | } 82 | 83 | #[derive(Clone, Debug)] 84 | pub enum Rule { 85 | Plain(PlainRule), 86 | Script(ScriptRule), 87 | RegExp(RegExpRule), 88 | } 89 | 90 | #[derive(Debug, Clone)] 91 | pub struct RegExpRule { 92 | pub nonterm: NTermID, 93 | pub hir: Hir, 94 | } 95 | 96 | impl RegExpRule { 97 | pub fn debug_show(&self, ctx: &Context) -> String { 98 | return format!("{} => {:?}", ctx.nt_id_to_s(self.nonterm), self.hir); 99 | } 100 | } 101 | 102 | #[derive(Debug)] 103 | pub struct ScriptRule { 104 | pub nonterm: NTermID, 105 | pub nonterms: Vec, 106 | pub script: PyObject, 107 | } 108 | 109 | impl ScriptRule { 110 | pub fn debug_show(&self, ctx: &Context) -> String { 111 | let args = self 112 | .nonterms 113 | .iter() 114 | .map(|nt| ctx.nt_id_to_s(*nt)) 115 | .collect::>() 116 | .join(", "); 117 | return format!("{} => func({})", ctx.nt_id_to_s(self.nonterm), args); 118 | } 119 | } 120 | 121 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] 122 | pub struct PlainRule { 123 | pub nonterm: NTermID, 124 | pub children: Vec, 125 | pub nonterms: Vec, 126 | } 127 | 128 | impl PlainRule { 129 | pub fn debug_show(&self, ctx: &Context) -> String { 130 | let args = self 131 | .children 132 | .iter() 133 | .map(|child| child.debug_show(ctx)) 134 | .collect::>() 135 | .join(", "); 136 | return format!("{} => {}", ctx.nt_id_to_s(self.nonterm), args); 137 | } 138 | } 139 | 140 | impl Clone for ScriptRule { 141 | fn clone(&self) -> Self { 142 | return Python::with_gil(|py| { 143 | ScriptRule { 144 | nonterm: self.nonterm.clone(), 145 | nonterms: self.nonterms.clone(), 146 | script: self.script.clone_ref(py), 147 | }}); 148 | } 149 | } 150 | 151 | impl Rule { 152 | pub fn from_script( 153 | ctx: &mut Context, 154 | nonterm: &str, 155 | nterms: Vec, 156 | script: PyObject, 157 | ) -> Self { 158 | return Self::Script(ScriptRule { 159 | nonterm: ctx.aquire_nt_id(nonterm), 160 | nonterms: nterms.iter().map(|s| ctx.aquire_nt_id(s)).collect(), 161 | script, 162 | }); 163 | } 164 | 165 | pub fn from_regex(ctx: &mut Context, nonterm: &str, regex: &str) -> Self { 166 | use regex_syntax::ParserBuilder; 167 | 168 | let mut parser = ParserBuilder::new() 169 | .unicode(true) 170 | .allow_invalid_utf8(true) 171 | .build(); 172 | 173 | let hir = parser.parse(regex).unwrap(); 174 | 175 | return Self::RegExp(RegExpRule { 176 | nonterm: ctx.aquire_nt_id(nonterm), 177 | hir, 178 | }); 179 | } 180 | 181 | pub fn debug_show(&self, ctx: &Context) -> String { 182 | match self { 183 | Self::Plain(r) => r.debug_show(ctx), 184 | Self::Script(r) => r.debug_show(ctx), 185 | Self::RegExp(r) => r.debug_show(ctx), 186 | } 187 | } 188 | 189 | pub fn from_format(ctx: &mut Context, nonterm: &str, format: &[u8]) -> Self { 190 | let children = Rule::tokenize(format, ctx); 191 | let nonterms = children 192 | .iter() 193 | .filter_map(|c| { 194 | if let &RuleChild::NTerm(n) = c { 195 | Some(n) 196 | } else { 197 | None 198 | } 199 | }) 200 | .collect(); 201 | return Self::Plain(PlainRule { 202 | nonterm: ctx.aquire_nt_id(nonterm), 203 | children, 204 | nonterms, 205 | }); 206 | } 207 | 208 | pub fn from_term(ntermid: NTermID, term: &Vec) -> Self { 209 | let children = vec![RuleChild::Term(term.to_vec())]; 210 | let nonterms = vec![]; 211 | return Self::Plain(PlainRule { 212 | nonterm: ntermid, 213 | children, 214 | nonterms, 215 | }); 216 | } 217 | 218 | fn unescape(bytes: &[u8]) -> Vec { 219 | if bytes.len() < 2 { 220 | return bytes.to_vec(); 221 | } 222 | let mut res = vec![]; 223 | let mut i = 0; 224 | while i < bytes.len() - 1 { 225 | if bytes[i] == 92 && bytes[i + 1] == 123 { 226 | // replace \{ with { 227 | res.push(123); 228 | i += 1; 229 | } else if bytes[i] == 92 && bytes[i + 1] == 125 { 230 | // replace \} with } 231 | res.push(125); 232 | i += 1; 233 | } else { 234 | res.push(bytes[i]); 235 | } 236 | i += 1; 237 | } 238 | if i < bytes.len() { 239 | res.push(bytes[bytes.len() - 1]); 240 | } 241 | return res; 242 | } 243 | 244 | fn tokenize(format: &[u8], ctx: &mut Context) -> Vec { 245 | lazy_static! { 246 | static ref TOKENIZER: regex::bytes::Regex = 247 | regex::bytes::RegexBuilder::new(r"(?-u)(\{[^}\\]+\})|((?:[^{\\]|\\\{|\\\}|\\)+)") 248 | .dot_matches_new_line(true) 249 | .build() 250 | .expect("RAND_994455541"); 251 | } //RegExp Changed from (\{[^}\\]+\})|((?:[^{\\]|\\\{|\\\}|\\\\)+) because of problems with \\ (\\ was not matched and therefore thrown away) 252 | 253 | return TOKENIZER 254 | .captures_iter(format) 255 | .map(|cap| { 256 | if let Some(sub) = cap.get(1) { 257 | //println!("cap.get(1): {}", sub.as_str()); 258 | RuleChild::from_nt( 259 | std::str::from_utf8(&sub.as_bytes()) 260 | .expect("nonterminals need to be valid strings"), 261 | ctx, 262 | ) 263 | } else if let Some(sub) = cap.get(2) { 264 | RuleChild::from_lit(&Self::unescape(sub.as_bytes())) 265 | } else { 266 | unreachable!() 267 | } 268 | }) 269 | .collect::>(); 270 | } 271 | 272 | pub fn nonterms(&self) -> &[NTermID] { 273 | return match self { 274 | Rule::Script(r) => &r.nonterms, 275 | Rule::Plain(r) => &r.nonterms, 276 | Rule::RegExp(_) => &[], 277 | }; 278 | } 279 | 280 | pub fn number_of_nonterms(&self) -> usize { 281 | return self.nonterms().len(); 282 | } 283 | 284 | pub fn nonterm(&self) -> NTermID { 285 | return match self { 286 | Rule::Script(r) => r.nonterm, 287 | Rule::Plain(r) => r.nonterm, 288 | Rule::RegExp(r) => r.nonterm, 289 | }; 290 | } 291 | 292 | pub fn generate(&self, tree: &mut Tree, ctx: &Context, len: usize) -> usize { 293 | // println!("Rhs: {:?}, len: {}", self.nonterms, len); 294 | // println!("Min needed len: {}", self.nonterms.iter().fold(0, |sum, nt| sum + ctx.get_min_len_for_nt(*nt) )); 295 | let minimal_needed_len = self 296 | .nonterms() 297 | .iter() 298 | .fold(0, |sum, nt| sum + ctx.get_min_len_for_nt(*nt)); 299 | assert!(minimal_needed_len <= len); 300 | let mut remaining_len = len; 301 | remaining_len -= minimal_needed_len; 302 | 303 | //if we have no further children, we consumed no len 304 | let mut total_size = 1; 305 | let paren = NodeID::from(tree.rules.len() - 1); 306 | //generate each childs tree from the left to the right. That way the only operation we ever 307 | //perform is to push another node to the end of the tree_vec 308 | 309 | for (i, nt) in self.nonterms().iter().enumerate() { 310 | //sample how much len this child can use up (e.g. how big can 311 | //let cur_child_max_len = Rule::get_random_len(remaining_nts, remaining_len) + ctx.get_min_len_for_nt(*nt); 312 | let mut cur_child_max_len; 313 | let mut new_nterms = Vec::new(); 314 | new_nterms.extend_from_slice(&self.nonterms()[i..]); 315 | if new_nterms.len() != 0 { 316 | cur_child_max_len = ctx.get_random_len(remaining_len, &new_nterms); 317 | } else { 318 | cur_child_max_len = remaining_len; 319 | } 320 | cur_child_max_len += ctx.get_min_len_for_nt(*nt); 321 | 322 | //get a rule that can be used with the remaining length 323 | let rid = ctx.get_random_rule_for_nt(*nt, cur_child_max_len); 324 | let rule_or_custom = match ctx.get_rule(rid) { 325 | Rule::Plain(_) => RuleIDOrCustom::Rule(rid), 326 | Rule::Script(_) => RuleIDOrCustom::Rule(rid), 327 | Rule::RegExp(RegExpRule { hir, .. }) => RuleIDOrCustom::Custom( 328 | rid, 329 | regex_mutator::generate(hir, thread_rng().gen::()), 330 | ), 331 | }; 332 | 333 | assert_eq!(tree.rules.len(), tree.sizes.len()); 334 | assert_eq!(tree.sizes.len(), tree.paren.len()); 335 | let offset = tree.rules.len(); 336 | 337 | tree.rules.push(rule_or_custom); 338 | tree.sizes.push(0); 339 | tree.paren.push(NodeID::from(0)); 340 | 341 | //generate the subtree for this rule, return the total consumed len 342 | let consumed_len = ctx.get_rule(rid).generate(tree, ctx, cur_child_max_len - 1); 343 | tree.sizes[offset] = consumed_len; 344 | tree.paren[offset] = paren; 345 | 346 | //println!("{}: min_needed_len: {}, Min-len: {} Consumed len: {} cur_child_max_len: {} remaining len: {}, total_size: {}, len: {}", ctx.nt_id_to_s(nt.clone()), minimal_needed_len, ctx.get_min_len_for_nt(*nt), consumed_len, cur_child_max_len, remaining_len, total_size, len); 347 | assert!(consumed_len <= cur_child_max_len); 348 | 349 | //println!("Rule: {}, min_len: {}", ctx.nt_id_to_s(nt.clone()), ctx.get_min_len_for_nt(*nt)); 350 | assert!(consumed_len >= ctx.get_min_len_for_nt(*nt)); 351 | 352 | //we can use the len that where not consumed by this iteration during the next iterations, 353 | //therefore it will be redistributed evenly amongst the other 354 | 355 | remaining_len += ctx.get_min_len_for_nt(*nt); 356 | 357 | remaining_len -= consumed_len; 358 | //add the consumed len to the total_len 359 | total_size += consumed_len; 360 | } 361 | //println!("Rule: {}, Size: {}", ctx.nt_id_to_s(self.nonterm.clone()), total_size); 362 | return total_size; 363 | } 364 | } 365 | -------------------------------------------------------------------------------- /fuzzer/src/fuzzer.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | extern crate time as othertime; 5 | use othertime::strftime; 6 | 7 | use std::collections::HashSet; 8 | use std::collections::VecDeque; 9 | use std::fs::File; 10 | use std::io::stdout; 11 | use std::io::Write; 12 | use std::sync::{Arc, Mutex}; 13 | use std::thread; 14 | use std::time::Instant; 15 | 16 | use forksrv::exitreason::ExitReason; 17 | use forksrv::newtypes::SubprocessError; 18 | use forksrv::ForkServer; 19 | use grammartec::context::Context; 20 | use grammartec::tree::TreeLike; 21 | use shared_state::GlobalSharedState; 22 | 23 | pub enum ExecutionReason { 24 | Havoc, 25 | HavocRec, 26 | Min, 27 | MinRec, 28 | Splice, 29 | Det, 30 | Gen, 31 | } 32 | 33 | pub struct Fuzzer { 34 | forksrv: ForkServer, 35 | last_tried_inputs: HashSet>, 36 | last_inputs_ring_buffer: VecDeque>, 37 | pub global_state: Arc>, 38 | pub target_path: String, 39 | pub target_args: Vec, 40 | pub execution_count: u64, 41 | pub average_executions_per_sec: f32, 42 | pub bits_found_by_havoc: u64, 43 | pub bits_found_by_havoc_rec: u64, 44 | pub bits_found_by_min: u64, 45 | pub bits_found_by_min_rec: u64, 46 | pub bits_found_by_splice: u64, 47 | pub bits_found_by_det: u64, 48 | pub bits_found_by_det_afl: u64, 49 | pub bits_found_by_gen: u64, 50 | pub asan_found_by_havoc: u64, 51 | pub asan_found_by_havoc_rec: u64, 52 | pub asan_found_by_min: u64, 53 | pub asan_found_by_min_rec: u64, 54 | pub asan_found_by_splice: u64, 55 | pub asan_found_by_det: u64, 56 | pub asan_found_by_det_afl: u64, 57 | pub asan_found_by_gen: u64, 58 | work_dir: String, 59 | } 60 | 61 | impl Fuzzer { 62 | pub fn new( 63 | path: String, 64 | args: Vec, 65 | global_state: Arc>, 66 | work_dir: String, 67 | timeout_in_millis: u64, 68 | bitmap_size: usize, 69 | ) -> Result { 70 | let fs = ForkServer::new( 71 | path.clone(), 72 | args.clone(), 73 | true, 74 | timeout_in_millis, 75 | bitmap_size, 76 | ); 77 | return Ok(Fuzzer { 78 | forksrv: fs, 79 | last_tried_inputs: HashSet::new(), 80 | last_inputs_ring_buffer: VecDeque::new(), 81 | global_state, 82 | target_path: path, 83 | target_args: args, 84 | execution_count: 0, 85 | average_executions_per_sec: 0.0, 86 | bits_found_by_havoc: 0, 87 | bits_found_by_havoc_rec: 0, 88 | bits_found_by_min: 0, 89 | bits_found_by_min_rec: 0, 90 | bits_found_by_splice: 0, 91 | bits_found_by_det: 0, 92 | bits_found_by_det_afl: 0, 93 | bits_found_by_gen: 0, 94 | asan_found_by_havoc: 0, 95 | asan_found_by_havoc_rec: 0, 96 | asan_found_by_min: 0, 97 | asan_found_by_min_rec: 0, 98 | asan_found_by_splice: 0, 99 | asan_found_by_det: 0, 100 | asan_found_by_det_afl: 0, 101 | asan_found_by_gen: 0, 102 | work_dir: work_dir, 103 | }); 104 | } 105 | 106 | pub fn run_on_with_dedup( 107 | &mut self, 108 | tree: &T, 109 | exec_reason: ExecutionReason, 110 | ctx: &Context, 111 | ) -> Result { 112 | let code: Vec = tree.unparse_to_vec(ctx); 113 | if self.input_is_known(&code) { 114 | return Ok(false); 115 | } 116 | self.run_on(&code, tree, exec_reason, ctx)?; 117 | return Ok(true); 118 | } 119 | 120 | pub fn run_on_without_dedup( 121 | &mut self, 122 | tree: &T, 123 | exec_reason: ExecutionReason, 124 | ctx: &Context, 125 | ) -> Result<(), SubprocessError> { 126 | let code = tree.unparse_to_vec(ctx); 127 | return self.run_on(&code, tree, exec_reason, ctx); 128 | } 129 | 130 | fn run_on( 131 | &mut self, 132 | code: &Vec, 133 | tree: &T, 134 | exec_reason: ExecutionReason, 135 | ctx: &Context, 136 | ) -> Result<(), SubprocessError> { 137 | let (new_bits, term_sig) = self.exec(code, tree, ctx)?; 138 | match term_sig { 139 | ExitReason::Normal(223) => { 140 | if new_bits.is_some() { 141 | //ASAN 142 | self.global_state 143 | .lock() 144 | .expect("RAND_3390206382") 145 | .total_found_asan += 1; 146 | self.global_state 147 | .lock() 148 | .expect("RAND_202860771") 149 | .last_found_asan = strftime("[%Y-%m-%d] %H:%M:%S", &othertime::now()) 150 | .expect("RAND_2888070412"); 151 | let mut file = File::create(format!( 152 | "{}/outputs/signaled/ASAN_{:09}_{}", 153 | self.work_dir, 154 | self.execution_count, 155 | thread::current().name().expect("RAND_4086695190") 156 | )) 157 | .expect("RAND_3096222153"); 158 | tree.unparse_to(ctx, &mut file); 159 | } 160 | } 161 | ExitReason::Normal(_) => { 162 | if new_bits.is_some() { 163 | match exec_reason { 164 | ExecutionReason::Havoc => { 165 | self.bits_found_by_havoc += 1; /*print!("Havoc+")*/ 166 | } 167 | ExecutionReason::HavocRec => { 168 | self.bits_found_by_havoc_rec += 1; /*print!("HavocRec+")*/ 169 | } 170 | ExecutionReason::Min => { 171 | self.bits_found_by_min += 1; /*print!("Min+")*/ 172 | } 173 | ExecutionReason::MinRec => { 174 | self.bits_found_by_min_rec += 1; /*print!("MinRec+")*/ 175 | } 176 | ExecutionReason::Splice => { 177 | self.bits_found_by_splice += 1; /*print!("Splice+")*/ 178 | } 179 | ExecutionReason::Det => { 180 | self.bits_found_by_det += 1; /*print!("Det+")*/ 181 | } 182 | ExecutionReason::Gen => { 183 | self.bits_found_by_gen += 1; /*print!("Gen+")*/ 184 | } 185 | } 186 | } 187 | } 188 | ExitReason::Timeouted => { 189 | self.global_state 190 | .lock() 191 | .expect("RAND_1706238230") 192 | .last_timeout = 193 | strftime("[%Y-%m-%d] %H:%M:%S", &othertime::now()).expect("RAND_1894162412"); 194 | let mut file = File::create(format!( 195 | "{}/outputs/timeout/{:09}", 196 | self.work_dir, self.execution_count 197 | )) 198 | .expect("RAND_452993103"); 199 | tree.unparse_to(ctx, &mut file); 200 | } 201 | ExitReason::Signaled(sig) => { 202 | if new_bits.is_some() { 203 | self.global_state 204 | .lock() 205 | .expect("RAND_1858328446") 206 | .total_found_sig += 1; 207 | self.global_state 208 | .lock() 209 | .expect("RAND_4287051369") 210 | .last_found_sig = 211 | strftime("[%Y-%m-%d] %H:%M:%S", &othertime::now()).expect("RAND_76391000"); 212 | let mut file = File::create(format!( 213 | "{}/outputs/signaled/{:?}_{:09}", 214 | self.work_dir, sig, self.execution_count 215 | )) 216 | .expect("RAND_3690294970"); 217 | tree.unparse_to(ctx, &mut file); 218 | } 219 | } 220 | ExitReason::Stopped(_sig) => {} 221 | } 222 | stdout().flush().expect("RAND_2937475131"); 223 | return Ok(()); 224 | } 225 | 226 | pub fn has_bits( 227 | &mut self, 228 | tree: &T, 229 | bits: &HashSet, 230 | exec_reason: ExecutionReason, 231 | ctx: &Context, 232 | ) -> Result { 233 | self.run_on_without_dedup(tree, exec_reason, ctx)?; 234 | let run_bitmap = self.forksrv.get_shared(); 235 | let mut found_all = true; 236 | for bit in bits.iter() { 237 | if run_bitmap[*bit] == 0 { 238 | //TODO: handle edge counts properly 239 | found_all = false; 240 | } 241 | } 242 | return Ok(found_all); 243 | } 244 | 245 | pub fn exec_raw<'a>(&'a mut self, code: &[u8]) -> Result<(ExitReason, u32), SubprocessError> { 246 | self.execution_count += 1; 247 | 248 | let start = Instant::now(); 249 | 250 | let exitreason = self.forksrv.run(&code)?; 251 | 252 | let execution_time = start.elapsed().subsec_nanos(); 253 | 254 | self.average_executions_per_sec = self.average_executions_per_sec * 0.9 255 | + ((1.0 / (execution_time as f32)) * 1000000000.0) * 0.1; 256 | 257 | return Ok((exitreason, execution_time)); 258 | } 259 | 260 | fn input_is_known(&mut self, code: &[u8]) -> bool { 261 | if self.last_tried_inputs.contains(code) { 262 | return true; 263 | } else { 264 | self.last_tried_inputs.insert(code.to_vec()); 265 | if self.last_inputs_ring_buffer.len() == 10000 { 266 | self.last_tried_inputs.remove( 267 | &self 268 | .last_inputs_ring_buffer 269 | .pop_back() 270 | .expect("No entry in last_inputs_ringbuffer"), 271 | ); 272 | } 273 | self.last_inputs_ring_buffer.push_front(code.to_vec()); 274 | } 275 | return false; 276 | } 277 | 278 | fn exec( 279 | &mut self, 280 | code: &[u8], 281 | tree_like: &T, 282 | ctx: &Context, 283 | ) -> Result<(Option>, ExitReason), SubprocessError> { 284 | let (exitreason, execution_time) = self.exec_raw(&code)?; 285 | 286 | let is_crash = match exitreason { 287 | ExitReason::Normal(223) => true, 288 | ExitReason::Signaled(_) => true, 289 | _ => false, 290 | }; 291 | 292 | let mut final_bits = None; 293 | if let Some(mut new_bits) = self.new_bits(is_crash) { 294 | //Only if not Timeout 295 | if exitreason != ExitReason::Timeouted { 296 | //Check for non deterministic bits 297 | let old_bitmap: Vec = self.forksrv.get_shared().to_vec(); 298 | self.check_deterministic_behaviour(&old_bitmap, &mut new_bits, &code)?; 299 | if new_bits.len() > 0 { 300 | final_bits = Some(new_bits); 301 | let tree = tree_like.to_tree(ctx); 302 | self.global_state 303 | .lock() 304 | .expect("RAND_2835014626") 305 | .queue 306 | .add(tree, old_bitmap, exitreason, ctx, execution_time); 307 | //println!("Entry added to queue! New bits: {:?}", bits.clone().expect("RAND_2243482569")); 308 | } 309 | } 310 | } 311 | return Ok((final_bits, exitreason)); 312 | } 313 | 314 | fn check_deterministic_behaviour( 315 | &mut self, 316 | old_bitmap: &[u8], 317 | new_bits: &mut Vec, 318 | code: &[u8], 319 | ) -> Result<(), SubprocessError> { 320 | for _ in 0..5 { 321 | let (_, _) = self.exec_raw(code)?; 322 | let run_bitmap = self.forksrv.get_shared(); 323 | for (i, &v) in old_bitmap.iter().enumerate() { 324 | if run_bitmap[i] != v { 325 | println!("found fucky bit {}", i); 326 | } 327 | } 328 | new_bits.retain(|&i| run_bitmap[i] != 0); 329 | } 330 | return Ok(()); 331 | } 332 | 333 | pub fn new_bits(&mut self, is_crash: bool) -> Option> { 334 | let mut res = vec![]; 335 | let run_bitmap = self.forksrv.get_shared(); 336 | let mut gstate_lock = self.global_state.lock().expect("RAND_2040280272"); 337 | let shared_bitmap = gstate_lock 338 | .bitmaps 339 | .get_mut(&is_crash) 340 | .expect("Bitmap missing! Maybe shared state was not initialized correctly?"); 341 | 342 | for (i, elem) in shared_bitmap.iter_mut().enumerate() { 343 | if (run_bitmap[i] != 0) && (*elem == 0) { 344 | *elem |= run_bitmap[i]; 345 | res.push(i); 346 | //println!("Added new bit to bitmap. Is Crash: {:?}; Added bit: {:?}", is_crash, i); 347 | } 348 | } 349 | 350 | if res.len() > 0 { 351 | //print!("New path found:\nNew bits: {:?}\n", res); 352 | return Some(res); 353 | } 354 | return None; 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /grammartec/src/context.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use std::collections::HashMap; 5 | 6 | 7 | use rand::{thread_rng, Rng}; 8 | use rand::seq::IteratorRandom; 9 | 10 | use newtypes::{NTermID, RuleID}; 11 | use pyo3::prelude::PyObject; 12 | use rule::{Rule, RuleIDOrCustom}; 13 | use tree::Tree; 14 | 15 | #[derive(Clone)] 16 | pub struct Context { 17 | rules: Vec, 18 | nts_to_rules: HashMap>, 19 | nt_ids_to_name: HashMap, 20 | names_to_nt_id: HashMap, 21 | rules_to_min_size: HashMap, 22 | 23 | nts_to_min_size: HashMap, 24 | 25 | rules_to_num_options: HashMap, 26 | nts_to_num_options: HashMap, 27 | max_len: usize, 28 | } 29 | 30 | impl Context { 31 | pub fn new() -> Self { 32 | return Context { 33 | rules: vec![], 34 | nts_to_rules: HashMap::new(), 35 | nt_ids_to_name: HashMap::new(), 36 | names_to_nt_id: HashMap::new(), 37 | 38 | rules_to_min_size: HashMap::new(), 39 | nts_to_min_size: HashMap::new(), 40 | 41 | rules_to_num_options: HashMap::new(), 42 | nts_to_num_options: HashMap::new(), 43 | max_len: 0, 44 | }; 45 | } 46 | 47 | pub fn initialize(&mut self, max_len: usize) { 48 | self.calc_min_len(); 49 | self.calc_num_options(); 50 | self.max_len = max_len + 2; 51 | } 52 | 53 | pub fn get_rule(&self, r: RuleID) -> &Rule { 54 | let id: usize = r.into(); 55 | return &self.rules[id]; 56 | } 57 | 58 | pub fn get_nt(&self, r: &RuleIDOrCustom) -> NTermID { 59 | return self.get_rule(r.id()).nonterm(); 60 | } 61 | 62 | pub fn get_num_children(&self, r: &RuleIDOrCustom) -> usize { 63 | return self.get_rule(r.id()).number_of_nonterms(); 64 | } 65 | 66 | pub fn add_rule(&mut self, nt: &str, format: &[u8]) -> RuleID { 67 | let rid = self.rules.len().into(); 68 | let rule = Rule::from_format(self, nt, format); 69 | let ntid = self.aquire_nt_id(nt); 70 | self.rules.push(rule); 71 | self.nts_to_rules 72 | .entry(ntid) 73 | .or_insert_with(|| vec![]) 74 | .push(rid); 75 | return rid; 76 | } 77 | 78 | pub fn add_script(&mut self, nt: &str, nts: Vec, script: PyObject) -> RuleID { 79 | let rid = self.rules.len().into(); 80 | let rule = Rule::from_script(self, nt, nts, script); 81 | let ntid = self.aquire_nt_id(nt); 82 | self.rules.push(rule); 83 | self.nts_to_rules 84 | .entry(ntid) 85 | .or_insert_with(|| vec![]) 86 | .push(rid); 87 | return rid; 88 | } 89 | 90 | pub fn add_regex(&mut self, nt: &str, regex: &str) -> RuleID { 91 | let rid = self.rules.len().into(); 92 | let rule = Rule::from_regex(self, nt, regex); 93 | let ntid = self.aquire_nt_id(nt); 94 | self.rules.push(rule); 95 | self.nts_to_rules 96 | .entry(ntid) 97 | .or_insert_with(|| vec![]) 98 | .push(rid); 99 | return rid; 100 | } 101 | 102 | pub fn add_term_rule(&mut self, nt: &str, term: &Vec) -> RuleID { 103 | let rid = self.rules.len().into(); 104 | let ntid = self.aquire_nt_id(nt); 105 | self.rules.push(Rule::from_term(ntid, term)); 106 | self.nts_to_rules 107 | .entry(ntid) 108 | .or_insert_with(|| vec![]) 109 | .push(rid); 110 | return rid; 111 | } 112 | 113 | pub fn aquire_nt_id(&mut self, nt: &str) -> NTermID { 114 | let next_id = self.nt_ids_to_name.len().into(); 115 | let id = self.names_to_nt_id.entry(nt.into()).or_insert(next_id); 116 | self.nt_ids_to_name.entry(*id).or_insert(nt.into()); 117 | return *id; 118 | } 119 | 120 | pub fn nt_id(&self, nt: &str) -> NTermID { 121 | return *self 122 | .names_to_nt_id 123 | .get(nt) 124 | .expect(&("no such nonterminal: ".to_owned() + nt)); 125 | } 126 | 127 | pub fn nt_id_to_s(&self, nt: NTermID) -> String { 128 | return self.nt_ids_to_name[&nt].clone(); 129 | } 130 | 131 | fn calc_min_len_for_rule(&self, r: RuleID) -> Option { 132 | let mut res = 1; 133 | for nt_id in self.get_rule(r).nonterms().iter() { 134 | if let Some(min) = self.nts_to_min_size.get(nt_id) { 135 | //println!("Calculating length for Rule(calc_min_len_for_rule): {}, current: {}, adding: {}, because of rule: {}", self.nt_id_to_s(self.get_rule(r).nonterm().clone()), res, min, self.nt_id_to_s(nt_id.clone())); 136 | res += *min; 137 | } else { 138 | return None; 139 | } 140 | } 141 | //println!("Calculated length for Rule(calc_min_len_for_rule): {}, Length: {}", self.nt_id_to_s(self.get_rule(r).nonterm().clone()), res); 142 | return Some(res); 143 | } 144 | 145 | pub fn calc_min_len(&mut self) { 146 | let mut something_changed = true; 147 | while something_changed == true { 148 | //TODO: find a better solution to prevent consumed_len >= ctx.get_min_len_for_nt(*nt)' Assertions 149 | let mut unknown_rules = (0..self.rules.len()) 150 | .map(|i| RuleID::from(i)) 151 | .collect::>(); 152 | something_changed = false; 153 | while unknown_rules.len() > 0 { 154 | let last_len = unknown_rules.len(); 155 | unknown_rules.retain(|rule| { 156 | if let Some(min) = self.calc_min_len_for_rule(*rule) { 157 | let nt = self.get_rule(*rule).nonterm(); 158 | //let name = self.nt_id_to_s(nt.clone()); //DEBUGGING 159 | let e = self.nts_to_min_size.entry(nt).or_insert(min); 160 | if *e > min { 161 | *e = min; 162 | something_changed = true; 163 | } 164 | //println!("Calculated length for Rule: {}, Length: {}, Min_length_of_nt: {}", name, min, *e); 165 | self.rules_to_min_size.insert(*rule, min); 166 | false 167 | } else { 168 | true 169 | } 170 | }); 171 | if last_len == unknown_rules.len() { 172 | println!("Found unproductive rules: (missing base/non recursive case?)"); 173 | for r in unknown_rules { 174 | println!("{}", self.get_rule(r).debug_show(&self)); 175 | } 176 | panic!("Broken Grammar"); 177 | } 178 | } 179 | } 180 | self.calc_rule_order(); 181 | } 182 | 183 | fn calc_num_options_for_rule(&self, r: RuleID) -> usize { 184 | let mut res = 1_usize; 185 | for nt_id in self.get_rule(r).nonterms().iter() { 186 | res = res.saturating_mul(*self.nts_to_num_options.get(nt_id).unwrap_or(&1)); 187 | } 188 | return res; 189 | } 190 | 191 | pub fn calc_num_options(&mut self) { 192 | for (nt, rules) in self.nts_to_rules.iter() { 193 | self.nts_to_num_options.entry(*nt).or_insert(rules.len()); 194 | } 195 | 196 | let mut something_changed = true; 197 | while something_changed == true { 198 | something_changed = false; 199 | 200 | for rid in (0..self.rules.len()).map(|i| RuleID::from(i)) { 201 | let num = self.calc_num_options_for_rule(rid); 202 | let nt = self.get_rule(rid).nonterm(); 203 | let e = self.nts_to_num_options.entry(nt).or_insert(num); 204 | if *e < num { 205 | *e = num; 206 | something_changed = true; 207 | } 208 | //println!("Calculated length for Rule: {}, Length: {}, Min_length_of_nt: {}", name, min, *e); 209 | self.rules_to_num_options.insert(rid, num); 210 | } 211 | } 212 | } 213 | 214 | fn calc_rule_order(&mut self) { 215 | let rules_to_min_size = &self.rules_to_min_size; 216 | for rules in self.nts_to_rules.values_mut() { 217 | (*rules).sort_by(|r1, r2| rules_to_min_size[r1].cmp(&rules_to_min_size[r2])); 218 | } 219 | } 220 | 221 | pub fn check_if_nterm_has_multiple_possiblities(&self, nt: &NTermID) -> bool { 222 | return self.get_rules_for_nt(*nt).len() > 1; 223 | } 224 | 225 | pub fn get_random_len(&self, len: usize, rhs_of_rule: &Vec) -> usize { 226 | return self.dumb_get_random_len(rhs_of_rule.len(), len); 227 | } 228 | 229 | //we need to get maximal sizes for all subtrees. To generate trees fairly, we want to split the 230 | //available size fairly to all nodes. (e.g. all children have the same expected size, 231 | //regardless of its index in the current rule. We use this version of the algorithm described 232 | //here: https://stackoverflow.com/a/8068956 to get the first value. 233 | fn dumb_get_random_len(&self, number_of_children: usize, total_remaining_len: usize) -> usize { 234 | let mut res = total_remaining_len; 235 | let iters = (number_of_children as i32) - 1; 236 | for _ in 0..iters { 237 | let proposal = thread_rng().gen_range(0, total_remaining_len + 1); 238 | if proposal < res { 239 | res = proposal 240 | } 241 | } 242 | return res; 243 | } 244 | 245 | pub fn get_min_len_for_nt(&self, nt: NTermID) -> usize { 246 | return self.nts_to_min_size[&nt]; 247 | } 248 | 249 | pub fn get_random_rule_for_nt(&self, nt: NTermID, len: usize) -> RuleID { 250 | return self.dumb_get_random_rule_for_nt(nt, len); 251 | } 252 | 253 | pub fn get_applicable_rules( 254 | &self, 255 | max_len: usize, 256 | nt: NTermID, 257 | p_include_short_rules: usize, 258 | ) -> impl Iterator { 259 | return self.nts_to_rules[&nt] 260 | .iter() 261 | .take_while(move |r| self.rules_to_min_size[r] <= max_len) 262 | .filter(move |r| { 263 | self.rules_to_num_options[r] > 1 264 | || (thread_rng().gen::() % 100) <= p_include_short_rules 265 | }); 266 | } 267 | 268 | fn dumb_get_random_rule_for_nt(&self, nt: NTermID, max_len: usize) -> RuleID { 269 | let p_include_short_rules = if self.nts_to_num_options[&nt] < 10 { 270 | 100 * 0 271 | } else if max_len > 100 { 272 | 2 * 0 273 | } else if max_len > 20 { 274 | 50 * 0 275 | } else { 276 | 100 * 0 277 | }; 278 | 279 | if let Some(opt) = self.get_applicable_rules(max_len, nt, p_include_short_rules).choose(&mut thread_rng()) { 280 | *opt 281 | } else if let Some(opt) = self.get_applicable_rules(max_len, nt, 100).choose(&mut thread_rng()) { 282 | *opt 283 | } else { 284 | panic!( 285 | "there is no way to derive {} within {} steps", 286 | self.nt_ids_to_name[&nt], max_len 287 | ) 288 | } 289 | } 290 | 291 | pub fn get_random_len_for_ruleid(&self, _rule_id: &RuleID) -> usize { 292 | return self.max_len; //TODO????? 293 | } 294 | 295 | pub fn get_random_len_for_nt(&self, _nt: &NTermID) -> usize { 296 | return self.max_len; 297 | } 298 | 299 | pub fn get_rules_for_nt(&self, nt: NTermID) -> &Vec { 300 | return &self.nts_to_rules[&nt]; 301 | } 302 | 303 | pub fn generate_tree_from_nt(&self, nt: NTermID, max_len: usize) -> Tree { 304 | return self.generate_tree_from_rule(self.get_random_rule_for_nt(nt, max_len), max_len - 1); 305 | } 306 | 307 | pub fn generate_tree_from_rule(&self, r: RuleID, len: usize) -> Tree { 308 | let mut tree = Tree::from_rule_vec(vec![], self); 309 | tree.generate_from_rule(r, len, self); 310 | return tree; 311 | } 312 | } 313 | 314 | #[cfg(test)] 315 | mod tests { 316 | use context::Context; 317 | use rule::{Rule, RuleChild, RuleIDOrCustom}; 318 | use tree::{Tree, TreeLike}; 319 | 320 | #[test] 321 | fn simple_context() { 322 | let mut ctx = Context::new(); 323 | let r = Rule::from_format(&mut ctx, "F", b"foo{A:a}\\{bar\\}{B:b}asd{C}"); 324 | let soll = vec![ 325 | RuleChild::from_lit(b"foo"), 326 | RuleChild::from_nt("{A:a}", &mut ctx), 327 | RuleChild::from_lit(b"{bar}"), 328 | RuleChild::from_nt("{B:b}", &mut ctx), 329 | RuleChild::from_lit(b"asd"), 330 | RuleChild::from_nt("{C}", &mut ctx), 331 | ]; 332 | if let Rule::Plain(rl) = &r { 333 | assert_eq!(&rl.children, &soll); 334 | } else { 335 | unreachable!(); 336 | } 337 | assert_eq!(r.nonterms()[0], ctx.nt_id("A")); 338 | assert_eq!(r.nonterms()[1], ctx.nt_id("B")); 339 | assert_eq!(r.nonterms()[2], ctx.nt_id("C")); 340 | } 341 | 342 | #[test] 343 | fn test_context() { 344 | let mut ctx = Context::new(); 345 | let r0 = ctx.add_rule("C", b"c{B}c"); 346 | let r1 = ctx.add_rule("B", b"b{A}b"); 347 | let _ = ctx.add_rule("A", b"a {A}"); 348 | let _ = ctx.add_rule("A", b"a {A}"); 349 | let _ = ctx.add_rule("A", b"a {A}"); 350 | let _ = ctx.add_rule("A", b"a {A}"); 351 | let _ = ctx.add_rule("A", b"a {A}"); 352 | let r3 = ctx.add_rule("A", b"a"); 353 | ctx.initialize(5); 354 | assert_eq!(ctx.get_min_len_for_nt(ctx.nt_id("A")), 1); 355 | assert_eq!(ctx.get_min_len_for_nt(ctx.nt_id("B")), 2); 356 | assert_eq!(ctx.get_min_len_for_nt(ctx.nt_id("C")), 3); 357 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 358 | tree.generate_from_nt(ctx.nt_id("C"), 3, &ctx); 359 | assert_eq!( 360 | tree.rules, 361 | vec![ 362 | RuleIDOrCustom::Rule(r0), 363 | RuleIDOrCustom::Rule(r1), 364 | RuleIDOrCustom::Rule(r3), 365 | ] 366 | ); 367 | let mut data: Vec = vec![]; 368 | tree.unparse_to(&ctx, &mut data); 369 | assert_eq!(String::from_utf8(data).expect("RAND_3377050372"), "cbabc"); 370 | } 371 | 372 | #[test] 373 | fn test_generate_len() { 374 | let mut ctx = Context::new(); 375 | let r0 = ctx.add_rule("E", b"({E}+{E})"); 376 | let r1 = ctx.add_rule("E", b"({E}*{E})"); 377 | let r2 = ctx.add_rule("E", b"({E}-{E})"); 378 | let r3 = ctx.add_rule("E", b"({E}/{E})"); 379 | let r4 = ctx.add_rule("E", b"1"); 380 | ctx.initialize(11); 381 | assert_eq!(ctx.get_min_len_for_nt(ctx.nt_id("E")), 1); 382 | 383 | for _ in 0..100 { 384 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 385 | tree.generate_from_nt(ctx.nt_id("E"), 9, &ctx); 386 | assert!(tree.rules.len() < 10); 387 | assert!(tree.rules.len() >= 1); 388 | } 389 | 390 | let rules = vec![r0, r1, r4, r4, r4] 391 | .iter() 392 | .map(|x| RuleIDOrCustom::Rule(*x)) 393 | .collect::>(); 394 | let tree = Tree::from_rule_vec(rules, &ctx); 395 | let mut data: Vec = vec![]; 396 | tree.unparse_to(&ctx, &mut data); 397 | assert_eq!( 398 | String::from_utf8(data).expect("RAND_3492562908"), 399 | "((1*1)+1)" 400 | ); 401 | 402 | let rules = vec![r0, r1, r2, r3, r4, r4, r4, r4, r4] 403 | .iter() 404 | .map(|x| RuleIDOrCustom::Rule(*x)) 405 | .collect::>(); 406 | let tree = Tree::from_rule_vec(rules, &ctx); 407 | let mut data: Vec = vec![]; 408 | tree.unparse_to(&ctx, &mut data); 409 | assert_eq!( 410 | String::from_utf8(data).expect("RAND_4245419893"), 411 | "((((1/1)-1)*1)+1)" 412 | ); 413 | } 414 | } 415 | -------------------------------------------------------------------------------- /grammars/lua.py: -------------------------------------------------------------------------------- 1 | ctx.rule(u'START',u'{PROGRAM}') 2 | ctx.rule(u'PROGRAM',u'{STATEMENT}\n{PROGRAM}') 3 | ctx.rule(u'PROGRAM',u'') 4 | ctx.rule(u'STATEMENT',u';') 5 | ctx.rule(u'STATEMENT',u'') 6 | ctx.rule(u'STATEMENT',u'break') 7 | ctx.rule(u'STATEMENT',u'{VAR} = {EXPR}') 8 | ctx.rule(u'STATEMENT',u'local {VARLIST} = {EXPRLIST}') 9 | ctx.rule(u'STATEMENT',u'{FUNCTION}') 10 | ctx.rule(u'STATEMENT',u'{COROUTINE}') 11 | ctx.rule(u'STATEMENT',u'{CONDITIONAL}') 12 | ctx.rule(u'STATEMENT',u'{LOOP}') 13 | ctx.rule(u'STATEMENT',u'return {EXPRLIST}') 14 | ctx.rule(u'STATEMENT',u'goto {LABELNAME}') 15 | ctx.rule(u'STATEMENT',u'::{LABELNAME}::') 16 | ctx.rule(u'LABELNAME',u'labela') 17 | ctx.rule(u'LABELNAME',u'labelb') 18 | ctx.rule(u'FUNCTION',u'{FUNCDEF} ({FUNCTION_ARGS}) {PROGRAM}\nend') 19 | ctx.rule(u'FUNCDEF',u'function {VAR}.{IDENTIFIER}') 20 | ctx.rule(u'FUNCDEF',u'function {VAR}:{IDENTIFIER}') 21 | ctx.rule(u'FUNCDEF',u'function {IDENTIFIER}') 22 | ctx.rule(u'LAMBDA',u'function ({FUNCTION_ARGS}) {PROGRAM} end') 23 | ctx.rule(u'FUNCTION_ARGS',u'') 24 | ctx.rule(u'FUNCTION_ARGS',u'{FUNCTION_ARGLIST}') 25 | ctx.rule(u'FUNCTION_ARGLIST',u'{VAR}, {FUNCTION_ARGLIST}') 26 | ctx.rule(u'FUNCTION_ARGLIST',u'{VAR}') 27 | ctx.rule(u'FUNCTION_ARGLIST',u'...') 28 | ctx.rule(u'COROUTINE',u'{VAR} = coroutine.create({LAMBDA})') 29 | ctx.rule(u'COROUTINE',u'{VAR} = coroutine.wrap({LAMBDA})') 30 | ctx.rule(u'COROUTINE',u'coroutine.resume({VAR}, {ARGS})') 31 | ctx.rule(u'COROUTINE',u'coroutine.yield({ARGS})') 32 | ctx.rule(u'FUNCTIONCALL',u'{IDENTIFIER} {ARGS}') 33 | ctx.rule(u'FUNCTIONCALL',u'{EXPR}:{IDENTIFIER} {ARGS}') 34 | ctx.rule(u'FUNCTIONCALL',u'{EXPR}.{IDENTIFIER} {ARGS}') 35 | ctx.rule(u'ARGS',u'({EXPRLIST})') 36 | ctx.rule(u'ARGS',u'{TABLECONSTRUCTOR}') 37 | ctx.rule(u'ARGS',u'{LITERALSTRING}') 38 | ctx.rule(u'CONDITIONAL',u'if {EXPR} then\n{PROGRAM}\nend') 39 | ctx.rule(u'CONDITIONAL',u'if {EXPR} then\n{PROGRAM}\nelse\n{PROGRAM}\nend') 40 | ctx.rule(u'LOOP',u'while ({EXPR})\ndo\n{PROGRAM}\nend') 41 | ctx.rule(u'LOOP',u'for {VAR}={EXPR}, {EXPR}, {EXPR}\ndo\n{PROGRAM}\nend') 42 | ctx.rule(u'LOOP',u'repeat\n{PROGRAM}\nuntil ({EXPR})') 43 | ctx.rule(u'EXPRLIST',u'{EXPR}, {EXPRLIST}') 44 | ctx.rule(u'EXPRLIST',u'{EXPR}') 45 | ctx.rule(u'EXPR',u'(nil)') 46 | ctx.rule(u'EXPR',u'(false)') 47 | ctx.rule(u'EXPR',u'(true)') 48 | ctx.rule(u'EXPR',u'({NUMERAL})') 49 | ctx.rule(u'EXPR',u'{LITERALSTRING}') 50 | ctx.rule(u'EXPR',u'{TABLECONSTRUCTOR}') 51 | ctx.rule(u'EXPR',u'({VAR}[{EXPR}])') 52 | ctx.rule(u'EXPR',u'({EXPR}{BINOP}{EXPR})') 53 | ctx.rule(u'EXPR',u'({UNOP}{EXPR})') 54 | ctx.rule(u'EXPR',u'{LAMBDA}') 55 | ctx.rule(u'EXPR',u'{VAR}') 56 | ctx.rule(u'EXPR',u'{FUNCTIONCALL}') 57 | ctx.rule(u'EXPR',u'({EXPR})') 58 | ctx.rule(u'EXPR',u'...') 59 | ctx.rule(u'BINOP',u'+') 60 | ctx.rule(u'BINOP',u'-') 61 | ctx.rule(u'BINOP',u'*') 62 | ctx.rule(u'BINOP',u'/') 63 | ctx.rule(u'BINOP',u'//') 64 | ctx.rule(u'BINOP',u'^') 65 | ctx.rule(u'BINOP',u'%') 66 | ctx.rule(u'BINOP',u'&') 67 | ctx.rule(u'BINOP',u'~') 68 | ctx.rule(u'BINOP',u'|') 69 | ctx.rule(u'BINOP',u'>>') 70 | ctx.rule(u'BINOP',u'<<') 71 | ctx.rule(u'BINOP',u' .. ') 72 | ctx.rule(u'BINOP',u'<') 73 | ctx.rule(u'BINOP',u'<=') 74 | ctx.rule(u'BINOP',u'>') 75 | ctx.rule(u'BINOP',u'>=') 76 | ctx.rule(u'BINOP',u'==') 77 | ctx.rule(u'BINOP',u'~=') 78 | ctx.rule(u'BINOP',u' and ') 79 | ctx.rule(u'BINOP',u' or ') 80 | ctx.rule(u'UNOP',u'-') 81 | ctx.rule(u'UNOP',u' not ') 82 | ctx.rule(u'UNOP',u'#') 83 | ctx.rule(u'UNOP',u'~') 84 | ctx.rule(u'TABLECONSTRUCTOR',u'\\{{FIELDLIST}\\}') 85 | ctx.rule(u'METATABLE',u'{VAR} = setmetatable({VAR}, {TABLECONSTRUCTOR})') 86 | ctx.rule(u'FIELDLIST',u'{FIELD},{FIELDLIST}') 87 | ctx.rule(u'FIELDLIST',u'{FIELD}') 88 | ctx.rule(u'FIELD',u'[{EXPR}]={EXPR}') 89 | ctx.rule(u'FIELD',u'{IDENTIFIER}={EXPR}') 90 | ctx.rule(u'FIELD',u'{EXPR}') 91 | ctx.rule(u'VARLIST',u'{VAR}, {VARLIST}') 92 | ctx.rule(u'VARLIST',u'{VAR}') 93 | ctx.rule(u'VAR',u'a') 94 | ctx.rule(u'VAR',u'b') 95 | ctx.rule(u'VAR',u'c') 96 | ctx.rule(u'VAR',u'd') 97 | ctx.rule(u'VAR',u'e') 98 | ctx.rule(u'VAR',u'coroutine') 99 | ctx.rule(u'VAR',u'debug') 100 | ctx.rule(u'VAR',u'math') 101 | ctx.rule(u'VAR',u'io') 102 | ctx.rule(u'VAR',u'os') 103 | ctx.rule(u'VAR',u'package') 104 | ctx.rule(u'VAR',u'string') 105 | ctx.rule(u'VAR',u'table') 106 | ctx.rule(u'VAR',u'utf8') 107 | ctx.rule(u'VAR',u'self') 108 | ctx.rule(u'LITERALSTRING',u'"{STRING}"') 109 | ctx.rule(u'LITERALSTRING',u'[[{STRING}]]') 110 | ctx.rule(u'STRING',u'') 111 | ctx.rule(u'STRING',u'{STRCHR}{STRING}') 112 | ctx.rule(u'STRCHR',u'\n') 113 | ctx.rule(u'STRCHR',u'\r') 114 | ctx.rule(u'STRCHR',u' ') 115 | ctx.rule(u'STRCHR',u'\t') 116 | ctx.rule(u'STRCHR',u'0') 117 | ctx.rule(u'STRCHR',u'a') 118 | ctx.rule(u'STRCHR',u'/') 119 | ctx.rule(u'STRCHR',u'.') 120 | ctx.rule(u'STRCHR',u'$') 121 | ctx.rule(u'STRCHR',u'{ESCAPESEQUENCE}') 122 | ctx.rule(u'ESCAPESEQUENCE',u'\\a') 123 | ctx.rule(u'ESCAPESEQUENCE',u'\\b') 124 | ctx.rule(u'ESCAPESEQUENCE',u'\\f') 125 | ctx.rule(u'ESCAPESEQUENCE',u'\\n') 126 | ctx.rule(u'ESCAPESEQUENCE',u'\\r') 127 | ctx.rule(u'ESCAPESEQUENCE',u'\\t') 128 | ctx.rule(u'ESCAPESEQUENCE',u'\\v') 129 | ctx.rule(u'ESCAPESEQUENCE',u'\\z') 130 | ctx.rule(u'ESCAPESEQUENCE',u'\n') 131 | ctx.rule(u'ESCAPESEQUENCE',u'\\x{HEXADECIMAL}') 132 | ctx.rule(u'ESCAPESEQUENCE',u'\\u\\{{HEXADECIMAL}\\}') 133 | ctx.rule(u'NUMERAL',u'{DECIMAL}') 134 | ctx.rule(u'NUMERAL',u'0x{HEXADECIMAL}') 135 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}') 136 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}e{DECIMALDIGIT}{DECIMALDIGITS}') 137 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}e-{DECIMALDIGIT}{DECIMALDIGITS}') 138 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}.{DECIMALDIGIT}{DECIMALDIGITS}') 139 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}.{DECIMALDIGIT}{DECIMALDIGITS}e{DECIMALDIGIT}{DECIMALDIGITS}') 140 | ctx.rule(u'DECIMAL',u'{DECIMALDIGIT}{DECIMALDIGITS}.{DECIMALDIGIT}{DECIMALDIGITS}e-{DECIMALDIGIT}{DECIMALDIGITS}') 141 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}') 142 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}p{HEXDIGIT}{HEXDIGITS}') 143 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}p-{HEXDIGIT}{HEXDIGITS}') 144 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}.{HEXDIGIT}{HEXDIGITS}') 145 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}.{HEXDIGIT}{HEXDIGITS}p{HEXDIGIT}{HEXDIGITS}') 146 | ctx.rule(u'HEXADECIMAL',u'{HEXDIGIT}{HEXDIGITS}.{HEXDIGIT}{HEXDIGITS}p-{HEXDIGIT}{HEXDIGITS}') 147 | ctx.rule(u'DECIMALDIGITS',u'{DECIMALDIGIT}{DECIMALDIGITS}') 148 | ctx.rule(u'DECIMALDIGITS',u'') 149 | ctx.rule(u'HEXDIGITS',u'{HEXDIGIT}{HEXDIGITS}') 150 | ctx.rule(u'HEXDIGITS',u'') 151 | ctx.rule(u'DECIMALDIGIT',u'0') 152 | ctx.rule(u'DECIMALDIGIT',u'1') 153 | ctx.rule(u'DECIMALDIGIT',u'2') 154 | ctx.rule(u'DECIMALDIGIT',u'3') 155 | ctx.rule(u'DECIMALDIGIT',u'4') 156 | ctx.rule(u'DECIMALDIGIT',u'5') 157 | ctx.rule(u'DECIMALDIGIT',u'6') 158 | ctx.rule(u'DECIMALDIGIT',u'7') 159 | ctx.rule(u'DECIMALDIGIT',u'8') 160 | ctx.rule(u'DECIMALDIGIT',u'9') 161 | ctx.rule(u'HEXDIGIT',u'a') 162 | ctx.rule(u'HEXDIGIT',u'b') 163 | ctx.rule(u'HEXDIGIT',u'c') 164 | ctx.rule(u'HEXDIGIT',u'd') 165 | ctx.rule(u'HEXDIGIT',u'e') 166 | ctx.rule(u'HEXDIGIT',u'f') 167 | ctx.rule(u'HEXDIGIT',u'A') 168 | ctx.rule(u'HEXDIGIT',u'B') 169 | ctx.rule(u'HEXDIGIT',u'C') 170 | ctx.rule(u'HEXDIGIT',u'D') 171 | ctx.rule(u'HEXDIGIT',u'E') 172 | ctx.rule(u'HEXDIGIT',u'F') 173 | ctx.rule(u'HEXDIGIT',u'{DECIMALDIGIT}') 174 | ctx.rule(u'IDENTIFIER',u'self') 175 | ctx.rule(u'IDENTIFIER',u'G') 176 | ctx.rule(u'IDENTIFIER',u'_VERSION') 177 | ctx.rule(u'IDENTIFIER',u'assert') 178 | ctx.rule(u'IDENTIFIER',u'collectgarbage') 179 | ctx.rule(u'IDENTIFIER',u'dofile') 180 | ctx.rule(u'IDENTIFIER',u'error') 181 | ctx.rule(u'IDENTIFIER',u'getmetatable') 182 | ctx.rule(u'IDENTIFIER',u'ipairs') 183 | ctx.rule(u'IDENTIFIER',u'load') 184 | ctx.rule(u'IDENTIFIER',u'loadfile') 185 | ctx.rule(u'IDENTIFIER',u'next') 186 | ctx.rule(u'IDENTIFIER',u'pairs') 187 | ctx.rule(u'IDENTIFIER',u'pcall') 188 | ctx.rule(u'IDENTIFIER',u'print') 189 | ctx.rule(u'IDENTIFIER',u'rawequal') 190 | ctx.rule(u'IDENTIFIER',u'rawget') 191 | ctx.rule(u'IDENTIFIER',u'rawlen') 192 | ctx.rule(u'IDENTIFIER',u'rawset') 193 | ctx.rule(u'IDENTIFIER',u'require') 194 | ctx.rule(u'IDENTIFIER',u'select') 195 | ctx.rule(u'IDENTIFIER',u'setmetatable') 196 | ctx.rule(u'IDENTIFIER',u'tonumber') 197 | ctx.rule(u'IDENTIFIER',u'tostring') 198 | ctx.rule(u'IDENTIFIER',u'type') 199 | ctx.rule(u'IDENTIFIER',u'xpcall') 200 | ctx.rule(u'IDENTIFIER',u'coroutine') 201 | ctx.rule(u'IDENTIFIER',u'create') 202 | ctx.rule(u'IDENTIFIER',u'isyieldable') 203 | ctx.rule(u'IDENTIFIER',u'resume') 204 | ctx.rule(u'IDENTIFIER',u'running') 205 | ctx.rule(u'IDENTIFIER',u'status') 206 | ctx.rule(u'IDENTIFIER',u'wrap') 207 | ctx.rule(u'IDENTIFIER',u'yield') 208 | ctx.rule(u'IDENTIFIER',u'debug') 209 | ctx.rule(u'IDENTIFIER',u'debug') 210 | ctx.rule(u'IDENTIFIER',u'gethook') 211 | ctx.rule(u'IDENTIFIER',u'getinfo') 212 | ctx.rule(u'IDENTIFIER',u'getlocal') 213 | ctx.rule(u'IDENTIFIER',u'getmetatable') 214 | ctx.rule(u'IDENTIFIER',u'getregistry') 215 | ctx.rule(u'IDENTIFIER',u'getupvalue') 216 | ctx.rule(u'IDENTIFIER',u'getuservalue') 217 | ctx.rule(u'IDENTIFIER',u'sethook') 218 | ctx.rule(u'IDENTIFIER',u'setlocal') 219 | ctx.rule(u'IDENTIFIER',u'setmetatable') 220 | ctx.rule(u'IDENTIFIER',u'setupvalue') 221 | ctx.rule(u'IDENTIFIER',u'setuservalue') 222 | ctx.rule(u'IDENTIFIER',u'traceback') 223 | ctx.rule(u'IDENTIFIER',u'upvalueid') 224 | ctx.rule(u'IDENTIFIER',u'upvaluejoin') 225 | ctx.rule(u'IDENTIFIER',u'io') 226 | ctx.rule(u'IDENTIFIER',u'close') 227 | ctx.rule(u'IDENTIFIER',u'flush') 228 | ctx.rule(u'IDENTIFIER',u'input') 229 | ctx.rule(u'IDENTIFIER',u'lines') 230 | ctx.rule(u'IDENTIFIER',u'open') 231 | ctx.rule(u'IDENTIFIER',u'output') 232 | ctx.rule(u'IDENTIFIER',u'popen') 233 | ctx.rule(u'IDENTIFIER',u'read') 234 | ctx.rule(u'IDENTIFIER',u'stderr') 235 | ctx.rule(u'IDENTIFIER',u'stdin') 236 | ctx.rule(u'IDENTIFIER',u'stdout') 237 | ctx.rule(u'IDENTIFIER',u'tmpfile') 238 | ctx.rule(u'IDENTIFIER',u'type') 239 | ctx.rule(u'IDENTIFIER',u'write') 240 | ctx.rule(u'IDENTIFIER',u'math') 241 | ctx.rule(u'IDENTIFIER',u'abs') 242 | ctx.rule(u'IDENTIFIER',u'acos') 243 | ctx.rule(u'IDENTIFIER',u'asin') 244 | ctx.rule(u'IDENTIFIER',u'atan') 245 | ctx.rule(u'IDENTIFIER',u'ceil') 246 | ctx.rule(u'IDENTIFIER',u'cos') 247 | ctx.rule(u'IDENTIFIER',u'deg') 248 | ctx.rule(u'IDENTIFIER',u'exp') 249 | ctx.rule(u'IDENTIFIER',u'floor') 250 | ctx.rule(u'IDENTIFIER',u'fmod') 251 | ctx.rule(u'IDENTIFIER',u'huge') 252 | ctx.rule(u'IDENTIFIER',u'log') 253 | ctx.rule(u'IDENTIFIER',u'max') 254 | ctx.rule(u'IDENTIFIER',u'maxinteger') 255 | ctx.rule(u'IDENTIFIER',u'min') 256 | ctx.rule(u'IDENTIFIER',u'mininteger') 257 | ctx.rule(u'IDENTIFIER',u'modf') 258 | ctx.rule(u'IDENTIFIER',u'pi') 259 | ctx.rule(u'IDENTIFIER',u'rad') 260 | ctx.rule(u'IDENTIFIER',u'random') 261 | ctx.rule(u'IDENTIFIER',u'randomseed') 262 | ctx.rule(u'IDENTIFIER',u'sin') 263 | ctx.rule(u'IDENTIFIER',u'sqrt') 264 | ctx.rule(u'IDENTIFIER',u'tan') 265 | ctx.rule(u'IDENTIFIER',u'tointeger') 266 | ctx.rule(u'IDENTIFIER',u'type') 267 | ctx.rule(u'IDENTIFIER',u'ult') 268 | ctx.rule(u'IDENTIFIER',u'os') 269 | ctx.rule(u'IDENTIFIER',u'clock') 270 | ctx.rule(u'IDENTIFIER',u'date') 271 | ctx.rule(u'IDENTIFIER',u'difftime') 272 | ctx.rule(u'IDENTIFIER',u'exit') 273 | ctx.rule(u'IDENTIFIER',u'getenv') 274 | ctx.rule(u'IDENTIFIER',u'remove') 275 | ctx.rule(u'IDENTIFIER',u'rename') 276 | ctx.rule(u'IDENTIFIER',u'setlocale') 277 | ctx.rule(u'IDENTIFIER',u'time') 278 | ctx.rule(u'IDENTIFIER',u'tmpname') 279 | ctx.rule(u'IDENTIFIER',u'package') 280 | ctx.rule(u'IDENTIFIER',u'config') 281 | ctx.rule(u'IDENTIFIER',u'cpath') 282 | ctx.rule(u'IDENTIFIER',u'loaded') 283 | ctx.rule(u'IDENTIFIER',u'loadlib') 284 | ctx.rule(u'IDENTIFIER',u'path') 285 | ctx.rule(u'IDENTIFIER',u'preload') 286 | ctx.rule(u'IDENTIFIER',u'searchers') 287 | ctx.rule(u'IDENTIFIER',u'searchpath') 288 | ctx.rule(u'IDENTIFIER',u'string') 289 | ctx.rule(u'IDENTIFIER',u'byte') 290 | ctx.rule(u'IDENTIFIER',u'char') 291 | ctx.rule(u'IDENTIFIER',u'dump') 292 | ctx.rule(u'IDENTIFIER',u'find') 293 | ctx.rule(u'IDENTIFIER',u'format') 294 | ctx.rule(u'IDENTIFIER',u'gmatch') 295 | ctx.rule(u'IDENTIFIER',u'gsub') 296 | ctx.rule(u'IDENTIFIER',u'len') 297 | ctx.rule(u'IDENTIFIER',u'lower') 298 | ctx.rule(u'IDENTIFIER',u'match') 299 | ctx.rule(u'IDENTIFIER',u'pack') 300 | ctx.rule(u'IDENTIFIER',u'packsize') 301 | ctx.rule(u'IDENTIFIER',u'rep') 302 | ctx.rule(u'IDENTIFIER',u'reverse') 303 | ctx.rule(u'IDENTIFIER',u'sub') 304 | ctx.rule(u'IDENTIFIER',u'unpack') 305 | ctx.rule(u'IDENTIFIER',u'upper') 306 | ctx.rule(u'IDENTIFIER',u'table') 307 | ctx.rule(u'IDENTIFIER',u'concat') 308 | ctx.rule(u'IDENTIFIER',u'insert') 309 | ctx.rule(u'IDENTIFIER',u'move') 310 | ctx.rule(u'IDENTIFIER',u'pack') 311 | ctx.rule(u'IDENTIFIER',u'remove') 312 | ctx.rule(u'IDENTIFIER',u'sort') 313 | ctx.rule(u'IDENTIFIER',u'unpack') 314 | ctx.rule(u'IDENTIFIER',u'utf8') 315 | ctx.rule(u'IDENTIFIER',u'char') 316 | ctx.rule(u'IDENTIFIER',u'charpattern') 317 | ctx.rule(u'IDENTIFIER',u'codepoint') 318 | ctx.rule(u'IDENTIFIER',u'codes') 319 | ctx.rule(u'IDENTIFIER',u'len') 320 | ctx.rule(u'IDENTIFIER',u'offset') 321 | ctx.rule(u'IDENTIFIER',u'create') 322 | ctx.rule(u'IDENTIFIER',u'isyieldable') 323 | ctx.rule(u'IDENTIFIER',u'resume') 324 | ctx.rule(u'IDENTIFIER',u'running') 325 | ctx.rule(u'IDENTIFIER',u'status') 326 | ctx.rule(u'IDENTIFIER',u'wrap') 327 | ctx.rule(u'IDENTIFIER',u'yield') 328 | ctx.rule(u'IDENTIFIER',u'debug') 329 | ctx.rule(u'IDENTIFIER',u'gethook') 330 | ctx.rule(u'IDENTIFIER',u'getinfo') 331 | ctx.rule(u'IDENTIFIER',u'getlocal') 332 | ctx.rule(u'IDENTIFIER',u'getmetatable') 333 | ctx.rule(u'IDENTIFIER',u'getregistry') 334 | ctx.rule(u'IDENTIFIER',u'getupvalue') 335 | ctx.rule(u'IDENTIFIER',u'getuservalue') 336 | ctx.rule(u'IDENTIFIER',u'sethook') 337 | ctx.rule(u'IDENTIFIER',u'setlocal') 338 | ctx.rule(u'IDENTIFIER',u'setmetatable') 339 | ctx.rule(u'IDENTIFIER',u'setupvalue') 340 | ctx.rule(u'IDENTIFIER',u'setuservalue') 341 | ctx.rule(u'IDENTIFIER',u'traceback') 342 | ctx.rule(u'IDENTIFIER',u'upvalueid') 343 | ctx.rule(u'IDENTIFIER',u'upvaluejoin') 344 | ctx.rule(u'IDENTIFIER',u'close') 345 | ctx.rule(u'IDENTIFIER',u'flush') 346 | ctx.rule(u'IDENTIFIER',u'input') 347 | ctx.rule(u'IDENTIFIER',u'lines') 348 | ctx.rule(u'IDENTIFIER',u'open') 349 | ctx.rule(u'IDENTIFIER',u'output') 350 | ctx.rule(u'IDENTIFIER',u'popen') 351 | ctx.rule(u'IDENTIFIER',u'read') 352 | ctx.rule(u'IDENTIFIER',u'stderr') 353 | ctx.rule(u'IDENTIFIER',u'stdin') 354 | ctx.rule(u'IDENTIFIER',u'stdout') 355 | ctx.rule(u'IDENTIFIER',u'tmpfile') 356 | ctx.rule(u'IDENTIFIER',u'type') 357 | ctx.rule(u'IDENTIFIER',u'write') 358 | ctx.rule(u'IDENTIFIER',u'close') 359 | ctx.rule(u'IDENTIFIER',u'flush') 360 | ctx.rule(u'IDENTIFIER',u'lines') 361 | ctx.rule(u'IDENTIFIER',u'read') 362 | ctx.rule(u'IDENTIFIER',u'seek') 363 | ctx.rule(u'IDENTIFIER',u'setvbuf') 364 | ctx.rule(u'IDENTIFIER',u'write') 365 | ctx.rule(u'IDENTIFIER',u'abs') 366 | ctx.rule(u'IDENTIFIER',u'acos') 367 | ctx.rule(u'IDENTIFIER',u'asin') 368 | ctx.rule(u'IDENTIFIER',u'atan') 369 | ctx.rule(u'IDENTIFIER',u'ceil') 370 | ctx.rule(u'IDENTIFIER',u'cos') 371 | ctx.rule(u'IDENTIFIER',u'deg') 372 | ctx.rule(u'IDENTIFIER',u'exp') 373 | ctx.rule(u'IDENTIFIER',u'floor') 374 | ctx.rule(u'IDENTIFIER',u'fmod') 375 | ctx.rule(u'IDENTIFIER',u'huge') 376 | ctx.rule(u'IDENTIFIER',u'log') 377 | ctx.rule(u'IDENTIFIER',u'max') 378 | ctx.rule(u'IDENTIFIER',u'maxinteger') 379 | ctx.rule(u'IDENTIFIER',u'min') 380 | ctx.rule(u'IDENTIFIER',u'mininteger') 381 | ctx.rule(u'IDENTIFIER',u'modf') 382 | ctx.rule(u'IDENTIFIER',u'pi') 383 | ctx.rule(u'IDENTIFIER',u'rad') 384 | ctx.rule(u'IDENTIFIER',u'random') 385 | ctx.rule(u'IDENTIFIER',u'randomseed') 386 | ctx.rule(u'IDENTIFIER',u'sin') 387 | ctx.rule(u'IDENTIFIER',u'sqrt') 388 | ctx.rule(u'IDENTIFIER',u'tan') 389 | ctx.rule(u'IDENTIFIER',u'tointeger') 390 | ctx.rule(u'IDENTIFIER',u'type') 391 | ctx.rule(u'IDENTIFIER',u'ult') 392 | ctx.rule(u'IDENTIFIER',u'clock') 393 | ctx.rule(u'IDENTIFIER',u'date') 394 | ctx.rule(u'IDENTIFIER',u'difftime') 395 | ctx.rule(u'IDENTIFIER',u'exit') 396 | ctx.rule(u'IDENTIFIER',u'getenv') 397 | ctx.rule(u'IDENTIFIER',u'remove') 398 | ctx.rule(u'IDENTIFIER',u'rename') 399 | ctx.rule(u'IDENTIFIER',u'setlocale') 400 | ctx.rule(u'IDENTIFIER',u'time') 401 | ctx.rule(u'IDENTIFIER',u'tmpname') 402 | ctx.rule(u'IDENTIFIER',u'config') 403 | ctx.rule(u'IDENTIFIER',u'cpath') 404 | ctx.rule(u'IDENTIFIER',u'loaded') 405 | ctx.rule(u'IDENTIFIER',u'loadlib') 406 | ctx.rule(u'IDENTIFIER',u'path') 407 | ctx.rule(u'IDENTIFIER',u'preload') 408 | ctx.rule(u'IDENTIFIER',u'searchers') 409 | ctx.rule(u'IDENTIFIER',u'searchpath') 410 | ctx.rule(u'IDENTIFIER',u'byte') 411 | ctx.rule(u'IDENTIFIER',u'char') 412 | ctx.rule(u'IDENTIFIER',u'dump') 413 | ctx.rule(u'IDENTIFIER',u'find') 414 | ctx.rule(u'IDENTIFIER',u'format') 415 | ctx.rule(u'IDENTIFIER',u'gmatch') 416 | ctx.rule(u'IDENTIFIER',u'gsub') 417 | ctx.rule(u'IDENTIFIER',u'len') 418 | ctx.rule(u'IDENTIFIER',u'lower') 419 | ctx.rule(u'IDENTIFIER',u'match') 420 | ctx.rule(u'IDENTIFIER',u'pack') 421 | ctx.rule(u'IDENTIFIER',u'packsize') 422 | ctx.rule(u'IDENTIFIER',u'rep') 423 | ctx.rule(u'IDENTIFIER',u'reverse') 424 | ctx.rule(u'IDENTIFIER',u'sub') 425 | ctx.rule(u'IDENTIFIER',u'unpack') 426 | ctx.rule(u'IDENTIFIER',u'upper') 427 | ctx.rule(u'IDENTIFIER',u'concat') 428 | ctx.rule(u'IDENTIFIER',u'insert') 429 | ctx.rule(u'IDENTIFIER',u'move') 430 | ctx.rule(u'IDENTIFIER',u'pack') 431 | ctx.rule(u'IDENTIFIER',u'remove') 432 | ctx.rule(u'IDENTIFIER',u'sort') 433 | ctx.rule(u'IDENTIFIER',u'unpack') 434 | ctx.rule(u'IDENTIFIER',u'char') 435 | ctx.rule(u'IDENTIFIER',u'charpattern') 436 | ctx.rule(u'IDENTIFIER',u'codepoint') 437 | ctx.rule(u'IDENTIFIER',u'codes') 438 | ctx.rule(u'IDENTIFIER',u'len') 439 | ctx.rule(u'IDENTIFIER',u'offset') 440 | ctx.rule(u'IDENTIFIER',u'__index') 441 | ctx.rule(u'IDENTIFIER',u'__newindex') 442 | ctx.rule(u'IDENTIFIER',u'__add') 443 | ctx.rule(u'IDENTIFIER',u'__sub') 444 | ctx.rule(u'IDENTIFIER',u'__mul') 445 | ctx.rule(u'IDENTIFIER',u'__div') 446 | ctx.rule(u'IDENTIFIER',u'__mod') 447 | ctx.rule(u'IDENTIFIER',u'__unm') 448 | ctx.rule(u'IDENTIFIER',u'__concat') 449 | ctx.rule(u'IDENTIFIER',u'__eq') 450 | ctx.rule(u'IDENTIFIER',u'__lt') 451 | ctx.rule(u'IDENTIFIER',u'__le') 452 | ctx.rule(u'IDENTIFIER',u'__call') 453 | ctx.rule(u'IDENTIFIER',u'__tostring') 454 | -------------------------------------------------------------------------------- /grammartec/src/tree.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | use std::cmp; 5 | use std::collections::HashSet; 6 | use std::io; 7 | use std::io::Write; 8 | use std::marker::Sized; 9 | 10 | use context::Context; 11 | use newtypes::{NTermID, NodeID, RuleID}; 12 | use pyo3::prelude::{PyObject, PyResult, Python}; 13 | use pyo3::types::{PyBytes, PyString, PyTuple}; 14 | use pyo3::FromPyObject; 15 | use recursion_info::RecursionInfo; 16 | use rule::{PlainRule, Rule, RuleChild, RuleIDOrCustom, ScriptRule, RegExpRule}; 17 | use rand::thread_rng; 18 | use rand::Rng; 19 | 20 | enum UnparseStep<'dat> { 21 | Term(&'dat [u8]), 22 | Nonterm(NTermID), 23 | Script(usize, PyObject), 24 | PushBuffer(), 25 | } 26 | 27 | struct Unparser<'data, 'tree: 'data, 'ctx: 'data, W: Write, T: TreeLike> { 28 | tree: &'tree T, 29 | stack: Vec>, 30 | buffers: Vec>>, 31 | w: W, 32 | i: usize, 33 | ctx: &'ctx Context, 34 | } 35 | 36 | impl<'data, 'tree: 'data, 'ctx: 'data, W: Write, T: TreeLike> Unparser<'data, 'tree, 'ctx, W, T> { 37 | fn new(nid: NodeID, w: W, tree: &'tree T, ctx: &'ctx Context) -> Self { 38 | let i = nid.to_i(); 39 | let nt = tree.get_rule(NodeID::from(i), ctx).nonterm(); 40 | let op = UnparseStep::<'data>::Nonterm(nt); 41 | let stack = vec![op]; 42 | return Self { 43 | stack, 44 | buffers: vec![], 45 | w, 46 | tree, 47 | i, 48 | ctx, 49 | }; 50 | } 51 | 52 | fn unparse_step(&mut self) -> bool { 53 | match self.stack.pop() { 54 | Some(UnparseStep::Term(data)) => self.write(data), 55 | Some(UnparseStep::Nonterm(nt)) => self.nonterm(nt), 56 | Some(UnparseStep::Script(num, expr)) => self.unwrap_script(num, expr), 57 | Some(UnparseStep::PushBuffer()) => self.push_buffer(), 58 | None => return false, 59 | }; 60 | return true; 61 | } 62 | 63 | fn write(&mut self, data: &[u8]) { 64 | if let Some(buff) = self.buffers.last_mut() { 65 | buff.write(data).unwrap(); 66 | } else { 67 | self.w.write(data).unwrap(); 68 | } 69 | } 70 | 71 | fn nonterm(&mut self, nt: NTermID) { 72 | self.next_rule(nt); 73 | } 74 | fn unwrap_script(&mut self, num: usize, expr: PyObject) { 75 | Python::with_gil(|py| { 76 | self.script(py, num, expr) 77 | .map_err(|e| e.print_and_set_sys_last_vars(py)) 78 | .unwrap(); 79 | }); 80 | } 81 | fn script(&mut self, py: Python, num: usize, expr: PyObject) -> PyResult<()> { 82 | use pyo3::PyRef; 83 | let bufs = self.buffers.split_off(self.buffers.len() - num); 84 | let bufs = bufs 85 | .into_iter() 86 | .map(|cur| cur.into_inner()) 87 | .collect::>(); 88 | let byte_arrays = bufs.iter().map(|b| PyBytes::new(py, b)); 89 | let res = expr.call1(py, PyTuple::new(py, byte_arrays))?; 90 | if let Ok(s) = res.extract::<&str>(py){ 91 | self.write(s.as_bytes()); 92 | } else if let Ok(s) = res.extract::<&[u8]>(py) { 93 | self.write(&s); 94 | } else { 95 | return Err(pyo3::exceptions::PyValueError::new_err( 96 | "script function should return string or bytes", 97 | )); 98 | } 99 | return Ok(()); 100 | } 101 | 102 | fn push_buffer(&mut self) { 103 | self.buffers.push(std::io::Cursor::new(vec![])); 104 | } 105 | 106 | fn next_rule(&mut self, nt: NTermID) { 107 | let nid = NodeID::from(self.i); 108 | let rule: &'ctx Rule = self.tree.get_rule(nid, self.ctx); 109 | assert_eq!(nt, rule.nonterm()); 110 | self.i += 1; 111 | match rule { 112 | Rule::Plain(r) => self.next_plain(r), 113 | Rule::Script(r) => self.next_script(r), 114 | Rule::RegExp(_) => self.next_regexp(self.tree.get_custom_rule_data(nid)), 115 | } 116 | } 117 | 118 | fn next_plain(&mut self, r: &'ctx PlainRule) { 119 | for rule_child in r.children.iter().rev() { 120 | let op = match rule_child { 121 | RuleChild::Term(data) => UnparseStep::<'data>::Term(&data), 122 | RuleChild::NTerm(id) => UnparseStep::<'data>::Nonterm(*id), 123 | }; 124 | self.stack.push(op); 125 | } 126 | } 127 | 128 | fn next_script(&mut self, r: &ScriptRule) { 129 | { 130 | Python::with_gil(|py|{ 131 | self.stack.push(UnparseStep::Script( 132 | r.nonterms.len(), 133 | r.script.clone_ref(py), 134 | )); 135 | }); 136 | } 137 | for nterm in r.nonterms.iter().rev() { 138 | self.stack.push(UnparseStep::Nonterm(*nterm)); 139 | self.stack.push(UnparseStep::PushBuffer()); 140 | } 141 | } 142 | 143 | fn next_regexp(&mut self, data: &'tree [u8]) { 144 | self.stack.push(UnparseStep::<'data>::Term(&data)); 145 | } 146 | 147 | fn unparse(&mut self) -> NodeID { 148 | while self.unparse_step() {} 149 | return NodeID::from(self.i); 150 | } 151 | } 152 | 153 | pub trait TreeLike 154 | where 155 | Self: Sized, 156 | { 157 | fn get_rule_id(&self, n: NodeID) -> RuleID; 158 | fn size(&self) -> usize; 159 | fn to_tree(&self, _: &Context) -> Tree; 160 | fn get_rule<'c>(&self, n: NodeID, ctx: &'c Context) -> &'c Rule; 161 | fn get_rule_or_custom(&self, n: NodeID) -> &RuleIDOrCustom; 162 | fn get_custom_rule_data(&self, n: NodeID) -> &[u8]; 163 | fn get_nonterm_id(&self, n: NodeID, ctx: &Context) -> NTermID { 164 | self.get_rule(n, ctx).nonterm() 165 | } 166 | 167 | fn unparse(&self, id: NodeID, ctx: &Context, mut w: &mut W) { 168 | Unparser::new(id, &mut w, self, ctx).unparse(); 169 | } 170 | 171 | fn unparse_to(&self, ctx: &Context, w: &mut W) { 172 | self.unparse(NodeID::from(0), ctx, w); 173 | } 174 | 175 | fn unparse_to_vec(&self, ctx: &Context) -> Vec { 176 | self.unparse_node_to_vec(NodeID::from(0), ctx) 177 | } 178 | 179 | fn unparse_node_to_vec(&self, n: NodeID, ctx: &Context) -> Vec { 180 | let mut data = vec![]; 181 | self.unparse(n, ctx, &mut data); 182 | return data; 183 | } 184 | 185 | fn unparse_print(&self, ctx: &Context) { 186 | self.unparse_to(ctx, &mut io::stdout()); 187 | } 188 | } 189 | 190 | #[derive(Clone, Debug, Serialize, Deserialize)] 191 | pub struct Tree { 192 | pub rules: Vec, 193 | pub sizes: Vec, 194 | pub paren: Vec, 195 | } 196 | 197 | impl TreeLike for Tree { 198 | fn get_rule_id(&self, n: NodeID) -> RuleID { 199 | self.rules[n.to_i()].id() 200 | } 201 | 202 | fn size(&self) -> usize { 203 | return self.rules.len(); 204 | } 205 | 206 | fn to_tree(&self, _ctx: &Context) -> Tree { 207 | return self.clone(); 208 | } 209 | 210 | fn get_rule<'c>(&self, n: NodeID, ctx: &'c Context) -> &'c Rule { 211 | return ctx.get_rule(self.get_rule_id(n)); 212 | } 213 | fn get_custom_rule_data(&self, n: NodeID) -> &[u8] { 214 | self.rules[n.to_i()].data() 215 | } 216 | fn get_rule_or_custom(&self, n: NodeID) -> &RuleIDOrCustom { 217 | &self.rules[n.to_i()] 218 | } 219 | } 220 | 221 | impl Tree { 222 | pub fn from_rule_vec(rules: Vec, ctx: &Context) -> Self { 223 | let sizes = vec![0; rules.len()]; 224 | let paren = vec![NodeID::from(0); rules.len()]; 225 | let mut res = Tree { 226 | rules, 227 | sizes, 228 | paren, 229 | }; 230 | if res.rules.len() > 0 { 231 | res.calc_subtree_sizes_and_parents(ctx); 232 | } 233 | return res; 234 | } 235 | 236 | pub fn get_rule_id(&self, n: NodeID) -> RuleID { 237 | return self.rules[n.to_i()].id(); 238 | } 239 | 240 | fn get_rule_or_custom(&self, n: NodeID) -> &RuleIDOrCustom { 241 | &self.rules[n.to_i()] 242 | } 243 | 244 | pub fn subtree_size(&self, n: NodeID) -> usize { 245 | return self.sizes[n.to_i()]; 246 | } 247 | 248 | pub fn mutate_replace_from_tree<'a>( 249 | &'a self, 250 | n: NodeID, 251 | other: &'a Tree, 252 | other_node: NodeID, 253 | ) -> TreeMutation<'a> { 254 | let old_size = self.subtree_size(n); 255 | let new_size = other.subtree_size(other_node); 256 | return TreeMutation { 257 | prefix: self.slice(0.into(), n), 258 | repl: other.slice(other_node, other_node + new_size), 259 | postfix: self.slice(n + old_size, self.rules.len().into()), 260 | }; 261 | } 262 | 263 | fn calc_subtree_sizes_and_parents(&mut self, ctx: &Context) { 264 | self.calc_parents(ctx); 265 | self.calc_sizes(); 266 | } 267 | 268 | fn calc_parents(&mut self, ctx: &Context) { 269 | if self.size() == 0 { 270 | return; 271 | } 272 | let mut stack: Vec<(NTermID, NodeID)> = Vec::new(); 273 | stack.push(( 274 | self.get_rule(NodeID::from(0), ctx).nonterm(), 275 | NodeID::from(0), 276 | )); 277 | for i in 0..self.size() { 278 | let node_id = NodeID::from(i); 279 | let nonterm = self.get_rule(node_id, ctx).nonterm(); 280 | //sanity check 281 | let (nterm_id, node) = stack.pop().expect("Not a valid tree for unparsing!"); 282 | if nterm_id != nonterm { 283 | panic!("Not a valid tree for unparsing!"); 284 | } else { 285 | self.paren[i] = node; 286 | } 287 | let rule = self.get_rule(node_id, ctx); 288 | for nonterm in rule.nonterms().iter().rev() { 289 | stack.push((*nonterm, node_id)); 290 | } 291 | } 292 | } 293 | 294 | fn calc_sizes(&mut self) { 295 | //Initiate with 1 296 | for size in self.sizes.iter_mut() { 297 | *size = 1; 298 | } 299 | for i in (1..self.size()).rev() { 300 | self.sizes[self.paren[i].to_i()] += self.sizes[i]; 301 | } 302 | } 303 | 304 | fn slice(&self, from: NodeID, to: NodeID) -> &[RuleIDOrCustom] { 305 | return &self.rules[from.into()..to.into()]; 306 | } 307 | 308 | pub fn get_parent(&self, n: NodeID) -> Option { 309 | if n != NodeID::from(0) { 310 | return Some(self.paren[n.to_i()]); 311 | } 312 | return None; 313 | } 314 | 315 | pub fn truncate(&mut self) { 316 | self.rules.truncate(0); 317 | self.sizes.truncate(0); 318 | self.paren.truncate(0); 319 | } 320 | 321 | pub fn generate_from_nt(&mut self, start: NTermID, len: usize, ctx: &Context) { 322 | let ruleid = ctx.get_random_rule_for_nt(start, len); 323 | self.generate_from_rule(ruleid, len - 1, ctx); 324 | } 325 | 326 | pub fn generate_from_rule(&mut self, ruleid: RuleID, max_len: usize, ctx: &Context) { 327 | match ctx.get_rule(ruleid) { 328 | Rule::Plain(..) | Rule::Script(..) => { 329 | self.truncate(); 330 | self.rules.push(RuleIDOrCustom::Rule(ruleid)); 331 | self.sizes.push(0); 332 | self.paren.push(NodeID::from(0)); 333 | ctx.get_rule(ruleid).generate(self, &ctx, max_len); 334 | self.sizes[0] = self.rules.len(); 335 | } 336 | Rule::RegExp(RegExpRule { hir, .. }) => { 337 | let rid = RuleIDOrCustom::Custom( 338 | ruleid, 339 | regex_mutator::generate(hir, thread_rng().gen::()), 340 | ); 341 | self.truncate(); 342 | self.rules.push(rid); 343 | self.sizes.push(0); 344 | self.paren.push(NodeID::from(0)); 345 | self.sizes[0] = self.rules.len(); 346 | } 347 | } 348 | } 349 | 350 | pub fn calc_recursions(&self, ctx: &Context) -> Option> { 351 | let mut ret = Vec::new(); 352 | let mut done_nterms = HashSet::new(); 353 | for rule in &self.rules { 354 | let nterm = ctx.get_nt(&rule); 355 | if !done_nterms.contains(&nterm) { 356 | match RecursionInfo::new(self, nterm, ctx) { 357 | Some(rec_info) => ret.push(rec_info), 358 | None => {} 359 | } 360 | done_nterms.insert(nterm); 361 | } 362 | } 363 | if ret.is_empty() { 364 | return None; 365 | } 366 | return Some(ret); 367 | } 368 | 369 | fn find_recursions_iter(&self, ctx: &Context) -> Vec<(NodeID, NodeID)> { 370 | let mut found_recursions = Vec::new(); 371 | //Only search for iterations for up to 10000 nodes 372 | for i in 1..cmp::min(self.size(), 10000) { 373 | let node_id = NodeID::from(self.size() - i); 374 | let current_nterm: NTermID = self.get_rule(node_id, ctx).nonterm(); 375 | let mut current_node_id = self.paren[node_id.to_i()]; 376 | let mut depth = 0; 377 | while current_node_id != NodeID::from(0) { 378 | if self.get_rule(current_node_id, ctx).nonterm() == current_nterm { 379 | found_recursions.push((current_node_id, node_id)); 380 | } 381 | current_node_id = self.paren[current_node_id.to_i()]; 382 | if depth > 15 { 383 | break; 384 | } 385 | depth += 1; 386 | } 387 | } 388 | return found_recursions; 389 | } 390 | } 391 | 392 | pub struct TreeMutation<'a> { 393 | pub prefix: &'a [RuleIDOrCustom], 394 | pub repl: &'a [RuleIDOrCustom], 395 | pub postfix: &'a [RuleIDOrCustom], 396 | } 397 | 398 | impl<'a> TreeMutation<'a> { 399 | pub fn get_at(&self, n: NodeID) -> &'a RuleIDOrCustom { 400 | let i = n.to_i(); 401 | let end0 = self.prefix.len(); 402 | let end1 = end0 + self.repl.len(); 403 | let end2 = end1 + self.postfix.len(); 404 | if i < end0 { 405 | return &self.prefix[i]; 406 | } 407 | if i < end1 { 408 | return &self.repl[i - end0]; 409 | } 410 | if i < end2 { 411 | return &self.postfix[i - end1]; 412 | } 413 | panic!("index out of bound for rule access"); 414 | } 415 | } 416 | 417 | impl<'a> TreeLike for TreeMutation<'a> { 418 | fn get_rule_id(&self, n: NodeID) -> RuleID { 419 | return self.get_at(n).id(); 420 | } 421 | 422 | fn size(&self) -> usize { 423 | return self.prefix.len() + self.repl.len() + self.postfix.len(); 424 | } 425 | fn get_rule_or_custom(&self, n: NodeID) -> &RuleIDOrCustom { 426 | self.get_at(n) 427 | } 428 | 429 | fn to_tree(&self, ctx: &Context) -> Tree { 430 | let mut vec = vec![]; 431 | vec.extend_from_slice(&self.prefix); 432 | vec.extend_from_slice(&self.repl); 433 | vec.extend_from_slice(&self.postfix); 434 | return Tree::from_rule_vec(vec, ctx); 435 | } 436 | 437 | fn get_rule<'c>(&self, n: NodeID, ctx: &'c Context) -> &'c Rule { 438 | return ctx.get_rule(self.get_rule_id(n)); 439 | } 440 | fn get_custom_rule_data(&self, n: NodeID) -> &[u8] { 441 | self.get_at(n).data() 442 | } 443 | } 444 | 445 | #[cfg(test)] 446 | mod tests { 447 | use super::*; 448 | use context::Context; 449 | use newtypes::NodeID; 450 | 451 | fn calc_subtree_sizes_and_parents_rec_test(tree: &mut Tree, n: NodeID, ctx: &Context) -> usize { 452 | let mut cur = n + 1; 453 | let mut size = 1; 454 | for _ in 0..tree.get_rule(n, ctx).number_of_nonterms() { 455 | tree.paren[cur.to_i()] = n; 456 | let sub_size = calc_subtree_sizes_and_parents_rec_test(tree, cur, ctx); 457 | cur = cur + sub_size; 458 | size += sub_size; 459 | } 460 | tree.sizes[n.to_i()] = size; 461 | return size; 462 | } 463 | 464 | #[test] 465 | fn check_calc_sizes_iter() { 466 | let mut ctx = Context::new(); 467 | let _ = ctx.add_rule("C", b"c{B}c3"); 468 | let _ = ctx.add_rule("B", b"b{A}b23"); 469 | let _ = ctx.add_rule("A", b"aasdf {A}"); 470 | let _ = ctx.add_rule("A", b"a2 {A}"); 471 | let _ = ctx.add_rule("A", b"a sdf{A}"); 472 | let _ = ctx.add_rule("A", b"a 34{A}"); 473 | let _ = ctx.add_rule("A", b"adfe {A}"); 474 | let _ = ctx.add_rule("A", b"a32"); 475 | ctx.initialize(50); 476 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 477 | for _ in 0..100 { 478 | tree.truncate(); 479 | tree.generate_from_nt(ctx.nt_id("C"), 50, &ctx); 480 | calc_subtree_sizes_and_parents_rec_test(&mut tree, NodeID::from(0), &ctx); 481 | let vec1 = tree.sizes.clone(); 482 | tree.calc_sizes(); 483 | let vec2 = tree.sizes.clone(); 484 | assert_eq!(vec1, vec2); 485 | } 486 | } 487 | 488 | #[test] 489 | fn check_calc_paren_iter() { 490 | let mut ctx = Context::new(); 491 | let _ = ctx.add_rule("C", b"c{B}c3"); 492 | let _ = ctx.add_rule("B", b"b{A}b23"); 493 | let _ = ctx.add_rule("A", b"aasdf {A}"); 494 | let _ = ctx.add_rule("A", b"a2 {A}"); 495 | let _ = ctx.add_rule("A", b"a sdf{A}"); 496 | let _ = ctx.add_rule("A", b"a 34{A}"); 497 | let _ = ctx.add_rule("A", b"adfe {A}"); 498 | let _ = ctx.add_rule("A", b"a32"); 499 | ctx.initialize(50); 500 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 501 | for _ in 0..100 { 502 | tree.truncate(); 503 | tree.generate_from_nt(ctx.nt_id("C"), 50, &ctx); 504 | calc_subtree_sizes_and_parents_rec_test(&mut tree, NodeID::from(0), &ctx); 505 | let vec1 = tree.paren.clone(); 506 | tree.calc_parents(&ctx); 507 | let vec2 = tree.paren.clone(); 508 | assert_eq!(vec1, vec2); 509 | } 510 | } 511 | 512 | #[test] 513 | fn check_unparse_iter() { 514 | let mut ctx = Context::new(); 515 | let _ = ctx.add_rule("C", b"c{B}c3"); 516 | let _ = ctx.add_rule("B", b"b{A}b23"); 517 | let _ = ctx.add_rule("A", b"aasdf {A}"); 518 | let _ = ctx.add_rule("A", b"a2 {A}"); 519 | let _ = ctx.add_rule("A", b"a sdf{A}"); 520 | let _ = ctx.add_rule("A", b"a 34{A}"); 521 | let _ = ctx.add_rule("A", b"adfe {A}"); 522 | let _ = ctx.add_rule("A", b"a32"); 523 | ctx.initialize(50); 524 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 525 | for _ in 0..100 { 526 | tree.truncate(); 527 | tree.generate_from_nt(ctx.nt_id("C"), 50, &ctx); 528 | let mut vec1 = vec![]; 529 | let mut vec2 = vec![]; 530 | tree.unparse(NodeID::from(0), &ctx, &mut vec1); 531 | tree.unparse(NodeID::from(0), &ctx, &mut vec2); 532 | assert_eq!(vec1, vec2); 533 | } 534 | } 535 | 536 | #[test] 537 | fn check_find_recursions() { 538 | let mut ctx = Context::new(); 539 | let _ = ctx.add_rule("C", b"c{B}c"); 540 | let _ = ctx.add_rule("B", b"b{A}b"); 541 | let _ = ctx.add_rule("A", b"a {A}"); 542 | let _ = ctx.add_rule("A", b"a {A}"); 543 | let _ = ctx.add_rule("A", b"a {A}"); 544 | let _ = ctx.add_rule("A", b"a {A}"); 545 | let _ = ctx.add_rule("A", b"a {A}"); 546 | let _ = ctx.add_rule("A", b"a"); 547 | ctx.initialize(20); 548 | let mut tree = Tree::from_rule_vec(vec![], &ctx); 549 | let mut some_recursions = false; 550 | for _ in 0..100 { 551 | tree.truncate(); 552 | tree.generate_from_nt(ctx.nt_id("C"), 20, &ctx); 553 | if let Some(recursions) = tree.calc_recursions(&ctx) { 554 | assert_ne!(recursions.len(), 0); 555 | for recursion_info in recursions { 556 | for offset in 0..recursion_info.get_number_of_recursions() { 557 | let tuple = recursion_info.get_recursion_pair_by_offset(offset); 558 | some_recursions = true; 559 | assert!(tuple.0.to_i() < tuple.1.to_i()); 560 | } 561 | } 562 | } 563 | } 564 | assert!(some_recursions); 565 | } 566 | } 567 | -------------------------------------------------------------------------------- /fuzzer/src/main.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | extern crate forksrv; 5 | extern crate grammartec; 6 | extern crate serde_json; 7 | extern crate time as othertime; 8 | #[macro_use] 9 | extern crate serde_derive; 10 | extern crate clap; 11 | extern crate pyo3; 12 | extern crate ron; 13 | 14 | mod config; 15 | mod fuzzer; 16 | mod python_grammar_loader; 17 | mod queue; 18 | mod shared_state; 19 | mod state; 20 | 21 | use config::Config; 22 | use forksrv::newtypes::SubprocessError; 23 | use fuzzer::Fuzzer; 24 | use grammartec::chunkstore::ChunkStoreWrapper; 25 | use grammartec::context::Context; 26 | use queue::{InputState, QueueItem}; 27 | use shared_state::GlobalSharedState; 28 | use state::FuzzingState; 29 | 30 | use clap::{App, Arg}; 31 | use std::fs; 32 | use std::fs::File; 33 | use std::io::Read; 34 | use std::path::Path; 35 | use std::sync::atomic::Ordering; 36 | use std::sync::{Arc, Mutex}; 37 | use std::time::Instant; 38 | use std::{thread, time}; 39 | 40 | fn process_input( 41 | state: &mut FuzzingState, 42 | inp: &mut QueueItem, 43 | config: &Config, 44 | ) -> Result<(), SubprocessError> { 45 | match inp.state { 46 | InputState::Init(start_index) => { 47 | let end_index = start_index + 200; 48 | 49 | if state.minimize(inp, start_index, end_index)? { 50 | inp.state = InputState::Det((0, 0)); 51 | } else { 52 | inp.state = InputState::Init(end_index); 53 | } 54 | } 55 | InputState::Det((cycle, start_index)) => { 56 | let end_index = start_index + 1; 57 | if state.deterministic_tree_mutation(inp, start_index, end_index)? { 58 | if cycle == config.number_of_deterministic_mutations { 59 | inp.state = InputState::Random; 60 | } else { 61 | inp.state = InputState::Det((cycle + 1, 0)); 62 | } 63 | } else { 64 | inp.state = InputState::Det((cycle, end_index)); 65 | } 66 | state.splice(inp)?; 67 | state.havoc(inp)?; 68 | state.havoc_recursion(inp)?; 69 | } 70 | InputState::Random => { 71 | state.splice(inp)?; 72 | state.havoc(inp)?; 73 | state.havoc_recursion(inp)?; 74 | } 75 | } 76 | return Ok(()); 77 | } 78 | 79 | fn fuzzing_thread( 80 | global_state: Arc>, 81 | config: Config, 82 | ctx: Context, 83 | cks: Arc, 84 | ) { 85 | let path_to_bin_target = config.path_to_bin_target.to_owned(); 86 | let args = config.arguments.clone(); 87 | 88 | let fuzzer = Fuzzer::new( 89 | path_to_bin_target.clone(), 90 | args, 91 | global_state.clone(), 92 | config.path_to_workdir.clone(), 93 | config.timeout_in_millis.clone(), 94 | config.bitmap_size.clone(), 95 | ) 96 | .expect("RAND_3617502350"); 97 | let mut state = FuzzingState::new(fuzzer, config.clone(), cks.clone()); 98 | state.ctx = ctx.clone(); 99 | let mut old_execution_count = 0; 100 | let mut old_executions_per_sec = 0; 101 | //Normal mode 102 | loop { 103 | let inp = global_state.lock().expect("RAND_2191486322").queue.pop(); 104 | if let Some(mut inp) = inp { 105 | //If subprocess died restart forkserver 106 | if process_input(&mut state, &mut inp, &config).is_err() { 107 | let args = vec![]; 108 | let fuzzer = Fuzzer::new( 109 | path_to_bin_target.clone(), 110 | args, 111 | global_state.clone(), 112 | config.path_to_workdir.clone(), 113 | config.timeout_in_millis.clone(), 114 | config.bitmap_size.clone(), 115 | ) 116 | .expect("RAND_3077320530"); 117 | state = FuzzingState::new(fuzzer, config.clone(), cks.clone()); 118 | state.ctx = ctx.clone(); 119 | old_execution_count = 0; 120 | old_executions_per_sec = 0; 121 | } 122 | global_state 123 | .lock() 124 | .expect("RAND_788470278") 125 | .queue 126 | .finished(inp); 127 | } else { 128 | for _ in 0..config.number_of_generate_inputs { 129 | //If subprocess dies restart forkserver 130 | if state.generate_random("START").is_err() { 131 | let args = vec![]; 132 | let fuzzer = Fuzzer::new( 133 | path_to_bin_target.clone(), 134 | args, 135 | global_state.clone(), 136 | config.path_to_workdir.clone(), 137 | config.timeout_in_millis.clone(), 138 | config.bitmap_size.clone(), 139 | ) 140 | .expect("RAND_357619639"); 141 | state = FuzzingState::new(fuzzer, config.clone(), cks.clone()); 142 | state.ctx = ctx.clone(); 143 | old_execution_count = 0; 144 | old_executions_per_sec = 0; 145 | } 146 | } 147 | global_state 148 | .lock() 149 | .expect("RAND_2035137253") 150 | .queue 151 | .new_round(); 152 | } 153 | let mut stats = global_state.lock().expect("RAND_2403514078"); 154 | stats.execution_count += state.fuzzer.execution_count - old_execution_count; 155 | old_execution_count = state.fuzzer.execution_count; 156 | stats.average_executions_per_sec += state.fuzzer.average_executions_per_sec as u32; 157 | stats.average_executions_per_sec -= old_executions_per_sec; 158 | old_executions_per_sec = state.fuzzer.average_executions_per_sec as u32; 159 | if state.fuzzer.bits_found_by_havoc > 0 { 160 | stats.bits_found_by_havoc += state.fuzzer.bits_found_by_havoc; 161 | state.fuzzer.bits_found_by_havoc = 0; 162 | } 163 | if state.fuzzer.bits_found_by_gen > 0 { 164 | stats.bits_found_by_gen += state.fuzzer.bits_found_by_gen; 165 | state.fuzzer.bits_found_by_gen = 0; 166 | } 167 | if state.fuzzer.bits_found_by_min > 0 { 168 | stats.bits_found_by_min += state.fuzzer.bits_found_by_min; 169 | state.fuzzer.bits_found_by_min = 0; 170 | } 171 | if state.fuzzer.bits_found_by_det > 0 { 172 | stats.bits_found_by_det += state.fuzzer.bits_found_by_det; 173 | state.fuzzer.bits_found_by_det = 0; 174 | } 175 | if state.fuzzer.bits_found_by_splice > 0 { 176 | stats.bits_found_by_splice += state.fuzzer.bits_found_by_splice; 177 | state.fuzzer.bits_found_by_splice = 0; 178 | } 179 | if state.fuzzer.bits_found_by_havoc_rec > 0 { 180 | stats.bits_found_by_havoc_rec += state.fuzzer.bits_found_by_havoc_rec; 181 | state.fuzzer.bits_found_by_havoc_rec = 0; 182 | } 183 | if state.fuzzer.bits_found_by_min_rec > 0 { 184 | stats.bits_found_by_min_rec += state.fuzzer.bits_found_by_min_rec; 185 | state.fuzzer.bits_found_by_min_rec = 0; 186 | } 187 | } 188 | } 189 | 190 | fn main() { 191 | 192 | pyo3::prepare_freethreaded_python(); 193 | 194 | //Parse parameters 195 | let matches = App::new("nautilus") 196 | .about("Grammar fuzzer") 197 | .setting(clap::AppSettings::TrailingVarArg) 198 | .arg( 199 | Arg::with_name("config") 200 | .short("c") 201 | .value_name("CONFIG") 202 | .takes_value(true) 203 | .help("Path to configuration file") 204 | .default_value("config.ron"), 205 | ) 206 | .arg( 207 | Arg::with_name("grammar") 208 | .short("g") 209 | .takes_value(true) 210 | .help("Overwrite the grammar file specified in the CONFIG"), 211 | ) 212 | .arg( 213 | Arg::with_name("workdir") 214 | .short("o") 215 | .takes_value(true) 216 | .help("Overwrite the workdir specified in the CONFIG"), 217 | ) 218 | .arg(Arg::with_name("cmdline").multiple(true)) 219 | .get_matches(); 220 | 221 | let config_file_path = matches 222 | .value_of("config") 223 | .expect("the path to the configuration file has a default value"); 224 | 225 | println!( 226 | "{} Starting Fuzzing...", 227 | othertime::now() 228 | .strftime("[%Y-%m-%d] %H:%M:%S") 229 | .expect("RAND_1939191497") 230 | ); 231 | 232 | //Set Config 233 | let mut config_file = File::open(&config_file_path).expect("cannot read config file"); 234 | let mut config_file_contents = String::new(); 235 | config_file 236 | .read_to_string(&mut config_file_contents) 237 | .expect("RAND_1413661228"); 238 | let mut config: Config = 239 | ron::de::from_str(&config_file_contents).expect("Failed to deserialize"); 240 | 241 | let workdir = matches 242 | .value_of("workdir") 243 | .unwrap_or(&config.path_to_workdir) 244 | .to_string(); 245 | config.path_to_workdir = workdir; 246 | 247 | //Check if specified workdir exists: 248 | if !Path::new(&config.path_to_workdir).exists() { 249 | panic!( 250 | "Specified working directory does not exist!\nGiven path: {}", 251 | config.path_to_workdir 252 | ); 253 | } 254 | 255 | if let Some(mut cmdline) = matches.values_of("cmdline") { 256 | if cmdline.len() > 0 { 257 | config.path_to_bin_target = cmdline.next().unwrap().to_string(); 258 | config.arguments = cmdline.map(|x| x.to_string()).collect(); 259 | } 260 | } 261 | //Check if target binary exists: 262 | if !Path::new(&config.path_to_bin_target).exists() { 263 | panic!( 264 | "Target binary does not exist!\nGiven path: {}", 265 | config.path_to_bin_target 266 | ); 267 | } 268 | 269 | let shared = Arc::new(Mutex::new(GlobalSharedState::new( 270 | config.path_to_workdir.clone(), 271 | config.bitmap_size, 272 | ))); 273 | let shared_chunkstore = Arc::new(ChunkStoreWrapper::new(config.path_to_workdir.clone())); 274 | 275 | let mut my_context; 276 | let grammar_path = matches 277 | .value_of("grammar") 278 | .unwrap_or(&config.path_to_grammar) 279 | .to_owned(); 280 | 281 | //Check if grammar file exists: 282 | if !Path::new(&grammar_path).exists() { 283 | panic!("Grammar does not exist!\nGiven path: {}", grammar_path); 284 | } 285 | 286 | //Generate rules using a grammar 287 | my_context = Context::new(); 288 | if grammar_path.ends_with(".json") { 289 | let gf = File::open(grammar_path).expect("cannot read grammar file"); 290 | let rules: Vec> = 291 | serde_json::from_reader(&gf).expect("cannot parse grammar file"); 292 | let root = "{".to_string() + &rules[0][0] + "}"; 293 | my_context.add_rule("START", root.as_bytes()); 294 | for rule in rules { 295 | my_context.add_rule(&rule[0], rule[1].as_bytes()); 296 | } 297 | } else if grammar_path.ends_with(".py") { 298 | my_context = python_grammar_loader::load_python_grammar(&grammar_path); 299 | } else { 300 | panic!("Unknown grammar type"); 301 | } 302 | 303 | my_context.initialize(config.max_tree_size); 304 | 305 | //Create output folder 306 | let folders = [ 307 | "/outputs/signaled", 308 | "/outputs/queue", 309 | "/outputs/timeout", 310 | "/outputs/chunks", 311 | ]; 312 | for f in folders.iter() { 313 | fs::create_dir_all(format!("{}/{}", config.path_to_workdir, f)) 314 | .expect("Could not create folder in workdir"); 315 | } 316 | 317 | //Start fuzzing threads 318 | let mut thread_number = 0; 319 | let threads = (0..config.number_of_threads).map(|_| { 320 | let state = shared.clone(); 321 | let config = config.clone(); 322 | let ctx = my_context.clone(); 323 | let cks = shared_chunkstore.clone(); 324 | thread_number += 1; 325 | thread::Builder::new() 326 | .name(format!("fuzzer_{}", thread_number)) 327 | .stack_size(config.thread_size) 328 | .spawn(move || fuzzing_thread(state, config, ctx, cks)) 329 | }); 330 | 331 | //Start status thread 332 | let status_thread = { 333 | let global_state = shared.clone(); 334 | let shared_cks = shared_chunkstore.clone(); 335 | thread::Builder::new() 336 | .name("status_thread".to_string()) 337 | .spawn(move || { 338 | let start_time = Instant::now(); 339 | thread::sleep(time::Duration::from_secs(1)); 340 | print!("{}[2J", 27 as char); 341 | print!("{}[H", 27 as char); 342 | loop { 343 | let execution_count; 344 | let average_executions_per_sec; 345 | let queue_len; 346 | let bits_found_by_gen; 347 | let bits_found_by_min; 348 | let bits_found_by_min_rec; 349 | let bits_found_by_det; 350 | let bits_found_by_splice; 351 | let bits_found_by_havoc; 352 | let bits_found_by_havoc_rec; 353 | let last_found_asan; 354 | let last_found_sig; 355 | let last_timeout; 356 | let total_found_asan; 357 | let total_found_sig; 358 | { 359 | let shared_state = global_state.lock().expect("RAND_597319831"); 360 | execution_count = shared_state.execution_count; 361 | average_executions_per_sec = shared_state.average_executions_per_sec; 362 | queue_len = shared_state.queue.len(); 363 | bits_found_by_gen = shared_state.bits_found_by_gen; 364 | bits_found_by_min = shared_state.bits_found_by_min; 365 | bits_found_by_min_rec = shared_state.bits_found_by_min_rec; 366 | bits_found_by_det = shared_state.bits_found_by_det; 367 | bits_found_by_splice = shared_state.bits_found_by_splice; 368 | bits_found_by_havoc = shared_state.bits_found_by_havoc; 369 | bits_found_by_havoc_rec = shared_state.bits_found_by_havoc_rec; 370 | last_found_asan = shared_state.last_found_asan.clone(); 371 | last_found_sig = shared_state.last_found_sig.clone(); 372 | last_timeout = shared_state.last_timeout.clone(); 373 | total_found_asan = shared_state.total_found_asan; 374 | total_found_sig = shared_state.total_found_sig; 375 | } 376 | let secs = start_time.elapsed().as_secs(); 377 | let minutes = secs / 60; 378 | let hours = minutes / 60; 379 | let days = hours / 24; 380 | 381 | print!("{}[H", 27 as char); 382 | 383 | println!(" _ _ _ _ _ "); 384 | println!(" | \\ | | | | (_) | "); 385 | println!(" | \\| | __ _ _ _| |_ _| |_ _ ___ "); 386 | println!(" | . ` |/ _` | | | | __| | | | | / __| "); 387 | println!(" | |\\ | (_| | |_| | |_| | | |_| \\__ \\ "); 388 | println!(" |_| \\_|\\__,_|\\__,_|\\__|_|_|\\__,_|___/ "); 389 | println!(" "); 390 | 391 | println!("------------------------------------------------------ "); 392 | println!( 393 | "Run Time: {} days, {} hours, {} minutes, {} seconds ", 394 | days, 395 | hours % 24, 396 | minutes % 60, 397 | secs % 60 398 | ); 399 | println!( 400 | "Execution Count: {} ", 401 | execution_count 402 | ); 403 | println!( 404 | "Executions per Sec: {} ", 405 | average_executions_per_sec 406 | ); 407 | println!( 408 | "Left in queue: {} ", 409 | queue_len 410 | ); 411 | let now = Instant::now(); 412 | while shared_cks.is_locked.load(Ordering::SeqCst) { 413 | if now.elapsed().as_secs() > 30 { 414 | panic!("Printing thread starved!"); 415 | } 416 | } 417 | println!( 418 | "Trees in Chunkstore: {} ", 419 | shared_cks 420 | .chunkstore 421 | .read() 422 | .expect("RAND_351823021") 423 | .trees() 424 | ); 425 | println!("------------------------------------------------------ "); 426 | println!( 427 | "Last ASAN crash: {} ", 428 | last_found_asan 429 | ); 430 | println!( 431 | "Last SIG crash: {} ", 432 | last_found_sig 433 | ); 434 | println!( 435 | "Last Timeout: {} ", 436 | last_timeout 437 | ); 438 | println!( 439 | "Total ASAN crashes: {} ", 440 | total_found_asan 441 | ); 442 | println!( 443 | "Total SIG crashes: {} ", 444 | total_found_sig 445 | ); 446 | println!("------------------------------------------------------ "); 447 | println!( 448 | "New paths found by Gen: {} ", 449 | bits_found_by_gen 450 | ); 451 | println!( 452 | "New paths found by Min: {} ", 453 | bits_found_by_min 454 | ); 455 | println!( 456 | "New paths found by Min Rec: {} ", 457 | bits_found_by_min_rec 458 | ); 459 | println!( 460 | "New paths found by Det: {} ", 461 | bits_found_by_det 462 | ); 463 | println!( 464 | "New paths found by Splice: {} ", 465 | bits_found_by_splice 466 | ); 467 | println!( 468 | "New paths found by Havoc: {} ", 469 | bits_found_by_havoc 470 | ); 471 | println!( 472 | "New paths found by Havoc Rec: {} ", 473 | bits_found_by_havoc_rec 474 | ); 475 | println!("------------------------------------------------------ "); 476 | //println!("Global bitmap: {:?}", global_state.lock().expect("RAND_1887203473").bitmaps.get(&false).expect("RAND_1887203473")); 477 | thread::sleep(time::Duration::from_secs(1)); 478 | } 479 | }) 480 | .expect("RAND_3541874337") 481 | }; 482 | 483 | for t in threads.collect::>().into_iter() { 484 | t.expect("RAND_2698731594").join().expect("RAND_2698731594"); 485 | } 486 | status_thread.join().expect("RAND_399292929"); 487 | } 488 | -------------------------------------------------------------------------------- /grammartec/src/mutator.rs: -------------------------------------------------------------------------------- 1 | // Nautilus 2 | // Copyright (C) 2024 Daniel Teuchert, Cornelius Aschermann, Sergej Schumilo 3 | 4 | extern crate rand; 5 | 6 | use rand::Rng; 7 | use rand::seq::SliceRandom; 8 | 9 | use std::collections::HashSet; 10 | use std::mem; 11 | 12 | use chunkstore::ChunkStore; 13 | use context::Context; 14 | use forksrv::newtypes::SubprocessError; 15 | use newtypes::NodeID; 16 | use recursion_info::RecursionInfo; 17 | use rule::RuleIDOrCustom; 18 | use tree::{Tree, TreeLike, TreeMutation}; 19 | 20 | pub struct Mutator { 21 | scratchpad: Tree, 22 | } 23 | 24 | impl Mutator { 25 | pub fn new(ctx: &Context) -> Self { 26 | return Mutator { 27 | scratchpad: Tree::from_rule_vec(vec![], ctx), 28 | }; 29 | } 30 | 31 | //Return value indicates if minimization is complete: true: complete, false: not complete 32 | pub fn minimize_tree( 33 | &mut self, 34 | tree: &mut Tree, 35 | bits: &HashSet, 36 | ctx: &Context, 37 | start_index: usize, 38 | end_index: usize, 39 | tester: &mut F, 40 | ) -> Result 41 | where 42 | F: FnMut(&TreeMutation, &HashSet, &Context) -> Result, 43 | { 44 | let mut i = start_index; 45 | while i < tree.size() { 46 | let n = NodeID::from(i); 47 | let nt = tree.get_rule(n, ctx).nonterm(); 48 | if tree.subtree_size(n) > ctx.get_min_len_for_nt(nt) { 49 | self.scratchpad 50 | .generate_from_nt(nt, ctx.get_min_len_for_nt(nt), &ctx); 51 | if let Some(t) = Mutator::test_and_convert( 52 | tree, 53 | n, 54 | &self.scratchpad, 55 | NodeID::from(0), 56 | ctx, 57 | bits, 58 | tester, 59 | )? { 60 | mem::replace(tree, t); 61 | } 62 | } 63 | i += 1; 64 | if i == end_index { 65 | return Ok(false); 66 | } 67 | } 68 | return Ok(true); 69 | } 70 | 71 | //Return value indicates if minimization is complete: true: complete, false: not complete 72 | pub fn minimize_rec( 73 | &mut self, 74 | tree: &mut Tree, 75 | bits: &HashSet, 76 | ctx: &Context, 77 | start_index: usize, 78 | end_index: usize, 79 | tester: &mut F, 80 | ) -> Result 81 | where 82 | F: FnMut(&TreeMutation, &HashSet, &Context) -> Result, 83 | { 84 | let mut i = start_index; 85 | while i < tree.size() { 86 | let n = NodeID::from(i); 87 | if let Some(parent) = Mutator::find_parent_with_nt(tree, n, ctx) { 88 | if let Some(t) = 89 | Mutator::test_and_convert(tree, parent, tree, n, ctx, bits, tester)? 90 | { 91 | mem::replace(tree, t); 92 | i = parent.into(); 93 | } 94 | } 95 | i += 1; 96 | if i == end_index { 97 | return Ok(false); 98 | } 99 | } 100 | return Ok(true); 101 | } 102 | 103 | pub fn mut_rules( 104 | &mut self, 105 | tree: &Tree, 106 | ctx: &Context, 107 | start_index: usize, 108 | end_index: usize, 109 | tester: &mut F, 110 | ) -> Result 111 | where 112 | F: FnMut(&TreeMutation, &Context) -> Result<(), SubprocessError>, 113 | { 114 | for i in start_index..end_index { 115 | if i == tree.size() { 116 | return Ok(true); 117 | } 118 | let n = NodeID::from(i); 119 | let old_rule_id = tree.get_rule_id(n); 120 | let rule_ids = ctx 121 | .get_rules_for_nt(ctx.get_nt(&RuleIDOrCustom::Rule(old_rule_id))) 122 | .to_vec(); //TODO: Maybe find a better solution 123 | for new_rule_id in rule_ids { 124 | if old_rule_id != new_rule_id { 125 | let random_size = ctx.get_random_len_for_ruleid(&new_rule_id); 126 | self.scratchpad 127 | .generate_from_rule(new_rule_id, random_size, ctx); 128 | let repl = tree.mutate_replace_from_tree(n, &self.scratchpad, NodeID::from(0)); 129 | tester(&repl, ctx)?; 130 | } 131 | } 132 | } 133 | return Ok(false); 134 | } 135 | 136 | pub fn mut_splice( 137 | &mut self, 138 | tree: &Tree, 139 | ctx: &Context, 140 | cks: &ChunkStore, 141 | tester: &mut F, 142 | ) -> Result<(), SubprocessError> 143 | where 144 | F: FnMut(&TreeMutation, &Context) -> Result<(), SubprocessError>, 145 | { 146 | let n = NodeID::from(rand::thread_rng().gen_range(0, tree.size())); 147 | let old_rule_id = tree.get_rule_id(n); 148 | if let Some((repl_tree, repl_node)) = cks.get_alternative_to(old_rule_id, ctx) { 149 | let repl = tree.mutate_replace_from_tree(n, repl_tree, repl_node); 150 | tester(&repl, ctx)?; 151 | } 152 | return Ok(()); 153 | } 154 | 155 | //pub fn rec_splice( 156 | // &mut self, 157 | // tree: &Tree, 158 | // ctx: &Context, 159 | // cks: &ChunkStore, 160 | // tester: &mut F 161 | // )-> Result<(), SubprocessError> 162 | //where 163 | // F: FnMut(&TreeMutation, &Context) -> Result<(), SubprocessError>, 164 | //{ 165 | // let n = NodeID::from(rand::thread_rng().gen_range(0, tree.size())); 166 | // if let Some(old_rule_id) = tree.get_rule_id(n){ 167 | // let nterm_id = ctx.get_rule(old_rule).nonterm(); 168 | // if let Some((repl_tree, repl_node)) = cks.get_alternative_to(old_rule_id, ctx) { 169 | // let repl = tree.mutate_replace_from_tree(n, repl_tree, repl_node); 170 | // tester(&repl, ctx)?; 171 | // } 172 | // } 173 | // 174 | // return Ok(()); 175 | //} 176 | 177 | pub fn mut_random( 178 | &mut self, 179 | tree: &Tree, 180 | ctx: &Context, 181 | tester: &mut F, 182 | ) -> Result<(), SubprocessError> 183 | where 184 | F: FnMut(&TreeMutation, &Context) -> Result<(), SubprocessError>, 185 | { 186 | let n = NodeID::from(rand::thread_rng().gen_range(0, tree.size())); 187 | let nterm = tree.get_rule(n, ctx).nonterm(); 188 | if ctx.check_if_nterm_has_multiple_possiblities(&nterm) { 189 | let len = ctx.get_random_len_for_nt(&nterm); 190 | self.scratchpad.generate_from_nt(nterm, len, ctx); 191 | let repl = tree.mutate_replace_from_tree(n, &self.scratchpad, NodeID::from(0)); 192 | tester(&repl, ctx)?; 193 | } 194 | return Ok(()); 195 | } 196 | 197 | pub fn mut_random_recursion( 198 | &mut self, 199 | tree: &Tree, 200 | recursions: &mut Vec, 201 | ctx: &Context, 202 | tester: &mut F, 203 | ) -> Result<(), SubprocessError> 204 | where 205 | F: FnMut(&TreeMutation, &Context) -> Result<(), SubprocessError>, 206 | { 207 | let max_len_of_recursions = 2 << rand::thread_rng().gen_range(1, 11); 208 | if let Some(recursion_info) = recursions.choose_mut(&mut rand::thread_rng()) { 209 | let recursion = recursion_info.get_random_recursion_pair(); 210 | let recursion_len_pre = recursion.1.to_i() - recursion.0.to_i(); 211 | let recursion_len_total = 212 | tree.subtree_size(recursion.0) - tree.subtree_size(recursion.1); 213 | let recursion_len_post = recursion_len_total - recursion_len_pre; 214 | let num_of_recursions = max_len_of_recursions / recursion_len_total; 215 | //Insert pre recursion 216 | let postfix = tree.subtree_size(recursion.1); 217 | let mut rules_new = Vec::with_capacity( 218 | recursion_len_pre * num_of_recursions 219 | + postfix 220 | + recursion_len_post * num_of_recursions, 221 | ); 222 | let mut sizes_new = Vec::with_capacity( 223 | recursion_len_pre * num_of_recursions 224 | + postfix 225 | + recursion_len_post * num_of_recursions, 226 | ); 227 | for i in 0..num_of_recursions * recursion_len_pre { 228 | rules_new.push( 229 | tree.get_rule_or_custom(recursion.0 + (i % recursion_len_pre)) 230 | .clone(), 231 | ); 232 | sizes_new.push(tree.sizes[recursion.0.to_i() + (i % recursion_len_pre)]); 233 | } 234 | 235 | //Append ending of original tree 236 | for i in 0..postfix { 237 | rules_new.push(tree.get_rule_or_custom(recursion.1 + i).clone()); 238 | sizes_new.push(tree.sizes[recursion.1.to_i() + i]); 239 | } 240 | 241 | //Adjust the sizes 242 | for i in 0..num_of_recursions * recursion_len_pre { 243 | if sizes_new[i] >= recursion_len_pre { 244 | sizes_new[i] += 245 | (num_of_recursions - i / recursion_len_pre - 1) * recursion_len_total; 246 | } 247 | } 248 | 249 | //Append post recursion 250 | for i in 0..num_of_recursions * recursion_len_post { 251 | rules_new.push( 252 | tree.get_rule_or_custom(recursion.1 + postfix + (i % recursion_len_post)) 253 | .clone(), 254 | ); 255 | sizes_new.push(tree.sizes[recursion.1.to_i() + postfix + (i % recursion_len_post)]); 256 | } 257 | 258 | let recursion_tree = Tree { 259 | rules: rules_new, 260 | sizes: sizes_new, 261 | paren: Vec::new(), /*paren_new*/ 262 | }; 263 | let repl = tree.mutate_replace_from_tree(recursion.1, &recursion_tree, NodeID::from(0)); 264 | 265 | tester(&repl, ctx)?; 266 | } 267 | return Ok(()); 268 | } 269 | 270 | fn find_parent_with_nt(tree: &Tree, mut node: NodeID, ctx: &Context) -> Option { 271 | let nt = tree.get_rule(node, ctx).nonterm(); 272 | while let Some(parent) = tree.get_parent(node) { 273 | if tree.get_rule(parent, ctx).nonterm() == nt { 274 | return Some(parent); 275 | } 276 | node = parent; 277 | } 278 | return None; 279 | } 280 | 281 | fn test_and_convert( 282 | tree_a: &Tree, 283 | n_a: NodeID, 284 | tree_b: &Tree, 285 | n_b: NodeID, 286 | ctx: &Context, 287 | fresh_bits: &HashSet, 288 | tester: &mut F, 289 | ) -> Result, SubprocessError> 290 | where 291 | F: FnMut(&TreeMutation, &HashSet, &Context) -> Result, 292 | { 293 | let repl = tree_a.mutate_replace_from_tree(n_a, tree_b, n_b); 294 | if tester(&repl, &fresh_bits, ctx)? { 295 | return Ok(Some(repl.to_tree(ctx))); 296 | } 297 | return Ok(None); 298 | } 299 | } 300 | 301 | #[cfg(test)] 302 | mod tests { 303 | use chunkstore::ChunkStore; 304 | use context::Context; 305 | use mutator::Mutator; 306 | use newtypes::{NodeID, RuleID}; 307 | use rule::RuleIDOrCustom; 308 | use std::collections::HashSet; 309 | use std::str; 310 | use tree::{Tree, TreeLike, TreeMutation}; 311 | 312 | #[test] 313 | fn check_mut_random_recursion() { 314 | let r1 = RuleID::from(0); 315 | let r2 = RuleID::from(1); 316 | let r3 = RuleID::from(2); 317 | let r4 = RuleID::from(3); 318 | let r5 = RuleID::from(4); 319 | 320 | let mut ctx = Context::new(); 321 | ctx.add_rule("N1", b"r1{N2}{N3}{N4}"); 322 | ctx.add_rule("N2", b"r2"); 323 | ctx.add_rule("N3", b"r3{N1}"); 324 | ctx.add_rule("N1", b"r4"); 325 | ctx.add_rule("N4", b"r5"); 326 | 327 | let rules = vec![r1, r2, r3, r4, r5] 328 | .iter() 329 | .map(|x| RuleIDOrCustom::Rule(*x)) 330 | .collect::>(); 331 | let mut tree = Tree::from_rule_vec(rules, &ctx); 332 | 333 | println!("tree: {:?}", tree); 334 | let mut mutator = Mutator::new(&ctx); 335 | let mut tester = |tree_mut: &TreeMutation, _ctx: &Context| { 336 | println!("prefix: {:?}", tree_mut.prefix); 337 | println!("repl: {:?}", tree_mut.repl); 338 | println!("postfix: {:?}", tree_mut.postfix); 339 | println!("mutated tree: "); 340 | assert!( 341 | tree_mut.prefix 342 | == &[r1, r2, r3] 343 | .iter() 344 | .map(|x| RuleIDOrCustom::Rule(*x)) 345 | .collect::>()[..] 346 | ); 347 | assert!( 348 | tree_mut.postfix 349 | == &[r5] 350 | .iter() 351 | .map(|x| RuleIDOrCustom::Rule(*x)) 352 | .collect::>()[..] 353 | ); 354 | 355 | assert!( 356 | tree_mut.repl[0..3] 357 | == [r1, r2, r3] 358 | .iter() 359 | .map(|x| RuleIDOrCustom::Rule(*x)) 360 | .collect::>()[..] 361 | ); 362 | assert_eq!(tree_mut.repl.last(), Some(&RuleIDOrCustom::Rule(r5))); 363 | return Ok(()); 364 | }; 365 | let mut recursions = tree.calc_recursions(&ctx).expect("RAND_3407743327"); 366 | println!("Recursions:\n{:?}", recursions); 367 | mutator 368 | .mut_random_recursion(&mut tree, &mut recursions, &ctx, &mut tester) 369 | .expect("RAND_4227583404"); 370 | } 371 | 372 | #[test] 373 | fn check_minimize_tree() { 374 | let mut ctx = Context::new(); 375 | let r1 = ctx.add_rule("S", b"s1 {A}"); 376 | let _ = ctx.add_rule("S", b"s2"); 377 | let _ = ctx.add_rule("S", b"a1"); 378 | let r2 = ctx.add_rule("A", b"a1 {B}"); 379 | let _ = ctx.add_rule("A", b"a1"); 380 | let _ = ctx.add_rule("A", b"a2"); 381 | let r3 = ctx.add_rule("B", b"b1"); 382 | let _ = ctx.add_rule("B", b"b2"); 383 | let _ = ctx.add_rule("B", b"b3{B}"); 384 | ctx.initialize(10); 385 | for _ in 0..100 { 386 | let mut tree = Tree::from_rule_vec( 387 | vec![r1, r2, r3] 388 | .iter() 389 | .map(|x| RuleIDOrCustom::Rule(*x)) 390 | .collect::>(), 391 | &ctx, 392 | ); 393 | let mut mutator = Mutator::new(&ctx); 394 | { 395 | let mut tester = 396 | |tree_mut: &TreeMutation, _bits: &HashSet, ctx: &Context| { 397 | if String::from_utf8(tree_mut.unparse_to_vec(&ctx)) 398 | .expect("RAND_2486760939") 399 | .contains("a1") 400 | { 401 | return Ok(true); 402 | } else { 403 | return Ok(false); 404 | } 405 | }; 406 | let tree_size = tree.size(); 407 | mutator 408 | .minimize_tree(&mut tree, &HashSet::new(), &ctx, 0, tree_size, &mut tester) 409 | .expect("RAND_4046907857"); 410 | } 411 | let unparse = String::from_utf8(tree.unparse_to_vec(&ctx)).expect("RAND_380778776"); 412 | println!("unparse: {}", unparse); 413 | assert!(unparse.contains("a1")); 414 | 415 | assert!(!unparse.contains("a2")); 416 | assert!(!unparse.contains("b2")); 417 | assert!(!unparse.contains("b3")); 418 | } 419 | } 420 | 421 | #[test] 422 | fn check_minimize_rec() { 423 | let mut ctx = Context::new(); 424 | let r1 = ctx.add_rule("S", b"s1 {A}"); 425 | let _ = ctx.add_rule("S", b"s2"); 426 | let r2 = ctx.add_rule("A", b"a1 {B}"); 427 | let _ = ctx.add_rule("A", b"a1"); 428 | let _ = ctx.add_rule("A", b"a2"); 429 | let r3 = ctx.add_rule("B", b"b1"); 430 | let _ = ctx.add_rule("B", b"b2"); 431 | let _ = ctx.add_rule("B", b"b3{B}"); 432 | ctx.initialize(10); 433 | for _ in 0..100 { 434 | let mut tree = Tree::from_rule_vec( 435 | vec![r1, r2, r3] 436 | .iter() 437 | .map(|x| RuleIDOrCustom::Rule(*x)) 438 | .collect::>(), 439 | &ctx, 440 | ); 441 | let mut mutator = Mutator::new(&ctx); 442 | { 443 | let mut tester = 444 | |tree_mut: &TreeMutation, _bits: &HashSet, ctx: &Context| { 445 | if String::from_utf8(tree_mut.unparse_to_vec(&ctx)) 446 | .expect("RAND_1958219388") 447 | .contains("a1") 448 | { 449 | return Ok(true); 450 | } else { 451 | return Ok(false); 452 | } 453 | }; 454 | let tree_size = tree.size(); 455 | mutator 456 | .minimize_rec(&mut tree, &HashSet::new(), &ctx, 0, tree_size, &mut tester) 457 | .expect("RAND_1814454842"); 458 | } 459 | let unparse = String::from_utf8(tree.unparse_to_vec(&ctx)).expect("RAND_3329325316"); 460 | println!("unparse: {}", unparse); 461 | assert!(unparse.contains("a1")); 462 | 463 | assert!(!unparse.contains("a2")); 464 | assert!(!unparse.contains("b2")); 465 | assert!(!unparse.contains("b3")); 466 | } 467 | } 468 | 469 | #[test] 470 | fn deterministic_rule() { 471 | let mut ctx = Context::new(); 472 | let r1 = ctx.add_rule("A", b"a {A:a}"); 473 | let _ = ctx.add_rule("A", b"b {A:a}"); 474 | let _ = ctx.add_rule("A", b"a"); 475 | ctx.initialize(101); 476 | for _ in 0..100 { 477 | let tree = ctx.generate_tree_from_rule(r1, 100); 478 | let mut mutator = Mutator::new(&ctx); 479 | let unparse = tree.unparse_to_vec(&ctx); 480 | let mut count = 0; 481 | { 482 | let mut tester = |tree_mut: &TreeMutation, ctx: &Context| { 483 | assert_ne!(tree_mut.unparse_to_vec(&ctx), unparse); 484 | count += 1; 485 | return Ok(()); 486 | }; 487 | mutator 488 | .mut_rules(&tree, &ctx, 0, tree.size(), &mut tester) 489 | .expect("RAND_3708258673"); 490 | } 491 | assert!(count > 2); 492 | } 493 | } 494 | 495 | #[test] 496 | fn deterministic_splice() { 497 | let mut ctx = Context::new(); 498 | let mut cks = ChunkStore::new("/tmp/".to_string()); 499 | let r1 = ctx.add_rule("A", b"a {A:a}"); 500 | let _ = ctx.add_rule("A", b"b {A:a}"); 501 | let r3 = ctx.add_rule("A", b"c {A:a}"); 502 | let _ = ctx.add_rule("A", b"a"); 503 | ctx.initialize(101); 504 | let tree = ctx.generate_tree_from_rule(r3, 100); 505 | cks.add_tree(tree, &ctx); 506 | for _ in 0..100 { 507 | let tree = ctx.generate_tree_from_rule(r1, 100); 508 | let mut mutator = Mutator::new(&ctx); 509 | let unparse = tree.unparse_to_vec(&ctx); 510 | let mut tester = |tree_mut: &TreeMutation, ctx: &Context| { 511 | assert_ne!(tree_mut.unparse_to_vec(&ctx), unparse); 512 | return Ok(()); 513 | }; 514 | mutator 515 | .mut_splice(&tree, &ctx, &cks, &mut tester) 516 | .expect("RAND_236145345"); 517 | } 518 | } 519 | 520 | #[test] 521 | fn check_det_rules_values() { 522 | let mut ctx = Context::new(); 523 | let r1 = ctx.add_rule("S", b"s1 {A}"); 524 | let _ = ctx.add_rule("S", b"s2 {A}"); 525 | let r2 = ctx.add_rule("A", b"a1 {B}"); 526 | let _ = ctx.add_rule("A", b"a2 {B}"); 527 | let r3 = ctx.add_rule("B", b"b1"); 528 | let _ = ctx.add_rule("B", b"b2"); 529 | ctx.initialize(10); 530 | for _ in 0..100 { 531 | let tree = Tree::from_rule_vec( 532 | vec![r1, r2, r3] 533 | .iter() 534 | .map(|x| RuleIDOrCustom::Rule(*x)) 535 | .collect::>(), 536 | &ctx, 537 | ); 538 | let mut mutator = Mutator::new(&ctx); 539 | let mut unparses = HashSet::new(); 540 | { 541 | let mut tester = |tree_mut: &TreeMutation, ctx: &Context| { 542 | unparses.insert(tree_mut.unparse_to_vec(&ctx)); 543 | return Ok(()); 544 | }; 545 | mutator 546 | .mut_rules(&tree, &ctx, 0, tree.size(), &mut tester) 547 | .expect("RAND_3954705736"); 548 | } 549 | print!( 550 | "{:?}\n", 551 | unparses 552 | .iter() 553 | .map(|v| str::from_utf8(v).expect("RAND_3927087882")) 554 | .collect::>() 555 | ); 556 | assert!(unparses.contains("s1 a1 b2".as_bytes())); 557 | 558 | assert!( 559 | unparses.contains("s1 a2 b1".as_bytes()) 560 | || unparses.contains("s1 a2 b2".as_bytes()) 561 | ); 562 | 563 | assert!( 564 | unparses.contains("s2 a1 b1".as_bytes()) 565 | || unparses.contains("s2 a2 b2".as_bytes()) 566 | || unparses.contains("s2 a1 b2".as_bytes()) 567 | || unparses.contains("s2 a2 b1".as_bytes()) 568 | ); 569 | } 570 | } 571 | } 572 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "0.7.10" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "ansi_term" 16 | version = "0.11.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 19 | dependencies = [ 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "argparse" 25 | version = "0.2.2" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "3f8ebf5827e4ac4fd5946560e6a99776ea73b596d80898f357007317a7141e47" 28 | 29 | [[package]] 30 | name = "atty" 31 | version = "0.2.14" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 34 | dependencies = [ 35 | "hermit-abi", 36 | "libc", 37 | "winapi", 38 | ] 39 | 40 | [[package]] 41 | name = "autocfg" 42 | version = "1.0.0" 43 | source = "registry+https://github.com/rust-lang/crates.io-index" 44 | checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" 45 | 46 | [[package]] 47 | name = "base64" 48 | version = "0.10.1" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" 51 | dependencies = [ 52 | "byteorder", 53 | ] 54 | 55 | [[package]] 56 | name = "bitflags" 57 | version = "1.2.1" 58 | source = "registry+https://github.com/rust-lang/crates.io-index" 59 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 60 | 61 | [[package]] 62 | name = "bitflags" 63 | version = "2.5.0" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" 66 | 67 | [[package]] 68 | name = "byteorder" 69 | version = "1.3.4" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" 72 | 73 | [[package]] 74 | name = "cc" 75 | version = "1.0.50" 76 | source = "registry+https://github.com/rust-lang/crates.io-index" 77 | checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" 78 | 79 | [[package]] 80 | name = "cfg-if" 81 | version = "0.1.10" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 84 | 85 | [[package]] 86 | name = "cfg-if" 87 | version = "1.0.0" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 90 | 91 | [[package]] 92 | name = "clap" 93 | version = "2.33.1" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129" 96 | dependencies = [ 97 | "ansi_term", 98 | "atty", 99 | "bitflags 1.2.1", 100 | "strsim", 101 | "textwrap", 102 | "unicode-width", 103 | "vec_map", 104 | ] 105 | 106 | [[package]] 107 | name = "doc-comment" 108 | version = "0.3.3" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" 111 | 112 | [[package]] 113 | name = "forksrv" 114 | version = "0.1.0" 115 | dependencies = [ 116 | "byteorder", 117 | "lazy_static", 118 | "libc", 119 | "nix 0.15.0", 120 | "quick-error", 121 | "rand", 122 | "regex", 123 | "serde", 124 | "serde_derive", 125 | "snafu", 126 | "subprocess", 127 | "tempfile", 128 | "time", 129 | "timeout-readwrite", 130 | ] 131 | 132 | [[package]] 133 | name = "fuzzer" 134 | version = "0.1.0" 135 | dependencies = [ 136 | "argparse", 137 | "clap", 138 | "forksrv", 139 | "grammartec", 140 | "libc", 141 | "nix 0.17.0", 142 | "pyo3", 143 | "ron", 144 | "serde", 145 | "serde_derive", 146 | "serde_json", 147 | "subprocess", 148 | "time", 149 | ] 150 | 151 | [[package]] 152 | name = "getrandom" 153 | version = "0.1.14" 154 | source = "registry+https://github.com/rust-lang/crates.io-index" 155 | checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" 156 | dependencies = [ 157 | "cfg-if 0.1.10", 158 | "libc", 159 | "wasi", 160 | ] 161 | 162 | [[package]] 163 | name = "grammartec" 164 | version = "0.1.0" 165 | dependencies = [ 166 | "forksrv", 167 | "hash_by_ref", 168 | "lazy_static", 169 | "loaded_dice", 170 | "num", 171 | "pyo3", 172 | "rand", 173 | "regex", 174 | "regex-syntax", 175 | "regex_mutator", 176 | "serde", 177 | "serde_derive", 178 | ] 179 | 180 | [[package]] 181 | name = "hash_by_ref" 182 | version = "0.1.0" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "0e7664adf57099e954c4407111eb26e4d9e090c7cabde79b548213d3f950a854" 185 | 186 | [[package]] 187 | name = "heck" 188 | version = "0.4.1" 189 | source = "registry+https://github.com/rust-lang/crates.io-index" 190 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 191 | 192 | [[package]] 193 | name = "hermit-abi" 194 | version = "0.1.8" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "1010591b26bbfe835e9faeabeb11866061cc7dcebffd56ad7d0942d0e61aefd8" 197 | dependencies = [ 198 | "libc", 199 | ] 200 | 201 | [[package]] 202 | name = "indoc" 203 | version = "2.0.5" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" 206 | 207 | [[package]] 208 | name = "itoa" 209 | version = "0.4.5" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" 212 | 213 | [[package]] 214 | name = "lazy_static" 215 | version = "1.4.0" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 218 | 219 | [[package]] 220 | name = "libc" 221 | version = "0.2.155" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" 224 | 225 | [[package]] 226 | name = "loaded_dice" 227 | version = "0.2.1" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "b681c42d1864f738b445708fe465e1598be89d77de99c4ea68797d879b4702f0" 230 | dependencies = [ 231 | "rand", 232 | ] 233 | 234 | [[package]] 235 | name = "lock_api" 236 | version = "0.4.6" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" 239 | dependencies = [ 240 | "scopeguard", 241 | ] 242 | 243 | [[package]] 244 | name = "memchr" 245 | version = "2.3.3" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" 248 | 249 | [[package]] 250 | name = "memoffset" 251 | version = "0.9.1" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" 254 | dependencies = [ 255 | "autocfg", 256 | ] 257 | 258 | [[package]] 259 | name = "nix" 260 | version = "0.15.0" 261 | source = "registry+https://github.com/rust-lang/crates.io-index" 262 | checksum = "3b2e0b4f3320ed72aaedb9a5ac838690a8047c7b275da22711fddff4f8a14229" 263 | dependencies = [ 264 | "bitflags 1.2.1", 265 | "cc", 266 | "cfg-if 0.1.10", 267 | "libc", 268 | "void", 269 | ] 270 | 271 | [[package]] 272 | name = "nix" 273 | version = "0.17.0" 274 | source = "registry+https://github.com/rust-lang/crates.io-index" 275 | checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" 276 | dependencies = [ 277 | "bitflags 1.2.1", 278 | "cc", 279 | "cfg-if 0.1.10", 280 | "libc", 281 | "void", 282 | ] 283 | 284 | [[package]] 285 | name = "num" 286 | version = "0.2.1" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" 289 | dependencies = [ 290 | "num-bigint", 291 | "num-complex", 292 | "num-integer", 293 | "num-iter", 294 | "num-rational", 295 | "num-traits", 296 | ] 297 | 298 | [[package]] 299 | name = "num-bigint" 300 | version = "0.2.6" 301 | source = "registry+https://github.com/rust-lang/crates.io-index" 302 | checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" 303 | dependencies = [ 304 | "autocfg", 305 | "num-integer", 306 | "num-traits", 307 | ] 308 | 309 | [[package]] 310 | name = "num-complex" 311 | version = "0.2.4" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" 314 | dependencies = [ 315 | "autocfg", 316 | "num-traits", 317 | ] 318 | 319 | [[package]] 320 | name = "num-integer" 321 | version = "0.1.42" 322 | source = "registry+https://github.com/rust-lang/crates.io-index" 323 | checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" 324 | dependencies = [ 325 | "autocfg", 326 | "num-traits", 327 | ] 328 | 329 | [[package]] 330 | name = "num-iter" 331 | version = "0.1.40" 332 | source = "registry+https://github.com/rust-lang/crates.io-index" 333 | checksum = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00" 334 | dependencies = [ 335 | "autocfg", 336 | "num-integer", 337 | "num-traits", 338 | ] 339 | 340 | [[package]] 341 | name = "num-rational" 342 | version = "0.2.4" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" 345 | dependencies = [ 346 | "autocfg", 347 | "num-bigint", 348 | "num-integer", 349 | "num-traits", 350 | ] 351 | 352 | [[package]] 353 | name = "num-traits" 354 | version = "0.2.11" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" 357 | dependencies = [ 358 | "autocfg", 359 | ] 360 | 361 | [[package]] 362 | name = "once_cell" 363 | version = "1.19.0" 364 | source = "registry+https://github.com/rust-lang/crates.io-index" 365 | checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" 366 | 367 | [[package]] 368 | name = "parking_lot" 369 | version = "0.12.3" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" 372 | dependencies = [ 373 | "lock_api", 374 | "parking_lot_core", 375 | ] 376 | 377 | [[package]] 378 | name = "parking_lot_core" 379 | version = "0.9.10" 380 | source = "registry+https://github.com/rust-lang/crates.io-index" 381 | checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" 382 | dependencies = [ 383 | "cfg-if 1.0.0", 384 | "libc", 385 | "redox_syscall 0.5.2", 386 | "smallvec", 387 | "windows-targets", 388 | ] 389 | 390 | [[package]] 391 | name = "portable-atomic" 392 | version = "1.6.0" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" 395 | 396 | [[package]] 397 | name = "ppv-lite86" 398 | version = "0.2.6" 399 | source = "registry+https://github.com/rust-lang/crates.io-index" 400 | checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" 401 | 402 | [[package]] 403 | name = "proc-macro2" 404 | version = "1.0.85" 405 | source = "registry+https://github.com/rust-lang/crates.io-index" 406 | checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" 407 | dependencies = [ 408 | "unicode-ident", 409 | ] 410 | 411 | [[package]] 412 | name = "pyo3" 413 | version = "0.21.2" 414 | source = "registry+https://github.com/rust-lang/crates.io-index" 415 | checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" 416 | dependencies = [ 417 | "cfg-if 1.0.0", 418 | "indoc", 419 | "libc", 420 | "memoffset", 421 | "parking_lot", 422 | "portable-atomic", 423 | "pyo3-build-config", 424 | "pyo3-ffi", 425 | "pyo3-macros", 426 | "unindent", 427 | ] 428 | 429 | [[package]] 430 | name = "pyo3-build-config" 431 | version = "0.21.2" 432 | source = "registry+https://github.com/rust-lang/crates.io-index" 433 | checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" 434 | dependencies = [ 435 | "once_cell", 436 | "target-lexicon", 437 | ] 438 | 439 | [[package]] 440 | name = "pyo3-ffi" 441 | version = "0.21.2" 442 | source = "registry+https://github.com/rust-lang/crates.io-index" 443 | checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" 444 | dependencies = [ 445 | "libc", 446 | "pyo3-build-config", 447 | ] 448 | 449 | [[package]] 450 | name = "pyo3-macros" 451 | version = "0.21.2" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" 454 | dependencies = [ 455 | "proc-macro2", 456 | "pyo3-macros-backend", 457 | "quote", 458 | "syn 2.0.66", 459 | ] 460 | 461 | [[package]] 462 | name = "pyo3-macros-backend" 463 | version = "0.21.2" 464 | source = "registry+https://github.com/rust-lang/crates.io-index" 465 | checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" 466 | dependencies = [ 467 | "heck", 468 | "proc-macro2", 469 | "pyo3-build-config", 470 | "quote", 471 | "syn 2.0.66", 472 | ] 473 | 474 | [[package]] 475 | name = "quick-error" 476 | version = "1.2.3" 477 | source = "registry+https://github.com/rust-lang/crates.io-index" 478 | checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" 479 | 480 | [[package]] 481 | name = "quote" 482 | version = "1.0.36" 483 | source = "registry+https://github.com/rust-lang/crates.io-index" 484 | checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" 485 | dependencies = [ 486 | "proc-macro2", 487 | ] 488 | 489 | [[package]] 490 | name = "rand" 491 | version = "0.7.3" 492 | source = "registry+https://github.com/rust-lang/crates.io-index" 493 | checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" 494 | dependencies = [ 495 | "getrandom", 496 | "libc", 497 | "rand_chacha", 498 | "rand_core", 499 | "rand_hc", 500 | ] 501 | 502 | [[package]] 503 | name = "rand_chacha" 504 | version = "0.2.2" 505 | source = "registry+https://github.com/rust-lang/crates.io-index" 506 | checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" 507 | dependencies = [ 508 | "ppv-lite86", 509 | "rand_core", 510 | ] 511 | 512 | [[package]] 513 | name = "rand_core" 514 | version = "0.5.1" 515 | source = "registry+https://github.com/rust-lang/crates.io-index" 516 | checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" 517 | dependencies = [ 518 | "getrandom", 519 | ] 520 | 521 | [[package]] 522 | name = "rand_hc" 523 | version = "0.2.0" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" 526 | dependencies = [ 527 | "rand_core", 528 | ] 529 | 530 | [[package]] 531 | name = "redox_syscall" 532 | version = "0.1.56" 533 | source = "registry+https://github.com/rust-lang/crates.io-index" 534 | checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" 535 | 536 | [[package]] 537 | name = "redox_syscall" 538 | version = "0.5.2" 539 | source = "registry+https://github.com/rust-lang/crates.io-index" 540 | checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" 541 | dependencies = [ 542 | "bitflags 2.5.0", 543 | ] 544 | 545 | [[package]] 546 | name = "regex" 547 | version = "1.3.7" 548 | source = "registry+https://github.com/rust-lang/crates.io-index" 549 | checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" 550 | dependencies = [ 551 | "aho-corasick", 552 | "memchr", 553 | "regex-syntax", 554 | "thread_local", 555 | ] 556 | 557 | [[package]] 558 | name = "regex-syntax" 559 | version = "0.6.17" 560 | source = "registry+https://github.com/rust-lang/crates.io-index" 561 | checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" 562 | 563 | [[package]] 564 | name = "regex_mutator" 565 | version = "0.1.0" 566 | dependencies = [ 567 | "regex-syntax", 568 | ] 569 | 570 | [[package]] 571 | name = "remove_dir_all" 572 | version = "0.5.2" 573 | source = "registry+https://github.com/rust-lang/crates.io-index" 574 | checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" 575 | dependencies = [ 576 | "winapi", 577 | ] 578 | 579 | [[package]] 580 | name = "ron" 581 | version = "0.5.1" 582 | source = "registry+https://github.com/rust-lang/crates.io-index" 583 | checksum = "2ece421e0c4129b90e4a35b6f625e472e96c552136f5093a2f4fa2bbb75a62d5" 584 | dependencies = [ 585 | "base64", 586 | "bitflags 1.2.1", 587 | "serde", 588 | ] 589 | 590 | [[package]] 591 | name = "ryu" 592 | version = "1.0.3" 593 | source = "registry+https://github.com/rust-lang/crates.io-index" 594 | checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" 595 | 596 | [[package]] 597 | name = "scopeguard" 598 | version = "1.1.0" 599 | source = "registry+https://github.com/rust-lang/crates.io-index" 600 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 601 | 602 | [[package]] 603 | name = "serde" 604 | version = "1.0.104" 605 | source = "registry+https://github.com/rust-lang/crates.io-index" 606 | checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" 607 | dependencies = [ 608 | "serde_derive", 609 | ] 610 | 611 | [[package]] 612 | name = "serde_derive" 613 | version = "1.0.104" 614 | source = "registry+https://github.com/rust-lang/crates.io-index" 615 | checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" 616 | dependencies = [ 617 | "proc-macro2", 618 | "quote", 619 | "syn 1.0.16", 620 | ] 621 | 622 | [[package]] 623 | name = "serde_json" 624 | version = "1.0.48" 625 | source = "registry+https://github.com/rust-lang/crates.io-index" 626 | checksum = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" 627 | dependencies = [ 628 | "itoa", 629 | "ryu", 630 | "serde", 631 | ] 632 | 633 | [[package]] 634 | name = "smallvec" 635 | version = "1.13.2" 636 | source = "registry+https://github.com/rust-lang/crates.io-index" 637 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" 638 | 639 | [[package]] 640 | name = "snafu" 641 | version = "0.6.8" 642 | source = "registry+https://github.com/rust-lang/crates.io-index" 643 | checksum = "c7f5aed652511f5c9123cf2afbe9c244c29db6effa2abb05c866e965c82405ce" 644 | dependencies = [ 645 | "doc-comment", 646 | "snafu-derive", 647 | ] 648 | 649 | [[package]] 650 | name = "snafu-derive" 651 | version = "0.6.8" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "ebf8f7d5720104a9df0f7076a8682024e958bba0fe9848767bb44f251f3648e9" 654 | dependencies = [ 655 | "proc-macro2", 656 | "quote", 657 | "syn 1.0.16", 658 | ] 659 | 660 | [[package]] 661 | name = "strsim" 662 | version = "0.8.0" 663 | source = "registry+https://github.com/rust-lang/crates.io-index" 664 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 665 | 666 | [[package]] 667 | name = "subprocess" 668 | version = "0.2.4" 669 | source = "registry+https://github.com/rust-lang/crates.io-index" 670 | checksum = "5109e3c3215ce5339ca9766a6db7f39144ccc7d9bad53b96d65cfb14ae6c2985" 671 | dependencies = [ 672 | "libc", 673 | "winapi", 674 | ] 675 | 676 | [[package]] 677 | name = "syn" 678 | version = "1.0.16" 679 | source = "registry+https://github.com/rust-lang/crates.io-index" 680 | checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" 681 | dependencies = [ 682 | "proc-macro2", 683 | "quote", 684 | "unicode-xid", 685 | ] 686 | 687 | [[package]] 688 | name = "syn" 689 | version = "2.0.66" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" 692 | dependencies = [ 693 | "proc-macro2", 694 | "quote", 695 | "unicode-ident", 696 | ] 697 | 698 | [[package]] 699 | name = "target-lexicon" 700 | version = "0.12.14" 701 | source = "registry+https://github.com/rust-lang/crates.io-index" 702 | checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" 703 | 704 | [[package]] 705 | name = "tempfile" 706 | version = "3.1.0" 707 | source = "registry+https://github.com/rust-lang/crates.io-index" 708 | checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" 709 | dependencies = [ 710 | "cfg-if 0.1.10", 711 | "libc", 712 | "rand", 713 | "redox_syscall 0.1.56", 714 | "remove_dir_all", 715 | "winapi", 716 | ] 717 | 718 | [[package]] 719 | name = "textwrap" 720 | version = "0.11.0" 721 | source = "registry+https://github.com/rust-lang/crates.io-index" 722 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 723 | dependencies = [ 724 | "unicode-width", 725 | ] 726 | 727 | [[package]] 728 | name = "thread_local" 729 | version = "1.0.1" 730 | source = "registry+https://github.com/rust-lang/crates.io-index" 731 | checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" 732 | dependencies = [ 733 | "lazy_static", 734 | ] 735 | 736 | [[package]] 737 | name = "time" 738 | version = "0.1.42" 739 | source = "registry+https://github.com/rust-lang/crates.io-index" 740 | checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" 741 | dependencies = [ 742 | "libc", 743 | "redox_syscall 0.1.56", 744 | "winapi", 745 | ] 746 | 747 | [[package]] 748 | name = "timeout-readwrite" 749 | version = "0.3.1" 750 | source = "registry+https://github.com/rust-lang/crates.io-index" 751 | checksum = "15f9e83663e1c312cbc7980611fa82ed60d622a947de3546350b549e5d69f9bc" 752 | dependencies = [ 753 | "nix 0.17.0", 754 | ] 755 | 756 | [[package]] 757 | name = "unicode-ident" 758 | version = "1.0.12" 759 | source = "registry+https://github.com/rust-lang/crates.io-index" 760 | checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" 761 | 762 | [[package]] 763 | name = "unicode-width" 764 | version = "0.1.7" 765 | source = "registry+https://github.com/rust-lang/crates.io-index" 766 | checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" 767 | 768 | [[package]] 769 | name = "unicode-xid" 770 | version = "0.2.0" 771 | source = "registry+https://github.com/rust-lang/crates.io-index" 772 | checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" 773 | 774 | [[package]] 775 | name = "unindent" 776 | version = "0.2.3" 777 | source = "registry+https://github.com/rust-lang/crates.io-index" 778 | checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" 779 | 780 | [[package]] 781 | name = "vec_map" 782 | version = "0.8.1" 783 | source = "registry+https://github.com/rust-lang/crates.io-index" 784 | checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" 785 | 786 | [[package]] 787 | name = "void" 788 | version = "1.0.2" 789 | source = "registry+https://github.com/rust-lang/crates.io-index" 790 | checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" 791 | 792 | [[package]] 793 | name = "wasi" 794 | version = "0.9.0+wasi-snapshot-preview1" 795 | source = "registry+https://github.com/rust-lang/crates.io-index" 796 | checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" 797 | 798 | [[package]] 799 | name = "winapi" 800 | version = "0.3.8" 801 | source = "registry+https://github.com/rust-lang/crates.io-index" 802 | checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" 803 | dependencies = [ 804 | "winapi-i686-pc-windows-gnu", 805 | "winapi-x86_64-pc-windows-gnu", 806 | ] 807 | 808 | [[package]] 809 | name = "winapi-i686-pc-windows-gnu" 810 | version = "0.4.0" 811 | source = "registry+https://github.com/rust-lang/crates.io-index" 812 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 813 | 814 | [[package]] 815 | name = "winapi-x86_64-pc-windows-gnu" 816 | version = "0.4.0" 817 | source = "registry+https://github.com/rust-lang/crates.io-index" 818 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 819 | 820 | [[package]] 821 | name = "windows-targets" 822 | version = "0.52.5" 823 | source = "registry+https://github.com/rust-lang/crates.io-index" 824 | checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" 825 | dependencies = [ 826 | "windows_aarch64_gnullvm", 827 | "windows_aarch64_msvc", 828 | "windows_i686_gnu", 829 | "windows_i686_gnullvm", 830 | "windows_i686_msvc", 831 | "windows_x86_64_gnu", 832 | "windows_x86_64_gnullvm", 833 | "windows_x86_64_msvc", 834 | ] 835 | 836 | [[package]] 837 | name = "windows_aarch64_gnullvm" 838 | version = "0.52.5" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" 841 | 842 | [[package]] 843 | name = "windows_aarch64_msvc" 844 | version = "0.52.5" 845 | source = "registry+https://github.com/rust-lang/crates.io-index" 846 | checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" 847 | 848 | [[package]] 849 | name = "windows_i686_gnu" 850 | version = "0.52.5" 851 | source = "registry+https://github.com/rust-lang/crates.io-index" 852 | checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" 853 | 854 | [[package]] 855 | name = "windows_i686_gnullvm" 856 | version = "0.52.5" 857 | source = "registry+https://github.com/rust-lang/crates.io-index" 858 | checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" 859 | 860 | [[package]] 861 | name = "windows_i686_msvc" 862 | version = "0.52.5" 863 | source = "registry+https://github.com/rust-lang/crates.io-index" 864 | checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" 865 | 866 | [[package]] 867 | name = "windows_x86_64_gnu" 868 | version = "0.52.5" 869 | source = "registry+https://github.com/rust-lang/crates.io-index" 870 | checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" 871 | 872 | [[package]] 873 | name = "windows_x86_64_gnullvm" 874 | version = "0.52.5" 875 | source = "registry+https://github.com/rust-lang/crates.io-index" 876 | checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" 877 | 878 | [[package]] 879 | name = "windows_x86_64_msvc" 880 | version = "0.52.5" 881 | source = "registry+https://github.com/rust-lang/crates.io-index" 882 | checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" 883 | --------------------------------------------------------------------------------