├── helpers ├── .gitignore ├── Cargo.toml └── src │ ├── hash_by_ref.rs │ └── lib.rs ├── rust_fuzzer ├── .gitignore ├── deploy.sh ├── Cargo.toml └── src │ ├── romu.rs │ ├── input.rs │ ├── bitmap.rs │ ├── runner.rs │ ├── main.rs │ └── queue.rs ├── logo.png ├── rust_fuzzer_debug ├── .gitignore ├── Cargo.toml └── src │ └── main.rs ├── structured_fuzzer ├── src │ ├── primitive_mutator │ │ ├── .gitignore │ │ ├── mod.rs │ │ ├── mutations.txt │ │ ├── size_changing_mutation.rs │ │ ├── inplace_mutation.rs │ │ └── mutator.rs │ ├── random │ │ ├── mod.rs │ │ ├── choices.rs │ │ ├── romu.rs │ │ └── distributions.rs │ ├── graph_mutator │ │ ├── mod.rs │ │ ├── generators.rs │ │ ├── newtypes.rs │ │ ├── regex_generator.rs │ │ ├── graph_iter.rs │ │ ├── spec_loader.rs │ │ ├── spec.rs │ │ ├── atomic_data.rs │ │ ├── graph_builder.rs │ │ └── graph_storage.rs │ ├── custom_dict.rs │ ├── data_buff.rs │ ├── lib.rs │ └── mutator.rs ├── .gitignore ├── Cargo.toml └── examples │ ├── display.rs │ └── gen.rs ├── nyx-fuzzer.gif ├── .gitmodules ├── setup.sh └── README.md /helpers/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /rust_fuzzer/.gitignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyx-fuzz/spec-fuzzer/HEAD/logo.png -------------------------------------------------------------------------------- /rust_fuzzer_debug/.gitignore: -------------------------------------------------------------------------------- 1 | debug/ 2 | target/ 3 | 4 | Cargo.lock 5 | -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /nyx-fuzzer.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyx-fuzz/spec-fuzzer/HEAD/nyx-fuzzer.gif -------------------------------------------------------------------------------- /structured_fuzzer/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock 4 | interpreter/.vscode 5 | -------------------------------------------------------------------------------- /structured_fuzzer/src/random/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod choices; 2 | pub mod distributions; 3 | pub mod romu; 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libnyx"] 2 | path = libnyx 3 | url = https://github.com/nyx-fuzz/libnyx.git 4 | -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod inplace_mutation; 2 | pub mod mutator; 3 | pub mod size_changing_mutation; 4 | -------------------------------------------------------------------------------- /rust_fuzzer/deploy.sh: -------------------------------------------------------------------------------- 1 | cargo build --release 2 | ssh kafl_02 'pkill -9 qemu' 3 | ssh kafl_02 'pkill -9 fuzz' 4 | scp target/release/rust_fuzzer kafl_02:/tmp/fuzz 5 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod atomic_data; 2 | pub mod graph_builder; 3 | pub mod graph_iter; 4 | pub mod graph_storage; 5 | pub mod newtypes; 6 | pub mod spec; 7 | pub mod spec_loader; 8 | pub mod generators; 9 | pub mod regex_generator; -------------------------------------------------------------------------------- /helpers/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "helpers" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | nix="0.17.0" 11 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | set -e 4 | 5 | echo "[?] Checking submodules ..." 6 | git submodule init 7 | git submodule update 8 | 9 | echo "[?] Checking rust_fuzzer ..." 10 | cd rust_fuzzer 11 | cargo build --release 12 | cd - 13 | 14 | echo "[?] Checking rust_fuzzer_debug ..." 15 | cd rust_fuzzer_debug 16 | cargo build --release 17 | cd - 18 | 19 | echo "[*] Done ... " -------------------------------------------------------------------------------- /structured_fuzzer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "structured_fuzzer" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | rand = "0.7.3" 11 | rand_core="0.5.1" 12 | serde ="1.0.104" 13 | serde_derive ="1.0.104" 14 | rmp-serde ="0.14.3" 15 | itertools="0.9.0" 16 | regex-syntax="0.6.18" 17 | 18 | [dev-dependencies] 19 | clap="2.33.0" 20 | -------------------------------------------------------------------------------- /structured_fuzzer/src/random/choices.rs: -------------------------------------------------------------------------------- 1 | use rand::distributions::weighted::alias_method::WeightedIndex; 2 | use rand::prelude::*; 3 | 4 | pub struct Choices { 5 | weights: WeightedIndex, 6 | options: Vec, 7 | } 8 | 9 | impl Choices { 10 | pub fn new(weights: Vec, options: Vec) -> Self { 11 | let weights = WeightedIndex::new(weights).unwrap(); 12 | return Self { weights, options }; 13 | } 14 | 15 | pub fn sample<'a, R: Rng>(&'a self, rng: &mut R) -> &'a T { 16 | let i = self.weights.sample(rng); 17 | return &self.options[i]; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /rust_fuzzer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust_fuzzer" 3 | version = "0.1.0" 4 | authors = ["coco "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | nix = "0.17.0" 11 | regex = "1.3.6" 12 | libnyx={path="../libnyx/libnyx"} 13 | structured_fuzzer={path="../structured_fuzzer"} 14 | helpers={path="../helpers"} 15 | core_affinity="0.5.10" 16 | serde ="1.0.104" 17 | serde_derive ="1.0.104" 18 | rmp-serde ="0.14.3" 19 | ron="0.6.2" 20 | clap="2.33.0" 21 | rand = "0.7.3" 22 | glob="0.3.0" 23 | colored = "2.0.0" 24 | lazy_static = "1.4.0" 25 | hex="0.4.2" 26 | -------------------------------------------------------------------------------- /rust_fuzzer_debug/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust_fuzzer_debug" 3 | version = "0.1.0" 4 | authors = ["Sergej Schumilo "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | fuzz_runner={path="../libnyx/fuzz_runner"} 11 | config={path="../libnyx/config"} 12 | structured_fuzzer={path="../structured_fuzzer"} 13 | helpers={path="../helpers"} 14 | core_affinity="0.5.10" 15 | serde ="1.0.104" 16 | serde_derive ="1.0.104" 17 | rmp-serde ="0.14.3" 18 | ron = "0.5.1" 19 | clap="2.33.0" 20 | rand = "0.7.3" 21 | glob="0.3.0" 22 | colored = "2.0.0" 23 | walkdir = "2.3.1" 24 | filetime = "0.2" 25 | -------------------------------------------------------------------------------- /helpers/src/hash_by_ref.rs: -------------------------------------------------------------------------------- 1 | use std::rc::Rc; 2 | use std::ptr; 3 | use std::hash::{Hash,Hasher}; 4 | 5 | #[derive(Clone,Debug)] 6 | pub struct HashAsRef(Rc); 7 | 8 | impl Hash for HashAsRef { 9 | fn hash(&self, state: &mut H) { 10 | let ptr: *const T = &*self.0; 11 | ptr::hash(ptr, state); 12 | } 13 | } 14 | 15 | impl PartialEq for HashAsRef { 16 | fn eq(&self, other: &Self) -> bool { 17 | Rc::ptr_eq(&self.0, &other.0) 18 | } 19 | } 20 | impl Eq for HashAsRef {} 21 | 22 | impl HashAsRef{ 23 | pub fn new(rc: Rc) -> Self{ 24 | return Self(rc); 25 | } 26 | pub fn into_rc(self) -> Rc{ 27 | self.0 28 | } 29 | pub fn as_usize(&self) -> usize{ 30 | let ptr: *const T = &*self.0; 31 | return ptr as usize; 32 | } 33 | } -------------------------------------------------------------------------------- /rust_fuzzer/src/romu.rs: -------------------------------------------------------------------------------- 1 | pub struct RomuPrng { 2 | xstate: u64, 3 | ystate: u64, 4 | } 5 | 6 | impl RomuPrng { 7 | pub fn new(xstate: u64, ystate: u64) -> Self { 8 | return Self { xstate, ystate }; 9 | } 10 | 11 | /* 12 | pub fn range(&mut self, min: usize, max: usize) -> usize { 13 | return ((self.next_u64() as usize) % (max - min)) + min; 14 | } 15 | */ 16 | 17 | pub fn new_from_u64(seed: u64) -> Self { 18 | return Self::new(seed, seed ^ 0xec77152282650854); 19 | } 20 | 21 | /* 22 | pub fn next_u32(&mut self) -> u32 { 23 | self.next_u64() as u32 24 | } 25 | */ 26 | 27 | pub fn next_u64(&mut self) -> u64 { 28 | let xp = self.xstate; 29 | self.xstate = 15241094284759029579u64.wrapping_mul(self.ystate); 30 | self.ystate = self.ystate.wrapping_sub(xp); 31 | self.ystate = self.ystate.rotate_left(27); 32 | return xp; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /helpers/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate nix; 2 | 3 | use core::ffi::c_void; 4 | use nix::sys::mman::*; 5 | use std::fs::{File, OpenOptions}; 6 | use std::os::unix::io::IntoRawFd; 7 | 8 | mod hash_by_ref; 9 | pub use hash_by_ref::HashAsRef; 10 | 11 | pub fn make_shared_data_from_path(path: &str, size: usize) -> &'static mut[u8] { 12 | let data_shm_f = OpenOptions::new() 13 | .create(true) 14 | .read(true) 15 | .write(true) 16 | .open(path) 17 | .expect("couldn't open input file"); 18 | data_shm_f.set_len(size as u64).unwrap(); 19 | return make_shared_data_from_file(data_shm_f, size); 20 | } 21 | 22 | pub fn make_shared_data_from_file(file: File, size: usize) -> &'static mut [u8] { 23 | let prot = ProtFlags::PROT_READ | ProtFlags::PROT_WRITE; 24 | let flags = MapFlags::MAP_SHARED; 25 | unsafe { 26 | let ptr = mmap(0 as *mut c_void, size, prot, flags, file.into_raw_fd(), 0).unwrap(); 27 | 28 | let data = std::slice::from_raw_parts_mut(ptr as *mut u8, size); 29 | return data; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/mutations.txt: -------------------------------------------------------------------------------- 1 | Text based mutations 2 | 3 | 4 | size_t Mutate_Custom(uint8_t *Data, size_t Size, size_t MaxSize); 5 | /// Mutates data by invoking user-provided crossover. 6 | size_t Mutate_CustomCrossOver(uint8_t *Data, size_t Size, size_t MaxSize); 7 | /// Mutates data by shuffling bytes. 8 | size_t Mutate_ShuffleBytes(uint8_t *Data, size_t Size, size_t MaxSize); 9 | /// Mutates data by erasing bytes. 10 | size_t Mutate_EraseBytes(uint8_t *Data, size_t Size, size_t MaxSize); 11 | /// Mutates data by inserting a byte. 12 | size_t Mutate_InsertByte(uint8_t *Data, size_t Size, size_t MaxSize); 13 | /// Mutates data by inserting several repeated bytes. 14 | size_t Mutate_InsertRepeatedBytes(uint8_t *Data, size_t Size, size_t MaxSize); 15 | /// Mutates data by chanding one byte. 16 | size_t Mutate_ChangeByte(uint8_t *Data, size_t Size, size_t MaxSize); 17 | /// Mutates data by chanding one bit. 18 | size_t Mutate_ChangeBit(uint8_t *Data, size_t Size, size_t MaxSize); 19 | /// Mutates data by copying/inserting a part of data into a different place. 20 | size_t Mutate_CopyPart(uint8_t *Data, size_t Size, size_t MaxSize); -------------------------------------------------------------------------------- /structured_fuzzer/src/random/romu.rs: -------------------------------------------------------------------------------- 1 | use rand::SeedableRng; 2 | use rand_core::{impls, Error, RngCore}; 3 | 4 | pub struct RomuPrng { 5 | xstate: u64, 6 | ystate: u64, 7 | } 8 | 9 | impl RomuPrng { 10 | pub fn new(xstate: u64, ystate: u64) -> Self { 11 | return Self { xstate, ystate }; 12 | } 13 | } 14 | 15 | impl SeedableRng for RomuPrng { 16 | type Seed = [u8; 16]; 17 | 18 | fn from_seed(seed: [u8; 16]) -> RomuPrng { 19 | if seed == [0; 16] { 20 | return RomuPrng::new(0x0DDB_1A5E_5BAD_5EED_u64, 0x519f_b20c_e6a1_99bb_u64); 21 | } 22 | let x = u64::from_le_bytes([ 23 | seed[0], seed[1], seed[2], seed[3], seed[4], seed[5], seed[6], seed[7], 24 | ]); 25 | let y = u64::from_le_bytes([ 26 | seed[8], seed[9], seed[10], seed[11], seed[12], seed[13], seed[14], seed[15], 27 | ]); 28 | return RomuPrng::new(x, y); 29 | } 30 | } 31 | 32 | impl RngCore for RomuPrng { 33 | fn next_u32(&mut self) -> u32 { 34 | self.next_u64() as u32 35 | } 36 | 37 | fn next_u64(&mut self) -> u64 { 38 | let xp = self.xstate; 39 | self.xstate = 15241094284759029579u64.wrapping_mul(self.ystate); 40 | self.ystate = self.ystate.wrapping_sub(xp); 41 | self.ystate = self.ystate.rotate_left(27); 42 | return xp; 43 | } 44 | 45 | fn fill_bytes(&mut self, dest: &mut [u8]) { 46 | impls::fill_bytes_via_next(self, dest) 47 | } 48 | 49 | fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { 50 | Ok(self.fill_bytes(dest)) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /rust_fuzzer/src/input.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | //use std::sync::RwLock; 3 | use std::time::Duration; 4 | 5 | use crate::structured_fuzzer::custom_dict::CustomDict; 6 | use crate::bitmap::{Bitmap, StorageReason}; 7 | 8 | use super::runner::ExitReason; 9 | use crate::structured_fuzzer::graph_mutator::graph_storage::VecGraph; 10 | use crate::structured_fuzzer::mutator::MutationStrategy; 11 | 12 | #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] 13 | pub struct InputID(usize); 14 | 15 | impl InputID { 16 | pub fn new(a: usize) -> Self { 17 | Self(a) 18 | } 19 | pub fn invalid() -> Self { 20 | Self(std::usize::MAX) 21 | } 22 | pub fn as_usize(&self) -> usize { 23 | self.0 24 | } 25 | } 26 | 27 | #[derive(Clone)] 28 | pub enum InputState { 29 | Minimize, 30 | Havoc, 31 | } 32 | 33 | #[derive(Clone)] 34 | pub struct Input { 35 | pub id: InputID, 36 | pub data: Arc, 37 | pub bitmap: Bitmap, 38 | pub exit_reason: ExitReason, 39 | pub ops_used: usize, 40 | pub time: Duration, 41 | pub storage_reasons: Vec, 42 | pub found_by: MutationStrategy, 43 | pub state: InputState, 44 | pub custom_dict: CustomDict, 45 | } 46 | 47 | impl Input { 48 | pub fn new( 49 | data: VecGraph, 50 | found_by: MutationStrategy, 51 | storage_reasons: Vec, 52 | bitmap: Bitmap, 53 | exit_reason: ExitReason, 54 | ops_used: usize, 55 | time: Duration, 56 | ) -> Self { 57 | return Self { 58 | id: InputID::invalid(), 59 | data: Arc::new(data), 60 | bitmap, 61 | storage_reasons, 62 | exit_reason, 63 | time, 64 | state: InputState::Minimize, 65 | found_by, 66 | ops_used, 67 | custom_dict: CustomDict::new(), 68 | }; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/size_changing_mutation.rs: -------------------------------------------------------------------------------- 1 | use crate::data_buff::DataBuff; 2 | use std::ops::Range; 3 | 4 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] 5 | pub enum SizeChangingMutationType { 6 | DeleteT, 7 | InsertChunkT, 8 | InsertFixedT, 9 | InsertRandomT, 10 | } 11 | 12 | impl SizeChangingMutationType { 13 | pub fn min_size(&self) -> usize { 14 | use SizeChangingMutationType::*; 15 | match self { 16 | DeleteT => return 8, 17 | InsertChunkT | InsertFixedT | InsertRandomT => return 1, 18 | } 19 | } 20 | pub fn min_available(&self) -> usize { 21 | use SizeChangingMutationType::*; 22 | match self { 23 | DeleteT => return 0, 24 | InsertChunkT | InsertFixedT | InsertRandomT => return 16, 25 | } 26 | } 27 | } 28 | 29 | #[derive(Clone, PartialEq, Eq, Hash, Debug)] 30 | pub enum SizeChangingMutation { 31 | Delete { block: Range }, 32 | InsertChunk { src: Range, dst: usize }, 33 | InsertFixed { amount: usize, val: u8, dst: usize }, 34 | InsertRandom { data: Vec, dst: usize }, 35 | } 36 | 37 | impl SizeChangingMutation { 38 | pub fn apply(&self, buff: &mut DataBuff) { 39 | use SizeChangingMutation::*; 40 | match self { 41 | Delete { block } => buff.drop(block), 42 | InsertChunk { src, dst } => { 43 | buff.shift_every_after(*dst, src.end - src.start); 44 | buff.copy_within(&src, *dst); 45 | } 46 | InsertFixed { dst, amount, val } => { 47 | buff.shift_every_after(*dst, *amount); 48 | buff.memset(&(*dst..dst + amount), *val); 49 | } 50 | InsertRandom { data, dst } => { 51 | buff.shift_every_after(*dst, data.len()); 52 | buff.copy_from(data, *dst); 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /structured_fuzzer/examples/display.rs: -------------------------------------------------------------------------------- 1 | //extern crate structured_fuzzer; 2 | extern crate rand; 3 | 4 | use std::fs::File; 5 | 6 | use structured_fuzzer::graph_mutator::graph_storage::{VecGraph}; 7 | use structured_fuzzer::graph_mutator::spec_loader; 8 | use structured_fuzzer::GraphStorage; 9 | use clap::{App, Arg}; 10 | 11 | fn main() { 12 | 13 | let matches = App::new("generator") 14 | .about("Generate strings using a grammar. This can also be used to generate a corpus") 15 | .arg(Arg::with_name("spec_path") 16 | .short("s") 17 | .value_name("SPEC") 18 | .takes_value(true) 19 | .required(true) 20 | .help("Path to the spec.msgp")) 21 | .arg(Arg::with_name("input_path") 22 | .short("i") 23 | .value_name("IN_PATH") 24 | .takes_value(true) 25 | .help("Which .bin file to read")) 26 | .arg(Arg::with_name("output_path") 27 | .short("o") 28 | .value_name("OUT_PATH") 29 | .takes_value(true) 30 | .help("Where to store outputs")) 31 | .arg(Arg::with_name("svg") 32 | .value_name("SVG") 33 | .help("dump output as svg")) 34 | .get_matches(); 35 | 36 | let spec_path = matches 37 | .value_of("spec_path") 38 | .expect("spec_path is a required parameter") 39 | .to_string(); 40 | let input_path = matches.value_of("input_path").expect("input path is a reuqired parameter").to_string(); 41 | let output_path = matches.value_of("output_path").expect("output path is a reuqired parameter").to_string(); 42 | let svg = matches.is_present("svg"); 43 | 44 | let file = File::open(spec_path).unwrap(); 45 | let gs = spec_loader::load_spec_from_read(file); 46 | 47 | let graph = VecGraph::new_from_bin_file(&input_path, &gs); 48 | if svg { 49 | graph.to_svg(&output_path, &gs); 50 | } else { 51 | println!("{}", graph.to_script(&gs)); 52 | graph.write_to_script_file(&output_path, &gs); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/generators.rs: -------------------------------------------------------------------------------- 1 | use crate::random::distributions::Distributions; 2 | use crate::graph_mutator; 3 | use crate::graph_mutator::regex_generator; 4 | //use crate::data_buff::DataBuff; 5 | 6 | extern crate regex_syntax; 7 | 8 | use regex_syntax::hir::{ 9 | //Class, ClassBytesRange, ClassUnicodeRange, 10 | Hir, 11 | //Literal, RepetitionKind, RepetitionRange, 12 | }; 13 | 14 | #[derive(Debug, PartialEq, Deserialize, Serialize, Clone)] 15 | #[serde(tag = "type")] 16 | pub enum IntGenerator{ 17 | Options{opts: Vec}, 18 | Flags{opts: Vec}, 19 | Limits{range:(u64, u64), align: u64}, 20 | } 21 | 22 | impl IntGenerator{ 23 | pub fn generate(&self, dist: &Distributions) -> (u64,bool){ 24 | use IntGenerator::*; 25 | match self { 26 | Options{opts} => (opts[dist.gen_range(0,opts.len())],false), 27 | Flags{opts} => (opts[dist.gen_range(0,opts.len())],false), 28 | Limits{range,align} => { 29 | let mut val = dist.gen_range(range.0,range.1); 30 | val = val-(val%align); 31 | if val < range.0 {val+=align} 32 | if val > range.1 {val-=align} 33 | (val,false) 34 | }, 35 | } 36 | } 37 | } 38 | #[derive(Debug, PartialEq, Deserialize, Serialize, Clone)] 39 | #[serde(tag = "type")] 40 | pub enum VecGeneratorLoader{ 41 | Regex{r: String}, 42 | } 43 | impl VecGeneratorLoader{ 44 | pub fn load(&self, data: &graph_mutator::spec::AtomicSpec) -> VecGenerator { 45 | match self{ 46 | VecGeneratorLoader::Regex{r} => { 47 | assert!(data.atomic_type.is_u8(),"Regex Generators are only valid for Vec"); 48 | use regex_syntax::ParserBuilder; 49 | 50 | let mut parser = ParserBuilder::new() 51 | .unicode(false) 52 | .allow_invalid_utf8(true) 53 | .build(); 54 | 55 | let hir = parser.parse(r).unwrap(); 56 | VecGenerator::Regex(hir) 57 | } 58 | } 59 | } 60 | } 61 | 62 | pub enum VecGenerator{ 63 | Regex(Hir) 64 | } 65 | 66 | impl VecGenerator{ 67 | pub fn generate(&self, max_len: u64, dist: &Distributions) -> Vec{ 68 | use VecGenerator::*; 69 | match self { 70 | Regex(hir) => { 71 | let mut vec = regex_generator::generate(hir, max_len, dist); 72 | if vec.len() > max_len as usize { 73 | vec.resize(max_len as usize,0); 74 | } 75 | return vec; 76 | }, 77 | } 78 | } 79 | } -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/newtypes.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 2 | #[repr(transparent)] 3 | pub struct PortID(u16); 4 | impl PortID { 5 | pub fn new(x: u16) -> Self { 6 | Self(x) 7 | } 8 | pub fn next(self) -> Self { 9 | Self(self.0 + 1) 10 | } 11 | pub fn as_u16(self) -> u16 { 12 | self.0 13 | } 14 | pub fn as_usize(self) -> usize { 15 | self.0 as usize 16 | } 17 | } 18 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 19 | #[repr(transparent)] 20 | pub struct OpIndex(usize); 21 | impl OpIndex { 22 | pub fn new(x: usize) -> Self { 23 | Self(x) 24 | } 25 | pub fn as_usize(self) -> usize { 26 | return self.0; 27 | } 28 | } 29 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 30 | #[repr(transparent)] 31 | pub struct NodeTypeID(u16); 32 | impl NodeTypeID { 33 | pub fn new(x: u16) -> Self { 34 | Self(x) 35 | } 36 | pub fn as_u16(self) -> u16 { 37 | return self.0; 38 | } 39 | pub fn as_usize(self) -> usize { 40 | return self.0 as usize; 41 | } 42 | } 43 | 44 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 45 | #[repr(transparent)] 46 | pub struct AtomicTypeID(usize); 47 | impl AtomicTypeID { 48 | pub fn new(x: usize) -> Self { 49 | Self(x) 50 | } 51 | pub fn as_usize(self) -> usize { 52 | return self.0; 53 | } 54 | } 55 | 56 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 57 | #[repr(transparent)] 58 | pub struct ValueTypeID(u16); 59 | impl ValueTypeID { 60 | pub fn new(x: u16) -> Self { 61 | Self(x) 62 | } 63 | pub fn as_usize(self) -> usize { 64 | return self.0 as usize; 65 | } 66 | } 67 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 68 | #[repr(transparent)] 69 | pub struct ConnectorID(u16); 70 | impl ConnectorID { 71 | pub fn new(x: u16) -> Self { 72 | Self(x) 73 | } 74 | pub fn next(self) -> Self { 75 | Self(self.0 + 1) 76 | } 77 | pub fn as_u16(self) -> u16 { 78 | return self.0; 79 | } 80 | } 81 | 82 | #[derive(Debug)] 83 | pub enum GraphError { 84 | UnknownNodeType(NodeTypeID), 85 | UnknownValueType(ValueTypeID), 86 | UnknownDataType(AtomicTypeID), 87 | UnknownID(ConnectorID), 88 | MissingInput(), 89 | InvalidSpecs, 90 | } 91 | 92 | #[derive(Eq, PartialEq, Debug, Clone)] 93 | pub struct SrcVal { 94 | pub id: OpIndex, 95 | pub port: PortID, 96 | } 97 | impl SrcVal { 98 | pub fn new(id: OpIndex, port: PortID) -> Self { 99 | Self { id, port } 100 | } 101 | } 102 | #[derive(Eq, PartialEq, Debug, Clone)] 103 | pub struct DstVal { 104 | pub id: OpIndex, 105 | pub port: PortID, 106 | } 107 | impl DstVal { 108 | pub fn new(id: OpIndex, port: PortID) -> Self { 109 | Self { id, port } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nyx Spec-Fuzzer 2 | 3 | Nyx Spec-Fuzzer is one fuzzing frontend building on Nyx's Snapshot capabilities. It features the ability to fuzz complex interactive sequences of actions, such as netowork packets, hypercalls, syscalls, api-calls or gui interactions, by expressing them in a specification. The specification describes a set of functions with affine types (e.g. very similar to rust functions). The fuzzer chains & combinaes those functions to obtain sequences of validly typed function calls. The spec compiles to a simple header-only bytecode interpreter that can be used to execute the fuzzing inputs. All you need to do is to fill in the function definitions with your concrete semantics. This format also allows this fuzzer to make use of incremental snapshots to speed up fuzzing. 4 | 5 | More details can be found in our papers. 6 | 7 |

8 | 9 |

10 | 11 | ## Usage 12 | 13 | Getting this fuzzer running is quite simple: run `./setup.sh`, move to the `rust_fuzzer/` folder and run `cargo run --release -- -h` to get the following describtion on all available parameters: 14 | 15 | ``` 16 | Fuzz EVERYTHING! 17 | 18 | USAGE: 19 | rust_fuzzer [FLAGS] [OPTIONS] 20 | 21 | FLAGS: 22 | --exit_after_first_crash terminate fuzzing after the first crash was found 23 | -h, --help Prints help information 24 | -V, --version Prints version information 25 | 26 | OPTIONS: 27 | -c, --cpu overrides the config value for the first CPU to pin threads to 28 | --seed runs the fuzzer with a specific seed, if not give, a seed is generated from 29 | a secure prng 30 | -s, --sharedir path to the sharedir 31 | -p, --placement overrides the config value for snapshot placement strategy (options: 32 | aggressive / balanced) 33 | -t, --threads overrides the config value for the number of parallel fuzzing threads to run 34 | -w, --workdir overrides the workdir path in the config 35 | ``` 36 | 37 | This fuzzer expects the target to be converted to a so-called "sharedir" format. This can be done by using the packer tool provided by the Nyx framework to convert your target application into this specific format. 38 | 39 | Moreover, there is also a debugger available to run single executions by providing the specific payload. You can find this tool in `rust_fuzzer_debug/`. 40 | 41 | 42 | 43 | ## Bug Reports and Contributions 44 | 45 | If you found and fixed a bug on your own: We are very open to patches, please create a pull request! 46 | 47 | ### License 48 | 49 | This tool is provided under **AGPL license**. 50 | 51 | **Free Software Hell Yeah!** 52 | 53 | Proudly provided by: 54 | * [Sergej Schumilo](http://schumilo.de) - sergej@schumilo.de / [@ms_s3c](https://twitter.com/ms_s3c) 55 | * [Cornelius Aschermann](https://hexgolems.com) - cornelius@hexgolems.com / [@is_eqv](https://twitter.com/is_eqv) 56 | -------------------------------------------------------------------------------- /structured_fuzzer/src/custom_dict.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use crate::data_buff::DataBuff; 4 | use crate::random::distributions::Distributions; 5 | 6 | #[derive(Clone)] 7 | pub enum DictEntry{ 8 | Replace(Vec, Vec) 9 | } 10 | 11 | #[derive(Clone)] 12 | pub struct CustomDict{ 13 | groups: Vec>, 14 | lhs_to_groups: HashMap,Vec>>, 15 | } 16 | 17 | impl CustomDict{ 18 | pub fn new() -> Self{ 19 | return Self{groups: vec!(), lhs_to_groups: HashMap::new()} 20 | } 21 | 22 | pub fn new_from_groups(groups: Vec>) -> Self{ 23 | let mut lhs_to_groups = HashMap::new(); 24 | for group in groups.iter() { 25 | for entry in group.iter() { 26 | match entry { 27 | DictEntry::Replace(lhs, rhs) => { 28 | let entry = lhs_to_groups.entry(lhs.clone()).or_insert_with(|| vec!()); 29 | entry.push(rhs.clone()); 30 | } 31 | } 32 | } 33 | } 34 | return Self{groups, lhs_to_groups} 35 | } 36 | 37 | pub fn len(&self) -> usize { 38 | return self.groups.len(); 39 | } 40 | 41 | pub fn mutate(&self, buff: &mut DataBuff, dist: &Distributions) -> bool { 42 | if let Some(rhs) = self.sample_rhs(buff, dist) { 43 | if dist.gen(){ 44 | buff.copy_from(&rhs, 0); 45 | return dist.gen::(); 46 | } 47 | } 48 | if let Some(entry) = self.sample_entry(buff,dist){ 49 | match entry{ 50 | DictEntry::Replace(lhs, rhs) => { 51 | if let Some(pos) = self.find_pos(buff, &lhs, dist){ 52 | buff.copy_from(&rhs, pos); 53 | } 54 | return dist.gen::(); 55 | } 56 | } 57 | } 58 | return true; 59 | } 60 | 61 | pub fn sample_rhs(&self, buff: &DataBuff, dist: &Distributions) -> Option<&Vec> { 62 | if self.lhs_to_groups.contains_key(buff.as_slice()) { 63 | let opts = &self.lhs_to_groups[buff.as_slice()]; 64 | assert!(opts.len() > 0); 65 | return Some(&opts[dist.gen_range(0, opts.len())]); 66 | } 67 | return None 68 | } 69 | 70 | pub fn sample_entry(&self, _buff: &DataBuff, dist: &Distributions) -> Option<&DictEntry> { 71 | if self.groups.len() == 0 {return None} 72 | let group = &self.groups[dist.gen_range(0,self.groups.len())]; 73 | return Some(&group[dist.gen_range(0,group.len())]); 74 | } 75 | 76 | pub fn find_pos(&self, buff: &DataBuff, lhs: &[u8], dist: &Distributions) -> Option{ 77 | let mut offsets = vec!(); 78 | for (i,win) in buff.as_slice().windows(lhs.len()).enumerate() { 79 | if win == lhs { 80 | offsets.push(i); 81 | } 82 | } 83 | if offsets.len() > 0 { 84 | return Some(offsets[dist.gen_range(0,offsets.len())]); 85 | } 86 | if buff.len() >= lhs.len(){ 87 | return Some(dist.gen_range(0,buff.len()-lhs.len()+1)); 88 | } 89 | return None 90 | } 91 | } 92 | 93 | -------------------------------------------------------------------------------- /structured_fuzzer/examples/gen.rs: -------------------------------------------------------------------------------- 1 | //extern crate structured_fuzzer; 2 | extern crate rand; 3 | 4 | use std::fs::File; 5 | use std::io::prelude::*; 6 | 7 | use structured_fuzzer::graph_mutator::graph_storage::{VecGraph}; 8 | use structured_fuzzer::graph_mutator::spec_loader; 9 | use structured_fuzzer::mutator::{Mutator,MutatorSnapshotState}; 10 | use structured_fuzzer::GraphStorage; 11 | use structured_fuzzer::random::distributions::Distributions; 12 | use structured_fuzzer::custom_dict::CustomDict; 13 | 14 | use clap::{App, Arg, value_t}; 15 | 16 | fn main() { 17 | 18 | let matches = App::new("generator") 19 | .about("Generate strings using a grammar. This can also be used to generate a corpus") 20 | .arg(Arg::with_name("spec_path") 21 | .short("s") 22 | .value_name("SPEC") 23 | .takes_value(true) 24 | .required(true) 25 | .help("Path to the spec.msgp")) 26 | .arg(Arg::with_name("length") 27 | .short("n") 28 | .value_name("LENGTH") 29 | .takes_value(true) 30 | .help("Length of the generated scripts [default: 10]")) 31 | .arg(Arg::with_name("number_of_mutations") 32 | .short("m") 33 | .value_name("NUMBER") 34 | .takes_value(true) 35 | .help("Number of mutations to generate [default: 0]")) 36 | .arg(Arg::with_name("output_path") 37 | .short("o") 38 | .value_name("OUT_PATH") 39 | .takes_value(true) 40 | .help("Where to store outputs")) 41 | .get_matches(); 42 | 43 | let spec_path = matches 44 | .value_of("spec_path") 45 | .expect("spec_path is a required parameter") 46 | .to_string(); 47 | let length = value_t!(matches, "length", usize).unwrap_or(10); 48 | let number_of_mutations = value_t!(matches, "number_of_mutations", usize).unwrap_or(0); 49 | let out_path = matches.value_of("output_path").unwrap_or("./").to_string(); 50 | 51 | 52 | let file = File::open(spec_path).unwrap(); 53 | let gs = spec_loader::load_spec_from_read(file); 54 | //let ops = Box::leak(Box::new([0u16; 512])); 55 | //let data = Box::leak(Box::new([0u8; 512])); 56 | //let mut storage = RefGraph::new(ops, data, Box::leak(Box::new(0)), Box::leak(Box::new(0))); 57 | let mut storage = VecGraph::empty(); 58 | let mut mutator = Mutator::new(gs); 59 | let dist = Distributions::new(vec!()); 60 | mutator.generate(length, &MutatorSnapshotState::none(), &mut storage, &dist); 61 | let graph = mutator.dump_graph(&storage); 62 | let mut file = File::create(&format!("{}/out.dot",out_path)).unwrap(); 63 | graph.to_svg(&format!("{}/out.svg",out_path), &mutator.spec); 64 | file.write_all(&graph.to_dot(&mutator.spec).as_bytes()).unwrap(); 65 | 66 | let queue = vec!(graph); 67 | let graph = &queue[0]; 68 | 69 | let cnt = number_of_mutations; 70 | for i in 0..cnt { 71 | if cnt > 100 && i % (cnt / 100) == 0 { 72 | println!("mutating {}%...", i / (cnt / 100)); 73 | } 74 | let strategy = mutator.mutate(graph, graph.node_len(&mutator.spec) &CustomDict::new(), &MutatorSnapshotState::none(), &queue, &mut storage, &dist); 75 | let g2 = mutator.dump_graph(&storage); 76 | let mut file = File::create(format!("{}/out_mut_{}_{}.dot", out_path, strategy.name(), i)).unwrap(); 77 | file.write_all(&g2.to_dot(&mutator.spec).as_bytes()).unwrap(); 78 | g2.to_svg(&format!("{}/out_mut_{}_{}.svg",out_path,strategy.name(), i), &mutator.spec); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /rust_fuzzer/src/bitmap.rs: -------------------------------------------------------------------------------- 1 | use super::runner::ExitReason; 2 | 3 | 4 | #[derive(Debug, Clone, Eq, PartialEq, Hash)] 5 | pub enum StorageReason{ 6 | Bitmap(BitmapStorageReason), 7 | IjonMax(IjonMaxStorageReason), 8 | Imported, 9 | } 10 | 11 | impl StorageReason{ 12 | pub fn still_valid(&self, bitmap: &[u8], ijon_max: &[u64]) -> bool{ 13 | match self{ 14 | Self::Bitmap(r) => bitmap[r.index] > r.old, 15 | Self::IjonMax(r) => ijon_max[r.index] > r.old, 16 | Self::Imported => true, 17 | } 18 | } 19 | 20 | pub fn has_new_byte(&self) -> bool { 21 | match self{ 22 | Self::Bitmap(r) => r.old == 0, 23 | Self::IjonMax(_r) => true, 24 | Self::Imported => false, 25 | } 26 | } 27 | } 28 | 29 | #[derive(Debug, Clone, Eq, PartialEq, Hash)] 30 | pub struct BitmapStorageReason { 31 | pub index: usize, 32 | pub old: u8, 33 | pub new: u8, 34 | } 35 | 36 | #[derive(Debug, Clone, Eq, PartialEq, Hash)] 37 | pub struct IjonMaxStorageReason { 38 | pub index: usize, 39 | pub old: u64, 40 | pub new: u64, 41 | } 42 | 43 | 44 | pub struct BitmapHandler { 45 | normal: Bitmap, 46 | crash: Bitmap, 47 | timeout: Bitmap, 48 | invalid_write_to_payload: Bitmap, 49 | size: usize, 50 | } 51 | 52 | impl BitmapHandler { 53 | pub fn new(size: usize) -> Self { 54 | return Self { 55 | normal: Bitmap::new(size), 56 | crash: Bitmap::new(size), 57 | timeout: Bitmap::new(size), 58 | invalid_write_to_payload: Bitmap::new(size), 59 | size, 60 | }; 61 | } 62 | 63 | pub fn check_new_bytes( 64 | &mut self, 65 | run_bitmap: &[u8], 66 | ijon_max_map: &[u64], 67 | etype: &ExitReason, 68 | ) -> Option> { 69 | match etype { 70 | ExitReason::Normal(_) => return self.normal.check_new_bytes(run_bitmap, ijon_max_map), 71 | ExitReason::Crash(_) => return self.crash.check_new_bytes(run_bitmap, ijon_max_map), 72 | ExitReason::Timeout => return self.timeout.check_new_bytes(run_bitmap, ijon_max_map), 73 | ExitReason::InvalidWriteToPayload(_) => { 74 | return self.invalid_write_to_payload.check_new_bytes(run_bitmap, ijon_max_map) 75 | } 76 | _ => return None, 77 | } 78 | } 79 | 80 | pub fn size(&self) -> usize { 81 | self.size 82 | } 83 | 84 | pub fn normal_bitmap(&self) -> &Bitmap{ 85 | return &self.normal 86 | } 87 | } 88 | 89 | #[derive(Clone)] 90 | pub struct Bitmap { 91 | bits: Vec, 92 | ijon_max: Vec, 93 | size: usize, 94 | } 95 | 96 | impl Bitmap { 97 | pub fn new(size: usize) -> Self { 98 | const IJON_MAX_SIZE: usize=256usize; 99 | return Self { 100 | bits: vec![0; size], 101 | ijon_max: vec![0; IJON_MAX_SIZE], 102 | size: size, 103 | }; 104 | } 105 | 106 | pub fn new_from_buffer(buff: &[u8], ijon_buff: &[u64], size: usize) -> Self { 107 | return Self { 108 | bits: buff.to_vec(), 109 | ijon_max: ijon_buff.to_vec(), 110 | size: size, 111 | }; 112 | } 113 | 114 | pub fn check_new_bytes(&mut self, run_bitmap: &[u8], run_ijon: &[u64]) -> Option> { 115 | assert_eq!(self.bits.len(), self.size); 116 | let mut res = None; 117 | for (i, (old, new)) in self.bits.iter_mut().zip(run_bitmap.iter()).enumerate() { 118 | if *new > *old && *old == 0{ 119 | if res.is_none() { 120 | res = Some(vec![]); 121 | } 122 | res.as_mut().unwrap().push(StorageReason::Bitmap(BitmapStorageReason { 123 | index: i, 124 | old: *old, 125 | new: *new, 126 | })); 127 | *old = *new; 128 | } 129 | } 130 | for (i, (old, new)) in self.ijon_max.iter_mut().zip(run_ijon.iter()).enumerate() { 131 | if *new > *old { 132 | if res.is_none() { 133 | res = Some(vec![]); 134 | } 135 | res.as_mut().unwrap().push(StorageReason::IjonMax(IjonMaxStorageReason { 136 | index: i, 137 | old: *old, 138 | new: *new, 139 | })); 140 | *old = *new; 141 | } 142 | } 143 | return res; 144 | } 145 | 146 | pub fn bits(&self) -> &[u8] { 147 | return &self.bits; 148 | } 149 | 150 | pub fn ijon_max_vals(&self) -> &[u64] { 151 | return &self.ijon_max; 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/regex_generator.rs: -------------------------------------------------------------------------------- 1 | extern crate regex_syntax; 2 | 3 | //use crate::data_buff::DataBuff; 4 | use crate::random::distributions::Distributions; 5 | 6 | use regex_syntax::hir::{ 7 | Class, ClassBytesRange, ClassUnicodeRange, Hir, Literal, RepetitionKind, RepetitionRange, 8 | }; 9 | 10 | pub struct RegexScript<'a> { 11 | rng: &'a Distributions, 12 | remaining: usize, 13 | } 14 | 15 | impl<'a> RegexScript<'a> { 16 | pub fn new(max_len: u64, rng: &'a Distributions) -> Self { 17 | let len: u64 = if rng.gen::() % 256 == 0 { 18 | (rng.gen::() % 0xffff) % max_len 19 | } else { 20 | let len = 1 << (rng.gen::() % 8); 21 | (rng.gen::() % len) % max_len 22 | }; 23 | RegexScript { 24 | rng, 25 | remaining: len as usize, 26 | } 27 | } 28 | 29 | pub fn get_mod(&mut self, val: usize) -> usize { 30 | if self.remaining == 0 { 31 | return 0; 32 | } 33 | return self.rng.gen::() % val; 34 | } 35 | 36 | pub fn get_range(&mut self, min: usize, max: usize) -> usize { 37 | return self.get_mod(max - min) + min; 38 | } 39 | } 40 | 41 | fn append_char(res: &mut Vec, chr: char) { 42 | let mut buf = [0; 4]; 43 | res.extend_from_slice(chr.encode_utf8(&mut buf).as_bytes()) 44 | } 45 | 46 | fn append_lit(res: &mut Vec, lit: &Literal) { 47 | use regex_syntax::hir::Literal::*; 48 | 49 | match lit { 50 | Unicode(chr) => append_char(res, *chr), 51 | Byte(b) => res.push(*b), 52 | } 53 | } 54 | 55 | fn append_unicode_range(res: &mut Vec, scr: &mut RegexScript, cls: &ClassUnicodeRange) { 56 | let mut chr_a_buf = [0; 4]; 57 | let mut chr_b_buf = [0; 4]; 58 | cls.start().encode_utf8(&mut chr_a_buf); 59 | cls.end().encode_utf8(&mut chr_b_buf); 60 | let a = u32::from_le_bytes(chr_a_buf); 61 | let b = u32::from_le_bytes(chr_b_buf); 62 | let c = scr.get_range(a as usize, (b + 1) as usize) as u32; 63 | append_char(res, std::char::from_u32(c).unwrap()); 64 | } 65 | 66 | fn append_byte_range(res: &mut Vec, scr: &mut RegexScript, cls: &ClassBytesRange) { 67 | res.push(scr.get_range(cls.start() as usize, cls.end() as usize + 1) as u8); 68 | } 69 | 70 | fn append_class(res: &mut Vec, scr: &mut RegexScript, cls: &Class) { 71 | use regex_syntax::hir::Class::*; 72 | match cls { 73 | Unicode(cls) => { 74 | let rngs = cls.ranges(); 75 | let rng = rngs[scr.get_mod(rngs.len())]; 76 | append_unicode_range(res, scr, &rng); 77 | } 78 | Bytes(cls) => { 79 | let rngs = cls.ranges(); 80 | let rng = rngs[scr.get_mod(rngs.len())]; 81 | append_byte_range(res, scr, &rng); 82 | } 83 | } 84 | } 85 | 86 | fn get_length(scr: &mut RegexScript) -> usize { 87 | let bits = scr.get_mod(8); 88 | return scr.get_mod(2 << bits); 89 | } 90 | 91 | fn get_repetition_range(rep: &RepetitionRange, scr: &mut RegexScript) -> usize { 92 | use regex_syntax::hir::RepetitionRange::*; 93 | match rep { 94 | Exactly(a) => return *a as usize, 95 | AtLeast(a) => return get_length(scr) + (*a as usize), 96 | Bounded(a, b) => return scr.get_range(*a as usize, *b as usize), 97 | } 98 | } 99 | 100 | fn get_repetitions(rep: &RepetitionKind, scr: &mut RegexScript) -> usize { 101 | use regex_syntax::hir::RepetitionKind::*; 102 | match rep { 103 | ZeroOrOne => return scr.get_mod(2), 104 | ZeroOrMore => return get_length(scr), 105 | OneOrMore => return 1 + get_length(scr), 106 | Range(rng) => get_repetition_range(rng, scr), 107 | } 108 | } 109 | 110 | pub fn generate(hir: &Hir, max_len: u64, dist: &Distributions) -> Vec { 111 | use regex_syntax::hir::HirKind::*; 112 | //println!("generating on {:?}",hir); 113 | let mut scr = RegexScript::new(max_len, dist); 114 | let mut stack = vec![hir]; 115 | let mut res = vec![]; 116 | while stack.len() > 0 { 117 | match stack.pop().unwrap().kind() { 118 | Empty => {} 119 | Literal(lit) => append_lit(&mut res, lit), 120 | Class(cls) => append_class(&mut res, &mut scr, cls), 121 | Anchor(_) => {} 122 | WordBoundary(_) => {} 123 | Repetition(rep) => { 124 | let num = get_repetitions(&rep.kind, &mut scr); 125 | for _ in 0..num { 126 | stack.push(&rep.hir); 127 | } 128 | } 129 | Group(grp) => stack.push(&grp.hir), 130 | Concat(hirs) => hirs.iter().rev().for_each(|h| stack.push(h)), 131 | Alternation(hirs) => stack.push(&hirs[scr.get_mod(hirs.len())]), 132 | } 133 | } 134 | return res; 135 | } 136 | -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/inplace_mutation.rs: -------------------------------------------------------------------------------- 1 | use crate::data_buff::DataBuff; 2 | use std::ops::Range; 3 | 4 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] 5 | pub enum InplaceMutationType { 6 | FlipBitT, 7 | AddU8T, 8 | AddU16T, 9 | AddU32T, 10 | AddU64T, 11 | InterestingU8T, 12 | InterestingU16T, 13 | InterestingU32T, 14 | InterestingU64T, 15 | OverwriteRandomByteT, 16 | OverwriteChunkT, 17 | OverwriteRandomT, 18 | OverwriteFixedT, 19 | } 20 | 21 | impl InplaceMutationType { 22 | pub fn min_size(&self) -> usize { 23 | use InplaceMutationType::*; 24 | match self { 25 | AddU8T | InterestingU8T | FlipBitT | OverwriteRandomByteT | OverwriteRandomT 26 | | OverwriteFixedT => return 1, 27 | AddU16T | InterestingU16T | OverwriteChunkT => return 2, 28 | AddU32T | InterestingU32T => return 4, 29 | AddU64T | InterestingU64T => return 8, 30 | } 31 | } 32 | } 33 | 34 | #[derive(Clone, PartialEq, Eq, Hash, Debug)] 35 | pub enum InplaceMutation { 36 | FlipBit { 37 | offset: usize, 38 | bit: usize, 39 | }, 40 | AddU8 { 41 | offset: usize, 42 | val: u8, 43 | }, 44 | AddU16 { 45 | offset: usize, 46 | val: u16, 47 | flip_endian: bool, 48 | }, 49 | AddU32 { 50 | offset: usize, 51 | val: u32, 52 | flip_endian: bool, 53 | }, 54 | AddU64 { 55 | offset: usize, 56 | val: u64, 57 | flip_endian: bool, 58 | }, 59 | InterestingU8 { 60 | offset: usize, 61 | val: u8, 62 | }, 63 | InterestingU16 { 64 | offset: usize, 65 | val: u16, 66 | flip_endian: bool, 67 | }, 68 | InterestingU32 { 69 | offset: usize, 70 | val: u32, 71 | flip_endian: bool, 72 | }, 73 | InterestingU64 { 74 | offset: usize, 75 | val: u64, 76 | flip_endian: bool, 77 | }, 78 | OverwriteRandomByte { 79 | offset: usize, 80 | val: u8, 81 | }, 82 | OverwriteChunk { 83 | src: Range, 84 | dst: usize, 85 | }, 86 | OverwriteRandom { 87 | data: Vec, 88 | dst: usize, 89 | }, 90 | OverwriteFixed { 91 | block: Range, 92 | val: u8, 93 | }, 94 | } 95 | 96 | impl InplaceMutation { 97 | pub fn apply(&self, buff: &mut DataBuff) { 98 | use InplaceMutation::*; 99 | match self { 100 | FlipBit { offset, bit } => { 101 | let new = buff.read_u8(*offset) ^ (1u8 << bit); 102 | buff.write_u8(*offset, new) 103 | } 104 | AddU8 { offset, val } => { 105 | let new = buff.read_u8(*offset).wrapping_add(*val); 106 | buff.write_u8(*offset, new) 107 | } 108 | AddU16 { 109 | offset, 110 | val, 111 | flip_endian, 112 | } => { 113 | let new = buff.read_u16(*offset, *flip_endian).wrapping_add(*val); 114 | buff.write_u16(*offset, *flip_endian, new) 115 | } 116 | AddU32 { 117 | offset, 118 | val, 119 | flip_endian, 120 | } => { 121 | let new = buff.read_u32(*offset, *flip_endian).wrapping_add(*val); 122 | buff.write_u32(*offset, *flip_endian, new) 123 | } 124 | AddU64 { 125 | offset, 126 | val, 127 | flip_endian, 128 | } => { 129 | let new = buff.read_u64(*offset, *flip_endian).wrapping_add(*val); 130 | buff.write_u64(*offset, *flip_endian, new) 131 | } 132 | InterestingU8 { offset, val } => buff.write_u8(*offset, *val), 133 | InterestingU16 { 134 | offset, 135 | val, 136 | flip_endian, 137 | } => buff.write_u16(*offset, *flip_endian, *val), 138 | InterestingU32 { 139 | offset, 140 | val, 141 | flip_endian, 142 | } => buff.write_u32(*offset, *flip_endian, *val), 143 | InterestingU64 { 144 | offset, 145 | val, 146 | flip_endian, 147 | } => buff.write_u64(*offset, *flip_endian, *val), 148 | OverwriteRandomByte { offset, val } => buff.write_u8(*offset, *val), 149 | OverwriteRandom { data, dst } => buff.copy_from(&data, *dst), 150 | OverwriteFixed { block, val } => buff.memset(block, *val), 151 | OverwriteChunk { src, dst } => { 152 | buff.copy_within(src, *dst); 153 | } 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/graph_iter.rs: -------------------------------------------------------------------------------- 1 | use crate::graph_mutator::atomic_data::AtomicSize; 2 | use crate::graph_mutator::newtypes::{ConnectorID, NodeTypeID, ValueTypeID}; 3 | use crate::graph_mutator::spec::{GraphSpec, NodeSpec}; 4 | 5 | pub struct GraphNode<'a> { 6 | pub id: NodeTypeID, 7 | pub op_i: usize, 8 | pub ops: &'a [u16], 9 | pub data: &'a [u8], 10 | pub spec: &'a GraphSpec, 11 | } 12 | 13 | impl<'a> GraphNode<'a> { 14 | pub fn iter_ops(&'a self) -> OpIter<'a> { 15 | return OpIter::new(self.ops, self.spec); 16 | } 17 | } 18 | 19 | pub struct NodeIter<'a> { 20 | graph_spec: &'a GraphSpec, 21 | op_i: usize, 22 | data_i: usize, 23 | op_frag: &'a [u16], 24 | data_frag: &'a [u8], 25 | } 26 | 27 | impl<'a> NodeIter<'a> { 28 | pub fn new(op_frag: &'a [u16], data_frag: &'a [u8], graph_spec: &'a GraphSpec) -> Self { 29 | return Self { 30 | op_frag, 31 | data_frag, 32 | op_i: 0, 33 | data_i: 0, 34 | graph_spec, 35 | }; 36 | } 37 | } 38 | 39 | impl<'a> Iterator for NodeIter<'a> { 40 | type Item = GraphNode<'a>; 41 | 42 | fn next(&mut self) -> Option> { 43 | if self.op_i >= self.op_frag.len() { 44 | return None; 45 | } 46 | 47 | let id = NodeTypeID::new(self.op_frag[self.op_i]); 48 | let op_i = self.op_i; 49 | let n_type = self 50 | .graph_spec 51 | .get_node(id) 52 | .expect("invalid graph, couldn't get node type"); 53 | let ops_len = 1 + n_type.size(); 54 | let ops = &self.op_frag[self.op_i..self.op_i + ops_len]; 55 | self.op_i += ops_len; 56 | 57 | let d_type = n_type.data.map(|did| { 58 | self.graph_spec 59 | .get_data(did) 60 | .expect("invalid spec, couldn't get data type") 61 | }); 62 | let data_len = match d_type.map(|d| d.atomic_type.size()) { 63 | None => 0, 64 | Some(AtomicSize::Fixed(x)) => x, 65 | Some(AtomicSize::Dynamic()) => { 66 | let size = self.data_frag[self.data_i] as usize 67 | | ((self.data_frag[self.data_i + 1] as usize) << 8); 68 | size + 2 69 | } 70 | }; 71 | ( 72 | "self.data_i {}, data_len {}, self.data_frag.len() {}", 73 | self.data_i, 74 | data_len, 75 | self.data_frag.len(), 76 | ); 77 | assert!(self.data_i + data_len <= self.data_frag.len()); 78 | let data = &self.data_frag[self.data_i..self.data_i + data_len]; 79 | self.data_i += data_len; 80 | return Some(GraphNode { 81 | id, 82 | op_i, 83 | ops, 84 | data, 85 | spec: &self.graph_spec, 86 | }); 87 | } 88 | } 89 | 90 | #[derive(Eq, PartialEq, Debug, Clone, Hash)] 91 | pub enum GraphOp { 92 | Get(ValueTypeID, ConnectorID), 93 | Pass(ValueTypeID, ConnectorID), 94 | Set(ValueTypeID, ConnectorID), 95 | Node(NodeTypeID), 96 | } 97 | 98 | impl GraphOp { 99 | pub fn node(&self) -> Option { 100 | if let GraphOp::Node(id) = self { 101 | return Some(*id); 102 | } 103 | return None; 104 | } 105 | } 106 | 107 | pub struct OpIter<'a> { 108 | frag: &'a [u16], 109 | graph_spec: &'a GraphSpec, 110 | i: usize, 111 | node_spec: Option, 112 | inputs: usize, 113 | passthroughs: usize, 114 | outputs: usize, 115 | } 116 | 117 | impl<'a> OpIter<'a> { 118 | pub fn new(frag: &'a [u16], graph_spec: &'a GraphSpec) -> Self { 119 | return Self { 120 | frag, 121 | i: 0, 122 | graph_spec, 123 | node_spec: None, 124 | inputs: 0, 125 | passthroughs: 0, 126 | outputs: 0, 127 | }; 128 | } 129 | 130 | fn next_op(&mut self) -> GraphOp { 131 | if self.inputs > 0 { 132 | let node = self.get_node(); 133 | let inps = &node.inputs; 134 | let value_id = inps[inps.len() - self.inputs]; 135 | self.inputs -= 1; 136 | return GraphOp::Get(value_id, ConnectorID::new(self.frag[self.i])); 137 | } 138 | if self.passthroughs > 0 { 139 | let node = self.get_node(); 140 | let pass = &node.passthroughs; 141 | let value_id = pass[pass.len() - self.passthroughs]; 142 | self.passthroughs -= 1; 143 | return GraphOp::Pass(value_id, ConnectorID::new(self.frag[self.i])); 144 | } 145 | if self.outputs > 0 { 146 | let node = self.get_node(); 147 | let outs = &node.outputs; 148 | let value_id = outs[outs.len() - self.outputs]; 149 | self.outputs -= 1; 150 | return GraphOp::Set(value_id, ConnectorID::new(self.frag[self.i])); 151 | } 152 | return GraphOp::Node(NodeTypeID::new(self.frag[self.i])); 153 | } 154 | 155 | fn get_node(&self) -> &NodeSpec { 156 | return self.graph_spec.get_node(self.node_spec.unwrap()).unwrap(); 157 | } 158 | 159 | fn set_node(&mut self, n: NodeTypeID) { 160 | self.node_spec = Some(n); 161 | let node = self.graph_spec.get_node(n).unwrap(); 162 | self.inputs = node.inputs.len(); 163 | self.passthroughs = node.passthroughs.len(); 164 | self.outputs = node.outputs.len(); 165 | } 166 | } 167 | 168 | impl<'a> Iterator for OpIter<'a> { 169 | type Item = GraphOp; 170 | 171 | fn next(&mut self) -> Option { 172 | if self.i < self.frag.len() { 173 | let op = self.next_op(); 174 | if let GraphOp::Node(nt) = op { 175 | let node = self.graph_spec.get_node(nt).unwrap(); 176 | self.set_node(node.id); 177 | } 178 | self.i += 1; 179 | return Some(op); 180 | } 181 | return None; 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /structured_fuzzer/src/data_buff.rs: -------------------------------------------------------------------------------- 1 | use std::convert::TryInto; 2 | use std::ops::Range; 3 | 4 | use crate::random::distributions::Distributions; 5 | 6 | pub struct DataBuff<'a> { 7 | data: &'a mut [u8], 8 | used: usize, 9 | } 10 | 11 | impl<'a> DataBuff<'a> { 12 | pub fn new(data: &'a mut [u8], used: usize) -> DataBuff<'a> { 13 | return DataBuff { data, used }; 14 | } 15 | 16 | pub fn memset(&mut self, range: &Range, val: u8) { 17 | for i in range.clone() { 18 | self.data[i] = val; 19 | } 20 | } 21 | 22 | pub fn set_to_random(&mut self, len: usize, prng: &Distributions) { 23 | self.clear(); 24 | self.used = len; 25 | for i in 0..len { 26 | self.data[i] = prng.gen(); 27 | } 28 | } 29 | 30 | pub fn set_to_slice(&mut self, data: &[u8]) { 31 | self.clear(); 32 | self.used = data.len(); 33 | self.data[..data.len()].copy_from_slice(data); 34 | } 35 | 36 | pub fn expand_to(&mut self, size: usize) { 37 | assert!(size <= self.data.len()); 38 | if self.used < size { 39 | self.memset(&(self.used..size), 0); 40 | self.used = size; 41 | } 42 | } 43 | 44 | pub fn copy_within(&mut self, src: &Range, dst: usize) { 45 | self.expand_to(dst + src.len()); 46 | self.data.copy_within(src.clone(), dst); 47 | } 48 | 49 | pub fn copy_from(&mut self, data: &[u8], dst: usize) { 50 | self.expand_to(dst + data.len()); 51 | self.data[dst..dst + data.len()].copy_from_slice(data) 52 | } 53 | 54 | /// copies all bytes after offset by amount bytes to the right. 55 | /// The space between offset and offset+amount is unmodified (or initialized with zeros if previously unused) 56 | /// Example: ABC|DEFGH amount: 3 => ABCDEFDEFGH 57 | pub fn shift_every_after(&mut self, offset: usize, amount: usize) { 58 | self.copy_within(&(offset..self.len()), offset + amount); 59 | } 60 | 61 | pub fn append(&mut self, data: &[u8]) { 62 | self.data[self.used..self.used + data.len()].copy_from_slice(data); 63 | } 64 | 65 | /// removes the bytes at the given range. 66 | /// Example: ABC[DEFG]HIJ => ABCHIJ 67 | pub fn drop(&mut self, src: &Range) { 68 | let rest = self.used - src.end; 69 | self.copy_within(&(src.end..self.used), src.start); 70 | self.shrink(src.start + rest); 71 | } 72 | 73 | pub fn shrink(&mut self, new_size: usize) { 74 | assert!(new_size <= self.used); 75 | self.used = new_size; 76 | } 77 | 78 | pub fn clear(&mut self) { 79 | self.shrink(0); 80 | } 81 | 82 | /// returns the number of bytes that are currently in use 83 | pub fn len(&self) -> usize { 84 | return self.used; 85 | } 86 | 87 | pub fn is_empty(&self) -> bool{ 88 | return self.used == 0; 89 | } 90 | 91 | /// returns the number of bytes in the backend data store 92 | pub fn capacity(&self) -> usize { 93 | return self.data.len(); 94 | } 95 | 96 | // returns the number of bytes currently not in use 97 | pub fn available(&self) -> usize { 98 | return self.capacity() - self.len(); 99 | } 100 | 101 | pub fn as_slice(&self) -> &[u8] { 102 | return &self.data[..self.used]; 103 | } 104 | 105 | pub fn as_mut_slice(&mut self) -> &mut [u8] { 106 | return &mut self.data[..self.used]; 107 | } 108 | 109 | pub fn write_u8(&mut self, offset: usize, val: u8) { 110 | assert!(offset < self.used); 111 | self.data[offset] = val; 112 | } 113 | pub fn read_u8(&self, offset: usize) -> u8 { 114 | assert!(offset < self.used); 115 | return self.data[offset]; 116 | } 117 | 118 | pub fn read_u16(&self, offset: usize, flip_endianess: bool) -> u16 { 119 | assert!(offset + 2 <= self.used); 120 | let v: [u8; 2] = self.data[offset..offset + 2].try_into().unwrap(); 121 | let res = u16::from_le_bytes(v); 122 | return if flip_endianess { 123 | res.swap_bytes() 124 | } else { 125 | res 126 | }; 127 | } 128 | 129 | pub fn write_u16(&mut self, offset: usize, flip_endianess: bool, val: u16) { 130 | assert!(offset + 2 <= self.used); 131 | let res = if flip_endianess { 132 | val.swap_bytes() 133 | } else { 134 | val 135 | }; 136 | self.data[offset..offset + 2].copy_from_slice(&res.to_le_bytes()); 137 | } 138 | 139 | pub fn read_u32(&self, offset: usize, flip_endianess: bool) -> u32 { 140 | assert!(offset + 4 <= self.used); 141 | let v: [u8; 4] = self.data[offset..offset + 4].try_into().unwrap(); 142 | let res = u32::from_le_bytes(v); 143 | return if flip_endianess { 144 | res.swap_bytes() 145 | } else { 146 | res 147 | }; 148 | } 149 | 150 | pub fn write_u32(&mut self, offset: usize, flip_endianess: bool, val: u32) { 151 | assert!(offset + 4 <= self.used); 152 | let res = if flip_endianess { 153 | val.swap_bytes() 154 | } else { 155 | val 156 | }; 157 | self.data[offset..offset + 4].copy_from_slice(&res.to_le_bytes()); 158 | } 159 | 160 | pub fn read_u64(&self, offset: usize, flip_endianess: bool) -> u64 { 161 | assert!(offset + 8 <= self.used); 162 | let v: [u8; 8] = self.data[offset..offset + 8].try_into().unwrap(); 163 | let res = u64::from_le_bytes(v); 164 | return if flip_endianess { 165 | res.swap_bytes() 166 | } else { 167 | res 168 | }; 169 | } 170 | 171 | pub fn write_u64(&mut self, offset: usize, flip_endianess: bool, val: u64) { 172 | assert!(offset + 8 <= self.used); 173 | let res = if flip_endianess { 174 | val.swap_bytes() 175 | } else { 176 | val 177 | }; 178 | self.data[offset..offset + 8].copy_from_slice(&res.to_le_bytes()); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/spec_loader.rs: -------------------------------------------------------------------------------- 1 | //use rmps::{Deserializer, Serializer}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | use std::io::Read; 5 | use std::sync::Arc; 6 | 7 | use crate::graph_mutator::atomic_data::{DataInt, DataStruct, DataVec}; 8 | use crate::graph_mutator::newtypes::{AtomicTypeID, ValueTypeID}; 9 | use crate::graph_mutator::spec::GraphSpec; 10 | use crate::graph_mutator::generators::{IntGenerator, VecGeneratorLoader}; 11 | 12 | #[derive(Debug, PartialEq, Deserialize, Serialize)] 13 | struct EdgeLoader { 14 | name: String, 15 | } 16 | 17 | #[derive(Debug, PartialEq, Deserialize, Serialize)] 18 | struct NodeLoader { 19 | name: String, 20 | atom_id: Option, 21 | inputs: Vec, 22 | borrows: Vec, 23 | outputs: Vec, 24 | is_interactive: bool, 25 | } 26 | 27 | #[derive(Debug, PartialEq, Deserialize, Serialize)] 28 | #[serde(tag = "type")] 29 | enum AtomLoader { 30 | Struct { 31 | name: String, 32 | fields: Vec<(String, usize)>, 33 | }, 34 | Int { 35 | name: String, 36 | size: usize, 37 | generators: Vec, 38 | }, 39 | Vec { 40 | name: String, 41 | size_range: (usize, usize), 42 | dtype: usize, 43 | generators: Vec, 44 | }, 45 | } 46 | 47 | #[derive(Debug, PartialEq, Deserialize, Serialize)] 48 | struct SpecLoader { 49 | checksum: u64, 50 | nodes: Vec, 51 | edges: Vec, 52 | atomics: Vec, 53 | } 54 | 55 | impl SpecLoader { 56 | fn to_graph_spec(mut self) -> GraphSpec { 57 | let mut g = GraphSpec::new(); 58 | g.checksum = self.checksum; 59 | self.atoms_to_graphspec(&mut g); 60 | self.edges_to_graphspec(&mut g); 61 | self.nodes_to_graphspec(&mut g); 62 | return g; 63 | } 64 | fn atoms_to_graphspec(&mut self, g: &mut GraphSpec) { 65 | for (i, atom) in self.atomics.iter().enumerate() { 66 | match atom { 67 | AtomLoader::Struct { name, fields } => { 68 | let fields = fields 69 | .into_iter() 70 | .map(|(n, id)| (n.clone(), AtomicTypeID::new(*id))) 71 | .collect::>(); 72 | let struct_def = DataStruct::new(fields, &g); 73 | let id = g.data_type(&name, Arc::new(struct_def)); 74 | assert_eq!(id.as_usize(), i); 75 | } 76 | AtomLoader::Int { name, size, generators } => { 77 | let id = g.data_type(&name, Arc::new(DataInt::new(*size, generators.to_vec()))); 78 | assert_eq!(id.as_usize(), i); 79 | } 80 | AtomLoader::Vec { 81 | name, 82 | size_range: rng, 83 | dtype, 84 | generators, 85 | } => { 86 | let dtype = AtomicTypeID::new(*dtype); 87 | let dspec = g 88 | .get_data(dtype) 89 | .expect(&format!("invalid data type id ({:?}) used in vec", dtype)); 90 | assert!(dspec.atomic_type.size().is_fixed()); 91 | let generators = generators.into_iter().map(|g| g.load(dspec)).collect(); 92 | let id = g.data_type(&name, Arc::new(DataVec::new(*rng, dtype, generators, &g))); 93 | assert_eq!(id.as_usize(), i); 94 | } 95 | } 96 | } 97 | } 98 | 99 | fn edges_to_graphspec(&mut self, g: &mut GraphSpec) { 100 | for (i, edge) in self.edges.iter().enumerate() { 101 | let id = g.value_type(&edge.name); 102 | assert_eq!(id.as_usize(), i); 103 | } 104 | } 105 | 106 | fn nodes_to_graphspec(&mut self, g: &mut GraphSpec) { 107 | for (i, node) in self.nodes.iter().enumerate() { 108 | let data = node.atom_id.map(|i| AtomicTypeID::new(i)); 109 | let inputs: Vec = 110 | node.inputs.iter().map(|i| ValueTypeID::new(*i)).collect(); 111 | let borrows: Vec = 112 | node.borrows.iter().map(|i| ValueTypeID::new(*i)).collect(); 113 | let outputs: Vec = 114 | node.outputs.iter().map(|i| ValueTypeID::new(*i)).collect(); 115 | let id = g.node_type(&node.name, data, inputs, borrows, outputs); 116 | assert_eq!(id.as_usize(), i); 117 | } 118 | } 119 | } 120 | 121 | pub fn load_spec_from_read(data: R) -> GraphSpec { 122 | let l: SpecLoader = rmp_serde::from_read(data).unwrap(); 123 | return l.to_graph_spec(); 124 | } 125 | 126 | #[cfg(test)] 127 | mod tests { 128 | use super::*; 129 | use crate::graph_mutator::graph_storage::VecGraph; 130 | use crate::primitive_mutator::mutator::PrimitiveMutator; 131 | use crate::GraphBuilder; 132 | use crate::mutator::MutatorSnapshotState; 133 | use std::fs::File; 134 | use std::rc::Rc; 135 | 136 | #[test] 137 | fn test_export() { 138 | let val = SpecLoader { 139 | atomics: vec![AtomLoader::Int { 140 | name: "foo".into(), 141 | size: 3, 142 | generators: vec!() 143 | }], 144 | checksum: 1337, 145 | edges: vec![], 146 | nodes: vec![], 147 | }; 148 | 149 | let mut buf = vec![]; 150 | //let mut buf = File::create("nsgpack_test.msgp").unwrap(); 151 | val.serialize(&mut Serializer::new(&mut buf)).unwrap(); 152 | 153 | let file = File::open("interpreter/build/spec.msgp").unwrap(); 154 | let l2: SpecLoader = rmp_serde::from_read(file).unwrap(); 155 | 156 | println!("{:?}", l2); 157 | } 158 | 159 | #[test] 160 | fn test_import() { 161 | let file = File::open("interpreter/build/spec.msgp").unwrap(); 162 | let g = load_spec_from_read(file); 163 | let d = crate::random::distributions::Distributions::new(vec!()); 164 | let mut gb = GraphBuilder::new(Rc::new(g),); 165 | let mutator = PrimitiveMutator::new(); 166 | let mut st = VecGraph::new(vec![], vec![]); 167 | gb.start(&mut st, &MutatorSnapshotState::none()); 168 | gb.append_random(10, &mutator, &mut st, &d).unwrap(); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/spec.rs: -------------------------------------------------------------------------------- 1 | use crate::graph_mutator::atomic_data::AtomicDataType; 2 | use crate::graph_mutator::newtypes::{AtomicTypeID, GraphError, NodeTypeID, ValueTypeID}; 3 | use std::collections::HashMap; 4 | use std::sync::Arc; 5 | 6 | #[derive(Default,Clone)] 7 | pub struct GraphSpec { 8 | pub checksum: u64, 9 | pub node_specs: Vec, 10 | pub value_specs: Vec, 11 | pub data_specs: Vec, 12 | pub snapshot_node_id: Option, 13 | max_data: usize, 14 | max_ops: usize, 15 | } 16 | 17 | impl GraphSpec { 18 | pub fn new() -> Self { 19 | return Self { 20 | checksum: 0, 21 | node_specs: Vec::new(), 22 | value_specs: Vec::new(), 23 | data_specs: Vec::new(), 24 | snapshot_node_id: None, 25 | max_data: 0, 26 | max_ops: 0, 27 | }; 28 | } 29 | 30 | pub fn biggest_data(&self) -> usize { return self.max_data; } 31 | pub fn biggest_ops(&self) -> usize { return self.max_ops; } 32 | 33 | pub fn get_node(&self, n: NodeTypeID) -> Result<&NodeSpec, GraphError> { 34 | return self 35 | .node_specs 36 | .get(n.as_usize()) 37 | .ok_or(GraphError::UnknownNodeType(n)); 38 | } 39 | 40 | pub fn get_value(&self, v: ValueTypeID) -> Result<&ValueSpec, GraphError> { 41 | return self 42 | .value_specs 43 | .get(v.as_usize()) 44 | .ok_or(GraphError::UnknownValueType(v)); 45 | } 46 | 47 | pub fn get_data(&self, v: AtomicTypeID) -> Result<&AtomicSpec, GraphError> { 48 | return self 49 | .data_specs 50 | .get(v.as_usize()) 51 | .ok_or(GraphError::UnknownDataType(v)); 52 | } 53 | 54 | pub fn get_node_size(&self, n: NodeTypeID) -> Result { 55 | return Ok(self.get_node(n)?.size()); 56 | } 57 | 58 | pub fn value_type(&mut self, name: &str) -> ValueTypeID { 59 | assert!(self.value_specs.len() < std::u16::MAX as usize); 60 | let new_id = ValueTypeID::new(self.value_specs.len() as u16); 61 | self.value_specs.push(ValueSpec::new(name, new_id)); 62 | return new_id; 63 | } 64 | 65 | pub fn data_type(&mut self, name: &str, atom: Arc) -> AtomicTypeID { 66 | let new_id = AtomicTypeID::new(self.data_specs.len()); 67 | if self.max_data < atom.min_data_size() {self.max_data = atom.min_data_size(); } 68 | let spec = AtomicSpec::new(name, new_id, atom); 69 | self.data_specs.push(spec); 70 | return new_id; 71 | } 72 | 73 | pub fn node_type( 74 | &mut self, 75 | name: &str, 76 | data: Option, 77 | inputs: Vec, 78 | passthrough: Vec, 79 | outputs: Vec, 80 | ) -> NodeTypeID { 81 | assert!(self.node_specs.len() < std::u16::MAX as usize); 82 | let new_id = NodeTypeID::new(self.node_specs.len() as u16); 83 | let ops_len = inputs.len() + passthrough.len() + outputs.len()+1; 84 | if self.max_ops < ops_len {self.max_ops = ops_len;} 85 | let spec = NodeSpec::new(name, new_id, data, inputs, passthrough, outputs); 86 | self.node_specs.push(spec); 87 | if name == "create_tmp_snapshot" { 88 | assert_eq!(ops_len,1); 89 | self.snapshot_node_id = Some(new_id); 90 | } 91 | return new_id; 92 | } 93 | 94 | pub fn node_data_inspect(&self,n: NodeTypeID, data:&[u8]) -> String{ 95 | let node = self.get_node(n).unwrap(); 96 | if let Some(atom_id) = node.data { 97 | let atom = self.get_data(atom_id).unwrap(); 98 | return format!("{}",atom.atomic_type.data_inspect(data, self) ); 99 | } 100 | return "".to_string(); 101 | } 102 | } 103 | 104 | #[derive(Clone)] 105 | pub struct AtomicSpec { 106 | pub name: String, 107 | pub id: AtomicTypeID, 108 | pub atomic_type: Arc, 109 | } 110 | 111 | impl AtomicSpec { 112 | pub fn new(name: &str, id: AtomicTypeID, atomic_type: Arc) -> Self { 113 | return AtomicSpec { 114 | name: name.to_string(), 115 | id, 116 | atomic_type, 117 | }; 118 | } 119 | } 120 | 121 | #[derive(Clone)] 122 | pub struct NodeSpec { 123 | pub name: String, 124 | pub id: NodeTypeID, 125 | pub inputs: Vec, 126 | pub outputs: Vec, 127 | pub passthroughs: Vec, 128 | pub required_values: HashMap, 129 | pub data: Option, 130 | pub generatable:bool, 131 | } 132 | 133 | impl NodeSpec { 134 | fn new( 135 | name: &str, 136 | id: NodeTypeID, 137 | data: Option, 138 | inputs: Vec, 139 | passthroughs: Vec, 140 | outputs: Vec, 141 | ) -> Self { 142 | let mut required_values = HashMap::new(); 143 | 144 | for pass in passthroughs.iter() { 145 | if *required_values.entry(*pass).or_insert(0) == 0 { 146 | *required_values.entry(*pass).or_insert(0) = 1; 147 | } 148 | } 149 | for inp in inputs.iter() { 150 | *required_values.entry(*inp).or_insert(0) += 1; 151 | } 152 | 153 | return Self { 154 | name: name.to_string(), 155 | id, 156 | inputs, 157 | outputs, 158 | passthroughs, 159 | required_values, 160 | data, 161 | generatable: name != "create_tmp_snapshot" 162 | }; 163 | } 164 | 165 | pub fn size(&self) -> usize { 166 | return self.inputs.len() + self.passthroughs.len() + self.outputs.len(); 167 | } 168 | 169 | pub fn min_data_size(&self, spec: &GraphSpec) -> usize { 170 | return self 171 | .data 172 | .map(|d| spec.get_data(d).unwrap()) 173 | .map(|spec| spec.atomic_type.min_data_size()) 174 | .unwrap_or(0); 175 | } 176 | } 177 | 178 | #[derive(Clone)] 179 | pub struct ValueSpec { 180 | pub id: ValueTypeID, 181 | pub name: String, 182 | } 183 | 184 | impl ValueSpec { 185 | pub fn new(name: &str, id: ValueTypeID) -> Self { 186 | return Self { 187 | name: name.to_string(), 188 | id, 189 | }; 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /structured_fuzzer/src/random/distributions.rs: -------------------------------------------------------------------------------- 1 | use crate::mutator::NodeMutationType; 2 | use crate::mutator::{GenerateTail, MutationStrategy}; 3 | use crate::primitive_mutator::inplace_mutation::InplaceMutationType; 4 | use crate::primitive_mutator::size_changing_mutation::SizeChangingMutationType; 5 | use crate::random::choices::Choices; 6 | use crate::random::romu::RomuPrng; 7 | 8 | use rand::distributions::uniform::SampleBorrow; 9 | use rand::distributions::uniform::SampleUniform; 10 | use rand::distributions::{Distribution, Standard}; 11 | use rand::prelude::*; 12 | use std::cell::RefCell; 13 | use std::ops::Range; 14 | 15 | pub struct Distributions { 16 | rng: RefCell, 17 | pub block_size: Choices<(usize, usize)>, 18 | pub inplace_mutations: Choices, 19 | pub size_changing_mutations: Choices, 20 | pub graph_mutations: Choices, 21 | pub endianess: f64, 22 | pub dict: Vec>, 23 | } 24 | 25 | impl Distributions { 26 | pub fn new(dict: Vec>) -> Self { 27 | let block_size = Choices::new(vec![10, 4, 1], vec![(1, 32), (32, 128), (128, 1500)]); 28 | 29 | use InplaceMutationType::*; 30 | let mut prob = vec![1; 13]; 31 | let opts = vec![ 32 | OverwriteRandomT, 33 | FlipBitT, 34 | AddU8T, 35 | AddU16T, 36 | AddU32T, 37 | AddU64T, 38 | InterestingU8T, 39 | InterestingU16T, 40 | InterestingU32T, 41 | InterestingU64T, 42 | OverwriteRandomByteT, 43 | OverwriteChunkT, 44 | OverwriteFixedT, 45 | ]; 46 | if !dict.is_empty() { 47 | assert_eq!(opts[0], OverwriteRandomT); 48 | prob[0] = 2; 49 | } 50 | let inplace_mutations = Choices::new(prob, opts); 51 | 52 | use SizeChangingMutationType::*; 53 | let size_changing_mutations = Choices::new( 54 | vec![1; 4], 55 | vec![DeleteT, InsertChunkT, InsertFixedT, InsertRandomT], 56 | ); 57 | 58 | use NodeMutationType::*; 59 | let graph_mutations = Choices::new( 60 | vec![10, 4, 1, 1], 61 | vec![CopyNode, MutateNodeData, DropNode, SkipAndGenerate], 62 | ); 63 | 64 | return Self { 65 | rng: RefCell::new(RomuPrng::from_entropy()), 66 | block_size, 67 | inplace_mutations, 68 | graph_mutations, 69 | size_changing_mutations, 70 | endianess: 0.5, 71 | dict, 72 | }; 73 | } 74 | 75 | pub fn set_seed(&mut self, seed: u64) { 76 | self.rng = RefCell::new(RomuPrng::seed_from_u64(seed)); 77 | } 78 | 79 | pub fn set_full_seed(&mut self, x: u64, y: u64) { 80 | self.rng = RefCell::new(RomuPrng::new(x, y)); 81 | } 82 | 83 | pub fn gen_inplace_mutation_type(&self) -> &InplaceMutationType { 84 | return self.inplace_mutations.sample(&mut *self.rng.borrow_mut()); 85 | } 86 | 87 | pub fn gen_size_changing_mutation_type(&self) -> &SizeChangingMutationType { 88 | return self 89 | .size_changing_mutations 90 | .sample(&mut *self.rng.borrow_mut()); 91 | } 92 | 93 | pub fn gen_block_size(&self) -> &(usize, usize) { 94 | let mut rng = self.rng.borrow_mut(); 95 | return self.block_size.sample(&mut *rng); 96 | } 97 | pub fn gen_endianess(&self) -> bool { 98 | self.rng.borrow_mut().gen_bool(self.endianess) 99 | } 100 | 101 | //Graph Muation Related stuff 102 | pub fn gen_number_of_random_nodes(&self) -> usize { 103 | return self.gen_range(1, 64); 104 | } 105 | 106 | fn gen_change_percentage(&self) -> usize { 107 | return 1 << self.gen_range(0, 6); 108 | } 109 | 110 | pub fn gen_random_overwrite_data(&self, dst: &Range) -> Vec { 111 | if !self.dict.is_empty() && self.gen::()&1 == 0{ 112 | let i = self.gen_range(0, self.dict.len()); 113 | if self.dict[i].len() <= (dst.end-dst.start){ 114 | return self.dict[i].clone(); 115 | } 116 | } 117 | return dst.clone().map(|_| self.gen()).collect(); 118 | } 119 | 120 | pub fn gen_mutation_strategy(&self, old_size: usize) -> MutationStrategy { 121 | return match self.gen_range(0, 15) { 122 | 0 => MutationStrategy::GenerateTail(self.gen_tail_generation_strategy(old_size)), 123 | 1 | 2 => MutationStrategy::Splice, 124 | 3 => MutationStrategy::SpliceRandom, 125 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 => MutationStrategy::DataOnly, 126 | 13 | 14 => MutationStrategy::Repeat, 127 | _ => unreachable!(), 128 | }; 129 | } 130 | 131 | pub fn gen_tail_generation_strategy(&self, old_size: usize) -> GenerateTail { 132 | let mut drop_last = (self.gen_change_percentage() * old_size) / 100; 133 | let gen_size_range = self.gen_block_size(); 134 | let mut generate = self.gen_range(gen_size_range.0, gen_size_range.1); 135 | if drop_last >= old_size { 136 | drop_last = 0; 137 | } 138 | if generate + old_size - drop_last < 16 && self.gen_range(0, 100) < 98 { 139 | generate += 16; 140 | } 141 | return GenerateTail { 142 | drop_last, 143 | generate, 144 | }; 145 | } 146 | 147 | pub fn gen_graph_mutation_type(&self) -> NodeMutationType { 148 | let mut rng = self.rng.borrow_mut(); 149 | return *self.graph_mutations.sample(&mut *rng); 150 | } 151 | 152 | pub fn gen_minimization_block_size( 153 | &self, 154 | i: usize, 155 | max_i: usize, 156 | graph_len: usize, 157 | ) -> std::ops::Range { 158 | let mut len = self.rng.borrow_mut().gen_range(0, graph_len / 2); 159 | if i > max_i / 4 { 160 | len = len / 2; 161 | } 162 | if i > max_i / 2 { 163 | len = len / 2; 164 | } 165 | if i > 3 * (max_i / 4) { 166 | len = len / 4; 167 | } 168 | if len < 1 { 169 | len = 1; 170 | } 171 | 172 | if graph_len == len { 173 | return 0..len; 174 | } 175 | 176 | let start = self.gen_range(0, graph_len - len); 177 | return start..start + len; 178 | } 179 | 180 | pub fn gen(&self) -> T 181 | where 182 | Standard: Distribution, 183 | { 184 | self.rng.borrow_mut().gen() 185 | } 186 | 187 | pub fn gen_range(&self, low: B1, high: B2) -> T 188 | where 189 | B1: SampleBorrow + Sized, 190 | B2: SampleBorrow + Sized, 191 | { 192 | self.rng.borrow_mut().gen_range(low, high) 193 | } 194 | 195 | pub fn choose_from_iter>(&self, iter: I) -> Option { 196 | iter.choose(&mut *self.rng.borrow_mut()) 197 | } 198 | 199 | pub fn should_mutate_splice_generator(&self) -> bool { 200 | return self.gen_range(0, 2) < 1; 201 | } 202 | 203 | pub fn should_mutate_data(&self, num_nodes: usize) -> bool { 204 | return self.gen_range(0, num_nodes + 1) <= 2; 205 | } 206 | 207 | pub fn should_mutate_dict(&self) -> bool { 208 | return self.gen_range(0, 3) == 0; // a change of 1/3 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /rust_fuzzer/src/runner.rs: -------------------------------------------------------------------------------- 1 | use libnyx::NyxProcess; 2 | use libnyx::NyxReturnValue; 3 | 4 | use std::error::Error; 5 | 6 | #[derive(Debug, Clone, PartialEq, Eq, Hash)] 7 | pub enum ExitReason { 8 | Normal(i32), 9 | Timeout, 10 | Crash(Vec), 11 | FuzzerError, 12 | InvalidWriteToPayload(Vec), 13 | } 14 | 15 | impl ExitReason { 16 | pub fn name(&self) -> &str{ 17 | use ExitReason::*; 18 | match self { 19 | Normal(_) => return "normal", 20 | Timeout => return "timeout", 21 | Crash(_) => return "crash", 22 | InvalidWriteToPayload(_) => return "invalid_write_to_payload_buffer", 23 | FuzzerError => unreachable!(), 24 | } 25 | } 26 | } 27 | 28 | 29 | #[derive(Debug, Copy, Clone)] 30 | #[repr(C, packed(1))] 31 | pub struct InterpreterData{ 32 | pub executed_opcode_num: u32 33 | } 34 | 35 | #[derive(Copy, Clone)] 36 | #[repr(C)] 37 | pub struct IjonData { 38 | pub max_data: [u64;256], 39 | } 40 | 41 | #[derive(Copy, Clone)] 42 | #[repr(C, packed(1))] 43 | pub struct SharedFeedbackData{ 44 | pub interpreter: InterpreterData, 45 | pad: [u8; 0x1000/2-std::mem::size_of::()], 46 | pub ijon: IjonData, 47 | } 48 | 49 | 50 | #[derive(Debug,Clone,Eq,PartialEq,Hash)] 51 | pub struct TestInfo { 52 | pub ops_used: u32, 53 | pub exitreason: ExitReason 54 | } 55 | 56 | #[derive(Debug,Clone,Eq,PartialEq,Hash)] 57 | pub enum RedqueenBPType{ 58 | Str, 59 | Cmp, 60 | Sub, 61 | } 62 | 63 | impl RedqueenBPType{ 64 | pub fn new(data:&str) -> Self { 65 | match data { 66 | "STR" => return Self::Str, 67 | "CMP" => return Self::Cmp, 68 | "SUB" => return Self::Sub, 69 | _ => panic!("unknown reqdueen bp type {}",data), 70 | } 71 | } 72 | } 73 | 74 | #[derive(Debug,Clone,Eq,PartialEq,Hash)] 75 | pub struct RedqueenEvent{ 76 | pub addr: u64, 77 | pub bp_type: RedqueenBPType, 78 | pub lhs: Vec, 79 | pub rhs: Vec, 80 | pub imm: bool, 81 | } 82 | 83 | impl RedqueenEvent{ 84 | pub fn new(line: &str) -> Self{ 85 | lazy_static! { 86 | static ref RE : regex::Regex = regex::Regex::new(r"([0-9a-fA-F]+)\s+(CMP|SUB|STR)\s+(\d+)\s+([0-9a-fA-F]+)-([0-9a-fA-F]+)(\sIMM)?").unwrap(); 87 | } 88 | if let Some(mat) = RE.captures(line){ 89 | let addr_s = mat.get(1).unwrap().as_str(); 90 | let type_s = mat.get(2).unwrap().as_str(); 91 | //let bits_s =mat.get(3); 92 | let lhs = mat.get(4).unwrap().as_str(); 93 | let rhs = mat.get(5).unwrap().as_str(); 94 | let imm = mat.get(6).map(|_x| true).unwrap_or(false); 95 | return Self{addr: u64::from_str_radix(addr_s, 16).unwrap(), bp_type: RedqueenBPType::new(type_s), lhs: hex::decode(lhs).unwrap(), rhs: hex::decode(rhs).unwrap(), imm}; 96 | } 97 | panic!("couldn't parse redqueen line {}",line); 98 | } 99 | } 100 | 101 | #[derive(Debug,Clone,Eq,PartialEq,Hash)] 102 | pub struct RedqueenInfo {pub bps: Vec} 103 | 104 | pub struct CFGInfo {} 105 | 106 | pub trait FuzzRunner { 107 | fn run_test(&mut self) -> Result>; 108 | fn run_redqueen(&mut self, workdir: &str, qemu_id: usize) -> Result>; 109 | fn run_cfg(&mut self) -> Result>; 110 | 111 | fn run_create_snapshot(&mut self) -> bool; 112 | fn delete_snapshot(&mut self) -> Result<(), Box>; 113 | 114 | fn shutdown(&mut self) -> Result<(), Box>; 115 | 116 | fn input_buffer(&mut self) -> &mut [u8]; 117 | fn bitmap_buffer(&self) -> &[u8]; 118 | fn bitmap_buffer_size(&self) -> usize; 119 | fn ijon_max_buffer(&self) -> &[u64]; 120 | 121 | fn set_timeout(&mut self, timeout: std::time::Duration); 122 | 123 | fn parse_redqueen_data(&self, data: &str) -> RedqueenInfo{ 124 | let bps = data.lines().map(|line| RedqueenEvent::new(line)).collect::>(); 125 | return RedqueenInfo{bps} 126 | } 127 | fn parse_redqueen_file(&self, path: &str) -> RedqueenInfo{ 128 | self.parse_redqueen_data(&std::fs::read_to_string(path).unwrap()) 129 | } 130 | } 131 | 132 | fn ijon_buffer(process: &NyxProcess) -> &SharedFeedbackData{ 133 | /* FML */ 134 | unsafe { 135 | (process.ijon_buffer().as_ptr() as *mut SharedFeedbackData).as_mut().unwrap() 136 | } 137 | } 138 | 139 | impl FuzzRunner for NyxProcess { 140 | fn run_test(&mut self) -> Result>{ 141 | 142 | let res = match self.exec(){ 143 | NyxReturnValue::Normal => ExitReason::Normal(0), 144 | NyxReturnValue::Crash => ExitReason::Crash(self.aux_misc()), 145 | NyxReturnValue::Timeout => ExitReason::Timeout, 146 | NyxReturnValue::InvalidWriteToPayload => ExitReason::InvalidWriteToPayload(self.aux_misc()), 147 | NyxReturnValue::Abort => panic!("Abort called!\n"), 148 | _ => ExitReason::FuzzerError, 149 | }; 150 | 151 | let feedback_buffer = ijon_buffer(self); 152 | let ops_used = feedback_buffer.interpreter.executed_opcode_num; 153 | Ok(TestInfo {ops_used, exitreason: res}) 154 | } 155 | 156 | fn run_redqueen(&mut self, workdir: &str, qemu_id: usize) -> Result>{ 157 | self.option_set_redqueen_mode(true); 158 | self.option_apply(); 159 | self.exec(); 160 | self.option_set_redqueen_mode(false); 161 | self.option_apply(); 162 | 163 | let rq_file = format!("{}/redqueen_workdir_{}/redqueen_results.txt", workdir, qemu_id); 164 | return Ok(self.parse_redqueen_file(&rq_file)); 165 | } 166 | 167 | fn run_cfg(&mut self) -> Result> { 168 | self.option_set_trace_mode(true); 169 | self.option_apply(); 170 | self.exec(); 171 | self.option_set_trace_mode(false); 172 | self.option_apply(); 173 | 174 | return Ok(CFGInfo {}); 175 | } 176 | 177 | fn run_create_snapshot(&mut self) -> bool{ 178 | assert_eq!(self.aux_tmp_snapshot_created(), false); 179 | self.exec(); 180 | //println!("=======> CREATED {}", self.aux_tmp_snapshot_created()); 181 | self.aux_tmp_snapshot_created() 182 | } 183 | 184 | fn delete_snapshot(&mut self) -> Result<(), Box>{ 185 | //println!("=======> DELETE"); 186 | if self.aux_tmp_snapshot_created() { 187 | self.option_set_delete_incremental_snapshot(true); 188 | self.option_apply(); 189 | self.exec(); 190 | if self.aux_tmp_snapshot_created() { 191 | println!("=======> ???"); 192 | assert!(false); 193 | //println!("AUX BUFFER {:#?}",self.aux); 194 | } 195 | assert_eq!(self.aux_tmp_snapshot_created(), false); 196 | } 197 | return Ok(()); 198 | } 199 | 200 | fn shutdown(&mut self) -> Result<(), Box>{ 201 | self.shutdown(); 202 | return Ok(()); 203 | } 204 | 205 | fn input_buffer(&mut self) -> &mut [u8]{ 206 | self.input_buffer_mut() 207 | } 208 | 209 | fn bitmap_buffer(&self) -> &[u8]{ 210 | self.bitmap_buffer() 211 | } 212 | 213 | fn bitmap_buffer_size(&self) -> usize{ 214 | self.bitmap_buffer_size() 215 | } 216 | 217 | fn ijon_max_buffer(&self) -> &[u64]{ 218 | let feedback_buffer = ijon_buffer(self); 219 | &feedback_buffer.ijon.max_data 220 | } 221 | 222 | fn parse_redqueen_data(&self, data: &str) -> RedqueenInfo{ 223 | let bps = data.lines().map(|line| RedqueenEvent::new(line)).collect::>(); 224 | return RedqueenInfo{bps} 225 | } 226 | 227 | fn parse_redqueen_file(&self, path: &str) -> RedqueenInfo{ 228 | self.parse_redqueen_data(&std::fs::read_to_string(path).unwrap()) 229 | } 230 | 231 | fn set_timeout(&mut self, timeout: std::time::Duration){ 232 | self.option_set_timeout(timeout.as_secs() as u8, timeout.subsec_micros() as u32); 233 | self.option_apply(); 234 | } 235 | 236 | } -------------------------------------------------------------------------------- /rust_fuzzer/src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate core_affinity; 2 | extern crate helpers; 3 | extern crate serde; 4 | extern crate structured_fuzzer; 5 | extern crate libnyx; 6 | extern crate nix; 7 | #[macro_use] extern crate lazy_static; 8 | extern crate regex; 9 | extern crate hex; 10 | 11 | #[macro_use] 12 | extern crate serde_derive; 13 | extern crate rmp_serde; 14 | extern crate ron; 15 | extern crate rand; 16 | extern crate glob; 17 | extern crate colored; 18 | 19 | use clap::{value_t, App, Arg}; 20 | 21 | use crate::queue::Queue; 22 | 23 | use std::fs::File; 24 | use std::thread; 25 | use std::time::Duration; 26 | 27 | mod bitmap; 28 | mod fuzzer; 29 | mod input; 30 | mod queue; 31 | mod romu; 32 | mod runner; 33 | 34 | use rand::thread_rng; 35 | use crate::rand::Rng; 36 | 37 | use fuzzer::StructFuzzer; 38 | 39 | use structured_fuzzer::graph_mutator::spec_loader; 40 | use crate::romu::*; 41 | 42 | use colored::*; 43 | 44 | use fuzzer::FuzzConfig; 45 | use fuzzer::SnapshotPlacement; 46 | use libnyx::NyxConfig; 47 | use libnyx::NyxProcess; 48 | 49 | 50 | fn main() { 51 | let matches = App::new("nyx") 52 | .about("Fuzz EVERYTHING!") 53 | .arg( 54 | Arg::with_name("sharedir") 55 | .short("s") 56 | .long("sharedir") 57 | .value_name("SHAREDIR_PATH") 58 | .takes_value(true) 59 | .help("path to the sharedir"), 60 | ) 61 | .arg( 62 | Arg::with_name("workdir") 63 | .short("w") 64 | .long("workdir") 65 | .value_name("WORKDIR_PATH") 66 | .takes_value(true) 67 | .help("overrides the workdir path in the config"), 68 | ) 69 | .arg( 70 | Arg::with_name("cpu_start") 71 | .short("c") 72 | .long("cpu") 73 | .value_name("CPU_START") 74 | .takes_value(true) 75 | .help("overrides the config value for the first CPU to pin threads to"), 76 | ) 77 | .arg( 78 | Arg::with_name("threads") 79 | .short("t") 80 | .long("threads") 81 | .value_name("THREADS") 82 | .takes_value(true) 83 | .help("overrides the config value for the number of parallel fuzzing threads to run"), 84 | ) 85 | .arg( 86 | Arg::with_name("seed") 87 | .long("seed") 88 | .value_name("SEED") 89 | .takes_value(true) 90 | .help("runs the fuzzer with a specific seed, if not give, a seed is generated from a secure prng"), 91 | ) 92 | .arg( 93 | Arg::with_name("snapshot_placement") 94 | .short("p") 95 | .long("placement") 96 | .value_name("SNAPSHOT_PLACEMENT") 97 | .takes_value(true) 98 | .help("overrides the config value for snapshot placement strategy (options: aggressive / balanced)") 99 | ) 100 | .arg( 101 | Arg::with_name("exit_after_first_crash") 102 | .long("exit_after_first_crash") 103 | .help("terminate fuzzing after the first crash was found") 104 | ) 105 | .get_matches(); 106 | 107 | let sharedir = matches 108 | .value_of("sharedir") 109 | .expect("need to specify sharedir (-s)") 110 | .to_string(); 111 | 112 | 113 | let mut nyx_config = NyxConfig::load(&sharedir).unwrap(); 114 | 115 | let cpu_start = if let Ok(start_cpu_id) = value_t!(matches, "cpu_start", usize) { 116 | start_cpu_id 117 | } 118 | else{ 119 | 0 120 | }; 121 | 122 | if let Some(path) = matches.value_of("workdir") { 123 | nyx_config.set_workdir_path(path.to_string()); 124 | } 125 | 126 | let snapshot_strategy = if let Some(snapshot_placement) = matches.value_of("snapshot_placement") { 127 | snapshot_placement.parse().unwrap() 128 | } 129 | else{ 130 | SnapshotPlacement::None 131 | }; 132 | 133 | let threads = if let Ok(threads) = value_t!(matches, "threads", usize) { 134 | threads 135 | } 136 | else{ 137 | 1 138 | }; 139 | 140 | let file = File::open(&nyx_config.spec_path()).expect(&format!( 141 | "couldn't open spec (File not found: {}", 142 | nyx_config.spec_path() 143 | )); 144 | 145 | /* Start the first Nyx process just before all other threads to retrieve 146 | * the final bitmap buffer size to allocate all global bitmap buffers. 147 | * This is done to fully support AFL-LTO executables, which report the 148 | * final bitmap size only during runtime. */ 149 | 150 | let core_ids = core_affinity::get_core_ids().unwrap(); 151 | println!("[!] fuzzer: spawning qemu instance #{}", 0); 152 | core_affinity::set_for_current(core_ids[(0 + cpu_start) % core_ids.len()].clone()); 153 | let init_runner = NyxProcess::from_config(&sharedir.clone(), &nyx_config, 0 as u32, threads > 1).unwrap(); 154 | let runtime_bitmap_size = init_runner.bitmap_buffer_size(); 155 | println!("[!] bitmap_buffer_size: {}", runtime_bitmap_size); 156 | 157 | let exit_after_first_crash = matches.is_present("exit_after_first_crash"); 158 | 159 | let spec = spec_loader::load_spec_from_read(file); 160 | let queue = Queue::new( nyx_config.workdir_path().to_string(), runtime_bitmap_size); 161 | 162 | let mut thread_handles = vec![]; 163 | let seed = value_t!(matches, "cpu_start", u64).unwrap_or(thread_rng().gen()); 164 | let mut rng = RomuPrng::new_from_u64(seed); 165 | let init_thread_seed = rng.next_u64(); 166 | 167 | { 168 | let nyx_config = nyx_config.clone(); 169 | let spec = spec.clone(); 170 | let queue = queue.clone(); 171 | 172 | let fuzzer_config = FuzzConfig{ 173 | cpu_start: cpu_start, 174 | snapshot_strategy: snapshot_strategy, 175 | thread_id: 0, 176 | exit_after_first_crash: exit_after_first_crash, 177 | }; 178 | 179 | thread_handles.push(thread::spawn(move || { 180 | let timeout = nyx_config.timeout().clone(); 181 | let mut fuzzer = StructFuzzer::new(init_runner, nyx_config, fuzzer_config, spec, queue, init_thread_seed); 182 | 183 | fuzzer.set_timeout(timeout); 184 | fuzzer.run(); 185 | fuzzer.shutdown(); 186 | 187 | })); 188 | } 189 | 190 | for i in 1..threads { 191 | let nyx_config = nyx_config.clone(); 192 | 193 | let spec = spec.clone(); 194 | let queue = queue.clone(); 195 | let core_id = core_ids[(i + cpu_start) % core_ids.len()].clone(); 196 | let thread_seed = rng.next_u64(); 197 | let sdir = sharedir.clone(); 198 | 199 | let fuzzer_config = FuzzConfig{ 200 | cpu_start: cpu_start, 201 | snapshot_strategy: snapshot_strategy, 202 | thread_id: i, 203 | exit_after_first_crash: exit_after_first_crash, 204 | }; 205 | 206 | std::thread::sleep(Duration::from_millis(100)); 207 | 208 | thread_handles.push(thread::spawn(move || { 209 | println!("[!] fuzzer: spawning qemu instance #{}", i); 210 | core_affinity::set_for_current(core_id); 211 | 212 | let runner = NyxProcess::from_config(&sdir, &nyx_config, i as u32, true).unwrap(); 213 | 214 | let timeout = nyx_config.timeout().clone(); 215 | let mut fuzzer = StructFuzzer::new(runner, nyx_config, fuzzer_config, spec, queue, thread_seed); 216 | 217 | fuzzer.set_timeout(timeout); 218 | fuzzer.run(); 219 | fuzzer.shutdown(); 220 | })); 221 | } 222 | thread_handles.push(thread::spawn(move || { 223 | let mut num_bits_last = 0; 224 | 225 | loop { 226 | let total_execs = queue.get_total_execs(); 227 | 228 | if total_execs > 0 { 229 | let num_bits = queue.num_bits(); 230 | 231 | if num_bits != num_bits_last { 232 | println!("[!] {}", format!("Execs/sec: {} / Bitmap: {}", total_execs as f32 / queue.get_runtime_as_secs_f32(), num_bits).yellow().bold()); 233 | num_bits_last = num_bits; 234 | } 235 | 236 | } 237 | std::thread::sleep(Duration::from_millis(1000*60)); 238 | } 239 | })); 240 | for t in thread_handles.into_iter() { 241 | t.join().unwrap(); 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /rust_fuzzer_debug/src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate config; 2 | extern crate core_affinity; 3 | extern crate fuzz_runner; 4 | extern crate helpers; 5 | extern crate serde; 6 | extern crate structured_fuzzer; 7 | extern crate serde_derive; 8 | extern crate rmp_serde; 9 | extern crate ron; 10 | extern crate rand; 11 | extern crate glob; 12 | 13 | use walkdir::{WalkDir}; 14 | 15 | use clap::{value_t, App, Arg, ArgMatches}; 16 | 17 | use std::fs::File; 18 | 19 | use fuzz_runner::nyx::qemu_process_new_from_kernel; 20 | use fuzz_runner::nyx::qemu_process_new_from_snapshot; 21 | use fuzz_runner::nyx::qemu_process::QemuProcess; 22 | 23 | use std::io::Read; 24 | use std::fs; 25 | 26 | use config::{Config, FuzzRunnerConfig}; 27 | 28 | use std::str; 29 | 30 | extern crate colored; // not needed in Rust 2018 31 | 32 | use colored::*; 33 | 34 | use std::path::Path; 35 | use filetime::FileTime; 36 | use fuzz_runner::nyx::aux_buffer::{NYX_CRASH, NYX_TIMEOUT, NYX_INPUT_WRITE, NYX_ABORT}; 37 | 38 | 39 | fn print_result(runner: &mut fuzz_runner::QemuProcess, target_file: &String){ 40 | println!("\n{} {} {}", "**************", target_file.green().bold(), "**************"); 41 | print!("{}", format!("{:#?}", runner.aux.result).yellow()); 42 | 43 | if runner.aux.result.exec_result_code == NYX_CRASH || 44 | runner.aux.result.exec_result_code == NYX_INPUT_WRITE || 45 | runner.aux.result.exec_result_code == NYX_TIMEOUT || 46 | runner.aux.result.exec_result_code == NYX_ABORT{ 47 | println!("{}", str::from_utf8(&runner.aux.misc.data).unwrap().red()); 48 | } 49 | println!(""); 50 | } 51 | 52 | fn execute_path(runner: &mut fuzz_runner::QemuProcess, target_path: String, dump_payload_folder: &Option, quite_mode: bool, workdir: &String) { 53 | runner.aux.config.timeout_sec += 1; 54 | //runner.aux.config.timeout_usec = 100_000; 55 | runner.aux.config.changed = 1; 56 | 57 | for entry in WalkDir::new(target_path) 58 | .max_depth(2) 59 | .into_iter() 60 | .filter_map(|v| v.ok()) { 61 | let final_path = entry.path(); 62 | let path_str = final_path.to_str().unwrap(); 63 | if path_str.ends_with("bin"){ 64 | //println!("path: {:?}", final_path); 65 | let mut f = File::open(path_str).expect("no file found"); 66 | f.read(runner.payload).expect("buffer overflow"); 67 | 68 | runner.send_payload().unwrap(); 69 | 70 | if !quite_mode{ 71 | print_result(runner, &format!("{:?}", final_path)); 72 | } 73 | 74 | match dump_payload_folder{ 75 | Some(x) => { 76 | let target = final_path.file_stem().unwrap().to_str().unwrap(); 77 | let target_str = format!("{}/{}.py", x, target); 78 | let source_str = format!("{}/dump/reproducer.py", workdir); 79 | println!("COPY: {} -> {}", source_str, target_str); 80 | fs::copy(source_str, target_str).expect("COPY FAILED!"); 81 | 82 | /* copy mtime */ 83 | let metadata = fs::metadata(final_path).unwrap(); 84 | let mtime = FileTime::from_last_modification_time(&metadata); 85 | filetime::set_file_mtime(format!("{}/{}.py", x, target), mtime).expect("cannot set mtime"); 86 | }, 87 | None => {} 88 | } 89 | } 90 | } 91 | 92 | runner.shutdown(); 93 | } 94 | 95 | fn execute_file(runner: &mut fuzz_runner::QemuProcess, target_file: &String, dump_payload_folder: &Option, quite_mode: bool, workdir: &String) { 96 | runner.aux.config.timeout_sec += 1; 97 | //runner.aux.config.timeout_usec = 100_000; 98 | runner.aux.config.changed = 1; 99 | 100 | let mut f = File::open(target_file).expect("no file found"); 101 | f.read(runner.payload).expect("buffer overflow"); 102 | runner.send_payload().unwrap(); 103 | 104 | if !quite_mode{ 105 | print_result(runner, target_file); 106 | } 107 | 108 | match dump_payload_folder{ 109 | Some(x) => { 110 | let target_path = Path::new(target_file); 111 | let target = target_path.file_stem().unwrap().to_str().unwrap(); 112 | let target_str = format!("{}/{}.py", x, target); 113 | let source_str = format!("{}/dump/reproducer.py", workdir); 114 | 115 | println!("COPY: {} -> {}", source_str, target_str); 116 | fs::copy(source_str, target_str).expect("COPY FAILED!"); 117 | 118 | /* copy mtime */ 119 | let metadata = fs::metadata(target_file).unwrap(); 120 | let mtime = FileTime::from_last_modification_time(&metadata); 121 | filetime::set_file_mtime(format!("{}/{}.py", x, target), mtime).expect("cannot set mtime"); 122 | }, 123 | None => {}, 124 | }; 125 | 126 | runner.shutdown(); 127 | } 128 | 129 | fn execute(runner: &mut fuzz_runner::QemuProcess, matches: &ArgMatches<'_>, quite_mode: bool, workdir: &String){ 130 | 131 | let dump_path = if matches.value_of("dump_payload_folder").is_none(){ 132 | None 133 | } 134 | else{ 135 | Some(matches.value_of("dump_payload_folder").unwrap().to_string()) 136 | }; 137 | 138 | if matches.value_of("target_file").is_some() { 139 | execute_file(runner, &matches.value_of("target_file").unwrap().to_string(), &dump_path, quite_mode, workdir); 140 | } 141 | if matches.value_of("target_path").is_some() { 142 | execute_path(runner, matches.value_of("target_path").unwrap().to_string(), &dump_path, quite_mode, workdir); 143 | } 144 | } 145 | 146 | fn main() { 147 | 148 | let matches = App::new("nyx") 149 | .about("Fuzz EVERYTHING!") 150 | .arg( 151 | Arg::with_name("sharedir") 152 | .short("s") 153 | .long("sharedir") 154 | .value_name("SHAREDIR_PATH") 155 | .takes_value(true) 156 | .help("path to the sharedir"), 157 | ) 158 | .arg( 159 | Arg::with_name("cpu_start") 160 | .short("c") 161 | .long("cpu") 162 | .value_name("CPU_START") 163 | .takes_value(true) 164 | .help("overrides the config value for the first CPU to pin threads to"), 165 | ) 166 | .arg( 167 | Arg::with_name("target_file") 168 | .short("f") 169 | .long("target_file") 170 | .value_name("TARGET") 171 | .takes_value(true) 172 | .help("specifies one target file"), 173 | ) 174 | .arg( 175 | Arg::with_name("target_path") 176 | .short("d") 177 | .long("target_path") 178 | .value_name("TARGET_PATH") 179 | .takes_value(true) 180 | .help("specifies path to a target folder"), 181 | ) 182 | .arg( 183 | Arg::with_name("dump_payload_folder") 184 | .short("t") 185 | .long("dump_payload_folder") 186 | .value_name("DUMP_PAYLOAD_PATH") 187 | .takes_value(true) 188 | .help("dump payload files to folder"), 189 | ) 190 | .arg( 191 | Arg::with_name("quiet") 192 | .short("q") 193 | .long("quiet") 194 | .takes_value(false) 195 | .help("quite mode - don't output aux buffer results"), 196 | ) 197 | .after_help("Example: cargo run --release -- -s -d /corpus/normal/ -t \n") 198 | .get_matches(); 199 | 200 | //println!("{:?}", matches); 201 | 202 | let sharedir = matches 203 | .value_of("sharedir") 204 | .expect("need to specify sharedir (-s)") 205 | .to_string(); 206 | 207 | if !matches.value_of("target_file").is_some() && !matches.value_of("target_path").is_some() { 208 | panic!("Neither a target_file nor a target_path has been specififed!"); 209 | } 210 | 211 | let cfg: Config = Config::new_from_sharedir(&sharedir).unwrap(); 212 | 213 | 214 | let mut config = cfg.fuzz; 215 | let runner_cfg = cfg.runner; 216 | 217 | if let Ok(start_cpu_id) = value_t!(matches, "cpu_start", usize) { 218 | config.cpu_pin_start_at = start_cpu_id; 219 | } 220 | 221 | let quite_mode = matches.is_present("quiet"); 222 | 223 | //println!("DUMP: {}", matches.value_of("dump_payload_folder").is_some()); 224 | config.dump_python_code_for_inputs = Some(matches.value_of("dump_payload_folder").is_some()); 225 | 226 | if config.dump_python_code_for_inputs.unwrap(){ 227 | fs::create_dir_all(matches.value_of("dump_payload_folder").unwrap()).unwrap(); 228 | } 229 | 230 | config.workdir_path = format!("/tmp/debug_workdir_{}/", config.cpu_pin_start_at); 231 | 232 | let sdir = sharedir.clone(); 233 | 234 | QemuProcess::prepare_workdir(&config.workdir_path, config.seed_path.clone()); 235 | 236 | 237 | match runner_cfg.clone() { 238 | FuzzRunnerConfig::QemuSnapshot(cfg) => { 239 | let mut runner = qemu_process_new_from_snapshot(sdir, &cfg, &config).unwrap(); 240 | execute(&mut runner, &matches, quite_mode, &config.workdir_path); 241 | } 242 | FuzzRunnerConfig::QemuKernel(cfg) => { 243 | let mut runner = qemu_process_new_from_kernel(sdir, &cfg, &config).unwrap(); 244 | //runner.aux.config.page_dump_mode = 1; 245 | //runner.aux.config.changed = 1; 246 | 247 | execute(&mut runner, &matches, quite_mode, &config.workdir_path); 248 | } 249 | //_ => unreachable!(), 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/atomic_data.rs: -------------------------------------------------------------------------------- 1 | use crate::graph_mutator::graph_storage::GraphMutationTarget; 2 | use crate::graph_mutator::newtypes::AtomicTypeID; 3 | use crate::graph_mutator::spec::GraphSpec; 4 | use crate::custom_dict::CustomDict; 5 | 6 | use crate::data_buff::DataBuff; 7 | use crate::primitive_mutator::mutator::PrimitiveMutator; 8 | use crate::graph_mutator::generators::{IntGenerator, VecGenerator}; 9 | use crate::random::distributions::Distributions; 10 | 11 | #[derive(Debug)] 12 | pub enum AtomicSize { 13 | Fixed(usize), 14 | Dynamic(), 15 | } 16 | 17 | impl AtomicSize { 18 | pub fn is_fixed(&self) -> bool { 19 | return match self { 20 | AtomicSize::Fixed(_) => true, 21 | _ => false, 22 | }; 23 | } 24 | 25 | pub fn as_usize(&self) -> usize { 26 | return match self { 27 | AtomicSize::Fixed(x) => *x, 28 | _ => panic!("can't call as_usize on dynamic sized atomic data"), 29 | }; 30 | } 31 | 32 | pub fn min_data_size(&self) -> usize { 33 | match self { 34 | AtomicSize::Fixed(x) => *x, 35 | AtomicSize::Dynamic() => 2, 36 | } 37 | } 38 | } 39 | 40 | //TODO try moving the S: GraphStorage into the append_generate / append_mutated 41 | pub trait AtomicDataType { 42 | fn size(&self) -> AtomicSize; 43 | fn append_generate( 44 | &self, 45 | storage: &mut dyn GraphMutationTarget, 46 | spec: &GraphSpec, 47 | mutator: &PrimitiveMutator, 48 | dist: &Distributions 49 | ); 50 | fn append_mutated<'a>( 51 | &'a self, 52 | _data: &[u8], 53 | dict: &CustomDict, 54 | storage: &mut dyn GraphMutationTarget, 55 | spec: &GraphSpec, 56 | mutator: &PrimitiveMutator, 57 | dist: &Distributions 58 | ); 59 | fn min_data_size(&self) -> usize { 60 | return self.size().min_data_size(); 61 | } 62 | 63 | fn data_inspect(&self, data:&[u8], spec: &GraphSpec) -> String; 64 | 65 | fn is_u8(&self) -> bool { 66 | return false; 67 | } 68 | } 69 | 70 | pub struct DataInt { 71 | size: usize, 72 | generators: Vec, 73 | } 74 | 75 | impl DataInt { 76 | pub fn new(size: usize, generators: Vec) -> Self { 77 | match size { 78 | 1 | 2 | 4 | 8 => return Self { size, generators }, 79 | _ => panic!("invalid integer size {}", size), 80 | } 81 | } 82 | 83 | fn generate_u64(&self, _mutator: &PrimitiveMutator, dist: &Distributions) -> (u64,bool){ 84 | if self.generators.len() == 0 { return (dist.gen::(), true) } 85 | let gen = &self.generators[dist.gen_range(0,self.generators.len())]; 86 | return gen.generate( dist ); 87 | } 88 | 89 | fn read_from_data(&self, data: &[u8]) -> u64 { 90 | use std::convert::TryInto; 91 | match self.size { 92 | 1 => return u8::from_le_bytes(data.try_into().unwrap()) as u64, 93 | 2 => return u16::from_le_bytes(data.try_into().unwrap()) as u64, 94 | 4 => return u32::from_le_bytes(data.try_into().unwrap()) as u64, 95 | 8 => return u64::from_le_bytes(data.try_into().unwrap()) as u64, 96 | _ => panic!("invalid integer size {}", self.size), 97 | } 98 | } 99 | } 100 | 101 | impl AtomicDataType for DataInt { 102 | fn append_generate( 103 | &self, 104 | storage: &mut dyn GraphMutationTarget, 105 | _spec: &GraphSpec, 106 | mutator: &PrimitiveMutator, 107 | dist: &Distributions 108 | ) { 109 | let (int,mutate_further) = self.generate_u64(mutator, dist); 110 | let data = storage.append_data(&int.to_le_bytes()[0..self.size]).unwrap(); 111 | if mutate_further { 112 | let mut buf = DataBuff::new(data, self.size); 113 | for _ in 0..dist.gen::() % 4 { 114 | mutator.mutate(&mut buf, None, dist); 115 | } 116 | } 117 | } 118 | fn size(&self) -> AtomicSize { 119 | return AtomicSize::Fixed(self.size); 120 | } 121 | fn is_u8(&self) -> bool { 122 | return self.size == 1; 123 | } 124 | fn append_mutated<'a>( 125 | &'a self, 126 | data: &[u8], 127 | dict: &CustomDict, 128 | storage: &mut dyn GraphMutationTarget, 129 | _spec: &GraphSpec, 130 | mutator: &PrimitiveMutator, 131 | dist: &Distributions 132 | ) { 133 | let copy = storage.append_data(data).unwrap(); 134 | let mut buf = DataBuff::new(copy, copy.len()); 135 | mutator.mutate(&mut buf, Some(dict), dist); 136 | } 137 | 138 | fn data_inspect(&self, data:&[u8], _spec: &GraphSpec) -> String{ 139 | match self.size{ 140 | 1 => return format!("0x{:02x}", self.read_from_data(data) ), 141 | 2 => return format!("0x{:04x}", self.read_from_data(data) ), 142 | 4 => return format!("0x{:08x}", self.read_from_data(data) ), 143 | 8 => return format!("0x{:016x}", self.read_from_data(data) ), 144 | _ => panic!("invalid integer size {}", self.size), 145 | } 146 | } 147 | } 148 | 149 | pub struct DataVec { 150 | size_range: (usize, usize), 151 | primitive_size: usize, 152 | dtype: AtomicTypeID, 153 | generators: Vec, 154 | } 155 | 156 | impl DataVec { 157 | pub fn new(size_range: (usize, usize), dtype: AtomicTypeID, generators: Vec, spec: &GraphSpec) -> Self { 158 | let primitive_size = spec.get_data(dtype).unwrap().atomic_type.size().as_usize(); 159 | return Self { 160 | size_range, 161 | dtype: dtype, 162 | primitive_size, 163 | generators 164 | }; 165 | } 166 | 167 | fn generate_raw_vec(&self, storage: &mut dyn GraphMutationTarget, mutator: &PrimitiveMutator, dist: &Distributions) { 168 | let num_elems = mutator.gen_num_array_elems( 169 | self.primitive_size, 170 | self.size_range.0, 171 | self.size_range.1, 172 | storage.data_available() - 2, 173 | dist 174 | ); 175 | let size = num_elems * self.primitive_size; 176 | assert!(size + 2 <= storage.data_available()); 177 | let data = storage.get_data(size + 2).unwrap(); 178 | data[0] = (size & 0xff) as u8; 179 | data[1] = ((size >> 8) & 0xff) as u8; 180 | assert!(data.len() == size + 2); 181 | if size > 0 { 182 | mutator.mutate(&mut DataBuff::new(&mut data[2..], size), None, dist); 183 | } 184 | } 185 | 186 | fn generate_vec(&self, storage: &mut dyn GraphMutationTarget, mutator: &PrimitiveMutator, dist: &Distributions) { 187 | if self.generators.len() == 0 { self.generate_raw_vec(storage, mutator, dist); return } 188 | let gen = &self.generators[dist.gen_range(0,self.generators.len())]; 189 | let max_gen_size = (storage.data_available()-2) as u64; 190 | let vec = gen.generate( max_gen_size, dist ); 191 | 192 | let size = vec.len(); 193 | assert!(size + 2 <= storage.data_available()); 194 | assert_eq!(size%self.primitive_size,0, "Vec Generator failed to produce a properly aligned input size"); 195 | let count = vec.len() / self.primitive_size; 196 | let data = storage.get_data(2).unwrap(); 197 | data[0] = (count & 0xff) as u8; 198 | data[1] = ((count >> 8) & 0xff) as u8; 199 | storage.append_data(&vec); 200 | } 201 | 202 | } 203 | 204 | impl AtomicDataType for DataVec { 205 | fn append_generate( 206 | &self, 207 | storage: &mut dyn GraphMutationTarget, 208 | _spec: &GraphSpec, 209 | mutator: &PrimitiveMutator, 210 | dist: &Distributions 211 | ) { 212 | self.generate_vec(storage, mutator, dist); 213 | } 214 | 215 | fn size(&self) -> AtomicSize { 216 | return AtomicSize::Dynamic(); 217 | } 218 | fn append_mutated<'a>( 219 | &'a self, 220 | data: &[u8], 221 | dict: &CustomDict, 222 | storage: &mut dyn GraphMutationTarget, 223 | _spec: &GraphSpec, 224 | mutator: &PrimitiveMutator, 225 | dist: &Distributions 226 | ) { 227 | assert_eq!(data[0] as usize + ((data[1] as usize) << 8) + 2, data.len()); 228 | let copy = storage.append_data(data).unwrap(); 229 | let len = copy.len() - 2; 230 | if !self.generators.is_empty() && dist.should_mutate_splice_generator(){ 231 | let gen = &self.generators[dist.gen_range(0,self.generators.len())]; 232 | if len > 1{ 233 | let max_gen_size = (len as u64)-1; 234 | let vec = gen.generate( max_gen_size, dist ); 235 | if vec.len() > 1{ 236 | let chunk_len : usize = dist.gen::()%(vec.len()-1); 237 | let offset = dist.gen_range(0, len-chunk_len); 238 | let chunk_start = dist.gen::()%(vec.len()-chunk_len); 239 | let chunk = &vec[chunk_start..chunk_start+chunk_len]; 240 | copy[(2+offset)..(2+offset+chunk_len)].copy_from_slice(chunk); 241 | } 242 | } 243 | } else { 244 | mutator.mutate(&mut DataBuff::new(&mut copy[2..], len), Some(dict), dist); 245 | } 246 | } 247 | 248 | fn data_inspect(&self, data:&[u8], spec: &GraphSpec) -> String{ 249 | let atom = spec.get_data(self.dtype).unwrap(); 250 | let vecsize = u16::from_le_bytes([data[0], data[1]]); 251 | let data = &data[2..]; 252 | if atom.atomic_type.is_u8() { 253 | use std::ascii::escape_default; 254 | use std::str; 255 | assert_eq!(vecsize as usize, data.len()); 256 | let mut visible = "\"".to_string(); 257 | for &b in data { 258 | let part: Vec = escape_default(b).collect(); 259 | visible.push_str(str::from_utf8(&part).unwrap()); 260 | } 261 | return visible+"\""; 262 | } else { 263 | let mut res = "[".to_string(); 264 | assert_eq!(data.len(), (vecsize as usize)*self.primitive_size); 265 | assert_eq!(data.len()%self.primitive_size, 0); 266 | 267 | for i in 0..data.len()/self.primitive_size{ 268 | res+= &format!("{}, ",atom.atomic_type.data_inspect(&data[i*self.primitive_size..i*self.primitive_size+self.primitive_size], spec)); 269 | } 270 | return res+"]"; 271 | } 272 | } 273 | } 274 | 275 | pub struct DataStruct { 276 | size: usize, 277 | fields: Vec<(String, AtomicTypeID)>, 278 | } 279 | 280 | impl DataStruct { 281 | pub fn new(fields: Vec<(String, AtomicTypeID)>, spec: &GraphSpec) -> Self { 282 | let size = fields 283 | .iter() 284 | .map(|(_, id)| { 285 | spec.get_data(*id) 286 | .expect("invalid field type in struct") 287 | .atomic_type 288 | .size() 289 | .as_usize() 290 | }) 291 | .sum(); 292 | return Self { 293 | size, 294 | fields: fields, 295 | }; 296 | } 297 | } 298 | 299 | impl AtomicDataType for DataStruct { 300 | fn append_generate( 301 | &self, 302 | storage: &mut dyn GraphMutationTarget, 303 | spec: &GraphSpec, 304 | mutator: &PrimitiveMutator, 305 | dist: &Distributions 306 | ) { 307 | for (_,dt) in self.fields.iter(){ 308 | spec.get_data(*dt).unwrap().atomic_type.append_generate(storage, spec, mutator, dist); 309 | } 310 | //let mut data = storage.get_data(self.size).unwrap(); 311 | //let len = data.len(); 312 | //mutator.mutate(&mut DataBuff::new(&mut data, len)); 313 | } 314 | fn size(&self) -> AtomicSize { 315 | return AtomicSize::Fixed(self.size); 316 | } 317 | fn append_mutated<'a>( 318 | &'a self, 319 | data: &[u8], 320 | dict: &CustomDict, 321 | storage: &mut dyn GraphMutationTarget, 322 | _spec: &GraphSpec, 323 | mutator: &PrimitiveMutator, 324 | dist: &Distributions 325 | ) { 326 | let mut copy = storage.append_data(data).unwrap(); 327 | let len = copy.len(); 328 | mutator.mutate(&mut DataBuff::new(&mut copy, len), Some(dict), dist); 329 | } 330 | fn data_inspect(&self, data:&[u8], spec: &GraphSpec) -> String{ 331 | let mut res = "{\\l".to_string(); 332 | let mut i = 0; 333 | for (name, atomic_id) in self.fields.iter() { 334 | let atom = spec.get_data(*atomic_id).unwrap(); 335 | let size = atom.atomic_type.size().as_usize(); 336 | res+= &format!("{}: {},\\l ",name, atom.atomic_type.data_inspect(&data[i..i+size], spec)); 337 | i+=size; 338 | } 339 | return res+"}"; 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /structured_fuzzer/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::needless_return)] 2 | extern crate rand; 3 | extern crate rand_core; 4 | 5 | extern crate serde; 6 | #[macro_use] 7 | extern crate serde_derive; 8 | extern crate rmp_serde as rmps; 9 | extern crate itertools; 10 | 11 | pub mod data_buff; 12 | pub mod graph_mutator; 13 | pub mod mutator; 14 | pub mod primitive_mutator; 15 | pub mod random; 16 | pub mod custom_dict; 17 | 18 | pub use graph_mutator::atomic_data; 19 | pub use graph_mutator::graph_builder::GraphBuilder; 20 | pub use graph_mutator::graph_storage::{GraphStorage, VecGraph}; 21 | pub use graph_mutator::newtypes::{AtomicTypeID, DstVal, OpIndex, PortID, SrcVal}; 22 | pub use graph_mutator::spec::{GraphSpec, NodeSpec, ValueSpec}; 23 | 24 | 25 | #[cfg(test)] 26 | mod graph_spec { 27 | use super::graph_mutator::graph_iter::GraphOp; 28 | use super::graph_mutator::newtypes::{ConnectorID, NodeTypeID, ValueTypeID}; 29 | use super::*; 30 | use super::mutator::MutatorSnapshotState; 31 | use std::rc::Rc; 32 | use std::sync::Arc; 33 | use super::random::distributions::Distributions; 34 | 35 | fn build_spec() -> GraphSpec { 36 | let mut gs = GraphSpec::new(); 37 | let va = gs.value_type("a"); 38 | assert_eq!(va, ValueTypeID::new(0)); 39 | let vb = gs.value_type("b"); 40 | assert_eq!(vb, ValueTypeID::new(1)); 41 | 42 | let data = None; 43 | let na = gs.node_type("na", data, vec![], vec![], vec![va]); 44 | assert_eq!(na, NodeTypeID::new(0)); 45 | let nb = gs.node_type("nb", data, vec![va], vec![], vec![vb]); 46 | assert_eq!(nb, NodeTypeID::new(1)); 47 | let nc = gs.node_type("nc", data, vec![va, vb], vec![], vec![]); 48 | assert_eq!(nc, NodeTypeID::new(2)); 49 | let nd = gs.node_type("nd", data, vec![], vec![va], vec![]); 50 | assert_eq!(nd, NodeTypeID::new(3)); 51 | return gs; 52 | } 53 | 54 | fn build_frag() -> Vec { 55 | return vec![ 56 | 0, //na 57 | 1337, //connection id for set(va) 58 | 0, //na 59 | 42, //connection if for set(va) 60 | 1, //nb 61 | 1337, //connection id get(va) 62 | 43, //connection id set(vb) 63 | 2, //nc 64 | 42, // connection id get(va) 65 | 43, // connection id get(vb) 66 | ]; 67 | } 68 | 69 | fn builder(dict: Vec>) -> (GraphBuilder, VecGraph) { 70 | let spec = build_spec(); 71 | let dist = crate::random::distributions::Distributions::new(dict); 72 | let mut gb = GraphBuilder::new(Rc::new(spec)); 73 | let mut storage = VecGraph::new(vec![], vec![]); 74 | gb.start(&mut storage, &MutatorSnapshotState::none()); 75 | gb.append_slice(&build_frag(), &mut storage, &dist); 76 | return (gb, storage); 77 | } 78 | 79 | #[test] 80 | fn test_spec() { 81 | let (gb, st) = builder(vec!()); 82 | let graph = gb.finalize(&st); 83 | 84 | assert_eq!( 85 | graph.ops_as_slice(), 86 | &vec!( 87 | 0, //na 88 | 1, //connection id for set(va) 89 | 0, //na 90 | 2, //connection if for set(va) 91 | 1, //nb 92 | 1, //connection id get(va) 93 | 1, //connection id set(vb) 94 | 2, //nc 95 | 2, // connection id get(va) 96 | 1, // connection id get(vb) 97 | )[..] 98 | ); 99 | 100 | let ops = graph.op_iter(&gb.spec).collect::>(); 101 | assert_eq!( 102 | ops, 103 | vec!( 104 | GraphOp::Node(NodeTypeID::new(0)), //0 105 | GraphOp::Set(ValueTypeID::new(0), ConnectorID::new(1)), 106 | GraphOp::Node(NodeTypeID::new(0)), //2 107 | GraphOp::Set(ValueTypeID::new(0), ConnectorID::new(2)), 108 | GraphOp::Node(NodeTypeID::new(1)), //4 109 | GraphOp::Get(ValueTypeID::new(0), ConnectorID::new(1)), 110 | GraphOp::Set(ValueTypeID::new(1), ConnectorID::new(1)), 111 | GraphOp::Node(NodeTypeID::new(2)), //7 112 | GraphOp::Get(ValueTypeID::new(0), ConnectorID::new(2)), 113 | GraphOp::Get(ValueTypeID::new(1), ConnectorID::new(1)), 114 | ) 115 | ) 116 | } 117 | 118 | #[test] 119 | fn test_edges() { 120 | let (gb, st) = builder(vec!()); 121 | let graph = gb.finalize(&st); 122 | let edges = graph.calc_edges(&gb.spec); 123 | assert_eq!( 124 | edges, 125 | vec!( 126 | ( 127 | SrcVal::new(OpIndex::new(0), PortID::new(0)), 128 | DstVal::new(OpIndex::new(4), PortID::new(0)) 129 | ), 130 | ( 131 | SrcVal::new(OpIndex::new(2), PortID::new(0)), 132 | DstVal::new(OpIndex::new(7), PortID::new(0)) 133 | ), 134 | ( 135 | SrcVal::new(OpIndex::new(4), PortID::new(0)), 136 | DstVal::new(OpIndex::new(7), PortID::new(1)) 137 | ), 138 | ) 139 | ) 140 | } 141 | 142 | fn build_redundant_graph(gb: &mut GraphBuilder, st: &mut VecGraph, dist: &Distributions) -> VecGraph { 143 | let frag = vec![ 144 | 0, //na 145 | 1337, //connection id for set(va) 146 | 3, //nd 147 | 1337, //connection id for pass(va) 148 | 3, //nd 149 | 1337, //connection id for pass(va) 150 | 1, //nb 151 | 1337, //connection id get(va) 152 | 43, //connection id set(vb) 153 | ]; 154 | let graph = VecGraph::new(frag, vec![]); 155 | gb.start(st,&MutatorSnapshotState::none()); 156 | gb.append_slice(&graph.ops_as_slice(), st, dist); 157 | return gb.finalize(st); 158 | } 159 | 160 | #[test] 161 | fn test_drop_at() { 162 | let spec = build_spec(); 163 | let dist = crate::random::distributions::Distributions::new(vec!()); 164 | let mut gb = GraphBuilder::new(Rc::new(spec)); 165 | let mut st = VecGraph::new(vec![], vec![]); 166 | let graph = build_redundant_graph(&mut gb, &mut st, &dist); 167 | gb.drop_node_at(&graph, 1, &mut st, &dist); 168 | assert_eq!( 169 | gb.finalize(&st).ops_as_slice(), 170 | &vec!(0, 1, 3, 1, 1, 1, 1)[..] 171 | ); 172 | gb.drop_node_at(&graph, 2, &mut st, &dist); 173 | assert_eq!( 174 | gb.finalize(&st).ops_as_slice(), 175 | &vec!(0, 1, 3, 1, 1, 1, 1)[..] 176 | ); 177 | gb.drop_node_at(&graph, 3, &mut st, &dist); 178 | assert_eq!(gb.finalize(&st).ops_as_slice(), &vec!(0, 1, 3, 1, 3, 1)[..]); 179 | gb.drop_node_at(&graph, 0, &mut st, &dist); 180 | } 181 | 182 | #[test] 183 | fn test_minimize() { 184 | let spec = build_spec(); 185 | let dist = crate::random::distributions::Distributions::new(vec!()); 186 | let mut gb = GraphBuilder::new(Rc::new(spec)); 187 | let mut st = VecGraph::new(vec![], vec![]); 188 | let graph = build_redundant_graph(&mut gb, &mut st, &dist); 189 | println!("FNORD {}", graph.to_dot(&gb.spec)); 190 | let min = gb.minimize(&graph, &mut st, &dist, |gr: &VecGraph, spec| { 191 | println!("test {}", gr.to_dot(spec)); 192 | gr.to_dot(spec).contains("nb") 193 | }); 194 | assert_eq!(min.ops_as_slice(), &vec!(0, 1, 1, 1, 1)[..]); 195 | } 196 | 197 | #[test] 198 | fn test_fuzz_minimize() { 199 | let mut gs = GraphSpec::new(); 200 | 201 | let t_path = gs.value_type("path"); 202 | let t_fd = gs.value_type("fd"); 203 | let t_mmap_buffer = gs.value_type("mmap_buffer"); 204 | 205 | let dt = Some(gs.data_type("none", Arc::new(atomic_data::DataInt::new(1, vec!())))); 206 | 207 | let _n_path = gs.node_type("path", dt, vec![], vec![], vec![t_path]); 208 | let _n_open = gs.node_type("open", dt, vec![], vec![t_path], vec![t_fd]); 209 | let _n_mmap = gs.node_type("mmap", dt, vec![], vec![t_fd], vec![t_mmap_buffer]); 210 | let _n_stdout = gs.node_type("fd:0", dt, vec![], vec![], vec![t_fd]); 211 | let _n_dup2 = gs.node_type("dup2", dt, vec![], vec![t_fd, t_fd], vec![]); 212 | let _n_close = gs.node_type("close", dt, vec![t_fd], vec![], vec![]); 213 | let dist = crate::random::distributions::Distributions::new(vec!()); 214 | let mut gb = GraphBuilder::new(Rc::new(gs)); 215 | let mut st = VecGraph::empty(); 216 | let mutator = primitive_mutator::mutator::PrimitiveMutator::new(); 217 | for _ in 0..10 { 218 | gb.start(&mut st, &MutatorSnapshotState::none()); 219 | gb.append_random(50, &mutator, &mut st,&dist).unwrap(); 220 | let graph = gb.finalize(&st); 221 | if graph.to_dot(&gb.spec).contains("dup2") { 222 | let min = gb.minimize(&graph, &mut st, &dist, |gr: &VecGraph, spec| { 223 | let res = gr.to_dot(spec).contains("dup2"); 224 | res 225 | }); 226 | assert!(min.op_len() <= 8); //TODO min.len() should be < 8, fix minimization! 227 | } 228 | } 229 | } 230 | 231 | fn example_io_spec() -> GraphSpec { 232 | let mut gs = GraphSpec::new(); 233 | 234 | let t_path = gs.value_type("path"); 235 | let t_fd = gs.value_type("fd"); 236 | let t_mmap_buffer = gs.value_type("mmap_buffer"); 237 | 238 | let dt = None; 239 | 240 | let _n_path = gs.node_type("path", dt, vec![], vec![], vec![t_path]); 241 | let _n_open = gs.node_type("open", dt, vec![], vec![t_path], vec![t_fd]); 242 | let _n_mmap = gs.node_type("mmap", dt, vec![], vec![t_fd], vec![t_mmap_buffer]); 243 | let _n_stdout = gs.node_type("fd:0", dt, vec![], vec![], vec![t_fd]); 244 | let _n_dup2 = gs.node_type("dup2", dt, vec![], vec![t_fd, t_fd], vec![]); 245 | let _n_close = gs.node_type("close", dt, vec![t_fd], vec![], vec![]); 246 | gs 247 | } 248 | 249 | #[test] 250 | pub fn test_drop_1() { 251 | let spec = example_io_spec(); 252 | let data = vec![0, 1, 3, 1, 1, 1, 2, 4, 1, 2]; 253 | let graph = VecGraph::new(data, vec![]); 254 | let dist = crate::random::distributions::Distributions::new(vec!()); 255 | let mut gb = GraphBuilder::new(Rc::new(spec)); 256 | let mut st = VecGraph::empty(); 257 | gb.drop_node_at(&graph, 2, &mut st, &dist); 258 | } 259 | 260 | #[test] 261 | fn test_to_dot() { 262 | let (gb, st) = builder(); 263 | let graph = gb.finalize(&st); 264 | let exp = "digraph{ 265 | rankdir=\"LR\"; 266 | { edge [style=invis weight=100];n0->n2->n4->n7} 267 | n0 [label=\"na []\"]; 268 | n2 [label=\"na []\"]; 269 | n4 [label=\"nb []\"]; 270 | n7 [label=\"nc []\"]; 271 | n0 -> n4 [label=\"a\"]; 272 | n2 -> n7 [label=\"a\"]; 273 | n4 -> n7 [label=\"b\"]; 274 | }"; 275 | assert_eq!(graph.to_dot(&gb.spec), exp); 276 | } 277 | 278 | #[test] 279 | fn test_data_generation() { 280 | let mut gs = GraphSpec::new(); 281 | let d_u8 = Some(gs.data_type("u8", Arc::new(atomic_data::DataInt::new(1, vec!())))); 282 | let _n_a = gs.node_type("u8_1", d_u8, vec![], vec![], vec![]); 283 | let _n_b = gs.node_type("u8_2", d_u8, vec![], vec![], vec![]); 284 | let dist = crate::random::distributions::Distributions::new(vec!()); 285 | let mut gb = GraphBuilder::new(Rc::new(gs)); 286 | let mutator = primitive_mutator::mutator::PrimitiveMutator::new(); 287 | let mut st = VecGraph::empty(); 288 | gb.start(&mut st, &MutatorSnapshotState::none()); 289 | gb.append_random(10, &mutator, &mut st, &dist).unwrap(); 290 | assert_eq!(gb.finalize(&st).data_len(), 10); 291 | } 292 | 293 | 294 | #[test] 295 | fn test_fuzz_splice_random_limited_data() { 296 | use std::fs::File; 297 | 298 | use crate::graph_mutator::graph_storage::RefGraph; 299 | use crate::graph_mutator::spec_loader; 300 | use crate::mutator::Mutator; 301 | 302 | let file = File::open("interpreter/build/spec.msgp").unwrap(); 303 | let gs = spec_loader::load_spec_from_read(file); 304 | let ops = Box::leak(Box::new([0u16; 512])); 305 | let data = Box::leak(Box::new([0u8; 512])); 306 | let dist = crate::random::distributions::Distributions::new(vec!()); 307 | let mut storage = RefGraph::new(ops, data, Box::leak(Box::new(0)), Box::leak(Box::new(0))); 308 | let mut mutator = Mutator::new(gs); 309 | mutator.generate(100, &mutator::MutatorSnapshotState::none(), &mut storage, &dist); 310 | let graph = mutator.dump_graph(&storage); 311 | let cnt = 1000; 312 | for _i in 0..cnt { 313 | mutator.splice_random(&graph, graph.node_len(&mutator.spec), &mutator::MutatorSnapshotState::none(), &custom_dict::CustomDict::new(), &mut storage, &dist); 314 | } 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /rust_fuzzer/src/queue.rs: -------------------------------------------------------------------------------- 1 | use crate::bitmap::{BitmapHandler, StorageReason}; 2 | 3 | use super::runner::ExitReason; 4 | use crate::structured_fuzzer::custom_dict::CustomDict; 5 | use crate::input::{Input, InputID, InputState}; 6 | use crate::structured_fuzzer::graph_mutator::graph_storage::{GraphStorage, VecGraph}; 7 | use crate::structured_fuzzer::mutator::InputQueue; 8 | use crate::structured_fuzzer::random::distributions::Distributions; 9 | use crate::structured_fuzzer::mutator::MutationStrategy; 10 | use crate::structured_fuzzer::GraphSpec; 11 | 12 | use std::collections::HashMap; 13 | 14 | use std::sync::Arc; 15 | use std::sync::RwLock; 16 | 17 | #[derive(Serialize)] 18 | pub struct QueueStats { 19 | num_inputs: usize, 20 | favqueue: Vec, 21 | } 22 | 23 | pub struct QueueData { 24 | bitmap_index_to_min_example: HashMap, 25 | //bitmap_index_to_max_example: HashMap, 26 | //ijon_index_to_max: HashMap, 27 | favqueue: Vec, 28 | inputs: Vec>>, 29 | input_to_iters_no_finds: Vec, 30 | bitmap_bits: Vec, 31 | bitmaps: BitmapHandler, 32 | next_input_id: usize, 33 | } 34 | 35 | #[derive(Clone)] 36 | pub struct Queue { 37 | workdir: String, 38 | start_time: std::time::Instant, 39 | total_execs: Arc>, 40 | data: Arc>, 41 | } 42 | 43 | impl<'a> InputQueue for Queue { 44 | fn sample_for_splicing(&self, dist: &Distributions) -> Arc { 45 | let data_lock = self.data.read().unwrap(); 46 | let inputs = &data_lock.inputs; 47 | let i = dist.gen_range(0, inputs.len()); 48 | let inp = inputs[i].read().unwrap().data.clone(); 49 | return inp; 50 | } 51 | } 52 | 53 | impl Queue { 54 | pub fn new(workdir: String, bitmap_size: usize) -> Self { 55 | return Self { 56 | workdir: workdir, 57 | start_time: std::time::Instant::now(), 58 | total_execs: Arc::new(RwLock::new(0_u64)), 59 | data: Arc::new(RwLock::new(QueueData { 60 | bitmap_index_to_min_example: HashMap::new(), 61 | //bitmap_index_to_max_example: HashMap::new(), 62 | //ijon_index_to_max: HashMap::new(), 63 | inputs: vec![], 64 | favqueue: vec![], 65 | input_to_iters_no_finds: vec![], 66 | bitmap_bits: vec![], 67 | bitmaps: BitmapHandler::new(bitmap_size), 68 | next_input_id: 0, 69 | })), 70 | }; 71 | } 72 | 73 | pub fn update_total_execs(&self, update: u64){ 74 | let mut w = self.total_execs.write().unwrap(); 75 | *w += update; 76 | } 77 | 78 | pub fn get_total_execs(&self) -> u64 { 79 | *self.total_execs.read().unwrap() 80 | } 81 | 82 | pub fn get_runtime_as_secs_f32(&self) -> f32 { 83 | (std::time::Instant::now() - self.start_time).as_secs_f32() 84 | } 85 | 86 | pub fn write_stats(&self) { 87 | use std::fs::File; 88 | use std::fs::OpenOptions; 89 | use std::io::prelude::*; 90 | let dat = self.data.read().unwrap(); 91 | let ser = QueueStats { 92 | num_inputs: dat.inputs.len(), 93 | favqueue: dat 94 | .favqueue 95 | .iter() 96 | .map(|id| id.as_usize()) 97 | .collect::>(), 98 | }; 99 | let mut file = File::create(format!("{}/queue_stats.msgp", &self.workdir)).unwrap(); 100 | rmp_serde::encode::write_named(&mut file, &ser).unwrap(); 101 | 102 | let mut file = OpenOptions::new() 103 | .append(true) 104 | .create(true) 105 | .open(format!("{}/bitmap_stats.txt", &self.workdir)) 106 | .unwrap(); 107 | file.write_fmt(format_args!( 108 | "{},{}\n", 109 | (std::time::Instant::now() - self.start_time).as_secs_f32(), 110 | dat.bitmaps.normal_bitmap().bits().iter().filter(|b| **b > 0).count() 111 | )) 112 | .unwrap(); 113 | } 114 | 115 | pub fn num_bits(&self) -> usize { 116 | return self.data.read().unwrap().bitmaps.normal_bitmap().bits().iter().filter(|b| **b > 0).count() 117 | } 118 | 119 | pub fn len(&self) -> usize { 120 | return self.data.read().unwrap().inputs.len(); 121 | } 122 | 123 | pub fn num_crashes(&self) -> usize{ 124 | self.data.read().unwrap().inputs.iter().filter(|i| matches!(i.read().unwrap().exit_reason, ExitReason::Crash(_)) ).count() 125 | } 126 | 127 | pub fn should_minimize(&self, inp: &Input) -> bool { 128 | // We generally don't want to minimize imported inputs - they might contain interesting state that's not obvious in coverage! 129 | if inp.found_by == MutationStrategy::SeedImport { 130 | return false; 131 | } 132 | match inp.exit_reason { 133 | ExitReason::Normal(_) => return inp.storage_reasons.iter().any(|rea| rea.has_new_byte() ), 134 | _ => return false, 135 | } 136 | } 137 | 138 | pub fn check_new_bytes( 139 | &mut self, 140 | run_bitmap: &[u8], 141 | ijon_max_map: &[u64], 142 | etype: &ExitReason, 143 | strat: MutationStrategy 144 | ) -> Option> { 145 | let mut res = self.data 146 | .write() 147 | .unwrap() 148 | .bitmaps 149 | .check_new_bytes(run_bitmap, ijon_max_map, etype); 150 | if strat == MutationStrategy::SeedImport && *etype != ExitReason::Timeout{ 151 | let mut reasons = res.take().unwrap_or(vec!()); 152 | reasons.push(StorageReason::Imported); 153 | res = Some(reasons); 154 | } 155 | return res; 156 | } 157 | 158 | pub fn set_state(&mut self, inp: &Input, state: InputState) { 159 | let data = self.data.write().unwrap(); 160 | assert!(inp.id.as_usize() < data.inputs.len()); 161 | data.inputs[inp.id.as_usize()].write().unwrap().state = state; 162 | } 163 | 164 | pub fn set_custom_dict(&mut self, inp: &Input, custom_dict: CustomDict){ 165 | let data = self.data.write().unwrap(); 166 | assert!(inp.id.as_usize() < data.inputs.len()); 167 | data.inputs[inp.id.as_usize()].write().unwrap().custom_dict = custom_dict; 168 | } 169 | 170 | pub fn register_best_input_for_bitmap(data: &mut std::sync::RwLockWriteGuard, bitmap_index: usize, input_id: InputID, spec: &GraphSpec, new_len: usize){ 171 | if !data.bitmap_index_to_min_example.contains_key(&bitmap_index) { 172 | data.bitmap_bits.push(bitmap_index); 173 | data.bitmap_index_to_min_example.insert(bitmap_index, input_id); 174 | } 175 | let old_entry = data 176 | .bitmap_index_to_min_example 177 | .get_mut(&bitmap_index) 178 | .unwrap() 179 | .as_usize(); 180 | if data.inputs[old_entry].read().unwrap().data.node_len(&spec) > new_len { 181 | data.bitmap_index_to_min_example.insert(bitmap_index, input_id); 182 | } 183 | } 184 | 185 | pub fn register_best_input_for_ijon_max(data: &mut std::sync::RwLockWriteGuard, ijon_index: usize, input_id: InputID){ 186 | data.bitmap_index_to_min_example.insert(ijon_index, input_id); 187 | } 188 | 189 | pub fn add(&mut self, mut input: Input, spec: &GraphSpec) -> Option { 190 | assert_eq!(input.id, InputID::invalid()); 191 | if input.data.node_len(spec) == 0 { 192 | return None; 193 | } 194 | let id; 195 | match input.exit_reason { 196 | ExitReason::Normal(_) | ExitReason::Crash(_) => { 197 | //let has_new_bytes = input.storage_reasons.iter().any(|r| r.has_new_byte() ); 198 | //let should_update_favs; 199 | { 200 | let mut data = self.data.write().unwrap(); 201 | //should_update_favs = has_new_bytes; 202 | id = InputID::new(data.inputs.len()); 203 | input.id = id; 204 | let new_len = input.data.node_len(&spec); 205 | let input = Arc::new(RwLock::new(input)); 206 | data.inputs.push(input.clone()); 207 | data.input_to_iters_no_finds.push(0); 208 | 209 | for r in input.read().unwrap().storage_reasons.iter() { 210 | match r { 211 | StorageReason::Bitmap(reason) => Self::register_best_input_for_bitmap(&mut data, reason.index, id, spec, new_len), 212 | StorageReason::IjonMax(reason) => Self::register_best_input_for_ijon_max(&mut data,reason.index, id), 213 | StorageReason::Imported => {}, 214 | } 215 | } 216 | } 217 | //if should_update_favs { 218 | self.calc_fav_bits(); 219 | //} 220 | self.write_stats(); 221 | } 222 | /* 223 | ExitReason::Crash(_) => return None, //println!("NEW crash found!"), 224 | */ 225 | _ => { 226 | //println!("ignoring input {:?}", input.exit_reason); 227 | return None; 228 | } 229 | } 230 | return Some(id); 231 | } 232 | 233 | pub fn calc_fav_bits(&mut self) { 234 | let mut favids = vec![]; 235 | let mut ijon_slot_to_fav = HashMap::::new(); 236 | //println!("==== update favbit queue store ===="); 237 | { 238 | //const IJON_MAX_SIZE: usize = 256; 239 | let data = self.data.read().unwrap(); 240 | let mut bits = vec![0u8; data.bitmaps.size()]; 241 | for input in data.inputs.iter().rev() { 242 | let inp = input.read().unwrap(); 243 | if inp.storage_reasons.iter().any(|s| s.has_new_byte() ) { 244 | //found new bytes 245 | let has_new_bits = inp 246 | .bitmap 247 | .bits() 248 | .iter() 249 | .enumerate() 250 | .any(|(i, v)| bits[i] == 0 && *v > 0); 251 | if has_new_bits { 252 | for (i, v) in inp.bitmap.bits().iter().enumerate() { 253 | if *v != 0 { 254 | bits[i] = 1; 255 | } 256 | } 257 | favids.push(inp.id); 258 | } 259 | for (i, v) in inp.bitmap.ijon_max_vals().iter().enumerate() { 260 | if *v != 0 && (!ijon_slot_to_fav.contains_key(&i) || ijon_slot_to_fav[&i].0<*v) { 261 | ijon_slot_to_fav.insert(i,(*v,inp.id)); 262 | } 263 | } 264 | } else if inp.found_by == MutationStrategy::SeedImport{ 265 | favids.push(inp.id); 266 | } 267 | } 268 | } 269 | for (i,(_val,id)) in ijon_slot_to_fav.iter(){ 270 | //use std::time::SystemTime; 271 | println!("[!] store ijon {:?} for {} => {:x}", 272 | //SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs(), 273 | id,i,_val); 274 | if !favids.contains(id) { //TODO FIX O(n) 275 | favids.push(*id); 276 | } 277 | } 278 | { 279 | let mut data = self.data.write().unwrap(); 280 | /* 281 | println!( 282 | "calc favbits ({}) out of {}", 283 | favids.len(), 284 | data.inputs.len() 285 | ); 286 | */ 287 | data.favqueue = favids; 288 | } 289 | } 290 | 291 | pub fn schedule(&self, rng: &Distributions) -> Arc> { 292 | let data = self.data.read().unwrap(); 293 | let id: usize; 294 | 295 | if data.favqueue.len() > 0 && rng.gen::() % 10 <= 6 { 296 | id = data.favqueue[rng.gen_range(0, data.favqueue.len())].as_usize(); 297 | } else if rng.gen::() % 8 <= 5 && data.bitmap_bits.len() != 0 { 298 | let bit = &data.bitmap_bits[rng.gen_range(0, data.bitmap_bits.len())]; 299 | id = data 300 | .bitmap_index_to_min_example 301 | .get(bit) 302 | .unwrap() 303 | .as_usize(); 304 | } else { 305 | id = rng.gen_range(0, data.inputs.len()); 306 | } 307 | 308 | //id = rng.gen_range(0, data.inputs.len()); 309 | data.inputs[id].clone() 310 | } 311 | 312 | pub fn next_id(&mut self) -> usize { 313 | let mut data = self.data.write().unwrap(); 314 | data.next_input_id += 1; 315 | return data.next_input_id; 316 | } 317 | 318 | pub fn get_input(&self, id: InputID) -> Arc> { 319 | let data = self.data.read().unwrap(); 320 | return data.inputs[id.as_usize()].clone(); 321 | } 322 | 323 | pub fn update_iters_no_finds(&self, id: InputID, has_new_finds: bool) -> usize { 324 | let mut data = self.data.write().unwrap(); 325 | let value = &mut data.input_to_iters_no_finds[id.as_usize()]; 326 | if has_new_finds { 327 | *value = 0; 328 | } else { 329 | *value += 1; 330 | } 331 | *value 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /structured_fuzzer/src/mutator.rs: -------------------------------------------------------------------------------- 1 | use std::rc::Rc; 2 | //use std::borrow::Borrow; 3 | use std::sync::Arc; 4 | 5 | use crate::graph_mutator::graph_builder::{GraphBuilder,GraphState}; 6 | use crate::graph_mutator::graph_iter::GraphNode; 7 | use crate::graph_mutator::graph_storage::GraphStorage; 8 | use crate::graph_mutator::graph_storage::VecGraph; 9 | use crate::graph_mutator::spec::GraphSpec; 10 | use crate::primitive_mutator::mutator::PrimitiveMutator; 11 | use crate::random::distributions::Distributions; 12 | use crate::custom_dict::CustomDict; 13 | 14 | #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] 15 | pub enum MutationStrategy{ 16 | GenerateTail(GenerateTail), 17 | SpliceRandom, 18 | Splice, 19 | DataOnly, 20 | Generate, 21 | Repeat, 22 | Minimize, 23 | MinimizeSplit, 24 | Import, 25 | SeedImport, 26 | } 27 | 28 | impl MutationStrategy{ 29 | pub fn name(&self) -> &str{ 30 | match self{ 31 | MutationStrategy::GenerateTail(_) => "generate_tail", 32 | MutationStrategy::SpliceRandom => "splice_random", 33 | MutationStrategy::Splice => "splice", 34 | MutationStrategy::DataOnly => "data_only", 35 | MutationStrategy::Generate => "generate", 36 | MutationStrategy::Repeat => "repeat", 37 | MutationStrategy::Minimize => "minimize", 38 | MutationStrategy::MinimizeSplit => "minimize_split", 39 | MutationStrategy::Import=>"import", 40 | MutationStrategy::SeedImport=>"seed_import" 41 | } 42 | } 43 | } 44 | 45 | #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] 46 | pub struct GenerateTail{ pub drop_last: usize, pub generate: usize } 47 | 48 | #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] 49 | pub enum NodeMutationType { 50 | CopyNode, 51 | MutateNodeData, 52 | DropNode, 53 | SkipAndGenerate, 54 | } 55 | 56 | pub struct MutatorSnapshotState{ 57 | pub skip_nodes: usize, 58 | pub skip_ops: usize, 59 | pub skip_data: usize, 60 | pub prefix_graph_state: Option, 61 | } 62 | 63 | impl MutatorSnapshotState{ 64 | pub fn none() -> Self { 65 | return Self{skip_data:0, skip_nodes: 0, skip_ops:0, prefix_graph_state: None} 66 | } 67 | } 68 | 69 | pub struct Mutator { 70 | pub spec: Rc, 71 | builder: GraphBuilder, 72 | mutator: PrimitiveMutator, 73 | } 74 | 75 | pub trait InputQueue { 76 | fn sample_for_splicing(&self, dist: &Distributions) -> Arc; 77 | } 78 | 79 | impl InputQueue for Vec{ 80 | fn sample_for_splicing(&self, dist: &Distributions) -> Arc{ 81 | assert!(self.len() > 0); 82 | return Arc::new(self[dist.gen_range(0,self.len())].clone()); 83 | } 84 | } 85 | 86 | impl Mutator { 87 | pub fn new(spec: GraphSpec) -> Self { 88 | let spec = Rc::new(spec); 89 | let mutator = PrimitiveMutator::new(); 90 | let builder = GraphBuilder::new(spec.clone()); 91 | 92 | return Self { 93 | spec, 94 | builder, 95 | mutator, 96 | }; 97 | } 98 | 99 | pub fn mutate(&mut self, orig: &VecGraph, ops_used: usize, dict: &CustomDict, snapshot: &MutatorSnapshotState, queue: &Q, storage: &mut S, dist: &Distributions) -> MutationStrategy{ 100 | let orig_len = ops_used as usize-snapshot.skip_nodes; 101 | 102 | if orig.op_len()== 0 || orig_len == 0 || ops_used == 0 { 103 | self.generate(50, snapshot, storage, dist); 104 | return MutationStrategy::Generate; 105 | } 106 | 107 | let strategy = dist.gen_mutation_strategy(orig_len); 108 | match strategy{ 109 | MutationStrategy::GenerateTail(args) => self.generate_tail(orig, ops_used, snapshot, args, storage, dist), 110 | MutationStrategy::SpliceRandom => self.splice_random(orig, ops_used, snapshot, dict, storage, dist), 111 | MutationStrategy::Splice => self.splice(orig, ops_used, snapshot,queue, storage, dist), 112 | MutationStrategy::DataOnly => self.mutate_data(orig, ops_used, snapshot, dict, storage, dist), 113 | MutationStrategy::Generate => self.generate(50, snapshot, storage, dist), 114 | MutationStrategy::Repeat => self.repeat(orig, ops_used, snapshot, dict, storage, dist), 115 | MutationStrategy::Minimize => unreachable!(), 116 | MutationStrategy::MinimizeSplit => unreachable!(), 117 | MutationStrategy::Import => unreachable!(), 118 | MutationStrategy::SeedImport => unreachable!(), 119 | } 120 | return strategy; 121 | } 122 | 123 | pub fn prepare_snapshot(&mut self, snapshot_cutoff: usize, data: &VecGraph, storage: &mut S, dist: &Distributions) -> MutatorSnapshotState{ 124 | self.builder.start(storage, &MutatorSnapshotState::none()); 125 | for n in data.node_iter(&self.spec).take(snapshot_cutoff){ 126 | self.builder.append_node(&n, storage, dist); 127 | } 128 | let prefix_graph_state = Some(self.builder.get_graph_state()); 129 | let skip_ops =storage.op_len(); 130 | let skip_data = storage.data_len(); 131 | storage.append_op(self.spec.snapshot_node_id.unwrap().as_u16()); 132 | return MutatorSnapshotState{skip_nodes: snapshot_cutoff, skip_ops, skip_data, prefix_graph_state}; 133 | } 134 | 135 | pub fn repeat(&mut self, orig: &VecGraph, ops_used: usize, snapshot: &MutatorSnapshotState, dict: &CustomDict, storage: &mut S, dist: &Distributions) { 136 | self.builder.start(storage, snapshot); 137 | 138 | let fragment_nodes = dist.gen_range(2, 16); 139 | let repeats = dist.gen_range(2, 6); 140 | let insert_pos = if ops_used-snapshot.skip_nodes-1 > 0 { 141 | dist.gen_range(0, ops_used-snapshot.skip_nodes-1) 142 | } else { 0 }; 143 | //println!("REPEAT {}..{} (out of {} with {} skipped) for {} times",insert_pos, insert_pos+fragment_nodes, orig.node_len(&self.spec), snapshot.skip_nodes, repeats); 144 | for n in orig.node_iter(&self.spec.clone()).skip(snapshot.skip_nodes).take(insert_pos) { 145 | if self.builder.is_full(storage){return;} 146 | self.builder.append_node(&n, storage, dist); 147 | } 148 | 149 | let origs = orig.node_iter(&self.spec).skip(snapshot.skip_nodes+insert_pos).take(fragment_nodes).collect::>(); 150 | assert!(origs.len() > 0); 151 | assert!(repeats > 1); 152 | for _ in 0..repeats { 153 | for n in origs.iter() { 154 | if self.builder.is_full(storage){return;} 155 | self.builder.append_node_mutated(&n, dict, &self.mutator, storage, dist); 156 | } 157 | } 158 | 159 | for n in orig.node_iter(&self.spec.clone()).skip(snapshot.skip_nodes+insert_pos) { 160 | if self.builder.is_full(storage){return;} 161 | self.builder.append_node(&n, storage, dist); 162 | } 163 | } 164 | 165 | pub fn generate_tail(&mut self, orig: &VecGraph, ops_used: usize, snapshot: &MutatorSnapshotState, args: GenerateTail, storage: &mut S, dist: &Distributions){ 166 | let orig_len = ops_used-snapshot.skip_nodes; 167 | 168 | self.builder.start(storage, snapshot); 169 | for n in orig.node_iter(&self.spec).skip(snapshot.skip_nodes).take(orig_len-args.drop_last) { 170 | if self.builder.is_full(storage){return;} 171 | self.builder.append_node(&n, storage, dist); 172 | } 173 | 174 | self.builder.append_random(args.generate, &self.mutator, storage, dist).unwrap(); 175 | } 176 | 177 | pub fn mutate_data(&mut self, orig: &VecGraph, ops_used: usize, snapshot: &MutatorSnapshotState, dict: &CustomDict, storage: &mut S, dist: &Distributions) { 178 | self.builder.start(storage, snapshot); 179 | for n in orig.node_iter(&self.spec.clone()).skip(snapshot.skip_nodes) { 180 | if self.builder.is_full(storage){return;} 181 | if dist.should_mutate_data(ops_used-snapshot.skip_nodes){ //TODO fix this with probability based on length 182 | self.builder.append_node_mutated(&n, dict, &self.mutator, storage, dist); 183 | } else { 184 | self.builder.append_node(&n, storage, dist); 185 | } 186 | } 187 | } 188 | 189 | 190 | pub fn splice_random(&mut self, orig: &VecGraph, ops_used: usize, snapshot: &MutatorSnapshotState, dict: &CustomDict, storage: &mut S, dist: &Distributions) { 191 | self.builder.start(storage, snapshot); 192 | for n in orig.node_iter(&self.spec.clone()).skip(snapshot.skip_nodes).take(ops_used) { 193 | if self.builder.is_full(storage){return;} 194 | let mutation = self.pick_op(&n, dist); 195 | self.apply_graph_node(mutation, &n, dict, storage, dist); 196 | } 197 | } 198 | 199 | pub fn pick_splice_points(&self, len: usize, dist: &Distributions) -> Vec{ 200 | use std::cmp::Reverse; 201 | let num = match len{ 202 | 0 => unreachable!(), 203 | 1..=3 => dist.gen_range(1,3), 204 | 4..=15 => dist.gen_range(1,5), 205 | _ => dist.gen_range(4,16), 206 | }; 207 | let mut res = (0..num).map(|_| dist.gen_range(0,len) ).collect::>(); 208 | res.sort_unstable(); 209 | res.sort_by_key(|x| (*x, Reverse(*x))); 210 | res.dedup(); 211 | return res; 212 | } 213 | 214 | pub fn splice< S: GraphStorage, Q:InputQueue >(&mut self, orig: &VecGraph, ops_used: usize, snapshot: &MutatorSnapshotState, queue: &Q, storage: &mut S, dist: &Distributions) { 215 | 216 | let orig_len = ops_used-snapshot.skip_nodes; 217 | let mut splice_points = self.pick_splice_points(orig_len, dist); 218 | //println!("splice with {:?} on graph with {}..{}/{}", splice_points, snapshot.skip_nodes, ops_used,orig.node_len(&self.spec)); 219 | self.builder.start(storage, snapshot); 220 | let mut spliced = false; 221 | for (i,n) in orig.node_iter(&self.spec.clone()).skip(snapshot.skip_nodes).enumerate() { 222 | if self.builder.is_full(storage){return;} 223 | if splice_points.len()>0{ 224 | //println!("{} vs {}",i,*splice_points.last().unwrap()); 225 | } 226 | if splice_points.len() > 0 && i == *splice_points.last().unwrap(){ 227 | splice_points.pop(); 228 | let other_lock = queue.sample_for_splicing(&dist); 229 | let other = other_lock.as_ref(); 230 | let other_len = other.node_len(&self.spec); 231 | let start = dist.gen_range(0,other_len); 232 | let mut len = dist.gen_range(1,16); 233 | if len > other_len-start { len = other_len-start } 234 | assert!(len>0); 235 | for nn in other.node_iter(&self.spec.clone()).skip(start).take(len){ 236 | self.builder.append_node(&nn, storage, dist); 237 | } 238 | spliced=true; 239 | } 240 | self.builder.append_node(&n, storage, dist); 241 | } 242 | assert!(spliced); 243 | } 244 | 245 | pub fn copy_all(&mut self, orig: &VecGraph, storage: &mut S, dist: &Distributions){ 246 | self.builder.start(storage, &MutatorSnapshotState::none()); 247 | for n in orig.node_iter(&self.spec){ 248 | self.builder.append_node(&n, storage, dist); 249 | } 250 | } 251 | 252 | pub fn generate(&mut self, n: usize, snapshot: &MutatorSnapshotState, storage: &mut S, dist: &Distributions) { 253 | self.builder.start(storage, snapshot); 254 | self.builder 255 | .append_random(n, &self.mutator, storage, dist) 256 | .unwrap(); 257 | } 258 | 259 | pub fn drop_range( 260 | &mut self, 261 | orig: &VecGraph, 262 | range: std::ops::Range, 263 | storage: &mut S, 264 | dist: &Distributions 265 | ) { 266 | self.builder.start(storage, &MutatorSnapshotState::none()); 267 | for (i, n) in orig.node_iter(&self.spec.clone()).enumerate() { 268 | if range.start <= i && i < range.end { 269 | continue; 270 | } 271 | self.builder.append_node(&n, storage, dist); 272 | } 273 | } 274 | 275 | //pub fn drop_node_at(&mut self, orig: &VecGraph, i: usize, storage: &mut S, dist: &Distributions) { 276 | // self.drop_range(orig, i..i + 1, storage, dist); 277 | //} 278 | 279 | pub fn dump_graph(&self, storage: &S) -> VecGraph { 280 | storage.as_vec_graph() 281 | } 282 | 283 | fn apply_graph_node( 284 | &mut self, 285 | op: NodeMutationType, 286 | n: &GraphNode, 287 | dict: &CustomDict, 288 | storage: &mut S, 289 | dist: &Distributions 290 | ) { 291 | use NodeMutationType::*; 292 | 293 | match op { 294 | CopyNode => { 295 | self.builder.append_node(&n, storage, dist); 296 | } 297 | MutateNodeData => self.builder.append_node_mutated(&n, dict, &self.mutator, storage, dist), 298 | DropNode => {} 299 | SkipAndGenerate => { 300 | let len = dist.gen_number_of_random_nodes(); 301 | self.builder 302 | .append_random(len, &self.mutator, storage, dist) 303 | .unwrap(); 304 | } 305 | } 306 | } 307 | 308 | fn pick_op(&self, _n: &GraphNode, dist: &Distributions) -> NodeMutationType { 309 | return dist.gen_graph_mutation_type(); 310 | } 311 | 312 | pub fn num_ops_used(&self, storage: &S) -> usize { 313 | return self.builder.num_ops_used(storage); 314 | } 315 | } -------------------------------------------------------------------------------- /structured_fuzzer/src/primitive_mutator/mutator.rs: -------------------------------------------------------------------------------- 1 | use std::i64; 2 | use std::ops::Range; 3 | //use std::rc::Rc; 4 | 5 | use crate::data_buff::DataBuff; 6 | use crate::custom_dict::CustomDict; 7 | use crate::primitive_mutator::inplace_mutation::{InplaceMutation, InplaceMutationType}; 8 | use crate::primitive_mutator::size_changing_mutation::{ 9 | SizeChangingMutation, SizeChangingMutationType, 10 | }; 11 | use crate::random::distributions::Distributions; 12 | 13 | const INTERESTING_U8: [u8; 9] = [(-128i8) as u8, (-1i8) as u8, 0, 1, 16, 32, 64, 100, 127]; 14 | const INTERESTING_U16: [u16; 19] = [ 15 | (-128i16) as u16, 16 | (-1i16) as u16, 17 | 0, 18 | 1, 19 | 16, 20 | 32, 21 | 64, 22 | 100, 23 | 127, //u8 24 | (-32768i16) as u16, 25 | (-129i16) as u16, 26 | 128, 27 | 255, 28 | 256, 29 | 512, 30 | 1000, 31 | 1024, 32 | 4096, 33 | 32767, 34 | ]; 35 | const INTERESTING_U32: [u32; 27] = [ 36 | (-128i32) as u32, 37 | (-1i32) as u32, 38 | 0, 39 | 1, 40 | 16, 41 | 32, 42 | 64, 43 | 100, 44 | 127, //u8 45 | (-32768i32) as u32, 46 | (-129i32) as u32, 47 | 128, 48 | 255, 49 | 256, 50 | 512, 51 | 1000, 52 | 1024, 53 | 4096, 54 | 32767, //u16 55 | (-2147483648i32) as u32, 56 | (-100663046i32) as u32, 57 | (-32769i32) as u32, 58 | 32768, 59 | 65535, 60 | 65536, 61 | 100663045, 62 | 2147483647, 63 | ]; 64 | const INTERESTING_U64: [u64; 30] = [ 65 | (-128i64) as u64, 66 | (-1i64) as u64, 67 | 0, 68 | 1, 69 | 16, 70 | 32, 71 | 64, 72 | 100, 73 | 127, //u8 74 | (-32768i32) as u64, 75 | (-129i64) as u64, 76 | 128, 77 | 255, 78 | 256, 79 | 512, 80 | 1000, 81 | 1024, 82 | 4096, 83 | 32767, //u16 84 | (-2147483648i64) as u64, 85 | (-100663046i64) as u64, 86 | (-32769i64) as u64, 87 | 32768, 88 | 65535, 89 | 65536, 90 | 100663045, 91 | 2147483647, //u32 92 | i64::MIN as u64, 93 | 0x7fffffffffffffff, 94 | 0x8080808080808080, 95 | ]; 96 | 97 | #[derive(Clone, Eq, Hash, PartialEq, Debug)] 98 | pub enum Mutation { 99 | Inplace(InplaceMutation), 100 | SizeChanging(SizeChangingMutation), 101 | } 102 | 103 | impl Mutation { 104 | pub fn apply(&self, buff: &mut DataBuff) { 105 | match self { 106 | Mutation::Inplace(x) => x.apply(buff), 107 | Mutation::SizeChanging(x) => x.apply(buff), 108 | } 109 | } 110 | } 111 | 112 | pub struct PrimitiveMutator {} 113 | 114 | impl PrimitiveMutator { 115 | pub fn new() -> Self { 116 | return Self { }; 117 | } 118 | 119 | pub fn mutate(&self, buff: &mut DataBuff, dict: Option<&CustomDict>, dist: &Distributions) { 120 | if buff.is_empty() { 121 | return; 122 | } 123 | if let Some(dict) = dict{ 124 | if dist.should_mutate_dict() { 125 | let continue_mutation = dict.mutate(buff, dist); 126 | if ! continue_mutation {return} 127 | } 128 | } 129 | //TODO add size changing mutations if buff.available != 0 130 | let mutation = self.gen_inplace_mutation(buff,dist); 131 | mutation.apply(buff); 132 | } 133 | 134 | fn gen_inplace_mutation_type(&self, buff: &DataBuff,dist: &Distributions) -> InplaceMutationType { 135 | assert!(!buff.is_empty()); 136 | for _ in 1..5 { 137 | let t = *dist.gen_inplace_mutation_type(); 138 | if t.min_size() <= buff.len() { 139 | return t; 140 | } 141 | } 142 | return InplaceMutationType::FlipBitT; 143 | } 144 | 145 | pub fn gen_inplace_mutation(&self, buff: &DataBuff, dist: &Distributions) -> InplaceMutation { 146 | use InplaceMutation::*; 147 | use InplaceMutationType::*; 148 | let m_type = self.gen_inplace_mutation_type(buff,dist); 149 | match m_type { 150 | FlipBitT => FlipBit { 151 | offset: self.gen_offset(1, buff, dist), 152 | bit: dist.gen_range(0, 8), 153 | }, 154 | AddU8T => AddU8 { 155 | offset: self.gen_offset(1, buff, dist), 156 | val: self.gen_arith_val(dist) as u8, 157 | }, 158 | AddU16T => AddU16 { 159 | offset: self.gen_offset(2, buff, dist), 160 | val: self.gen_arith_val(dist) as u16, 161 | flip_endian: dist.gen_endianess(), 162 | }, 163 | AddU32T => AddU32 { 164 | offset: self.gen_offset(4, buff, dist), 165 | val: self.gen_arith_val(dist) as u32, 166 | flip_endian: dist.gen_endianess(), 167 | }, 168 | AddU64T => AddU64 { 169 | offset: self.gen_offset(8, buff, dist), 170 | val: self.gen_arith_val(dist) as u64, 171 | flip_endian: dist.gen_endianess(), 172 | }, 173 | InterestingU8T => InterestingU8 { 174 | offset: self.gen_offset(1, buff, dist), 175 | val: self.gen_pick(&INTERESTING_U8, dist), 176 | }, 177 | InterestingU16T => InterestingU16 { 178 | offset: self.gen_offset(2, buff, dist), 179 | val: self.gen_pick(&INTERESTING_U16, dist), 180 | flip_endian: dist.gen_endianess(), 181 | }, 182 | InterestingU32T => InterestingU32 { 183 | offset: self.gen_offset(4, buff, dist), 184 | val: self.gen_pick(&INTERESTING_U32, dist), 185 | flip_endian: dist.gen_endianess(), 186 | }, 187 | InterestingU64T => InterestingU64 { 188 | offset: self.gen_offset(8, buff, dist), 189 | val: self.gen_pick(&INTERESTING_U64, dist), 190 | flip_endian: dist.gen_endianess(), 191 | }, 192 | OverwriteRandomByteT => { 193 | let offset = self.gen_offset(1, buff, dist); 194 | OverwriteRandomByte { 195 | offset, 196 | val: dist.gen_range(1, 0xff) ^ buff.read_u8(offset), 197 | } 198 | } 199 | OverwriteChunkT => { 200 | let src = self.gen_block_range(buff, dist); 201 | let len = src.end - src.start; 202 | let mut dst = self.gen_offset(len, buff, dist); 203 | while src.start == dst && len != buff.len() { 204 | dst = self.gen_offset(len, buff, dist); 205 | } 206 | OverwriteChunk { src, dst } 207 | } 208 | OverwriteRandomT => { 209 | let mut dst = self.gen_block_range(buff, dist); 210 | let data = dist.gen_random_overwrite_data(&dst); 211 | dst.end = dst.start+data.len(); 212 | OverwriteRandom { 213 | data, 214 | dst: dst.start, 215 | } 216 | } 217 | OverwriteFixedT => OverwriteFixed { 218 | block: self.gen_block_range(buff, dist), 219 | val: dist.gen(), 220 | }, 221 | } 222 | } 223 | 224 | fn gen_size_changing_mutation_type(&self, buff: &DataBuff,dist: &Distributions) -> SizeChangingMutationType { 225 | assert!(buff.capacity() > 16); 226 | assert!(!buff.is_empty()); 227 | for _ in 1..5 { 228 | let t = *dist.gen_size_changing_mutation_type(); 229 | if t.min_size() <= buff.len() && t.min_available() < buff.available() { 230 | return t; 231 | } 232 | } 233 | if buff.available() > SizeChangingMutationType::InsertRandomT.min_available() { 234 | return SizeChangingMutationType::InsertRandomT; 235 | } 236 | return SizeChangingMutationType::DeleteT; 237 | } 238 | 239 | pub fn gen_size_changing_mutation(&self, buff: &DataBuff,dist: &Distributions) -> SizeChangingMutation { 240 | use SizeChangingMutation::*; 241 | use SizeChangingMutationType::*; 242 | let m_type = self.gen_size_changing_mutation_type(buff, dist); 243 | match m_type { 244 | DeleteT => Delete { 245 | block: self.gen_block_range(buff, dist), 246 | }, 247 | InsertChunkT => InsertChunk { 248 | src: self.gen_insert_block_range(buff, dist), 249 | dst: self.gen_offset(1, buff, dist), 250 | }, 251 | InsertFixedT => InsertFixed { 252 | dst: self.gen_offset(1, buff, dist), 253 | amount: dist.gen_range(1, buff.capacity() - buff.len()), 254 | val: dist.gen(), 255 | }, 256 | InsertRandomT => { 257 | let max = dist.gen_range(1, buff.available()); 258 | InsertRandom { 259 | data: (0..max).map(|_| dist.gen()).collect(), 260 | dst: self.gen_offset(1, buff, dist), 261 | } 262 | } 263 | } 264 | } 265 | 266 | pub fn gen_mutation(&self, buff: &DataBuff,dist: &Distributions) -> Mutation { 267 | if dist.gen() { 268 | return Mutation::Inplace(self.gen_inplace_mutation(buff, dist)); 269 | } 270 | return Mutation::SizeChanging(self.gen_size_changing_mutation(buff, dist)); 271 | } 272 | 273 | fn gen_offset(&self, size: usize, buff: &DataBuff,dist: &Distributions) -> usize { 274 | assert!(buff.len() >= size); 275 | dist.gen_range(0, buff.len() - size + 1) 276 | } 277 | 278 | fn gen_arith_val(&self,dist: &Distributions) -> i8 { 279 | if dist.gen() { 280 | dist.gen_range(1, 35) 281 | } else { 282 | dist.gen_range(-35, -1) 283 | } 284 | } 285 | 286 | fn gen_pick(&self, data: &[T],dist: &Distributions) -> T { 287 | data[dist.gen_range(0, data.len())] 288 | } 289 | fn gen_insert_block_range(&self, buff: &DataBuff,dist: &Distributions) -> Range { 290 | return self.gen_block_max(0..buff.len(), buff.capacity() - buff.len(), dist); 291 | } 292 | fn gen_block_range(&self, buff: &DataBuff,dist: &Distributions) -> Range { 293 | return self.gen_block_max(0..buff.len(), buff.len(), dist); 294 | } 295 | fn gen_block_max(&self, range: Range, max_len: usize,dist: &Distributions) -> Range { 296 | if range.start == range.end { 297 | return range; 298 | } 299 | let (mut min, mut max) = *dist.gen_block_size(); 300 | if max > max_len { 301 | max = max_len; 302 | } 303 | let len = range.end - range.start; 304 | if max > len { 305 | max = len; 306 | } 307 | if min >= max { 308 | min = 1; 309 | } 310 | let size = dist.gen_range(min, max + 1); 311 | let start = dist.gen_range(0, range.end - size + 1); 312 | return start..start + size; 313 | } 314 | 315 | pub fn gen_num_array_elems( 316 | &self, 317 | elem_size: usize, 318 | min_elems: usize, 319 | max_elems: usize, 320 | max_data: usize, 321 | dist: &Distributions 322 | ) -> usize { 323 | if max_data / elem_size > min_elems { 324 | return dist 325 | .gen_range(min_elems, max_elems.min(max_data / elem_size)); 326 | } 327 | return max_data / elem_size; 328 | } 329 | } 330 | 331 | #[cfg(test)] 332 | mod tests { 333 | 334 | use super::*; 335 | 336 | #[test] 337 | fn test_inplace() { 338 | let dist = crate::random::distributions::Distributions::new(vec!()); 339 | let mutator = PrimitiveMutator::new(); 340 | let mut data = vec![0u8; 1024]; 341 | let mut buff = DataBuff::new(&mut data, 0); 342 | 343 | for _ in 0..1000 { 344 | let len = dist.gen_range(1, 1024); 345 | buff.set_to_random(len, &dist); 346 | let mutation = mutator.gen_inplace_mutation(&buff,&dist); 347 | 348 | mutation.apply(&mut buff); 349 | assert_eq!(buff.len(), len); 350 | } 351 | } 352 | 353 | #[test] 354 | fn test_sized() { 355 | let dist = crate::random::distributions::Distributions::new(vec!()); 356 | let mutator = PrimitiveMutator::new(); 357 | let mut data = vec![0u8; 1024]; 358 | let mut buff = DataBuff::new(&mut data, 0); 359 | 360 | for _ in 0..1000 { 361 | let len = dist.gen_range(1, 1024); 362 | buff.set_to_random(len, &dist); 363 | let mutation = mutator.gen_size_changing_mutation(&buff, &dist); 364 | println!("{:?} on buff of length {}", mutation, buff.len()); 365 | mutation.apply(&mut buff); 366 | } 367 | } 368 | 369 | use std::collections::HashMap; 370 | #[test] 371 | fn test_uniqueness() { 372 | let dist = crate::random::distributions::Distributions::new(vec!()); 373 | let mutator = PrimitiveMutator::new(); 374 | let mut data = vec![0u8; 1024]; 375 | let mut buff = DataBuff::new(&mut data, 0); 376 | let mut storage = HashMap::, usize>::new(); 377 | let mut infos = HashMap::, HashMap>::new(); 378 | let base = (0..256) 379 | .map(|_| dist.gen::()) 380 | .collect::>(); 381 | let iters = 10000; 382 | for _ in 0..iters { 383 | buff.set_to_slice(&base); 384 | let mutation = mutator.gen_mutation(&buff, &dist); 385 | mutation.apply(&mut buff); 386 | let inf = infos 387 | .entry(buff.as_slice().to_vec()) 388 | .or_insert_with(|| HashMap::new()); 389 | *(inf.entry(mutation).or_insert(0)) += 1; 390 | *storage.entry(buff.as_slice().to_vec()).or_insert(0) += 1; 391 | } 392 | println!("{} of {} iters produced uniq results", storage.len(), iters); 393 | let mut generated = storage.keys().collect::>(); 394 | generated.sort_unstable_by_key(|x| std::usize::MAX - storage.get(*x).unwrap()); 395 | for dat in generated.iter().take(20) { 396 | println!( 397 | "data was hit {:?} times, produced by {:?}", 398 | storage.get(*dat), 399 | infos.get(*dat) 400 | ); 401 | } 402 | assert!(storage.len() >= iters - (iters / 5)); //max 5% of the mutation should be duplicates 403 | } 404 | } 405 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/graph_builder.rs: -------------------------------------------------------------------------------- 1 | use crate::graph_mutator::graph_iter::{GraphNode, GraphOp, OpIter}; 2 | use crate::graph_mutator::graph_storage::{GraphStorage, VecGraph}; 3 | use crate::graph_mutator::newtypes::{ConnectorID, GraphError, NodeTypeID, ValueTypeID}; 4 | use crate::graph_mutator::spec::{GraphSpec, NodeSpec}; 5 | use crate::custom_dict::CustomDict; 6 | 7 | use crate::data_buff::DataBuff; 8 | use crate::primitive_mutator::mutator::PrimitiveMutator; 9 | use crate::random::distributions::Distributions; 10 | use crate::mutator::MutatorSnapshotState; 11 | 12 | use std::collections::HashMap; 13 | use std::rc::Rc; 14 | 15 | // Implements a set of available values, including the ability to insert, take and update values 16 | // quickly. This is used to construct a graph in the GraphBuilder by inserting nodes individually. 17 | // For every node inserted into the graph, the GraphState is used first to consume the inputs 18 | // needed to create the node, and then to make the outputs available for future nodes. As building 19 | // graphs includes renaming value ids, this also keeps track of the renaming. 20 | 21 | #[derive(Debug,Clone)] 22 | struct ValueState { 23 | cur_id: ConnectorID, 24 | new_ids_for_sampling: Vec, //used to quickly pick an random available id 25 | new_id_to_ids_index: HashMap, //used to remove entries from old_ids_for_sampling vec 26 | new_id_to_old_id: HashMap, 27 | old_id_to_new_id: HashMap, //used to look up renamed ids. Will not be cleared if new ids are used, if you have strange memory consumtion issues, look here. 28 | } 29 | 30 | impl ValueState { 31 | pub fn new() -> Self { 32 | return ValueState { 33 | cur_id: ConnectorID::new(0), 34 | new_ids_for_sampling: vec![], 35 | new_id_to_ids_index: HashMap::new(), 36 | new_id_to_old_id: HashMap::new(), 37 | old_id_to_new_id: HashMap::new(), 38 | }; 39 | } 40 | 41 | fn num_available(&self) -> usize { 42 | return self.new_ids_for_sampling.len(); 43 | } 44 | 45 | fn clear(&mut self) { 46 | self.cur_id = ConnectorID::new(0); 47 | self.new_ids_for_sampling.clear(); 48 | self.new_id_to_ids_index.clear(); 49 | self.old_id_to_new_id.clear(); 50 | } 51 | 52 | fn reset_to(&mut self, other: &ValueState){ 53 | self.cur_id = other.cur_id; 54 | self.new_id_to_ids_index = other.new_id_to_ids_index.clone(); 55 | self.new_ids_for_sampling = other.new_ids_for_sampling.clone(); 56 | self.old_id_to_new_id = other.old_id_to_new_id.clone(); 57 | } 58 | 59 | fn insert_new_id(&mut self) -> ConnectorID { 60 | self.cur_id = self.cur_id.next(); 61 | let new_id = self.cur_id; 62 | self.new_ids_for_sampling.push(new_id); 63 | self.new_id_to_ids_index 64 | .insert(new_id, self.new_ids_for_sampling.len() - 1); 65 | return new_id; 66 | } 67 | 68 | fn insert_old_id(&mut self, old_id: ConnectorID) -> ConnectorID { 69 | let new_id = self.insert_new_id(); 70 | self.new_id_to_old_id.insert(new_id, old_id); 71 | self.old_id_to_new_id.insert(old_id, new_id); 72 | return new_id; 73 | } 74 | 75 | fn take_new_input(&mut self, new_id: ConnectorID) -> ConnectorID { 76 | if let Some(i) = self.new_id_to_ids_index.remove(&new_id) { 77 | //new_id is no longer available for random sampling 78 | self.new_ids_for_sampling.swap_remove(i); 79 | if self.new_ids_for_sampling.len() > i { 80 | self.new_id_to_ids_index 81 | .insert(self.new_ids_for_sampling[i], i); 82 | } 83 | if let Some(old) = self.new_id_to_old_id.remove(&new_id) { 84 | self.old_id_to_new_id.remove(&old); 85 | } 86 | //this will not remove the old_id -> new_id mapping. If the old id is used without checking, this might cause problems. 87 | //also this might cause garbage old ids to be collected in the old_id_to_new_id hashmap. 88 | //Since the builder object is cleared regularly, I believe this will be no big issue. 89 | return new_id; 90 | } else { 91 | panic!("unknown new id: {:?}", new_id); 92 | } 93 | } 94 | 95 | fn get_renamed_old_input(&self, old_id: ConnectorID) -> Option { 96 | return self.old_id_to_new_id.get(&old_id).cloned(); 97 | } 98 | 99 | fn take_random_input(&mut self, rng: &Distributions) -> ConnectorID { 100 | let id = self.get_random_input(rng); 101 | return self.take_new_input(id); 102 | } 103 | 104 | fn get_random_input(&mut self, rng: &Distributions) -> ConnectorID { 105 | if self.new_ids_for_sampling.is_empty() { 106 | panic!("couldn get random input"); 107 | } 108 | let index = rng.gen_range(0, self.new_ids_for_sampling.len()); 109 | return self.new_ids_for_sampling[index]; 110 | } 111 | 112 | fn take_old_available_input( 113 | &mut self, 114 | old_id: ConnectorID, 115 | rng: &Distributions, 116 | ) -> ConnectorID { 117 | let id = self.get_old_available_input(old_id, rng); 118 | return self.take_new_input(id); 119 | } 120 | 121 | fn get_old_available_input(&mut self, old_id: ConnectorID, rng: &Distributions) -> ConnectorID { 122 | return self 123 | .get_renamed_old_input(old_id) 124 | .unwrap_or_else(|| self.get_random_input(rng)); 125 | } 126 | } 127 | 128 | #[derive(Clone)] 129 | pub struct GraphState { 130 | v_type_to_state: HashMap, 131 | } 132 | impl GraphState { 133 | fn new() -> Self { 134 | return Self { 135 | v_type_to_state: HashMap::new(), 136 | }; 137 | } 138 | 139 | fn get_state_mut(&mut self, spec: &ValueTypeID) -> &mut ValueState { 140 | if !self.v_type_to_state.contains_key(spec) { 141 | self.v_type_to_state.insert(*spec, ValueState::new()); 142 | } 143 | return self.v_type_to_state.get_mut(spec).unwrap(); 144 | } 145 | 146 | fn clear(&mut self) { 147 | for (_, s) in self.v_type_to_state.iter_mut() { 148 | s.clear(); 149 | } 150 | } 151 | 152 | fn reset_to(&mut self, other: &GraphState){ 153 | for (v_type, s) in self.v_type_to_state.iter_mut() { 154 | s.reset_to(&other.v_type_to_state[v_type]); 155 | } 156 | } 157 | 158 | pub fn info(&self, spec: &GraphSpec) -> String { 159 | let mut res = String::new(); 160 | for (vt, state) in self.v_type_to_state.iter() { 161 | res += &format!( 162 | "{}: {}\n", 163 | spec.get_value(*vt).unwrap().name, 164 | state.num_available() 165 | ); 166 | } 167 | return res; 168 | } 169 | 170 | fn is_available(&self, spec: &GraphSpec, n: NodeTypeID) -> bool { 171 | if let Ok(node_spec) = spec.get_node(n) { 172 | for (v_type, cnt) in node_spec.required_values.iter() { 173 | let is_available = self 174 | .v_type_to_state 175 | .get(v_type) 176 | .map(|state| state.num_available() >= *cnt) 177 | .unwrap_or(false); 178 | if !is_available { 179 | return false; 180 | } 181 | } 182 | return true; 183 | } 184 | return false; 185 | } 186 | } 187 | 188 | pub struct GraphBuilder { 189 | pub spec: Rc, 190 | state: GraphState, 191 | } 192 | 193 | // Used to store a graph, as it is build node by node 194 | impl GraphBuilder { 195 | pub fn new(spec: Rc) -> Self { 196 | return Self { 197 | spec, 198 | state: GraphState::new(), 199 | }; 200 | } 201 | 202 | pub fn start(&mut self, graph: &mut S, snapshot: &MutatorSnapshotState) { 203 | graph.truncate_to(snapshot.skip_ops, snapshot.skip_data); 204 | if let Some(ref state) = snapshot.prefix_graph_state{ 205 | self.state.reset_to(&state); 206 | } else { 207 | self.state.clear(); 208 | } 209 | } 210 | 211 | pub fn num_ops_used(&self, graph: &S) -> usize { 212 | return graph.ops_as_slice().len(); 213 | } 214 | 215 | fn pick_available_type(&mut self, dist: &Distributions) -> Result { 216 | let state = &self.state; 217 | let spec = &self.spec; 218 | let iter = self 219 | .spec 220 | .node_specs 221 | .iter() 222 | .filter(|n| state.is_available(spec, n.id) && n.generatable) 223 | .map(|n| n.id); 224 | return dist 225 | .choose_from_iter(iter) 226 | .ok_or(GraphError::InvalidSpecs); 227 | } 228 | 229 | pub fn rename_and_relink_op_val( 230 | state: &mut GraphState, 231 | rng: &Distributions, 232 | op: GraphOp, 233 | ) -> u16 { 234 | let newval = match op { 235 | GraphOp::Get(vtype, input_id) => state 236 | .get_state_mut(&vtype) 237 | .take_old_available_input(input_id, rng) 238 | .as_u16(), 239 | GraphOp::Set(vtype, con) => state.get_state_mut(&vtype).insert_old_id(con).as_u16(), 240 | GraphOp::Pass(vtype, input_id) => state 241 | .get_state_mut(&vtype) 242 | .get_old_available_input(input_id, rng) 243 | .as_u16(), 244 | GraphOp::Node(nt) => nt.as_u16(), 245 | }; 246 | return newval; 247 | } 248 | 249 | pub fn append_slice(&mut self, frag: &[u16], graph: &mut S, dist: &Distributions) { 250 | for op in OpIter::new(frag, &self.spec) { 251 | let new_val = Self::rename_and_relink_op_val(&mut self.state, &dist, op); 252 | graph.append_op(new_val); 253 | } 254 | } 255 | 256 | pub fn get_graph_state(&self) -> GraphState{ 257 | return self.state.clone(); 258 | } 259 | 260 | pub fn can_append_node(&self, node: &GraphNode, graph: &S) -> bool { 261 | if self.state.is_available(&self.spec, node.id) { 262 | if graph.can_append(node) { 263 | return true; 264 | } 265 | } 266 | return false; 267 | } 268 | 269 | pub fn append_node<'a, S: GraphStorage>( 270 | &mut self, 271 | node: &GraphNode, 272 | graph: &'a mut S, 273 | dist: &Distributions 274 | ) -> Option> { 275 | if self.can_append_node(node, graph) { 276 | self.append_slice(&node.ops, graph, dist); 277 | let data = graph.append_data(node.data).unwrap(); 278 | return Some(DataBuff::new(data, data.len())); 279 | } 280 | return None; 281 | } 282 | 283 | pub fn append_node_mutated( 284 | &mut self, 285 | node: &GraphNode, 286 | dict: &CustomDict, 287 | mutator: &PrimitiveMutator, 288 | graph: &mut S, 289 | dist: &Distributions 290 | ) { 291 | if self.can_append_node(node, graph) { 292 | self.append_slice(&node.ops, graph, dist); 293 | let ntype = self.spec.get_node(node.id).unwrap(); 294 | if let Some(dtype) = ntype.data { 295 | if let Ok(dat) = self.spec.get_data(dtype) { 296 | dat.atomic_type 297 | .append_mutated(node.data, dict, graph, &self.spec, mutator, dist); 298 | } else { 299 | panic!("Node {} has invalid data type {:?}", ntype.name, ntype.data); 300 | } 301 | } 302 | } 303 | } 304 | 305 | pub fn finalize(&self, graph: &S) -> VecGraph { 306 | return graph.as_vec_graph(); 307 | } 308 | 309 | pub fn drop_node_at(&mut self, frag: &VecGraph, index: usize, graph: &mut S, dist: &Distributions) { 310 | self.start(graph, &MutatorSnapshotState::none()); 311 | for (i, node) in frag.node_iter(&self.spec).enumerate() { 312 | if i != index { 313 | // basicall inlined append_node to satisfy the borrow checker... 314 | if self.can_append_node(&node, graph) { 315 | for op in OpIter::new(&node.ops, &self.spec) { 316 | let new_val = 317 | Self::rename_and_relink_op_val(&mut self.state, &dist, op); 318 | graph.append_op(new_val); 319 | } 320 | graph.append_data(node.data); 321 | } 322 | } 323 | } 324 | } 325 | 326 | pub fn is_full(&self, graph: &S) -> bool{ 327 | let data_full = self.spec.biggest_data() >= graph.data_available(); 328 | let ops_full = self.spec.biggest_ops() >= graph.ops_available(); 329 | return data_full || ops_full; 330 | } 331 | 332 | fn can_generate_node(&self, ntype: &NodeSpec, graph: &S) -> bool { 333 | if ntype.size() + 1 > graph.ops_available() { 334 | return false; 335 | } 336 | if ntype.min_data_size(&self.spec) > graph.data_available() { 337 | return false; 338 | } 339 | return true; 340 | } 341 | 342 | pub fn append_random_node( 343 | &mut self, 344 | node: NodeTypeID, 345 | mutator: &PrimitiveMutator, 346 | graph: &mut S, 347 | dist: &Distributions 348 | ) -> Result<(), GraphError> { 349 | let ntype = self.spec.get_node(node)?; 350 | if !self.can_generate_node(ntype, graph) { 351 | return Ok(()); 352 | } 353 | graph.append_op(node.as_u16()); 354 | for vtype in ntype.inputs.iter() { 355 | let id = self 356 | .state 357 | .get_state_mut(&vtype) 358 | .take_random_input(&dist); 359 | graph.append_op(id.as_u16()); 360 | } 361 | for vtype in ntype.passthroughs.iter() { 362 | let id = self 363 | .state 364 | .get_state_mut(&vtype) 365 | .get_random_input(&dist); 366 | graph.append_op(id.as_u16()); 367 | } 368 | for vtype in ntype.outputs.iter() { 369 | let id = self.state.get_state_mut(&vtype).insert_new_id(); 370 | graph.append_op(id.as_u16()); 371 | } 372 | if let Some(dtype) = ntype.data { 373 | if let Ok(dat) = self.spec.get_data(dtype) { 374 | dat.atomic_type.append_generate(graph, &self.spec, mutator, dist); 375 | } else { 376 | panic!( 377 | "Node {} {:?} has invalid data type {:?}", 378 | ntype.name, node, ntype.data 379 | ); 380 | } 381 | } 382 | return Ok(()); 383 | } 384 | 385 | pub fn minimize( 386 | &mut self, 387 | frag: &S1, 388 | storage: &mut S2, 389 | dist: &Distributions, 390 | mut tester: F, 391 | ) -> VecGraph 392 | where 393 | F: FnMut(&S2, &GraphSpec) -> bool, 394 | { 395 | let mut min_graph = frag.as_vec_graph(); 396 | let num_nodes = frag.node_len(&self.spec); 397 | for idx in (0..num_nodes).rev() { 398 | self.drop_node_at(&min_graph, idx, storage, dist); 399 | if tester(&storage, &self.spec) { 400 | min_graph = storage.as_vec_graph(); 401 | } 402 | } 403 | return min_graph; 404 | } 405 | 406 | pub fn append_random( 407 | &mut self, 408 | n: usize, 409 | mutator: &PrimitiveMutator, 410 | graph: &mut S, 411 | dist: &Distributions 412 | ) -> Result<(), GraphError> { 413 | for _i in 0..n { 414 | if self.is_full(graph){break;} 415 | let n_type = self.pick_available_type(dist)?; 416 | self.append_random_node(n_type, mutator, graph, dist)?; 417 | } 418 | return Ok(()); 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /structured_fuzzer/src/graph_mutator/graph_storage.rs: -------------------------------------------------------------------------------- 1 | use crate::graph_mutator::graph_iter::{GraphNode, GraphOp, NodeIter, OpIter}; 2 | use crate::graph_mutator::newtypes::{DstVal, NodeTypeID, ValueTypeID, OpIndex, PortID, SrcVal}; 3 | use crate::graph_mutator::spec::GraphSpec; 4 | 5 | use std::collections::HashMap; 6 | use std::fs::File; 7 | use std::io::prelude::*; 8 | 9 | //Needs to be used as a wrapper over Graph Storage, as Graph Storage can't be made into an object due to the Sized constraint 10 | pub trait GraphMutationTarget { 11 | fn append_op(&mut self, op: u16) -> Option<()>; 12 | fn append_data(&mut self, data: &[u8]) -> Option<&mut [u8]>; 13 | fn get_data(&mut self, size: usize) -> Option<&mut [u8]>; 14 | fn data_available(&self) -> usize; 15 | fn ops_available(&self) -> usize; 16 | } 17 | 18 | impl GraphMutationTarget for T { 19 | fn append_op(&mut self, op: u16) -> Option<()> { 20 | return self.append_op(op); 21 | } 22 | fn append_data(&mut self, data: &[u8]) -> Option<&mut [u8]> { 23 | return self.append_data(data); 24 | } 25 | fn get_data(&mut self, size: usize) -> Option<&mut [u8]> { 26 | return self.get_data(size); 27 | } 28 | fn data_available(&self) -> usize { 29 | return self.data_available(); 30 | } 31 | fn ops_available(&self) -> usize { 32 | return self.ops_available(); 33 | } 34 | } 35 | 36 | pub trait GraphStorage: Sized { 37 | fn clear(&mut self); 38 | fn truncate_to(&mut self, ops_i: usize, data_i: usize); 39 | fn append_op(&mut self, op: u16) -> Option<()>; 40 | fn append_data(&mut self, data: &[u8]) -> Option<&mut [u8]>; 41 | fn get_data(&mut self, size: usize) -> Option<&mut [u8]>; 42 | fn data_available(&self) -> usize; 43 | fn ops_available(&self) -> usize; 44 | fn can_append(&self, node: &GraphNode) -> bool; 45 | fn data_len(&self) -> usize; 46 | fn op_len(&self) -> usize; 47 | fn node_len(&self, spec:&GraphSpec) -> usize; 48 | fn is_empty(&self) -> bool { 49 | return self.op_len() == 0; 50 | } 51 | fn ops_as_slice(&self) -> &[u16]; 52 | fn data_as_slice(&self) -> &[u8]; 53 | 54 | fn as_vec_graph(&self) -> VecGraph { 55 | let mut ops = Vec::with_capacity(self.op_len()); 56 | let mut data = Vec::with_capacity(self.data_len()); 57 | ops.extend_from_slice(self.ops_as_slice()); 58 | data.extend_from_slice(self.data_as_slice()); 59 | let res = VecGraph::new(ops, data); 60 | return res; 61 | } 62 | 63 | fn copy_from(&mut self, graph: &VecGraph) { 64 | self.clear(); 65 | for op in graph.ops_as_slice(){ 66 | self.append_op(*op); 67 | } 68 | self.append_data(graph.data_as_slice()); 69 | } 70 | 71 | fn calc_edges(&self, spec: &GraphSpec) -> Vec<(SrcVal, DstVal)> { 72 | let mut res = vec![]; 73 | let mut id_to_src = HashMap::new(); 74 | let mut last_node = None; 75 | let mut last_out_port = PortID::new(0); 76 | let mut last_in_port = PortID::new(0); 77 | let mut last_pass_port = PortID::new(0); 78 | for (i, op) in self.op_iter(spec).enumerate() { 79 | match op { 80 | GraphOp::Node(_n_type_id) => { 81 | last_node = Some(OpIndex::new(i)); 82 | last_in_port = PortID::new(0); 83 | last_pass_port = PortID::new(0); 84 | last_out_port = PortID::new(0); 85 | } 86 | GraphOp::Set(vt, id) => { 87 | id_to_src.insert((vt, id), SrcVal::new(last_node.unwrap(), last_out_port)); 88 | last_out_port = last_out_port.next(); 89 | } 90 | GraphOp::Get(vt, id) => { 91 | res.push(( 92 | id_to_src.remove(&(vt, id)).unwrap(), 93 | DstVal::new(last_node.unwrap(), last_in_port), 94 | )); 95 | last_in_port = last_in_port.next(); 96 | } 97 | GraphOp::Pass(vt, id) => { 98 | res.push(( 99 | id_to_src.get(&(vt, id)).cloned().unwrap(), 100 | DstVal::new(last_node.unwrap(), last_pass_port), 101 | )); 102 | last_pass_port = last_pass_port.next(); 103 | } 104 | } 105 | } 106 | return res; 107 | } 108 | 109 | fn write_to_file(&self, path: &str, spec: &GraphSpec) { 110 | use std::io::BufWriter; 111 | use std::io::prelude::*; 112 | let mut file = BufWriter::new(File::create(path).unwrap_or_else(|_| panic!("couldn't open file to dump input {}",path))); 113 | file.write_all(&spec.checksum.to_le_bytes()).expect("couldn't write checksum"); 114 | file.write_all(&(self.ops_as_slice().len() as u64).to_le_bytes()).expect("couldn't write graph op len"); 115 | file.write_all(&(self.data_as_slice().len() as u64).to_le_bytes()).expect("couldn't write graph data len"); 116 | file.write_all(&(5*8_u64).to_le_bytes()).expect("couldn't write graph op offset"); 117 | file.write_all(&(5*8+(self.ops_as_slice().len() as u64)*2_u64).to_le_bytes()).expect("couldn't write graph data offset"); 118 | for b in self.ops_as_slice().iter(){ 119 | file.write_all(&b.to_le_bytes()).expect("couldn't write graph op"); 120 | } 121 | file.write_all(self.data_as_slice()).expect("couldn't write graph data"); 122 | } 123 | 124 | fn node_iter<'a>(&'a self, spec: &'a GraphSpec) -> NodeIter<'a> { 125 | return NodeIter::new(self.ops_as_slice(), self.data_as_slice(), spec); 126 | } 127 | fn op_iter<'a>(&'a self, spec: &'a GraphSpec) -> OpIter<'a> { 128 | return OpIter::new(self.ops_as_slice(), spec); 129 | } 130 | 131 | fn to_svg(&self, path: &str, spec: &GraphSpec) { 132 | use std::process::{Command, Stdio}; 133 | let dot = self.to_dot(spec); 134 | let mut child = Command::new("dot") 135 | .stdout(Stdio::inherit()) 136 | .stdin(Stdio::piped()) 137 | .arg("-Tsvg") 138 | .arg("-o") 139 | .arg(path) 140 | .spawn() 141 | .expect("failed to execute dot"); 142 | child 143 | .stdin 144 | .as_mut() 145 | .unwrap() 146 | .write_all(&dot.as_bytes()) 147 | .expect("failed to write dot graph"); 148 | child.wait().expect("failed to wait on dot"); 149 | } 150 | 151 | fn to_png(&self, path: &str, spec: &GraphSpec) { 152 | use std::process::{Command, Stdio}; 153 | let dot = self.to_dot(spec); 154 | let mut child = Command::new("dot") 155 | .stdout(Stdio::inherit()) 156 | .stdin(Stdio::piped()) 157 | .arg("-Tpng") 158 | .arg("-o") 159 | .arg(path) 160 | .spawn() 161 | .expect("failed to execute dot"); 162 | child 163 | .stdin 164 | .as_mut() 165 | .unwrap() 166 | .write_all(&dot.as_bytes()) 167 | .expect("failed to write dot graph"); 168 | child.wait().expect("failed to wait on dot"); 169 | } 170 | 171 | fn var_names(ops: &[u16], types: &[ValueTypeID], spec: &GraphSpec) -> String{ 172 | let iter = ops.iter().enumerate().map(|(i,id)| format!("v_{}_{}",spec.get_value(types[i]).unwrap().name, id)); 173 | return itertools::Itertools::intersperse(iter, ", ".to_string()).collect::(); 174 | } 175 | 176 | fn write_to_script_file(&self, path: &str, spec: &GraphSpec) { 177 | let mut file = File::create(path).unwrap(); 178 | file.write_all(self.to_script(spec).as_bytes()).unwrap() 179 | } 180 | 181 | fn to_script(&self, spec: &GraphSpec) -> String { 182 | let mut res = "".to_string(); 183 | for n in self.node_iter(spec){ 184 | let node = n.spec.get_node(n.id).unwrap(); 185 | let data = spec.node_data_inspect(n.id, n.data).replace("\\l",""); 186 | let i = 1+node.inputs.len(); 187 | let inputs = Self::var_names(&n.ops[1..i], &node.inputs, spec); 188 | let j = i + node.passthroughs.len(); 189 | let borrows = Self::var_names(&n.ops[i..j], &node.passthroughs, spec); 190 | let outputs = Self::var_names(&n.ops[j..], &node.outputs, spec); 191 | if !outputs.is_empty() { 192 | res += &outputs; 193 | res += " = "; 194 | } 195 | res+= &format!("{}( inputs=[{}], borrows=[{}], data={})\n",node.name, inputs, borrows, data) 196 | } 197 | //println!("{}",res); 198 | return res; 199 | } 200 | 201 | fn write_to_dot_file(&self, path: &str, spec: &GraphSpec) { 202 | let mut file = File::create(path).unwrap(); 203 | file.write_all(self.to_dot(spec).as_bytes()).unwrap() 204 | } 205 | 206 | fn to_dot(&self, spec: &GraphSpec) -> String { 207 | let mut res = "digraph{\n rankdir=\"LR\";\n { edge [style=invis weight=100];".to_string(); 208 | let edges = self.calc_edges(spec); 209 | let mut join = ""; 210 | for (_ntype, i) in self 211 | .op_iter(&spec) 212 | .enumerate() 213 | .filter_map(|(i, op)| op.node().map(|n| (n, i))) 214 | { 215 | res += &format!("{}n{}", join, i); 216 | join = "->"; 217 | } 218 | res += "}\n"; 219 | for node in self.node_iter(spec) { 220 | res += &format!( 221 | "n{} [label=\"{}{}\", shape=box];\n", 222 | node.op_i, 223 | spec.get_node(node.id).unwrap().name, 224 | spec.node_data_inspect(node.id, node.data), 225 | ); 226 | } 227 | for (src, dst) in edges.iter() { 228 | let node_type = NodeTypeID::new(self.ops_as_slice()[src.id.as_usize()]); 229 | let value_type = spec.get_node(node_type).unwrap().outputs[src.port.as_usize()]; 230 | let edge_type = &spec.get_value(value_type).unwrap().name; 231 | res += &format!( 232 | "n{} -> n{} [label=\"{}\"];\n", 233 | src.id.as_usize(), 234 | dst.id.as_usize(), 235 | edge_type 236 | ); 237 | } 238 | res += "}"; 239 | //println!("{}", res); 240 | return res; 241 | } 242 | } 243 | 244 | #[derive(Clone)] 245 | pub struct VecGraph { 246 | ops: Vec, 247 | data: Vec, 248 | } 249 | 250 | impl VecGraph { 251 | 252 | pub fn new_with_size(op_len: usize, data_len: usize) -> Self{ 253 | return Self::new(vec!(0; op_len), vec!(0; data_len)); 254 | } 255 | 256 | pub fn new(ops: Vec, data: Vec) -> Self { 257 | return Self { ops, data,}; 258 | } 259 | 260 | pub fn new_from_bin_file(path: &str, spec: &GraphSpec) -> VecGraph{ 261 | 262 | use std::io::BufReader; 263 | use std::convert::TryInto; 264 | 265 | let mut f = BufReader::new(File::open(path).expect("youldn't open .bin file for reading")); 266 | let mut buffer = [0; 8]; 267 | let n = f.read(&mut buffer[..]).expect("couldn't read checksum"); 268 | assert_eq!(n,8); 269 | let checksum = u64::from_le_bytes(buffer.try_into().unwrap()); 270 | assert_eq!(checksum, spec.checksum); 271 | 272 | let n = f.read(&mut buffer[..]).expect("couldn't read num_ops"); 273 | assert_eq!(n,8); 274 | let num_ops = u64::from_le_bytes(buffer.try_into().unwrap()); 275 | 276 | let n = f.read(&mut buffer[..]).expect("couldn't read num_data"); 277 | assert_eq!(n,8); 278 | let num_data = u64::from_le_bytes(buffer.try_into().unwrap()); 279 | 280 | let n = f.read(&mut buffer[..]).expect("couldn't read op_offset"); 281 | assert_eq!(n,8); 282 | let op_offset = u64::from_le_bytes(buffer.try_into().unwrap()); 283 | 284 | let n = f.read(&mut buffer[..]).expect("couldn't read data_offset"); 285 | assert_eq!(n,8); 286 | let data_offset = u64::from_le_bytes(buffer.try_into().unwrap()); 287 | 288 | f.seek(std::io::SeekFrom::Start(op_offset)).unwrap(); 289 | 290 | let mut res = VecGraph::empty(); 291 | for _i in 0..num_ops{ 292 | let n = f.read(&mut buffer[..2]).unwrap(); 293 | assert_eq!(n,2); 294 | let op = u16::from_le_bytes(buffer[..2].try_into().unwrap()); 295 | res.ops.push(op); 296 | } 297 | 298 | f.seek(std::io::SeekFrom::Start(data_offset)).unwrap(); 299 | res.data = vec!(0; num_data as usize); 300 | f.read_exact(&mut res.data[..]).unwrap(); 301 | return res; 302 | } 303 | 304 | pub fn empty() -> Self { 305 | return Self::new(vec![], vec![]); 306 | } 307 | 308 | pub fn as_ref_graph<'a>(&'a mut self, ops_i: &'a mut usize, data_i: &'a mut usize) -> RefGraph<'a>{ 309 | return RefGraph::new(&mut self.ops[..], &mut self.data[..], ops_i, data_i); 310 | } 311 | } 312 | 313 | impl GraphStorage for VecGraph { 314 | fn clear(&mut self) { 315 | self.ops.clear(); 316 | self.data.clear(); 317 | } 318 | fn truncate_to(&mut self, ops_i: usize, data_i: usize){ 319 | assert!(self.ops.len()>=ops_i); 320 | assert!(self.data.len()>=data_i); 321 | self.ops.resize(ops_i, 0); 322 | self.data.resize(data_i, 0); 323 | } 324 | fn append_op(&mut self, op: u16) -> Option<()> { 325 | self.ops.push(op); 326 | return Some(()); 327 | } 328 | 329 | fn get_data(&mut self, size: usize) -> Option<&mut [u8]> { 330 | let len = self.data.len(); 331 | self.data.resize(len + size, 0); 332 | return Some(&mut self.data[len..]); 333 | } 334 | 335 | fn append_data(&mut self, data: &[u8]) -> Option<&mut [u8]> { 336 | self.data.extend_from_slice(data); 337 | let len = self.data.len(); 338 | return Some(&mut self.data[len - data.len()..]); 339 | } 340 | 341 | fn data_available(&self) -> usize { 342 | return 0xffff_ffff; 343 | } 344 | 345 | fn ops_available(&self) -> usize { 346 | return 0xffff_ffff; 347 | } 348 | 349 | fn can_append(&self, _n: &GraphNode) -> bool { 350 | return true; 351 | } 352 | 353 | fn op_len(&self) -> usize { 354 | return self.ops.len(); 355 | } 356 | 357 | fn node_len(&self, spec:&GraphSpec) -> usize{ 358 | return self.node_iter(spec).count(); 359 | } 360 | 361 | fn data_len(&self) -> usize { 362 | return self.data.len(); 363 | } 364 | fn ops_as_slice(&self) -> &[u16] { 365 | return &self.ops[..]; 366 | } 367 | fn data_as_slice(&self) -> &[u8] { 368 | return &self.data[..]; 369 | } 370 | } 371 | 372 | pub struct RefGraph<'a> { 373 | ops: &'a mut [u16], 374 | ops_i: &'a mut usize, 375 | data: &'a mut [u8], 376 | data_i: &'a mut usize, 377 | } 378 | 379 | impl<'a> RefGraph<'a> { 380 | pub fn new( 381 | ops: &'a mut [u16], 382 | data: &'a mut [u8], 383 | ops_i: &'a mut usize, 384 | data_i: &'a mut usize, 385 | ) -> Self { 386 | return Self { 387 | ops, 388 | ops_i, 389 | data, 390 | data_i, 391 | }; 392 | } 393 | 394 | pub fn new_from_slice(payload: &mut [u8], checksum: u64) -> Self { 395 | let header_len = std::mem::size_of::() * 5; 396 | let data_len = payload.len() - header_len; 397 | assert_eq!(data_len % 8, 0); 398 | let buff_size = data_len / 2; 399 | 400 | unsafe { 401 | let ptr = payload.as_mut_ptr(); 402 | assert_eq!((ptr as usize) % std::mem::align_of::(), 0); 403 | let checksum_ptr = (ptr as *mut u64).add(0).as_mut().unwrap(); 404 | let ops_i_ptr = (ptr as *mut usize).add(1).as_mut().unwrap(); 405 | let data_i_ptr = (ptr as *mut usize).add(2).as_mut().unwrap(); 406 | assert_eq!(std::mem::size_of::(), std::mem::size_of::()); 407 | let graph_offset_ptr = (ptr as *mut u64).add(3).as_mut().unwrap(); 408 | let data_offset_ptr = (ptr as *mut u64).add(4).as_mut().unwrap(); 409 | 410 | assert_eq!(std::mem::size_of::(), std::mem::size_of::()); 411 | *checksum_ptr = checksum; 412 | *graph_offset_ptr = header_len as u64; 413 | *data_offset_ptr = (header_len + buff_size) as u64; 414 | 415 | let op_ptr = ptr.add(*graph_offset_ptr as usize) as *mut u16; 416 | let struct_buff = std::slice::from_raw_parts_mut(op_ptr, buff_size / 2); 417 | 418 | let data_ptr = ptr.add(*data_offset_ptr as usize) as *mut u8; 419 | let data_buff = std::slice::from_raw_parts_mut(data_ptr, buff_size); 420 | 421 | assert_eq!(*data_offset_ptr as usize + buff_size, payload.len()); 422 | 423 | return RefGraph::new(struct_buff, data_buff, ops_i_ptr, data_i_ptr); 424 | } 425 | } 426 | } 427 | 428 | impl<'a> GraphStorage for RefGraph<'a> { 429 | fn clear(&mut self) { 430 | *self.ops_i = 0; 431 | *self.data_i = 0; 432 | } 433 | fn truncate_to(&mut self, ops_i: usize, data_i: usize){ 434 | assert!(*self.ops_i>=ops_i); 435 | assert!(*self.data_i>=data_i); 436 | *self.ops_i = ops_i; 437 | *self.data_i = data_i; 438 | } 439 | 440 | fn append_op(&mut self, op: u16) -> Option<()> { 441 | if (*self.ops_i as usize) < self.ops.len() { 442 | self.ops[*self.ops_i as usize] = op; 443 | *self.ops_i += 1; 444 | return Some(()); 445 | } 446 | return None; 447 | } 448 | 449 | fn get_data(&mut self, size: usize) -> Option<&mut [u8]> { 450 | if *self.data_i + size <= self.data.len() { 451 | let range = *self.data_i..*self.data_i + size; 452 | *self.data_i += size; 453 | return Some(&mut self.data[range]); 454 | } 455 | return None; 456 | } 457 | 458 | fn append_data(&mut self, data: &[u8]) -> Option<&mut [u8]> { 459 | if *self.data_i + data.len() <= self.data.len() { 460 | let range = *self.data_i..(*self.data_i + data.len()); 461 | *self.data_i += data.len(); 462 | let buf = &mut self.data[range]; 463 | buf.copy_from_slice(data); 464 | return Some(buf); 465 | } 466 | return None; 467 | } 468 | 469 | fn data_available(&self) -> usize { 470 | return self.data.len() - *self.data_i; 471 | } 472 | fn ops_available(&self) -> usize { 473 | return self.ops.len() - *self.ops_i; 474 | } 475 | 476 | fn can_append(&self, n: &GraphNode) -> bool { 477 | return *self.data_i + n.data.len() < self.data.len(); 478 | } 479 | 480 | fn op_len(&self) -> usize { 481 | return *self.ops_i; 482 | } 483 | 484 | fn node_len(&self, spec:&GraphSpec) -> usize{ 485 | return self.node_iter(spec).count(); 486 | } 487 | 488 | fn data_len(&self) -> usize { 489 | return *self.data_i; 490 | } 491 | fn ops_as_slice(&self) -> &[u16] { 492 | return &self.ops[..*self.ops_i]; 493 | } 494 | fn data_as_slice(&self) -> &[u8] { 495 | return &self.data[..*self.data_i]; 496 | } 497 | } 498 | --------------------------------------------------------------------------------