├── .gitignore ├── LICENSE.txt ├── Package.resolved ├── Package.swift ├── Sources ├── CBuiltinsNotAvailableInSwift │ ├── include │ │ └── shims.h │ └── shims.c ├── FuzzCheck │ ├── Artifact.swift │ ├── CodeCoverageFeature.swift │ ├── CodeCoverageSensor.swift │ ├── CommandLineArguments.swift │ ├── Fuzzer.swift │ ├── FuzzerInput │ │ ├── ArrayFuzzerGenerator.swift │ │ ├── FuzzerInputGenerator.swift │ │ ├── FuzzerInputProperties.swift │ │ ├── IntegerFuzzerGenerator.swift │ │ └── StringFuzzerGenerator.swift │ ├── FuzzerPRNG.swift │ ├── FuzzerSensor.swift │ ├── InputPool.swift │ ├── SanitizerHooks.swift │ ├── SignalHandlers.swift │ └── World.swift └── FuzzCheckTool │ ├── ArgumentsParser.swift │ ├── Commands.swift │ └── main.swift ├── Tests ├── FuzzerTests │ ├── ArtifactsTest.swift │ ├── InputPoolTests.swift │ ├── Mock.swift │ └── RandomTests.swift └── GraphGeneratorTests │ └── MutationTests.swift └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | .build 2 | *.xcodeproj 3 | .DS_Store 4 | crashes 5 | Corpus/ -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Loïc Lecrenier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "Files", 6 | "repositoryURL": "https://github.com/JohnSundell/Files.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "06f95bd1bf4f8d5f50bc6bb30b808c253acd4c88", 10 | "version": "2.2.1" 11 | } 12 | }, 13 | { 14 | "package": "llbuild", 15 | "repositoryURL": "https://github.com/apple/swift-llbuild.git", 16 | "state": { 17 | "branch": "master", 18 | "revision": "ebeba84e562b588a231c910ff544b6957940f654", 19 | "version": null 20 | } 21 | }, 22 | { 23 | "package": "SwiftPM", 24 | "repositoryURL": "https://github.com/apple/swift-package-manager", 25 | "state": { 26 | "branch": null, 27 | "revision": "87523ce7e50d5fb0995682cd3c05558adbadfdfe", 28 | "version": null 29 | } 30 | } 31 | ] 32 | }, 33 | "version": 1 34 | } 35 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:4.2 2 | import PackageDescription 3 | 4 | let package = Package( 5 | name: "FuzzCheck", 6 | products: [ 7 | .library(name: "FuzzCheck", targets: ["FuzzCheck"]), 8 | .executable(name: "FuzzCheckTool", targets: ["FuzzCheckTool"]), 9 | ], 10 | dependencies: [ 11 | .package(url: "https://github.com/apple/swift-package-manager", .revision("87523ce7e50d5fb0995682cd3c05558adbadfdfe")), 12 | .package(url: "https://github.com/JohnSundell/Files.git", from: Version(2, 0, 0)) 13 | ], 14 | targets: [ 15 | .target(name: "CBuiltinsNotAvailableInSwift", dependencies: []), 16 | .target(name: "FuzzCheck", dependencies: ["Files", "Utility", "CBuiltinsNotAvailableInSwift"]), 17 | .target(name: "FuzzCheckTool", dependencies: ["Files", "FuzzCheck", "Utility"]), 18 | .testTarget(name: "FuzzerTests", dependencies: ["FuzzCheck"]), 19 | ] 20 | ) 21 | 22 | -------------------------------------------------------------------------------- /Sources/CBuiltinsNotAvailableInSwift/include/shims.h: -------------------------------------------------------------------------------- 1 | 2 | #import 3 | 4 | void* __return_address(); 5 | -------------------------------------------------------------------------------- /Sources/CBuiltinsNotAvailableInSwift/shims.c: -------------------------------------------------------------------------------- 1 | 2 | #import 3 | 4 | void* __return_address() { 5 | return __builtin_return_address(2); 6 | } 7 | 8 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/Artifact.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Artifact.swift 3 | // FuzzCheck 4 | // 5 | 6 | import Files 7 | import Foundation 8 | 9 | /* 10 | This file is longer than it is worth. 11 | 12 | Basically, it describes how to save the artifacts created by a fuzzer 13 | to disk. To do so, it needs to know how to construct the artifact 14 | filename and its content, which are parameterizable by the user. 15 | 16 | For example, a user might wish to save artifacts under the filename 17 | schema ?kind-?index-?hash and the extension .json, which might result in 18 | a filename "testFailure-3-fc3daa1056". 19 | 20 | Inside an artifact file, a user might wish to include the complexity 21 | of the test input and its score, but not its hash, etc. 22 | */ 23 | 24 | /** 25 | Description of an Artifact file generated by a command line fuzzer 26 | It contains the input and maybe some metadata associated to it. 27 | */ 28 | public struct Artifact { 29 | /// The partial name of the artifact file, to be completed by its index. 30 | let name: ArtifactNameWithoutIndex 31 | /// The content of the artifact file. 32 | let content: Content 33 | 34 | public struct Content { 35 | /// The test input that the artifact file describes 36 | let input: Input 37 | /// The features discovered by that test input. 38 | let features: [CodeCoverageSensor.Feature]? 39 | /// The score of the input, which can depend on the state 40 | /// of the fuzzer's input pool at the time the artifact was generated. 41 | let score: Double? 42 | let hash: Int? 43 | let complexity: Double? 44 | let kind: ArtifactKind? 45 | } 46 | } 47 | 48 | /** 49 | Describes the structure of the artifact filename and 50 | which information to include in the artifact file content. 51 | */ 52 | public struct ArtifactSchema { 53 | public var name: Name 54 | public var content: Content 55 | 56 | /** 57 | Describes the filename schema of an artifact. 58 | It consists of a list of “components” and an optional filename extension. 59 | */ 60 | public struct Name: Hashable { 61 | /** 62 | An ArtifactSchema.Name.Component is a part of the artifact filename. 63 | It can either be a string literal, or space for artifact-specific information. 64 | */ 65 | public enum Component: Hashable { 66 | case literal(String) 67 | case hash 68 | case complexity 69 | 70 | /// Space for a generated unique identifier to make 71 | /// the artifact filename unique in the file system 72 | case index 73 | 74 | case kind 75 | } 76 | public var components: [Component] 77 | 78 | /// The file name extension 79 | public var ext: String? 80 | } 81 | 82 | /** 83 | Describes the content schema of an artifact file. 84 | 85 | It contains a property for each optional piece of information that could 86 | be included in the artifact file. 87 | */ 88 | public struct Content { 89 | public let features: Bool 90 | public let score: Bool 91 | public let hash: Bool 92 | public let complexity: Bool 93 | public let kind: Bool 94 | 95 | public init(features: Bool, score: Bool, hash: Bool, complexity: Bool, kind: Bool) { 96 | self.features = features 97 | self.score = score 98 | self.hash = hash 99 | self.complexity = complexity 100 | self.kind = kind 101 | } 102 | } 103 | } 104 | 105 | extension Artifact.Content { 106 | init(schema: ArtifactSchema.Content, input: Input, features: [CodeCoverageSensor.Feature]?, score: Double?, hash: Int?, complexity: Double?, kind: ArtifactKind) { 107 | self.input = input 108 | self.features = schema.features ? features : nil 109 | self.score = schema.score ? score : nil 110 | self.hash = schema.hash ? hash : nil 111 | self.complexity = schema.complexity ? complexity : nil 112 | self.kind = schema.kind ? kind : nil 113 | } 114 | } 115 | 116 | /// Information that might be needed to construct the filename of an artifact. 117 | public struct ArtifactNameInfo: Hashable { 118 | let hash: Int 119 | let complexity: Double 120 | let kind: ArtifactKind 121 | } 122 | 123 | /// The kind of an artifact 124 | public enum ArtifactKind: String, Codable { 125 | /// The input did not result in any kind of failure, but may be interesting enough to save. 126 | case input 127 | /// The input caused the test function to take too long to executes 128 | case timeout 129 | /// The input caused the test function to crash 130 | case crash 131 | /// The input caused the test function to fail 132 | case testFailure 133 | } 134 | 135 | /** 136 | A resolved artifact filename whose only piece of missing information 137 | is its generated unique identifier (called `index`). 138 | */ 139 | public struct ArtifactNameWithoutIndex: Hashable { 140 | /// The filename that does not include the index 141 | let string: String 142 | /// The range inside `self.string` that should be replaced by the index 143 | let gapForIndex: Range 144 | 145 | func fillGap(with index: Int) -> String { 146 | var s = string 147 | s.replaceSubrange(gapForIndex, with: "\(index)") 148 | return s 149 | } 150 | 151 | /** 152 | Find an index such that `self.fillGap(with: index)` results in a 153 | string distinct from those in the given set, and return that string. 154 | */ 155 | func fillGapToBeUnique(from set: Set) -> String { 156 | // in practice very few loop iterations will be performed 157 | for i in 0... { 158 | let candidate = fillGap(with: i) 159 | if !set.contains(candidate) { return candidate } 160 | } 161 | fatalError() 162 | } 163 | 164 | init(string: String, gapForIndex: Range) { 165 | self.string = string 166 | self.gapForIndex = gapForIndex 167 | } 168 | 169 | init(schema: ArtifactSchema.Name, info: ArtifactNameInfo) { 170 | var name = "" 171 | var gapForIndex: Range? = nil 172 | for a in schema.components { 173 | switch a { 174 | case .literal(let s): 175 | name += s 176 | case .hash: 177 | name += hexString(info.hash) 178 | case .complexity: 179 | name += "\(Int(info.complexity.rounded()))" 180 | case .kind: 181 | name += "\(info.kind)" 182 | case .index: 183 | gapForIndex = name.endIndex ..< name.endIndex 184 | } 185 | } 186 | if gapForIndex == nil { 187 | gapForIndex = name.endIndex ..< name.endIndex 188 | } 189 | if let ext = schema.ext, !ext.isEmpty { 190 | name += ".\(ext)" 191 | } 192 | self.init(string: name, gapForIndex: gapForIndex!) 193 | } 194 | } 195 | 196 | extension ArtifactSchema.Name.Component: CustomStringConvertible { 197 | public var description: String { 198 | switch self { 199 | case .literal(let s): 200 | return s 201 | case .hash: 202 | return "?hash" 203 | case .complexity: 204 | return "?complexity" 205 | case .index: 206 | return "?index" 207 | case .kind: 208 | return "?kind" 209 | } 210 | 211 | } 212 | } 213 | 214 | extension ArtifactSchema.Name.Component { 215 | /** 216 | Consume a single artifact schema name component from the given substring. 217 | */ 218 | static func read(from s: inout Substring) -> ArtifactSchema.Name.Component? { 219 | guard let first = s.first else { 220 | return nil 221 | } 222 | let specials: [ArtifactSchema.Name.Component] = [.hash, .complexity, .index, .kind] 223 | for special in specials { 224 | if s.starts(with: special.description) { 225 | defer { s = s.dropFirst(special.description.count) } 226 | return special 227 | } 228 | } 229 | // No matches. Return a simple string 230 | guard let nextQMark = s.dropFirst().firstIndex(of: "?") else { 231 | defer { s = s[s.endIndex ..< s.endIndex] } 232 | return .literal(String(s)) 233 | } 234 | defer { s = s.suffix(from: nextQMark) } 235 | return .literal(String(s.prefix(upTo: nextQMark))) 236 | } 237 | 238 | /** 239 | Parses a list of artifact name schema from the given string. 240 | 241 | ## Syntax 242 | A question mark followed by the name of a known component result in a non-literal component. 243 | Everything else is a literal. 244 | 245 | ## Example 246 | 247 | "`?hash-?complexity and ?whatever`" is parsed as 248 | - `.hash` 249 | - `.literal("-")` 250 | - `.complexity` 251 | - `.literal("and ?whatever")` 252 | */ 253 | public static func read(from s: String) -> [ArtifactSchema.Name.Component] { 254 | var subs = Substring(s) 255 | var res: [ArtifactSchema.Name.Component] = [] 256 | while let a = read(from: &subs) { 257 | res.append(a) 258 | } 259 | return res 260 | } 261 | } 262 | 263 | extension Artifact.Content: Codable { 264 | enum CodingKey: Swift.CodingKey { 265 | case input 266 | case complexity 267 | case hash 268 | case score 269 | case features 270 | case kind 271 | } 272 | 273 | public func encode(to encoder: Encoder) throws { 274 | if self.complexity == nil, self.score == nil, self.hash == nil, self.kind == nil, self.features == nil { 275 | try input.encode(to: encoder) 276 | } else { 277 | var container = encoder.container(keyedBy: CodingKey.self) 278 | try container.encode(input, forKey: .input) 279 | try container.encodeIfPresent(complexity, forKey: .complexity) 280 | try container.encodeIfPresent(score, forKey: .score) 281 | try container.encodeIfPresent(hash, forKey: .hash) 282 | try container.encodeIfPresent(kind, forKey: .kind) 283 | try container.encodeIfPresent(features, forKey: .features) 284 | } 285 | } 286 | public init(from decoder: Decoder) throws { 287 | if let input = try? Input(from: decoder) { 288 | self.input = input 289 | self.complexity = nil 290 | self.score = nil 291 | self.hash = nil 292 | self.features = nil 293 | self.kind = nil 294 | } else { 295 | let container = try decoder.container(keyedBy: CodingKey.self) 296 | self.input = try container.decode(Input.self, forKey: .input) 297 | self.complexity = try container.decodeIfPresent(Double.self, forKey: .complexity) 298 | self.score = try container.decodeIfPresent(Double.self, forKey: .score) 299 | self.hash = try container.decodeIfPresent(Int.self, forKey: .hash) 300 | self.features = try container.decodeIfPresent(Array.self, forKey: .features) 301 | self.kind = try container.decodeIfPresent(ArtifactKind.self, forKey: .kind) 302 | } 303 | } 304 | } 305 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/CodeCoverageFeature.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Feature.swift 3 | // FuzzCheck 4 | // 5 | // Created by Loïc Lecrenier on 27/05/2018. 6 | // 7 | 8 | extension CodeCoverageSensor { 9 | public enum Feature: FuzzerSensorFeature, Equatable, Hashable { 10 | case indirect(Indirect) 11 | case edge(Edge) 12 | case comparison(Comparison) 13 | } 14 | } 15 | 16 | extension CodeCoverageSensor.Feature { 17 | public var score: Double { 18 | switch self { 19 | case .indirect(_): 20 | return 1 21 | case .edge(_): 22 | return 1 23 | case .comparison(_): 24 | return 0.5 25 | } 26 | } 27 | } 28 | 29 | 30 | func scoreFromCounter (_ counter: T) -> UInt8 { 31 | guard counter != T.max else { return UInt8(T.bitWidth) } 32 | if counter <= 3 { 33 | return UInt8(counter) 34 | } else { 35 | return UInt8(T.bitWidth - counter.leadingZeroBitCount) + 1 36 | } 37 | } 38 | 39 | 40 | extension CodeCoverageSensor.Feature { 41 | public struct Indirect: Equatable, Hashable { 42 | let caller: UInt 43 | let callee: UInt 44 | } 45 | public struct Edge: Equatable, Hashable { 46 | let pcguard: UInt 47 | let intensity: UInt8 48 | 49 | init(pcguard: UInt, intensity: UInt8) { 50 | self.pcguard = pcguard 51 | self.intensity = intensity 52 | } 53 | init(pcguard: UInt, counter: UInt16) { 54 | self.init(pcguard: pcguard, intensity: scoreFromCounter(counter)) 55 | } 56 | } 57 | 58 | public struct Comparison: Equatable, Hashable { 59 | let pc: UInt 60 | let argxordist: UInt8 61 | 62 | init(pc: UInt, argxordist: UInt8) { 63 | self.pc = pc 64 | self.argxordist = argxordist 65 | } 66 | init(pc: UInt, arg1: UInt64, arg2: UInt64) { 67 | self.init(pc: pc, argxordist: scoreFromCounter(UInt8((arg1 &- arg2).nonzeroBitCount))) 68 | } 69 | } 70 | } 71 | 72 | extension CodeCoverageSensor.Feature.Indirect: Comparable { 73 | public static func < (lhs: CodeCoverageSensor.Feature.Indirect, rhs: CodeCoverageSensor.Feature.Indirect) -> Bool { 74 | return (lhs.caller, lhs.callee) < (rhs.caller, rhs.callee) 75 | } 76 | } 77 | 78 | extension CodeCoverageSensor.Feature.Comparison: Comparable { 79 | public static func < (lhs: CodeCoverageSensor.Feature.Comparison, rhs: CodeCoverageSensor.Feature.Comparison) -> Bool { 80 | return (lhs.pc, lhs.argxordist) < (rhs.pc, rhs.argxordist) 81 | } 82 | } 83 | 84 | extension CodeCoverageSensor.Feature: Codable { 85 | enum Kind: String, Codable { 86 | case indirect 87 | case edge 88 | case comparison 89 | } 90 | 91 | enum CodingKey: Swift.CodingKey { 92 | case kind 93 | case pc 94 | case pcguard 95 | case intensity 96 | case argxordist 97 | case caller 98 | case callee 99 | } 100 | 101 | public init(from decoder: Decoder) throws { 102 | let container = try decoder.container(keyedBy: CodingKey.self) 103 | let kind = try container.decode(Kind.self, forKey: .kind) 104 | switch kind { 105 | case .indirect: 106 | let caller = try container.decode(UInt.self, forKey: .caller) 107 | let callee = try container.decode(UInt.self, forKey: .callee) 108 | self = .indirect(.init(caller: caller, callee: callee)) 109 | case .edge: 110 | let pcguard = try container.decode(UInt.self, forKey: .pcguard) 111 | let intensity = try container.decode(UInt8.self, forKey: .intensity) 112 | self = .edge(.init(pcguard: pcguard, intensity: intensity)) 113 | case .comparison: 114 | let pc = try container.decode(UInt.self, forKey: .pc) 115 | let argxordist = try container.decode(UInt8.self, forKey: .argxordist) 116 | self = .comparison(.init(pc: pc, argxordist: argxordist)) 117 | } 118 | } 119 | public func encode(to encoder: Encoder) throws { 120 | var container = encoder.container(keyedBy: CodingKey.self) 121 | switch self { 122 | case .indirect(let x): 123 | try container.encode(Kind.indirect, forKey: .kind) 124 | try container.encode(x.caller, forKey: .caller) 125 | try container.encode(x.callee, forKey: .callee) 126 | case .edge(let x): 127 | try container.encode(Kind.edge, forKey: .kind) 128 | try container.encode(x.pcguard, forKey: .pcguard) 129 | try container.encode(x.intensity, forKey: .intensity) 130 | case .comparison(let x): 131 | try container.encode(Kind.comparison, forKey: .kind) 132 | try container.encode(x.pc, forKey: .pc) 133 | try container.encode(x.argxordist, forKey: .argxordist) 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/CodeCoverageSensor.swift: -------------------------------------------------------------------------------- 1 | 2 | import CBuiltinsNotAvailableInSwift 3 | import Darwin 4 | import Foundation 5 | 6 | extension UnsafeMutableBufferPointer { 7 | static func allocateAndInitializeTo(_ x: Element, capacity: Int) -> UnsafeMutableBufferPointer { 8 | let b = UnsafeMutableBufferPointer.allocate(capacity: capacity) 9 | b.initialize(repeating: x) 10 | return b 11 | } 12 | } 13 | 14 | /// Program Counter: the index of an instruction in the program binary 15 | typealias PC = UInt 16 | 17 | 18 | /// A FuzzerSensor for recording code coverage. 19 | /// Please only use the instance defined by `CodeCoverageSensor.shared` 20 | public final class CodeCoverageSensor: FuzzerSensor { 21 | static let shared: CodeCoverageSensor = .init() 22 | /// The maximum number of instrumented code edges allowed by CodeCoverageSensor. 23 | static let maxNumGuards: Int = 1 << 21 24 | 25 | /// The number of instrumented code edges. 26 | var numGuards: Int = 0 27 | 28 | /// True iff `self` is currently recording the execution of the program 29 | public var isRecording = false 30 | 31 | /// The value at index `i` of this buffer holds the number of time that 32 | /// the code identified by the `pc_guard` of value `i` was visited. 33 | var eightBitCounters: UnsafeMutableBufferPointer = .allocateAndInitializeTo(0, capacity: 1) 34 | 35 | /// An array holding the `Feature`s describing indirect function calls. 36 | private var indirectFeatures: [Feature.Indirect] = [] 37 | 38 | /// An array holding the `Feature`s describing comparisons. 39 | private var cmpFeatures: [Feature.Comparison] = [] 40 | 41 | /// Handle a call to the sanitizer code coverage function `trace_pc_guard_init` 42 | /// It assigns a unique value to every pointer inside [start, stop). These values 43 | /// are the identifiers of the instrumented code edges. 44 | /// Reset and resize `edges` and `eightBitCounters` 45 | func handlePCGuardInit(start: UnsafeMutablePointer, stop: UnsafeMutablePointer) { 46 | guard start != stop && start.pointee == 0 else { return } 47 | 48 | let buffer = UnsafeMutableBufferPointer(start: start, count: stop - start) 49 | for i in buffer.indices { 50 | numGuards += 1 51 | precondition(numGuards < CodeCoverageSensor.maxNumGuards) 52 | buffer[i] = UInt32(numGuards) 53 | } 54 | // Not ideal, but oh well 55 | eightBitCounters.deallocate() 56 | eightBitCounters = .allocateAndInitializeTo(0, capacity: numGuards+1) 57 | } 58 | 59 | /// Handle a call to the sanitizer code coverage function `trace_pc_indir` 60 | func handlePCIndir(caller: NormalizedPC, callee: NormalizedPC) { 61 | let (caller, callee) = (caller.value, callee.value) 62 | let f = Feature.Indirect(caller: caller, callee: callee) 63 | // TODO: could this be guarded by a lock, or would it ruin performance? 64 | indirectFeatures.append(f) 65 | } 66 | 67 | /// Call the given function for every unique collected `Feature`. 68 | /// The features are passed in a deterministic order. Therefore, even if 69 | /// two different executions of the program trigger the same features but 70 | /// in a different order, `collectFeatures` will pass them in the same order. 71 | public func iterateOverCollectedFeatures(_ handle: (Feature) -> Void) { 72 | 73 | for i in eightBitCounters.indices where eightBitCounters[i] != 0 { 74 | let f = Feature.Edge(pcguard: UInt(i), counter: eightBitCounters[i]) 75 | handle(.edge(f)) 76 | } 77 | 78 | indirectFeatures.sort() 79 | cmpFeatures.sort() 80 | 81 | // Ensure we don't call `handle` with the same argument twice. 82 | // This works because the arrays are sorted. 83 | var last1: Feature.Indirect? = nil 84 | for f in indirectFeatures where last1 != f { 85 | handle(.indirect(f)) 86 | last1 = f 87 | } 88 | var last2: Feature.Comparison? = nil 89 | for f in cmpFeatures where last2 != f { 90 | handle(.comparison(f)) 91 | last2 = f 92 | } 93 | } 94 | 95 | func handleTraceCmp (pc: NormalizedPC, arg1: T, arg2: T) { 96 | let f = Feature.Comparison(pc: pc.value, arg1: numericCast(arg1), arg2: numericCast(arg2)) 97 | // TODO: could this be guarded by a lock, or would it ruin performance? 98 | cmpFeatures.append(f) 99 | } 100 | 101 | public func resetCollectedFeatures() { 102 | eightBitCounters.assign(repeating: 0) 103 | indirectFeatures.removeAll(keepingCapacity: true) 104 | cmpFeatures.removeAll(keepingCapacity: true) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/CommandLineArguments.swift: -------------------------------------------------------------------------------- 1 | // 2 | // CommandLineArguments.swift 3 | // FuzzCheck 4 | // 5 | // Created by Loïc Lecrenier on 04/06/2018. 6 | // 7 | 8 | import Files 9 | import Foundation 10 | import Utility 11 | 12 | let usage = "[options]" 13 | 14 | let overview = """ 15 | FuzzCheck is an evolutionary in-process fuzzer for testing Swift programs. 16 | Steps: 17 | - read sample inputs from a list of input folders 18 | - feed these inputs to a test Swift function 19 | - analyze the code coverage triggered by each input 20 | - the most interesting inputs are kept in an in-memory pool of inputs 21 | - then, repeatedly: 22 | - randomly select an input from the in-memory pool 23 | - apply random mutations to that input 24 | - feed the mutated input to the test function 25 | - analyze the code coverage again 26 | - evaluate the usefulness of the mutated input 27 | and maybe add it to the in-memory pool 28 | - repeat until a crash is found 29 | """ 30 | 31 | extension Double: ArgumentKind { 32 | public init(argument: String) throws { 33 | if let d = Double.init(argument) { 34 | self = d 35 | } else { 36 | throw ArgumentParserError.invalidValue(argument: argument, error: ArgumentConversionError.typeMismatch(value: argument, expectedType: Double.self)) 37 | } 38 | } 39 | public static var completion: ShellCompletion { 40 | return ShellCompletion.none 41 | } 42 | } 43 | 44 | extension UInt: ArgumentKind { 45 | public init(argument: String) throws { 46 | if let d = UInt(argument) { 47 | self = d 48 | } else { 49 | throw ArgumentParserError.invalidValue(argument: argument, error: ArgumentConversionError.typeMismatch(value: argument, expectedType: UInt.self)) 50 | } 51 | } 52 | public static var completion: ShellCompletion { 53 | return ShellCompletion.none 54 | } 55 | } 56 | 57 | extension Folder: ArgumentKind { 58 | public convenience init(argument: String) throws { 59 | try self.init(path: argument) 60 | } 61 | public static var completion: ShellCompletion { 62 | return ShellCompletion.filename 63 | } 64 | } 65 | extension File: ArgumentKind { 66 | public convenience init(argument: String) throws { 67 | try self.init(path: argument) 68 | } 69 | public static var completion: ShellCompletion { 70 | return ShellCompletion.filename 71 | } 72 | } 73 | 74 | extension Array: ArgumentKind where Element == ArtifactSchema.Name.Component { 75 | public init(argument: String) throws { 76 | self = ArtifactSchema.Name.Component.read(from: argument) 77 | } 78 | public static var completion: ShellCompletion { 79 | return ShellCompletion.none 80 | } 81 | } 82 | 83 | extension ArtifactSchema.Content: ArgumentKind { 84 | public init(argument: String) throws { 85 | self.init(features: argument.contains("features"), 86 | score: argument.contains("coverage"), 87 | hash: argument.contains("hash"), 88 | complexity: argument.contains("complexity"), 89 | kind: argument.contains("kind")) 90 | } 91 | public static var completion: ShellCompletion { 92 | return ShellCompletion.none 93 | } 94 | } 95 | 96 | extension FuzzerSettings.Command: ArgumentKind { 97 | public init(argument: String) throws { 98 | if let x = FuzzerSettings.Command.init(rawValue: argument) { 99 | self = x 100 | } else { 101 | throw ArgumentParserError.invalidValue(argument: argument, error: .unknown(value: argument)) 102 | } 103 | } 104 | public static var completion: ShellCompletion { 105 | return ShellCompletion.none 106 | } 107 | } 108 | 109 | extension CommandLineFuzzerWorldInfo { 110 | public static func argumentsParser() -> (ArgumentParser, ArgumentBinder, ArgumentBinder, ArgumentBinder) { 111 | let parser = ArgumentParser(usage: usage, overview: overview) 112 | let settingsBinder = ArgumentBinder() 113 | let worldBinder = ArgumentBinder() 114 | let managerSettingsBinder = ArgumentBinder() 115 | 116 | let maxNumberOfRuns = parser.add( 117 | option: "--max-number-of-runs", 118 | shortName: "-runs", 119 | kind: UInt.self, 120 | usage: "The number of fuzzer iterations to run before exiting", 121 | completion: nil 122 | ) 123 | let maxComplexity = parser.add( 124 | option: "--max-complexity", 125 | shortName: "-cplx", 126 | kind: Double.self, 127 | usage: "The upper bound complexity of the test inputs", 128 | completion: nil 129 | ) 130 | let mutationDepth = parser.add( 131 | option: "--mutation-depth", 132 | shortName: "-mut", 133 | kind: UInt.self, 134 | usage: "The number of consecutive mutations applied to an input in a single iteration", 135 | completion: nil 136 | ) 137 | let globalTimeout = parser.add( 138 | option: "--global-timeout", 139 | shortName: "-gtm", 140 | kind: UInt.self, 141 | usage: "The maximum number of seconds to run FuzzCheck before exiting", 142 | completion: nil 143 | ) 144 | let inputCorpora = parser.add( 145 | option: "--input-folders", 146 | shortName: "-in-f", 147 | kind: Array.self, 148 | usage: "List of folders containing JSON-encoded sample inputs to use as a starting point", 149 | completion: nil 150 | ) 151 | let outputCorpus = parser.add( 152 | option: "--output-folder", 153 | shortName: "-out-f", 154 | kind: Folder.self, 155 | usage: "Folder in which to store the interesting inputs generated during the fuzzing process" 156 | ) 157 | let artifactsFolder = parser.add( 158 | option: "--artifact-folder", 159 | shortName: "-art-f", 160 | kind: Folder.self, 161 | usage: "Folder in which to store the artifact generated at the end of the fuzzing process. Artifacts may be inputs that cause a crash, or inputs that took longer than milliseconds to be tested" 162 | ) 163 | let artifactFileName = parser.add( 164 | option: "--artifact-filename", 165 | shortName: "-art-name", 166 | kind: Array.self, 167 | usage: "The name of the artifact" 168 | ) 169 | let artifactFileExtension = parser.add( 170 | option: "--artifact-file-extension", 171 | shortName: "-art-ext", 172 | kind: String.self, 173 | usage: "The extension of the artifact" 174 | ) 175 | let noSaveArtifacts = parser.add( 176 | option: "--no-save-artifact", 177 | shortName: nil, 178 | kind: Bool.self, 179 | usage: "If set, does not save artifacts" 180 | ) 181 | let artifactContent = parser.add( 182 | option: "--artifact-content", 183 | shortName: nil, 184 | kind: ArtifactSchema.Content.self, 185 | usage: "A list of comma-separated fields that the JSON in the artifact file should contain. (e.g. hash,complexity,features)" 186 | ) 187 | 188 | let command = parser.add( 189 | option: "--command", 190 | shortName: nil, 191 | kind: FuzzerSettings.Command.self, 192 | usage: "The command to execute. (fuzz | minimize | read)" 193 | ) 194 | 195 | let inputFile = parser.add( 196 | option: "--input-file", 197 | shortName: nil, 198 | kind: File.self, 199 | usage: "Input file" 200 | ) 201 | 202 | let seed = parser.add( 203 | option: "--seed", 204 | shortName: nil, 205 | kind: UInt.self, 206 | usage: "Seed for the pseudo-random number generator" 207 | ) 208 | 209 | let target = parser.add( 210 | option: "--target", 211 | shortName: "-t", 212 | kind: String.self, 213 | usage: "The executable containing the fuzzer loop", 214 | completion: nil 215 | ) 216 | 217 | settingsBinder.bind(option: maxNumberOfRuns) { $0.maxNumberOfRuns = Int($1) } 218 | settingsBinder.bind(option: maxComplexity) { $0.maxInputComplexity = $1 } 219 | settingsBinder.bind(option: mutationDepth) { $0.mutateDepth = Int($1) } 220 | settingsBinder.bind(option: command) { $0.command = $1 } 221 | 222 | worldBinder.bind(option: inputCorpora) { $0.inputCorpora = $1 } 223 | worldBinder.bind(option: outputCorpus) { $0.outputCorpus = $1 } 224 | worldBinder.bind(option: artifactsFolder) { $0.artifactsFolder = $1 } 225 | worldBinder.bind(option: artifactFileName) { $0.artifactsNameSchema.components = $1 } 226 | worldBinder.bind(option: artifactFileExtension) { $0.artifactsNameSchema.ext = $1 } 227 | worldBinder.bind(option: seed) { $0.rand = FuzzerPRNG(seed: UInt32($1)) } 228 | worldBinder.bind(option: inputFile) { $0.inputFile = $1 } 229 | worldBinder.bind(option: noSaveArtifacts) { x, _ in x.artifactsFolder = nil } 230 | worldBinder.bind(option: artifactContent) { $0.artifactsContentSchema = $1 } 231 | 232 | managerSettingsBinder.bind(option: target) { $0.testExecutable = try getExecutableFile().parent!.file(named: $1) } 233 | managerSettingsBinder.bind(option: globalTimeout) { $0.globalTimeout = $1 } 234 | 235 | return (parser, settingsBinder, worldBinder, managerSettingsBinder) 236 | } 237 | } 238 | 239 | public struct FuzzerManagerSettings { 240 | public var testExecutable: File? = nil 241 | public var globalTimeout: UInt? = nil 242 | public init() { } 243 | } 244 | 245 | extension FuzzerManagerSettings { 246 | public var commandLineArguments: [String] { 247 | var args: [String] = [] 248 | if let exec = testExecutable { args += ["--target", exec.path] } 249 | if let gtm = globalTimeout { args += ["--global-timeout", "\(gtm)"] } 250 | return args 251 | } 252 | } 253 | 254 | extension FuzzerSettings { 255 | public var commandLineArguments: [String] { 256 | var args: [String] = [] 257 | args += ["--command", "\(command)"] 258 | args += ["--max-complexity", "\(maxInputComplexity)"] 259 | args += ["--max-number-of-runs", "\(maxNumberOfRuns)"] 260 | args += ["--mutation-depth", "\(mutateDepth)"] 261 | return args 262 | } 263 | } 264 | 265 | extension CommandLineFuzzerWorldInfo { 266 | public var commandLineArguments: [String] { 267 | var args: [String] = [] 268 | args += ["--seed", "\(rand.seed)"] 269 | 270 | if let artifactsFolder = artifactsFolder { 271 | args += ["--artifact-folder", "\(artifactsFolder.path)"] 272 | args += ["--artifact-filename", "\(artifactsNameSchema.components.map { $0.description }.joined())"] 273 | if let ext = artifactsNameSchema.ext { args += ["--artifact-file-extension", "\(ext)"] } 274 | if let out = outputCorpus { args += ["--output-folder", "\(out.path)"] } 275 | var contentSchema: [String] = [] 276 | if artifactsContentSchema.complexity { contentSchema.append("complexity") } 277 | if artifactsContentSchema.hash { contentSchema.append("hash") } 278 | if artifactsContentSchema.features { contentSchema.append("features") } 279 | if artifactsContentSchema.kind { contentSchema.append("kind") } 280 | if artifactsContentSchema.score { contentSchema.append("coverage") } 281 | if !contentSchema.isEmpty { 282 | args += ["--artifact-content", contentSchema.joined(separator: ",")] 283 | } 284 | } else { 285 | args += ["--no-save-artifact"] 286 | } 287 | 288 | if let f = inputFile { args += ["--input-file", "\(f.path)"] } 289 | if !inputCorpora.isEmpty { 290 | args.append("--input-folders") 291 | args += inputCorpora.map { $0.path } 292 | } 293 | 294 | return args 295 | } 296 | } 297 | 298 | func getExecutableFile() throws -> File { 299 | var cPath: UnsafeMutablePointer = UnsafeMutablePointer.allocate(capacity: 1) 300 | var size: UInt32 = 1 301 | defer { cPath.deallocate() } 302 | 303 | Loop: while true { 304 | let result = _NSGetExecutablePath(cPath, &size) 305 | switch result { 306 | case 0: 307 | break Loop 308 | case -1: 309 | cPath.deallocate() 310 | cPath = UnsafeMutablePointer.allocate(capacity: Int(size)) 311 | default: 312 | fatalError("Failed to get an executable path to the current process.") 313 | } 314 | } 315 | 316 | let path = String(cString: cPath) 317 | return try File(path: path) 318 | } 319 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/Fuzzer.swift: -------------------------------------------------------------------------------- 1 | 2 | import Basic 3 | import Darwin 4 | import Foundation 5 | 6 | /// The reason why the fuzzer terminated, to be passed as argument to `exit(..)` 7 | public enum FuzzerTerminationStatus: Int32 { 8 | case success = 0 9 | case crash = 1 10 | case testFailure = 2 11 | case unknown = 3 12 | } 13 | 14 | /** 15 | The state of the fuzzer: 16 | - in-memory pool of inputs 17 | - statistics 18 | - state of the external world 19 | - etc. 20 | */ 21 | public final class FuzzerState 22 | where 23 | World: FuzzerWorld, 24 | World.Input == Input, 25 | Sensor: FuzzerSensor, 26 | Sensor.Feature == World.Feature, 27 | Properties: FuzzerInputProperties, 28 | Properties.Input == Input 29 | { 30 | /// A collection of previously-tested inputs that are considered interesting 31 | let pool: InputPool = InputPool() 32 | 33 | var poolThreshold: Int = 64 34 | 35 | /// The current inputs that are being tested 36 | var inputs: [Input] 37 | 38 | /// The index of the input being tested 39 | var inputIndex: Int 40 | 41 | /// Various statistics about the current fuzzer run. 42 | var stats: FuzzerStats 43 | /// The initial settings passed to the fuzzer 44 | var settings: FuzzerSettings 45 | /// The time at which the fuzzer started. Used for computing the average execution speed. 46 | var processStartTime: UInt = 0 47 | /// 48 | var sensor: Sensor 49 | /** 50 | A property managing the effects and coeffects produced and needed by the fuzzer. 51 | 52 | It provides a random number generator, performs file operations, gives the current 53 | time, the memory consumption, etc. 54 | */ 55 | var world: World 56 | 57 | init(inputs: [Input], inputIndex: Int, settings: FuzzerSettings, world: World, sensor: Sensor) { 58 | self.inputs = inputs 59 | self.inputIndex = inputIndex 60 | self.stats = FuzzerStats() 61 | self.settings = settings 62 | self.world = world 63 | self.sensor = sensor 64 | } 65 | 66 | /// Gather statistics about the state of the fuzzer and store them in `self.stats`. 67 | func updateStats() { 68 | let now = world.clock() 69 | let seconds = Double(now - processStartTime) / 1_000_000 70 | stats.executionsPerSecond = Int((Double(stats.totalNumberOfRuns) / seconds).rounded()) 71 | stats.poolSize = pool.inputs.count 72 | stats.score = Int((pool.score * 10).rounded()) 73 | let avgCplx = pool.smallestInputComplexityForFeature.values.reduce(0, +) / Double(pool.smallestInputComplexityForFeature.count) 74 | stats.averageComplexity = Int((avgCplx * 100).rounded()) 75 | //stats.rss = Int(world.getPeakMemoryUsage()) 76 | } 77 | 78 | /// Handle the signal sent to the process and exit. 79 | func receive(signal: Signal) -> Never { 80 | world.reportEvent(.caughtSignal(signal), stats: stats) 81 | switch signal { 82 | case .illegalInstruction, .abort, .busError, .floatingPointException: 83 | var features: [Sensor.Feature] = [] 84 | sensor.iterateOverCollectedFeatures { features.append($0) } 85 | try! world.saveArtifact(input: inputs[inputIndex], features: features, score: pool.score, kind: .crash) 86 | exit(FuzzerTerminationStatus.crash.rawValue) 87 | 88 | case .interrupt: 89 | exit(FuzzerTerminationStatus.success.rawValue) 90 | 91 | default: 92 | exit(FuzzerTerminationStatus.unknown.rawValue) 93 | } 94 | } 95 | } 96 | 97 | /** 98 | A fuzzer can fuzz-test a function `test: (Input) -> Bool`. It finds values of 99 | `Input` for which `test` returns `false` or crashes. 100 | 101 | It is configurable by four generic type parameters: 102 | - `Generator` defines how to generate and evolve values of type `Input` 103 | - `Properties` defines how to compute essential properties of `Input` (such as their complexities or hash values) 104 | - `World` regulates the communication between the Fuzzer and the real-world, 105 | such as the file system, time, or random number generator. 106 | - `Sensor` collects, from a test function execution, the measurements to optimize (e.g. code coverage) 107 | 108 | This type is a bit too complex to make part of the public API and end users 109 | should only used (partly) specialized versions of it, like CommandLineFuzzer. 110 | */ 111 | final class Fuzzer 112 | where 113 | Generator: FuzzerInputGenerator, 114 | World: FuzzerWorld, 115 | Sensor: FuzzerSensor, 116 | World.Feature == Sensor.Feature, 117 | Generator.Input == Input, 118 | World.Input == Input 119 | { 120 | 121 | typealias State = FuzzerState 122 | 123 | let state: State 124 | let generator: Generator 125 | let test: (Input) -> Bool 126 | let signalsHandler: SignalsHandler 127 | 128 | init(test: @escaping (Input) -> Bool, generator: Generator, settings: FuzzerSettings, world: World, sensor: Sensor) { 129 | self.generator = generator 130 | self.test = test 131 | self.state = State(inputs: [generator.baseInput], inputIndex: 0, settings: settings, world: world, sensor: sensor) 132 | 133 | let signals: [Signal] = [.segmentationViolation, .busError, .abort, .illegalInstruction, .floatingPointException, .interrupt, .softwareTermination, .fileSizeLimitExceeded] 134 | 135 | self.signalsHandler = SignalsHandler(signals: signals) { [state] signal in 136 | state.receive(signal: signal) 137 | } 138 | 139 | precondition(Foundation.Thread.isMainThread, "Fuzzer can only be initialized on the main thread") 140 | // :shame: please send help 141 | let idx = Foundation.Thread.callStackSymbols.firstIndex(where: { $0.contains(" main + ")})! 142 | let adr = Foundation.Thread.callStackReturnAddresses[idx].uintValue 143 | NormalizedPC.constant = adr 144 | } 145 | } 146 | 147 | // note: it is not a typealias because I feel bad for the typechecker 148 | public enum CommandLineFuzzer { 149 | public typealias Input = Generator.Input 150 | typealias SpecializedFuzzer = Fuzzer, CodeCoverageSensor> 151 | 152 | /// Execute the fuzzer command given by `Commandline.arguments` for the given test function and generator. 153 | public static func launch(test: @escaping (Input) -> Bool, generator: Generator) throws { 154 | let (parser, settingsBinder, worldBinder, _) = CommandLineFuzzerWorldInfo.argumentsParser() 155 | var settings: FuzzerSettings 156 | var world: CommandLineFuzzerWorldInfo 157 | do { 158 | let res = try parser.parse(Array(CommandLine.arguments.dropFirst())) 159 | settings = FuzzerSettings() 160 | try settingsBinder.fill(parseResult: res, into: &settings) 161 | world = CommandLineFuzzerWorldInfo() 162 | try worldBinder.fill(parseResult: res, into: &world) 163 | } catch let e { 164 | print(e) 165 | parser.printUsage(on: stdoutStream) 166 | return 167 | } 168 | let fuzzer = SpecializedFuzzer(test: test, generator: generator, settings: settings, world: CommandLineFuzzerWorld(info: world), sensor: .shared) 169 | switch fuzzer.state.settings.command { 170 | case .fuzz: 171 | try fuzzer.loop() 172 | case .minimize: 173 | try fuzzer.minimizeLoop() 174 | case .read: 175 | fuzzer.state.inputs = [try fuzzer.state.world.readInputFile()] 176 | try fuzzer.testCurrentInputs() 177 | } 178 | } 179 | } 180 | 181 | extension Fuzzer { 182 | 183 | /** 184 | Run and record the test function for the current test inputs. 185 | Exit and save the artifact if the test function failed. 186 | */ 187 | func testCurrentInputs() throws { 188 | for i in state.inputs { 189 | try testInput(i) 190 | } 191 | } 192 | 193 | func testInput(_ input: Input) throws { 194 | state.sensor.resetCollectedFeatures() 195 | 196 | state.sensor.isRecording = true 197 | let success = test(input) 198 | state.sensor.isRecording = false 199 | 200 | guard success else { 201 | state.world.reportEvent(.testFailure, stats: state.stats) 202 | var features: [Sensor.Feature] = [] 203 | state.sensor.iterateOverCollectedFeatures { features.append($0) } 204 | try state.world.saveArtifact(input: input, features: features, score: state.pool.score, kind: .testFailure) 205 | exit(FuzzerTerminationStatus.testFailure.rawValue) 206 | } 207 | 208 | state.stats.totalNumberOfRuns += 1 209 | } 210 | 211 | /** 212 | Analyze the recording of the last test function call. 213 | Return the current input along with its associated analysis data iff 214 | the current input is interesting and should be added to the input pool. 215 | */ 216 | func analyze() -> State.InputPool.Element? { 217 | 218 | // it is slow to recreate the array here each time 219 | // move them to FuzzerState to improve performance a bit 220 | var bestInputForFeatures: [Sensor.Feature] = [] 221 | var otherFeatures: [Sensor.Feature] = [] 222 | 223 | let currentInputComplexity = Generator.adjustedComplexity(of: state.inputs[state.inputIndex]) 224 | 225 | state.sensor.iterateOverCollectedFeatures { feature in 226 | guard let oldComplexity = state.pool.smallestInputComplexityForFeature[feature] else { 227 | bestInputForFeatures.append(feature) 228 | return 229 | } 230 | if currentInputComplexity < oldComplexity { 231 | bestInputForFeatures.append(feature) 232 | return 233 | } else { 234 | otherFeatures.append(feature) 235 | return 236 | } 237 | } 238 | 239 | guard !bestInputForFeatures.isEmpty else { 240 | return nil 241 | } 242 | 243 | return State.InputPool.Element( 244 | input: state.inputs[state.inputIndex], 245 | complexity: currentInputComplexity, 246 | features: bestInputForFeatures + otherFeatures 247 | ) 248 | } 249 | 250 | /** 251 | Run and record the test function for the current test input, 252 | analyze the recording, and update the input pool if needed. 253 | */ 254 | func processCurrentInputs() throws { 255 | var newPoolElements: [State.InputPool.Element] = [] 256 | for (input, i) in zip(state.inputs, state.inputs.indices) { 257 | state.inputIndex = i 258 | try testInput(input) 259 | let result = analyze() 260 | if let newPoolElement = result { 261 | newPoolElements.append(newPoolElement) 262 | } 263 | } 264 | guard !newPoolElements.isEmpty else { 265 | return 266 | } 267 | let effect = state.pool.add(newPoolElements) 268 | try effect(&state.world) 269 | 270 | state.updateStats() 271 | state.world.reportEvent(.new, stats: state.stats) 272 | } 273 | 274 | /** 275 | Change the current input to a selection from the input pool. 276 | Then repeatedly mutate and process the current input, up to `mutateDepth` times. 277 | */ 278 | func processNextInputs() throws { 279 | state.inputs = [] 280 | state.inputIndex = 0 281 | 282 | while state.inputs.count < 50 { 283 | let idx = state.pool.randomIndex(&state.world.rand) 284 | var newInput = state.pool[idx].input 285 | 286 | var complexity: Double = state.pool[idx].complexity - 1.0 287 | for _ in 0 ..< state.settings.mutateDepth { 288 | guard 289 | state.stats.totalNumberOfRuns < state.settings.maxNumberOfRuns, 290 | generator.mutate(&newInput, state.settings.maxInputComplexity - complexity, &state.world.rand) 291 | else { 292 | break 293 | } 294 | complexity = Generator.complexity(of: newInput) 295 | guard complexity < state.settings.maxInputComplexity else { 296 | continue 297 | } 298 | state.inputs.append(newInput) 299 | } 300 | } 301 | 302 | try processCurrentInputs() 303 | } 304 | 305 | /** 306 | Process the inputs in the input corpus, or process the initial inputs 307 | given by the generator if the input corpus is empty. 308 | */ 309 | func processInitialInputs() throws { 310 | var inputs = try state.world.readInputCorpus() 311 | if inputs.isEmpty { 312 | inputs += generator.initialInputs(maxComplexity: state.settings.maxInputComplexity, &state.world.rand) 313 | } 314 | // Filter the inputs that are too complex 315 | inputs = inputs.filter { Generator.complexity(of: $0) <= state.settings.maxInputComplexity } 316 | state.inputs = inputs 317 | state.inputIndex = 0 318 | try processCurrentInputs() 319 | 320 | } 321 | 322 | /** 323 | Launch the regular fuzzing loop, which processes inputs, starting from the initial ones, 324 | until either a bug is found or the maximum number of iterations have been executed. 325 | */ 326 | public func loop() throws { 327 | state.processStartTime = state.world.clock() 328 | state.world.reportEvent(.start, stats: state.stats) 329 | 330 | try processInitialInputs() 331 | state.world.reportEvent(.didReadCorpus, stats: state.stats) 332 | 333 | while state.stats.totalNumberOfRuns < state.settings.maxNumberOfRuns { 334 | // Note: resetting the pool is not really useful, 335 | // it is a feature I used for debugging purposes. 336 | if state.pool.inputs.count > state.poolThreshold { 337 | state.inputs = state.pool.inputs.map { $0.input } 338 | state.inputIndex = 0 339 | state.pool.empty() 340 | try processCurrentInputs() 341 | state.poolThreshold = (state.pool.inputs.count * 3) / 2 342 | // state.pool.verify() 343 | state.world.reportEvent(.didResetPool, stats: state.stats) 344 | } else { 345 | try processNextInputs() 346 | } 347 | } 348 | state.world.reportEvent(.done, stats: state.stats) 349 | } 350 | 351 | /** 352 | Launch the minimizing loop. It reads the input to minimize from the input file, then 353 | processes simpler variations of that input until either a bug is found or the maximum 354 | number of iterations have been executed. 355 | */ 356 | public func minimizeLoop() throws { 357 | state.processStartTime = state.world.clock() 358 | state.world.reportEvent(.start, stats: state.stats) 359 | let input = try state.world.readInputFile() 360 | let favoredInput = State.InputPool.Element( 361 | input: input, 362 | complexity: Generator.adjustedComplexity(of: input), 363 | features: [] 364 | ) 365 | state.pool.favoredInput = favoredInput 366 | let effect = state.pool.updateScores() 367 | try effect(&state.world) 368 | state.settings.maxInputComplexity = Generator.complexity(of: input).nextDown 369 | state.world.reportEvent(.didReadCorpus, stats: state.stats) 370 | while state.stats.totalNumberOfRuns < state.settings.maxNumberOfRuns { 371 | try processNextInputs() 372 | } 373 | state.world.reportEvent(.done, stats: state.stats) 374 | } 375 | } 376 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerInput/ArrayFuzzerGenerator.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ArrayFuzzerGenerator.swift 3 | // FuzzCheck 4 | // 5 | 6 | extension Double { 7 | static func randomRatioBiasedToZero (bias: UInt8, using r: inout R) -> Double { 8 | var result: Double = 1.0 9 | for _ in 0 ..< bias { 10 | result *= Double.random(in: 0 ..< 1.0, using: &r) 11 | } 12 | return result 13 | } 14 | } 15 | 16 | public struct ArrayFuzzerGenerator : FuzzerInputGenerator { 17 | public typealias Input = [G.Input] 18 | 19 | let elementGenerator: G 20 | var mutators: ArrayMutators 21 | 22 | public let baseInput: Input = [] 23 | 24 | public func newInput(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> Input { 25 | guard maxComplexity > 0 else { return [] } 26 | let targetComplexity = Double.random(in: 0 ..< maxComplexity, using: &rand) 27 | var a: Input = [] 28 | var currentComplexity = ArrayFuzzerGenerator.complexity(of: a) 29 | while true { 30 | _ = mutators.mutate(&a, with: .appendNew, spareComplexity: targetComplexity - currentComplexity, &rand) 31 | currentComplexity = ArrayFuzzerGenerator.complexity(of: a) 32 | 33 | while currentComplexity >= targetComplexity { 34 | _ = mutators.mutate(&a, with: .removeRandom, spareComplexity: 0, &rand) 35 | currentComplexity = ArrayFuzzerGenerator.complexity(of: a) 36 | if currentComplexity <= targetComplexity { 37 | a.shuffle() 38 | return a 39 | } 40 | } 41 | 42 | } 43 | } 44 | 45 | public init(_ elementGenerator: G) { 46 | self.elementGenerator = elementGenerator 47 | 48 | self.mutators = ArrayMutators( 49 | initializeElement: { [elementGenerator] c, r in 50 | elementGenerator.newInput(maxComplexity: c, &r) 51 | }, 52 | mutateElement: elementGenerator.mutate 53 | ) 54 | } 55 | 56 | public func mutate(_ input: inout Input, _ spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 57 | return mutators.mutate(&input, spareComplexity, &rand) 58 | } 59 | 60 | public typealias CodableInput = [G.CodableInput] 61 | 62 | public static func complexity(of input: Input) -> Double { 63 | return input.reduce(0) { $0 + G.complexity(of: $1) } 64 | } 65 | public static func hash(_ input: Input, into hasher: inout Hasher) { 66 | for x in input { 67 | G.hash(x, into: &hasher) 68 | } 69 | } 70 | public static func convertToCodable(_ input: Input) -> CodableInput { 71 | return input.map(G.convertToCodable) 72 | } 73 | public static func convertFromCodable(_ codable: CodableInput) -> Input { 74 | return codable.map(G.convertFromCodable) 75 | } 76 | } 77 | 78 | public struct ArrayMutators : FuzzerInputMutatorGroup { 79 | 80 | public typealias Input = Array 81 | 82 | public let initializeElement: (Double, inout FuzzerPRNG) -> Element 83 | public let mutateElement: (inout Element, Double, inout FuzzerPRNG) -> Bool 84 | 85 | public init(initializeElement: @escaping (Double, inout FuzzerPRNG) -> Element, mutateElement: @escaping (inout Element, Double, inout FuzzerPRNG) -> Bool) { 86 | self.initializeElement = initializeElement 87 | self.mutateElement = mutateElement 88 | } 89 | 90 | public enum Mutator { 91 | case appendNew 92 | // TODO: appendRecycled should not exist. Instead, appendNew 93 | // should initialize its element from a pool that may include 94 | // recycled ones 95 | case appendRecycled 96 | case insertNew 97 | case insertRecycled 98 | // TODO: the probability of picking mutateElement should depend on 99 | // the size of the array. Or maybe even from the relative complexity 100 | // of the element. And maybe from the maximum allowed complexity. 101 | // This is something that we get for free when using binary 102 | // buffers with libFuzzer but is more difficult to achieve 103 | // for typed values in FuzzCheck 104 | case mutateElement 105 | // TODO: generalize to swapping subsequences 106 | case swap 107 | case removeLast 108 | case removeRandom 109 | // TODO: append/insert repeated? 110 | // TODO: duplicate subsequence? 111 | // TODO: rotate, partition, sort? 112 | // TODO: a way to configure the array mutators so I don't have to pick 113 | // one set of tradeoffs for all possible situations 114 | } 115 | 116 | public func mutate(_ input: inout Array, with mutator: Mutator, spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 117 | 118 | switch mutator { 119 | case .appendNew: 120 | let additionalComplexity = (Double.randomRatioBiasedToZero(bias: 4, using: &rand) * spareComplexity).rounded(.up) 121 | input.append(initializeElement(additionalComplexity, &rand)) 122 | return true 123 | case .appendRecycled: 124 | guard !input.isEmpty else { return false } 125 | let y = input.randomElement(using: &rand)! 126 | input.append(y) 127 | return true 128 | case .insertNew: 129 | guard !input.isEmpty else { 130 | return mutate(&input, with: .appendNew, spareComplexity: spareComplexity, &rand) 131 | } 132 | let additionalComplexity = (Double.randomRatioBiasedToZero(bias: 4, using: &rand) * spareComplexity).rounded(.up) 133 | let i = input.indices.randomElement(using: &rand)! 134 | input.insert(initializeElement(additionalComplexity, &rand), at: i) 135 | return true 136 | case .insertRecycled: 137 | guard !input.isEmpty else { return false } 138 | let y = input.randomElement(using: &rand)! 139 | let i = input.indices.randomElement(using: &rand)! 140 | input.insert(y, at: i) 141 | return true 142 | case .mutateElement: 143 | guard !input.isEmpty else { return false } 144 | let i = input.indices.randomElement(using: &rand)! 145 | return mutateElement(&input[i], spareComplexity, &rand) 146 | case .swap: 147 | guard input.count > 1 else { return false } 148 | let (i, j) = (input.indices.randomElement(using: &rand)!, input.indices.randomElement(using: &rand)!) 149 | input.swapAt(i, j) 150 | return i != j 151 | case .removeLast: 152 | guard !input.isEmpty else { return false } 153 | input.removeLast() 154 | return true 155 | case .removeRandom: 156 | guard !input.isEmpty else { return false } 157 | input.remove(at: input.indices.randomElement(using: &rand)!) 158 | return true 159 | } 160 | } 161 | 162 | public let weightedMutators: [(Mutator, UInt)] = [ 163 | // TODO: this is completely arbitrary 164 | // I should find a better way to determine the 165 | // correct weights 166 | (.appendNew, 40), 167 | (.appendRecycled, 80), 168 | (.insertNew, 120), 169 | (.insertRecycled, 160), 170 | (.mutateElement, 300), 171 | (.swap, 380), 172 | (.removeLast, 420), 173 | (.removeRandom, 460) 174 | ] 175 | } 176 | 177 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerInput/FuzzerInputGenerator.swift: -------------------------------------------------------------------------------- 1 | // 2 | // FuzzerInputGenerator.swift 3 | // FuzzCheck 4 | // 5 | 6 | /// A protocol defining how to generate, mutate, analyze, and store values of type Input. 7 | public protocol FuzzerInputGenerator: FuzzerInputProperties { 8 | 9 | associatedtype Input 10 | 11 | /** 12 | The simplest value of `Input`. 13 | 14 | Having a perfect value for `baseInput` is not essential to FuzzCheck. 15 | 16 | ## Examples 17 | - the empty array 18 | - the number 0 19 | - an arbitrary value if `Input` doesn't have a “simplest” value 20 | */ 21 | var baseInput: Input { get } 22 | 23 | /** 24 | Return a new input to test. 25 | 26 | It can be completely random or drawn from a corpus of “special” inputs 27 | or generated in any other way that yields a wide variety of inputs. 28 | 29 | - Parameter maxComplexity: the maximum value of the generated input's complexity 30 | - Parameter rand: a random number generator 31 | - Returns: The new generated input 32 | */ 33 | func newInput(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> Input 34 | 35 | /** 36 | Returns an array of initial inputs to fuzz-test. 37 | 38 | The elements of the array should be different from each other, and 39 | each one of them should be interesting in its own way. 40 | 41 | For example, one could be an empty array, another one could be a sorted array, 42 | one a small array and one a large array, etc. 43 | 44 | Having a perfect list of initial elements is not essential to FuzzCheck, 45 | but it can help it start working on the right foot. 46 | 47 | - Parameter rand: a random number generator 48 | */ 49 | func initialInputs(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> [Input] 50 | 51 | /** 52 | Mutate the given input. 53 | 54 | FuzzCheck will call this method repeatedly in order to explore all the 55 | possible values of Input. It is therefore important that it is implemented 56 | efficiently. 57 | 58 | It should be theoretically possible to mutate any arbitrary input `u1` into any 59 | other arbitrary input `u2` by calling `mutate` repeatedly. 60 | 61 | Moreover, the result of `mutate` should try to be “interesting” to FuzzCheck. 62 | That is, it should be likely to trigger new code paths when passed to the 63 | test function. 64 | 65 | A good approach to implement this method is to use a `FuzzerInputMutatorGroup`. 66 | 67 | ## Examples 68 | - append a random element to an array 69 | - mutate a random element in an array 70 | - subtract a small constant from an integer 71 | - change an integer to Int.min or Int.max or 0 72 | - replace a substring by a keyword relevant to the test function 73 | 74 | - Parameter input: the input to mutate 75 | - Parameter spareComplexity: the additional complexity that can be added to the input 76 | - Parameter rand: a random number generator 77 | - Returns: true iff the input was actually mutated 78 | */ 79 | func mutate(_ input: inout Input, _ spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool 80 | } 81 | 82 | extension FuzzerInputGenerator { 83 | // Default implementation: generate 10 new inputs 84 | public func initialInputs(maxComplexity: Double, _ r: inout FuzzerPRNG) -> [Input] { 85 | return (0 ..< 10).map { _ in 86 | newInput(maxComplexity: maxComplexity, &r) 87 | } 88 | } 89 | } 90 | 91 | /** 92 | A type providing a list of weighted mutators, which is handy to implement 93 | the `mutate` method of a `FuzzerInputGenerator`. 94 | 95 | The weight of a mutator determines how often it should be used relative to 96 | the other mutators in the list. 97 | */ 98 | public protocol FuzzerInputMutatorGroup { 99 | associatedtype Input 100 | associatedtype Mutator 101 | 102 | /** 103 | Mutate the given input using the given mutator and random number generator. 104 | 105 | - Parameter input: the input to mutate 106 | - Parameter mutator: the mutator to use to mutate the input 107 | - Parameter rand: a random number generator 108 | - Returns: true iff the input was actually mutated 109 | */ 110 | func mutate(_ input: inout Input, with mutator: Mutator, spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool 111 | 112 | /** 113 | A list of mutators and their associated weight. 114 | 115 | # IMPORTANT 116 | The second component of the tuples in the array is the sum of the previous weight 117 | and the weight of the mutator itself. For example, for three mutators `(m1, m2, m3)` 118 | with relative weights `(120, 5, 56)`. Then `weightedMutators` should return 119 | `[(m1, 120), (m2, 125), (m3, 181)]`. 120 | */ 121 | var weightedMutators: [(Mutator, UInt)] { get } 122 | } 123 | 124 | extension FuzzerInputMutatorGroup { 125 | /** 126 | Choose a mutator from the list of weighted mutators and execute it on `input`. 127 | 128 | - Parameter input: the input to mutate 129 | - Parameter mutator: the mutator to use to mutate the input 130 | - Parameter rand: a random number generator 131 | - Returns: true iff the input was actually mutated 132 | */ 133 | public func mutate(_ input: inout Input, _ spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 134 | for _ in 0 ..< weightedMutators.count { 135 | let mutator = rand.weightedRandomElement(from: weightedMutators, minimum: 0) 136 | if mutate(&input, with: mutator, spareComplexity: spareComplexity, &rand) { return true } 137 | } 138 | return false 139 | } 140 | } 141 | 142 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerInput/FuzzerInputProperties.swift: -------------------------------------------------------------------------------- 1 | 2 | /// A protocol giving some information about values of type Input, such as its complexity or hash. 3 | public protocol FuzzerInputProperties { 4 | associatedtype Input 5 | 6 | /// A type isomorphic to Input that is Codable. This exists here so 7 | /// that non-nominal types can be inputs too 8 | associatedtype CodableInput: Codable = Input 9 | 10 | /** 11 | Returns the complexity of the given input. 12 | 13 | FuzzCheck will prefer using inputs with a smaller complexity. 14 | 15 | - Important: The return value must be >= 0.0 16 | 17 | ## Examples 18 | - an array might have a complexity equal to the sum of complexities of its elements 19 | - an integer might have a complexity equal to the number of bytes used to represent it 20 | */ 21 | static func complexity(of input: Input) -> Double 22 | 23 | static func hash(_ input: Input, into hasher: inout Hasher) 24 | 25 | static func convertToCodable(_ input: Input) -> CodableInput 26 | static func convertFromCodable(_ codable: CodableInput) -> Input 27 | } 28 | 29 | extension FuzzerInputProperties { 30 | internal static func adjustedComplexity(of input: Input) -> Double { 31 | let cplx = complexity(of: input) 32 | precondition(cplx >= 0.0) 33 | return cplx + 1.0 34 | } 35 | } 36 | 37 | extension FuzzerInputProperties { 38 | public static func hashValue(of input: Input) -> Int { 39 | var h = Hasher() 40 | hash(input, into: &h) 41 | return h.finalize() 42 | } 43 | } 44 | 45 | extension FuzzerInputProperties where Input: Hashable { 46 | public static func hash(_ input: Input, into hasher: inout Hasher) { 47 | input.hash(into: &hasher) 48 | } 49 | } 50 | 51 | extension FuzzerInputProperties where Input == CodableInput { 52 | public static func convertToCodable(_ input: Input) -> CodableInput { 53 | return input 54 | } 55 | public static func convertFromCodable(_ codable: CodableInput) -> Input { 56 | return codable 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerInput/IntegerFuzzerGenerator.swift: -------------------------------------------------------------------------------- 1 | 2 | public struct IntegerFuzzerInputMutators : FuzzerInputMutatorGroup { 3 | public typealias Input = T 4 | 5 | public enum Mutator { 6 | case nudge 7 | case random 8 | case special 9 | } 10 | 11 | let maxNudge: UInt 12 | let specialValues: [T] 13 | 14 | public func mutate(_ input: inout Input, with mutator: Mutator, spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 15 | switch mutator { 16 | case .nudge: 17 | return nudge(&input, &rand) 18 | case .random: 19 | return random(&input, &rand) 20 | case .special: 21 | return special(&input, &rand) 22 | } 23 | } 24 | 25 | func nudge(_ x: inout Input, _ r: inout FuzzerPRNG) -> Bool { 26 | let nudge = Input(r.next(upperBound: maxNudge)) 27 | let op: (Input, Input) -> Input = Bool.random(using: &r) ? (&-) : (&+) 28 | x = op(x, nudge) 29 | return true 30 | } 31 | 32 | func random(_ x: inout Input, _ r: inout FuzzerPRNG) -> Bool { 33 | x = T.random(using: &r) 34 | return true 35 | } 36 | 37 | func special(_ x: inout Input, _ r: inout FuzzerPRNG) -> Bool { 38 | let oldX = x 39 | x = specialValues.randomElement() ?? x 40 | return x != oldX 41 | } 42 | 43 | public init(specialValues: [T]) { 44 | self.maxNudge = 10 45 | self.specialValues = specialValues 46 | } 47 | 48 | public let weightedMutators: [(Mutator, UInt)] = [ 49 | (.special, 1), 50 | (.random, 11), 51 | (.nudge, 21), 52 | ] 53 | } 54 | 55 | public struct IntegerFuzzerGenerator : FuzzerInputGenerator { 56 | 57 | public let baseInput = 0 as T 58 | let mutators: IntegerFuzzerInputMutators 59 | 60 | public init(specialValues: [T]) { 61 | self.mutators = IntegerFuzzerInputMutators(specialValues: specialValues) 62 | } 63 | 64 | public func newInput(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> T { 65 | return T.random(using: &rand) 66 | } 67 | 68 | public func mutate(_ x: inout T, _ spareComplexity: Double, _ r: inout FuzzerPRNG) -> Bool { 69 | return mutators.mutate(&x, spareComplexity, &r) 70 | } 71 | 72 | public static func complexity(of: T) -> Double { 73 | return Double(T.bitWidth) / 8 74 | } 75 | } 76 | 77 | 78 | extension FixedWidthInteger where Self: UnsignedInteger { 79 | /** 80 | Return an array of “special” values of this UnsignedInteger type that 81 | deserve to be prioritized during fuzzing. 82 | 83 | For example: 0, Int8.max, Self.max, etc. 84 | */ 85 | fileprivate static func specialValues() -> [Self] { 86 | var result: [Self] = [] 87 | result.append(0) 88 | var i = 8 89 | while i <= bitWidth { 90 | defer { i *= 2 } 91 | let ones = max 92 | let zeros = min 93 | 94 | let umax = zeros | (ones >> (bitWidth - i)) 95 | let umax_lesser = umax / 2 96 | 97 | result.append(umax) 98 | result.append(umax_lesser) 99 | } 100 | return result 101 | } 102 | } 103 | 104 | extension FixedWidthInteger where Self: SignedInteger { 105 | /** 106 | Return an array of “special” values of this UnsignedInteger type that 107 | deserve to be prioritized during fuzzing. 108 | 109 | For example: 0, -1, Int8.min, Int8.max, Int16.min, Self.max, etc. 110 | */ 111 | fileprivate static func specialValues (_ initWithBitPattern: (U) -> Self) -> [Self] { 112 | var result: [Self] = [] 113 | result += [0, -1] 114 | var i = 8 115 | while i < bitWidth { 116 | defer { i *= 2 } 117 | let ones = U.max 118 | let zeros = U.min 119 | 120 | let umax = zeros | (ones >> (bitWidth - i)) 121 | let umin = zeros | (ones << i) 122 | 123 | let max = initWithBitPattern(umax) 124 | let lesser_max = max / 2 125 | let min = initWithBitPattern(umin) 126 | let lesser_min = min / 2 127 | 128 | result += [max, lesser_max, min, lesser_min] 129 | } 130 | result += [max, min] 131 | return result 132 | } 133 | } 134 | 135 | extension IntegerFuzzerGenerator where T: UnsignedInteger { 136 | public init() { 137 | self.init(specialValues: T.specialValues()) 138 | } 139 | } 140 | 141 | extension IntegerFuzzerGenerator where T == Int8 { 142 | public init() { 143 | self.init(specialValues: T.specialValues(T.init(bitPattern:))) 144 | } 145 | } 146 | 147 | extension IntegerFuzzerGenerator where T == Int16 { 148 | public init() { 149 | self.init(specialValues: T.specialValues(T.init(bitPattern:))) 150 | } 151 | } 152 | 153 | extension IntegerFuzzerGenerator where T == Int32 { 154 | public init() { 155 | self.init(specialValues: T.specialValues(T.init(bitPattern:))) 156 | } 157 | } 158 | 159 | extension IntegerFuzzerGenerator where T == Int64 { 160 | public init() { 161 | self.init(specialValues: T.specialValues(T.init(bitPattern:))) 162 | } 163 | } 164 | 165 | extension IntegerFuzzerGenerator where T == Int { 166 | public init() { 167 | self.init(specialValues: T.specialValues(T.init(bitPattern:))) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerInput/StringFuzzerGenerator.swift: -------------------------------------------------------------------------------- 1 | // 2 | // StringFuzzerGenerator.swift 3 | // FuzzCheck 4 | // 5 | 6 | public struct UnicodeScalarViewFuzzerGenerator: FuzzerInputGenerator, FuzzerInputProperties { 7 | public typealias Input = String 8 | 9 | public let baseInput: String = "" 10 | let mutators = UnicodeScalarViewFuzzerMutators() 11 | 12 | public func newInput(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> String { 13 | let targetComplexity = Double.random(in: 0 ..< maxComplexity, using: &rand) 14 | 15 | var s = "" 16 | var currentComplexity = UnicodeScalarViewFuzzerGenerator.complexity(of: s) 17 | while true { 18 | _ = mutators.mutate(&s, with: .appendRandom, spareComplexity: targetComplexity - currentComplexity, &rand) 19 | currentComplexity = UnicodeScalarViewFuzzerGenerator.complexity(of: s) 20 | 21 | while currentComplexity >= targetComplexity { 22 | _ = mutators.mutate(&s, with: .removeRandom, spareComplexity: 0, &rand) 23 | currentComplexity = UnicodeScalarViewFuzzerGenerator.complexity(of: s) 24 | if currentComplexity <= targetComplexity { 25 | return s 26 | } 27 | } 28 | 29 | } 30 | } 31 | 32 | public func mutate(_ input: inout String, _ spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 33 | return mutators.mutate(&input, spareComplexity, &rand) 34 | } 35 | public static func complexity(of input: String) -> Double { 36 | return Double(input.utf16.count) 37 | } 38 | } 39 | 40 | struct UnicodeScalarViewFuzzerMutators: FuzzerInputMutatorGroup { 41 | public typealias Input = String 42 | 43 | enum Mutator { 44 | case appendRandom 45 | case insert 46 | case mutateElement 47 | case removeLast 48 | case removeRandom 49 | } 50 | 51 | func randomScalar(_ rand: inout FuzzerPRNG) -> UnicodeScalar { 52 | switch UInt8.random(in: 0...10, using: &rand) { 53 | // basic ascii code points 54 | case 0...4: 55 | return UnicodeScalar(UInt8.random(in: 0x20 ..< 0x7E, using: &rand)) 56 | // common newlines/whitespace 57 | case 5: 58 | let code: UInt16 = [ 59 | 0x09, 0x10, 0x0B, 0x0D, 0xA0 60 | ].randomElement(using: &rand)! 61 | return UnicodeScalar(code)! 62 | // any 8-bit codepoint 63 | case 6: 64 | return UnicodeScalar(UInt8.random(using: &rand)) 65 | // any 16-bit codepoint 66 | case 7: 67 | return UnicodeScalar(UInt16.random(using: &rand)) ?? UnicodeScalar(UInt8.random(using: &rand)) 68 | // general punctuation 69 | case 8: 70 | let range: ClosedRange = 0x2000...0x206F 71 | return UnicodeScalar(UInt16.random(in: range, using: &rand))! 72 | // emoji, country codes, other less interesting things 73 | case 9: 74 | let range: ClosedRange = 0x1F100...0x1F9FF 75 | return UnicodeScalar(UInt32.random(in: range, using: &rand))! 76 | // any scalar 77 | case 10: 78 | while true { 79 | let code = UInt32.random(in: 0 ... 0x10FFFF, using: &rand) 80 | if let scalar = UnicodeScalar(code) { 81 | return scalar 82 | } 83 | } 84 | default: 85 | fatalError() 86 | } 87 | } 88 | 89 | func mutate(_ input: inout String, with mutator: Mutator, spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool { 90 | switch mutator { 91 | case .appendRandom: 92 | input.unicodeScalars.append(randomScalar(&rand)) 93 | return true 94 | case .insert: 95 | guard let idx = input.unicodeScalars.indices.randomElement(using: &rand) else { 96 | return mutate(&input, with: .appendRandom, spareComplexity: spareComplexity, &rand) 97 | } 98 | input.unicodeScalars.insert(randomScalar(&rand), at: idx) 99 | return true 100 | 101 | case .mutateElement: 102 | guard let idx = input.unicodeScalars.indices.randomElement(using: &rand) else { 103 | return false 104 | } 105 | input.unicodeScalars.replaceSubrange(idx ..< input.unicodeScalars.index(after: idx), with: CollectionOfOne(randomScalar(&rand))) 106 | return true 107 | 108 | case .removeLast: 109 | guard !input.isEmpty else { return false } 110 | input.unicodeScalars.removeLast() 111 | return true 112 | 113 | case .removeRandom: 114 | guard let idx = input.unicodeScalars.indices.randomElement(using: &rand) else { 115 | return false 116 | } 117 | input.unicodeScalars.remove(at: idx) 118 | return true 119 | } 120 | } 121 | 122 | public var weightedMutators: [(Mutator, UInt)] = [ 123 | (.appendRandom, 1), 124 | (.insert, 2), 125 | (.mutateElement, 3), 126 | (.removeLast, 4), 127 | (.removeRandom, 5), 128 | ] 129 | } 130 | 131 | 132 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerPRNG.swift: -------------------------------------------------------------------------------- 1 | 2 | // I didn't want to implement my own random number generator, but I didn't 3 | // find a single one that can be seeded manually. Send help plz. 4 | 5 | /// A pseudo-random number generator that can be seeded 6 | public struct FuzzerPRNG: RandomNumberGenerator { 7 | 8 | public var seed: UInt32 9 | 10 | public init(seed: UInt32) { 11 | self.seed = seed 12 | } 13 | 14 | /// Return an integer whose 31 lower bits are pseudo-random 15 | private mutating func next31() -> UInt32 { 16 | // https://software.intel.com/en-us/articles/fast-random-number-generator-on-the-intel-pentiumr-4-processor/ 17 | seed = 214013 &* seed &+ 2531011 18 | return seed >> 16 &* 0x7FFF 19 | } 20 | } 21 | 22 | extension FuzzerPRNG { 23 | public mutating func next() -> UInt16 { 24 | return UInt16(next31() & 0xFFFF) 25 | } 26 | 27 | public mutating func next() -> UInt32 { 28 | let l = next() as UInt16 29 | let r = next() as UInt16 30 | return (UInt32(l) << 16) | UInt32(r) 31 | } 32 | 33 | public mutating func next() -> UInt64 { 34 | let l = next() as UInt32 35 | let r = next() as UInt32 36 | return (UInt64(l) << 32) | UInt64(r) 37 | } 38 | } 39 | 40 | extension RandomNumberGenerator { 41 | /** 42 | Pick a random index from the `cumulativeWeights` collection where the probability of 43 | choosing an index is given by the distance between `cumulativeWeights[i]` and its 44 | predecessor. 45 | 46 | To be more precise, for a collection `c`, each index has a probability 47 | `(c[i] - c[i-1]) / (max(c) - minimum)` of being chosen, where `c[-1] = minimum`. 48 | 49 | ## Example 50 | ``` 51 | let xs = [2, 4, 8, 9] 52 | let idx = weightedRandomIndex(cumulativeWeights: xs, minimum: 1) 53 | 54 | idx is: 55 | - i with probability (xs[i] - xs[i-1]) / (max(xs) - minimum) 56 | - 0 with probability (2 - 1) / 8 == 1/8 57 | - 1 with probability (4 - 2) / 8 == 2/8 58 | - 2 with probability (8 - 4) / 8 == 4/8 59 | - 3 with probability (9 - 8) / 8 == 1/8 60 | ``` 61 | 62 | - Precondition: 63 | - `cumulativeWeights` is sorted 64 | - `minimum` <= min(cumulativeWeights) 65 | 66 | - Complexity: O(log(n)) with n = cumulativeWeights.count 67 | */ 68 | public mutating func weightedRandomIndex (cumulativeWeights: W, minimum: W.Element) -> W.Index where W.Element: RandomRangeInitializable { 69 | 70 | let randWeight = W.Element.random(in: minimum ..< cumulativeWeights.last!, using: &self) 71 | var index: W.Index = cumulativeWeights.startIndex 72 | switch cumulativeWeights.binarySearch(compare: { $0.compare(randWeight) }) { 73 | case .success(let i): 74 | index = min(cumulativeWeights.index(before: cumulativeWeights.endIndex), cumulativeWeights.index(after: i)) 75 | case .failure(_, let end): 76 | index = min(cumulativeWeights.index(before: cumulativeWeights.endIndex), end) 77 | } 78 | while index > cumulativeWeights.startIndex { 79 | let before = cumulativeWeights.index(before: index) 80 | if cumulativeWeights[before] == cumulativeWeights[index] { 81 | index = before 82 | } else { 83 | break 84 | } 85 | } 86 | return index 87 | } 88 | 89 | /** 90 | Pick a random index from the given collection using the same policy as 91 | `weightedRandomIndex`, and return `c[idx].0`. Unlike `weightedRandomIndex`, 92 | this method runs in O(c.count) time. 93 | 94 | Prefer using this method over `weightedRandomIndex` when the collection is expected 95 | to be very small and performance is important. 96 | 97 | ## Example 98 | ``` 99 | let xs = [("a", 2), ("b", 4), ("c", 8), ("d", 9)] 100 | let element = weightedRandomElement(from: xs, minimum: 1) 101 | 102 | element is: 103 | - "a" with probability (2 - 1) / 8 == 1/8 104 | - "b" with probability (4 - 2) / 8 == 2/8 105 | - "c" with probability (8 - 4) / 8 == 4/8 106 | - "d" with probability (9 - 8) / 8 == 1/8 107 | ``` 108 | 109 | - Precondition: 110 | Let w be c.map{$0.1}. 111 | - `w` is sorted 112 | - `minimum` <= min(w) 113 | 114 | - Complexity: O(n) with `n = c.count` 115 | */ 116 | public mutating func weightedRandomElement (from c: [(T, W)], minimum: W) -> T where W: RandomRangeInitializable { 117 | precondition(!c.isEmpty) 118 | // inelegant, but I needed that one to be fast 119 | return c.withUnsafeBufferPointer { b in 120 | var i = b.baseAddress! 121 | let randWeight = W.random(in: minimum ..< i.advanced(by: (b.count &- 1)).pointee.1, using: &self) 122 | let last = i.advanced(by: b.count) 123 | while i < last { 124 | if i.pointee.1 > randWeight { 125 | return i.pointee.0 126 | } 127 | i = i.advanced(by: 1) 128 | } 129 | fatalError() 130 | } 131 | } 132 | 133 | /// Return true with probability `odds` (e.g. odds = 0.33 -> will return true 1/3rd of the time) 134 | public mutating func bool(odds: Double) -> Bool { 135 | precondition(0 < odds && odds < 1) 136 | let x = Double.random(in: 0 ..< 1, using: &self) 137 | return x < odds 138 | } 139 | } 140 | 141 | extension Sequence { 142 | /** 143 | Return an array whose elements are given by `self.prefix(i+1).reduce(initial, acc)` 144 | with `i` being the index of the element. 145 | 146 | ## Example 147 | ``` 148 | let xs = [1, 0, 9, -1, 3] 149 | let sums = xs.scan(2, +) 150 | // sums == [3, 3, 12, 11, 14] 151 | ``` 152 | */ 153 | public func scan (_ initial: T, _ acc: (T, Element) -> T) -> [T] { 154 | var results: [T] = [] 155 | var t = initial 156 | for x in self { 157 | t = acc(t, x) 158 | results.append(t) 159 | } 160 | return results 161 | } 162 | } 163 | 164 | public enum BinarySearchOrdering { 165 | case less 166 | case match 167 | case greater 168 | } 169 | 170 | extension RandomAccessCollection { 171 | public func binarySearch(compare: (Element) -> BinarySearchOrdering) -> BinarySearchResult { 172 | var beforeBound = startIndex 173 | var startSearch = startIndex 174 | var endSearch = endIndex 175 | 176 | while startSearch != endSearch { 177 | let mid = self.index(startSearch, offsetBy: distance(from: startSearch, to: endSearch) / 2) 178 | let candidate = self[mid] 179 | switch compare(candidate) { 180 | case .less: 181 | beforeBound = mid 182 | startSearch = index(after: mid) 183 | case .match: 184 | return .success(mid) 185 | case .greater: 186 | endSearch = mid 187 | } 188 | } 189 | return .failure(start: beforeBound, end: endSearch) 190 | } 191 | } 192 | 193 | public enum BinarySearchResult { 194 | case success(Index) 195 | case failure(start: Index, end: Index) 196 | } 197 | 198 | extension Comparable { 199 | public func compare(_ element: Self) -> BinarySearchOrdering { 200 | if self > element { return .greater } 201 | else if self < element { return .less } 202 | else { return .match } 203 | } 204 | } 205 | 206 | public protocol RandomInitializable { 207 | static func random (using r: inout R) -> Self 208 | } 209 | public protocol RandomRangeInitializable : Comparable { 210 | static func random (in range: Range, using r: inout R) -> Self 211 | } 212 | 213 | extension FixedWidthInteger where Self: UnsignedInteger { 214 | public static func random (using r: inout R) -> Self { 215 | return r.next() 216 | } 217 | } 218 | 219 | extension UInt8: RandomInitializable, RandomRangeInitializable {} 220 | extension UInt16: RandomInitializable, RandomRangeInitializable {} 221 | extension UInt32: RandomInitializable, RandomRangeInitializable {} 222 | extension UInt64: RandomInitializable, RandomRangeInitializable {} 223 | extension UInt: RandomInitializable, RandomRangeInitializable {} 224 | 225 | extension Int8: RandomInitializable, RandomRangeInitializable { 226 | public static func random (using r: inout R) -> Int8 { 227 | return .init(bitPattern: r.next()) 228 | } 229 | } 230 | extension Int16: RandomInitializable, RandomRangeInitializable { 231 | public static func random (using r: inout R) -> Int16 { 232 | return .init(bitPattern: r.next()) 233 | } 234 | } 235 | extension Int32: RandomInitializable, RandomRangeInitializable { 236 | public static func random (using r: inout R) -> Int32 { 237 | return .init(bitPattern: r.next()) 238 | } 239 | } 240 | extension Int64: RandomInitializable, RandomRangeInitializable { 241 | public static func random (using r: inout R) -> Int64 { 242 | return .init(bitPattern: r.next()) 243 | } 244 | } 245 | extension Int: RandomInitializable, RandomRangeInitializable { 246 | public static func random (using r: inout R) -> Int { 247 | return .init(bitPattern: r.next()) 248 | } 249 | } 250 | extension Float: RandomRangeInitializable { } 251 | extension Double: RandomRangeInitializable { } 252 | extension Float80: RandomRangeInitializable { } 253 | 254 | extension Bool: RandomInitializable { } 255 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/FuzzerSensor.swift: -------------------------------------------------------------------------------- 1 | // 2 | // FuzzerSensor.swift 3 | // FuzzCheck 4 | // 5 | 6 | /** 7 | A `FuzzerSensor` collects measurements that the fuzzer should optimize (e.g. code coverage) 8 | 9 | These measurements are expressed in terms of “Feature” (see `FuzzerSensorFeature` protocol). 10 | */ 11 | public protocol FuzzerSensor { 12 | associatedtype Feature: FuzzerSensorFeature 13 | 14 | mutating func resetCollectedFeatures() 15 | 16 | var isRecording: Bool { get set } 17 | 18 | func iterateOverCollectedFeatures(_ handle: (Feature) -> Void) 19 | } 20 | 21 | /** 22 | A FuzzerSensorFeature describes a single property about a run of the test function. 23 | It will be collected by a FuzzerSensor and analyzed by a Fuzzer. 24 | 25 | For example, this property could be that a specific line of code was reached, 26 | or that a specific comparison operation failed, etc. 27 | 28 | Each feature has an associated “score”, which measures its relative importance compared 29 | to the other features. 30 | */ 31 | public protocol FuzzerSensorFeature: Hashable { 32 | var score: Double { get } 33 | } 34 | 35 | 36 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/InputPool.swift: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | The index for all FuzzerState.InputPool types. 4 | 5 | It points either to a regular element of the pool or to a “favored” element. 6 | */ 7 | enum InputPoolIndex: Equatable, Hashable { 8 | case normal(Int) 9 | case favored 10 | } 11 | 12 | extension FuzzerState { 13 | 14 | /** 15 | A `InputPool` is a collection of test inputs along with some of their 16 | analysis results. 17 | 18 | Each input in the `InputPool` is given a weight based on multiple factors 19 | (e.g. its features and its complexity). This weight determines the 20 | probability of being selected by the `randomIndex` method. 21 | 22 | The pool can also contain a “favored” input, which cannot be deleted and has 23 | a consistently high probability of being selected by `randomIndex`. 24 | 25 | Finally, the pool keeps track of the complexity of the simplest input that 26 | triggered each feature. It is used to filter out uninteresting inputs and 27 | to compute the score of each input. 28 | 29 | an inputPool can also have an alternate representation maintained by the 30 | Fuzzer’s world. For example, we could mirror the content of the pool in a 31 | folder in the file system. For that reason, some methods return a closure 32 | `(inout World) -> Void` that describes the action needed to maintain 33 | consistency between the pool and the world. 34 | */ 35 | final class InputPool { 36 | 37 | /** 38 | Represents an input in the pool along with its initial analysis and 39 | its score in the input pool. 40 | */ 41 | struct Element { 42 | let input: Input 43 | /// The complexity of the input 44 | let complexity: Double 45 | /// The features triggered by feeding the input to the test function 46 | let features: [Sensor.Feature] 47 | 48 | /** 49 | The relative score of the input in the pool. 50 | 51 | It depends on the other properties of InputPool.Element and the global 52 | state of the pool. 53 | 54 | It is not always set at initialization time and is not updated automatically 55 | after changes to the pool, so it should be kept in sync manually. 56 | */ 57 | var score: Double 58 | 59 | // Is only used because `collection.removeAt(indices:)` is not 60 | // implemented in the stdlib and it's not worth reimplementing here 61 | var flaggedForDeletion: Bool 62 | 63 | init(input: Input, complexity: Double, features: [Sensor.Feature]) { 64 | self.input = input 65 | self.complexity = complexity 66 | self.features = features 67 | self.score = -1 // uninitialized 68 | self.flaggedForDeletion = false 69 | } 70 | } 71 | 72 | /// The main content of the pool 73 | var inputs: [Element] = [] 74 | /// The favored input of the pool (optional) 75 | var favoredInput: Element? = nil 76 | 77 | /** 78 | `cumulativeWeights[idx]` is the sum of all the weights of the elements 79 | in `inputs[...idx]`. 80 | 81 | The weight of an input is an estimation of its relative importance 82 | compared to the other inputs in the pool. 83 | */ 84 | var cumulativeWeights: [Double] = [] 85 | 86 | /** 87 | The global score of all inputs in the pool. It should always be equal 88 | to the sum of each input’s score. 89 | */ 90 | var score: Double = 0 91 | 92 | /** 93 | A dictionary that keeps track of the complexity of the simplest input that 94 | triggered each feature. 95 | 96 | Every feature that has ever been recorded by the fuzzer should be in this dictionary. 97 | */ 98 | var smallestInputComplexityForFeature: [Sensor.Feature: Double] = [:] 99 | } 100 | } 101 | 102 | extension FuzzerState.InputPool { 103 | /** 104 | Access an input in the pool. It will crash if the index is invalid or 105 | if it is used to modify the favored input. 106 | */ 107 | subscript(idx: InputPoolIndex) -> Element { 108 | get { 109 | switch idx { 110 | case .normal(let idx): 111 | return inputs[idx] 112 | case .favored: 113 | return favoredInput! 114 | } 115 | } 116 | set { 117 | switch idx { 118 | case .normal(let idx): 119 | inputs[idx] = newValue 120 | case .favored: 121 | fatalError("Cannot assign new input info to favoredInput") 122 | } 123 | } 124 | } 125 | } 126 | 127 | extension FuzzerState.InputPool { 128 | 129 | func add(_ inputsInfo: [Element]) -> (inout World) throws -> Void { 130 | for inputInfo in inputsInfo { 131 | for f in inputInfo.features { 132 | let complexity = smallestInputComplexityForFeature[f] 133 | if complexity == nil || inputInfo.complexity < complexity! { 134 | smallestInputComplexityForFeature[f] = inputInfo.complexity 135 | } 136 | } 137 | inputs.append(inputInfo) 138 | } 139 | let worldUpdate1 = updateScores() 140 | cumulativeWeights = inputs.scan(0.0) { $0 + $1.score } 141 | 142 | return { w in 143 | try worldUpdate1(&w) 144 | for inputInfo in inputsInfo { 145 | try w.addToOutputCorpus(inputInfo.input) 146 | } 147 | } 148 | } 149 | 150 | /** 151 | Add the input to the input pool. Update the score and weight of each input 152 | accordingly. This might result in other inputs being removed from the pool. 153 | 154 | - Complexity: Proportional to the sum of `inputs[i].features.count` for each `i` in 155 | `inputs.indices` (i.e. expensive) 156 | - Returns: The mutating function to apply to the World to keep it in sync with the pool 157 | */ 158 | func add(_ inputInfo: Element) -> (inout World) throws -> Void { 159 | 160 | for f in inputInfo.features { 161 | let complexity = smallestInputComplexityForFeature[f] 162 | if complexity == nil || inputInfo.complexity < complexity! { 163 | smallestInputComplexityForFeature[f] = inputInfo.complexity 164 | } 165 | } 166 | inputs.append(inputInfo) 167 | let worldUpdate1 = updateScores() 168 | cumulativeWeights = inputs.scan(0.0) { $0 + $1.score } 169 | 170 | return { w in 171 | try worldUpdate1(&w) 172 | try w.addToOutputCorpus(inputInfo.input) 173 | } 174 | } 175 | } 176 | 177 | extension FuzzerState.InputPool { 178 | 179 | /** 180 | Update the score of every input in the pool 181 | - Complexity: Proportional to the sum of `inputs[i].features.count` for each `i` in 182 | `inputs.indices` (i.e. expensive) 183 | */ 184 | func updateScores() -> (inout World) throws -> Void { 185 | /* 186 | NOTE: the logic for computing the scores will probably change, but here 187 | is an explanation of the current behavior. 188 | 189 | The main ideas are: 190 | 1. Each feature has a score that is split among all inputs containing 191 | that feature. Because simpler inputs are considered better, they receive a 192 | larger share of the feature score. 193 | 2. an input's score is the sum of the scores associated with 194 | each of its features 195 | 196 | Example: 197 | The pool contains three inputs: u1, u2, u3, which have triggered these features: 198 | - u1: f1 f2 f3 199 | - u2: f2 f3 200 | - u3: f2 f4 201 | To keep it simple, let's assume that all features (f1, f2, f3. f4) have the same 202 | score of 1. 203 | Finally, we need to know the inputs' complexities: 204 | - u1: 10.0 205 | - u2: 5.0 206 | - u3: 5.0 207 | The final scores of the inputs will be: 208 | - u1: 1.31 209 | - u2: 1.24 210 | - u3: 1.44 211 | 212 | Let's first split the score of the feature f2 between u1, u2, and u3. 213 | 214 | (Notation: the share of an input `u`'s score given by a feature `fy` 215 | is written `u.fy_score`) 216 | 217 | We need to satisfy these conditions: 218 | 1. u1.f2_score + u2.f2_score + u3.f2_score == f2.score (== 2) 219 | 2. u1.f2_score < u3.f2_score (because u3 is a simpler, better input than u1) 220 | 3. u2.f2_score == u3.f2_score (because u2 and u3 have the same complexity) 221 | 4. u3.f2_score == u3.f2_score (obviously) 222 | 223 | There are many possible solutions to this system of equation, so we need to choose 224 | a specific ratio between u1.f1_score and u3.f1_score. I chose to use the squared ratio 225 | of the complexities of u1 and u3 (why? because I had to choose something and lack the 226 | empirical evidence needed to make an informed choice). 227 | So the equations are, more specifically: 228 | 1. u1.f2_score + u2.f2_score + u3.f2_score == f2.score (== 1) 229 | 2. u1.f2_score = (u3.complexity / u1.complexity)^2 * u3.f2_score 230 | 3. u2.f2_score = (u3.complexity / u2.complexity)^2 * u3.f2_score 231 | 4. u3.f2_score = (u3.complexity / u3.complexity)^2 * u3.f2_score 232 | 233 | We replace the terms in (1) with (2) and (3), abbreviating (u3.complexity / ux.complexity)^2 by ratio(ux) 234 | 1. ratio(u1) * u3.f2_score + ratio(u2) * u3.f2_score + ratio(u3) * u3.f2_score = f2.score 235 | 236 | which can be simplified to: 237 | 1. (ratio(u1) + ratio(u2) + ratio(u3)) * u3.f2_score = f2.score 238 | => u3.f2_score = f2.score / (ratio(u1) + ratio(u2) + ratio(u3)) [ref: #LHBasKGXc] 239 | which allows us to compute an unknown of the equation, finally! 240 | => u3.f2_score = 1.0 / (0.25 + 1.0 + 1.0) = 0.44 241 | 242 | the remaining scores can now be computed: 243 | 2. u1.f2_score = ratio(u1) * u3.f2_score = (5.0 / 10.0)^2 * 0.44 = 0.11 244 | 3. u2.f2_score = ratio(u2) * u3.f2_score = (5.0 / 5.0)^2 * 0.44 = 0.44 245 | 246 | It all checks out: 247 | 1. u1.f2_score + u2.f2_score + u3.f2_score == 0.11 + 0.44 + 0.44 = 1.0 = f2.score 248 | 249 | Repeat this procedure with the other features to find the total score of each unit. 250 | 251 | So, in order to compute the score of each unit, we need to sum the complexity ratio between 252 | a particular unit and all other units (see: #LHBasKGXc). We decide that this “particular unit” 253 | is the simplest unit containing the feature (as we already keep track of its complexity in 254 | `smallestInputComplexityForFeature`). 255 | 256 | This is what the code below does! 257 | 258 | Except not really! There is an exception. 259 | 260 | If an input is not the simplest one to contain any particular feature, we delete it from 261 | the pool, to avoid pool bloat. Maybe in the future we should use inputs' scores to decide 262 | whether to delete them from the pool, but that would be more complicated. 263 | */ 264 | 265 | func complexityRatio(simplest: Double, other: Double) -> Double { 266 | // the square of the ratio of complexities 267 | return { $0 * $0 }(simplest / other) 268 | } 269 | 270 | // I am sure this could be much faster. 271 | 272 | // First iterate over all inputs and features to initialize sumComplexityRatios 273 | var sumComplexityRatios: [Sensor.Feature: Double] = [:] // see: #LHBasKGXc 274 | for (input, idx) in zip(inputs, inputs.indices) { 275 | inputs[idx].flaggedForDeletion = true 276 | inputs[idx].score = 0 277 | for f in input.features { 278 | let simplestComplexity = smallestInputComplexityForFeature[f]! 279 | let ratio = complexityRatio(simplest: simplestComplexity, other: input.complexity) 280 | precondition(ratio <= 1) 281 | if simplestComplexity == input.complexity { inputs[idx].flaggedForDeletion = false } 282 | } 283 | guard inputs[idx].flaggedForDeletion == false else { 284 | continue 285 | } 286 | for f in input.features { 287 | let simplestComplexity = smallestInputComplexityForFeature[f]! 288 | let ratio = complexityRatio(simplest: simplestComplexity, other: input.complexity) 289 | sumComplexityRatios[f, default: 0.0] += ratio 290 | } 291 | } 292 | 293 | // Then use sumComplexityRatios to find the score of each feature for each unit and sum all the things 294 | for (input, idx) in zip(inputs, inputs.indices) where input.flaggedForDeletion == false { 295 | for f in input.features { 296 | let simplestComplexity = smallestInputComplexityForFeature[f]! 297 | let sumRatios = sumComplexityRatios[f]! 298 | let baseScore = f.score / sumRatios 299 | let ratio = complexityRatio(simplest: simplestComplexity, other: input.complexity) 300 | let score = baseScore * ratio 301 | inputs[idx].score += score 302 | } 303 | } 304 | 305 | // remove inputs that don't deserve to be in the pool anymore 306 | let inputsToDelete = inputs.filter { $0.flaggedForDeletion }.map { $0.input } 307 | let worldUpdate: (inout World) throws -> Void = { [inputsToDelete] w in 308 | for u in inputsToDelete { 309 | try w.removeFromOutputCorpus(u) 310 | } 311 | if !inputsToDelete.isEmpty { 312 | print("DELETE \(inputsToDelete.count)") // TODO: this shouldn't be here 313 | } 314 | } 315 | 316 | inputs.removeAll { $0.flaggedForDeletion } 317 | 318 | score = inputs.reduce(0) { $0 + $1.score } 319 | 320 | return worldUpdate 321 | } 322 | 323 | 324 | func randomIndex(_ r: inout FuzzerPRNG) -> InputPoolIndex { 325 | // maybe the odd of choosing the favoredInput should decrease as time goes on 326 | if favoredInput != nil, r.bool(odds: 0.25) { 327 | return .favored 328 | } else if inputs.isEmpty { 329 | return .favored 330 | } else { 331 | let x = r.weightedRandomIndex(cumulativeWeights: cumulativeWeights, minimum: 0) 332 | return .normal(x) 333 | } 334 | } 335 | 336 | func empty() { 337 | self.inputs.removeAll() 338 | self.score = 0.0 339 | self.cumulativeWeights.removeAll() 340 | self.smallestInputComplexityForFeature.removeAll() 341 | } 342 | 343 | func verify() { 344 | for i in inputs { 345 | var smallestForSomething = false 346 | for f in i.features { 347 | let cplx = smallestInputComplexityForFeature[f]! 348 | precondition(i.complexity >= cplx) 349 | if cplx == i.complexity { 350 | smallestForSomething = true 351 | } 352 | } 353 | precondition(smallestForSomething) 354 | } 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/SanitizerHooks.swift: -------------------------------------------------------------------------------- 1 | 2 | import Darwin 3 | import Foundation 4 | import CBuiltinsNotAvailableInSwift 5 | 6 | /** 7 | Normalize the value of a program counter so that it doesn't vary 8 | between runs of the same program. 9 | */ 10 | struct NormalizedPC { 11 | static var constant: PC = 0 12 | let raw: PC 13 | let value: PC 14 | init(_ raw: PC) { 15 | self.raw = raw 16 | self.value = NormalizedPC.constant &- raw 17 | } 18 | } 19 | 20 | @_cdecl("__sanitizer_cov_trace_pc_guard") 21 | func trace_pc_guard(g: UnsafeMutablePointer?) { 22 | guard CodeCoverageSensor.shared.isRecording, let g = g else { return } 23 | let idx = Int(g.pointee) 24 | // TODO: could this be guarded by a lock, or would it ruin performance? 25 | // TODO: is the overflow check important, or could I let it overflow? 26 | let (result, overflow) = CodeCoverageSensor.shared.eightBitCounters[idx].addingReportingOverflow(1) 27 | if !overflow { 28 | CodeCoverageSensor.shared.eightBitCounters[idx] = result 29 | } 30 | } 31 | 32 | @_cdecl("__sanitizer_cov_trace_pc_guard_init") 33 | func trace_pc_guard_init(start: UnsafeMutablePointer, stop: UnsafeMutablePointer) { 34 | CodeCoverageSensor.shared.handlePCGuardInit(start: start, stop: stop) 35 | } 36 | 37 | @_cdecl("__sanitizer_cov_trace_pc_indir") 38 | func trace_pc_indir(callee: PC) { 39 | guard CodeCoverageSensor.shared.isRecording else { return } 40 | 41 | let caller = PC(bitPattern: __return_address()) 42 | CodeCoverageSensor.shared.handlePCIndir(caller: NormalizedPC(caller), callee: NormalizedPC(callee)) 43 | } 44 | 45 | @_cdecl("__sanitizer_cov_trace_cmp8") 46 | func trace_cmp8(arg1: UInt64, arg2: UInt64) { 47 | guard CodeCoverageSensor.shared.isRecording else { return } 48 | let pc = NormalizedPC(PC(bitPattern: __return_address())) 49 | CodeCoverageSensor.shared.handleTraceCmp(pc: pc, arg1: arg1, arg2: arg2) 50 | } 51 | 52 | @_cdecl("__sanitizer_cov_trace_const_cmp8") 53 | func trace_const_cmp8(arg1: UInt64, arg2: UInt64) { 54 | guard CodeCoverageSensor.shared.isRecording else { return } 55 | let pc = PC(bitPattern: __return_address()) 56 | let x = NormalizedPC(pc) 57 | CodeCoverageSensor.shared.handleTraceCmp(pc: x, arg1: arg1, arg2: arg2) 58 | } 59 | 60 | @_cdecl("__sanitizer_cov_trace_cmp4") 61 | func trace_cmp4(arg1: UInt32, arg2: UInt32) { 62 | guard CodeCoverageSensor.shared.isRecording else { return } 63 | 64 | let pc = PC(bitPattern: __return_address()) 65 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 66 | } 67 | 68 | @_cdecl("__sanitizer_cov_trace_const_cmp4") 69 | func trace_const_cmp4(arg1: UInt32, arg2: UInt32) { 70 | guard CodeCoverageSensor.shared.isRecording else { return } 71 | 72 | let pc = PC(bitPattern: __return_address()) 73 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 74 | } 75 | 76 | @_cdecl("__sanitizer_cov_trace_cmp2") 77 | func trace_cmp2(arg1: UInt16, arg2: UInt16) { 78 | guard CodeCoverageSensor.shared.isRecording else { return } 79 | 80 | let pc = PC(bitPattern: __return_address()) 81 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 82 | } 83 | 84 | @_cdecl("__sanitizer_cov_trace_const_cmp2") 85 | func trace_const_cmp2(arg1: UInt16, arg2: UInt16) { 86 | guard CodeCoverageSensor.shared.isRecording else { return } 87 | 88 | let pc = PC(bitPattern: __return_address()) 89 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 90 | } 91 | 92 | @_cdecl("__sanitizer_cov_trace_cmp1") 93 | func trace_cmp1(arg1: UInt8, arg2: UInt8) { 94 | guard CodeCoverageSensor.shared.isRecording else { return } 95 | 96 | let pc = PC(bitPattern: __return_address()) 97 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 98 | } 99 | 100 | @_cdecl("__sanitizer_cov_trace_const_cmp1") 101 | func trace_const_cmp1(arg1: UInt8, arg2: UInt8) { 102 | guard CodeCoverageSensor.shared.isRecording else { return } 103 | let pc = PC(bitPattern: __return_address()) 104 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: arg1, arg2: arg2) 105 | } 106 | 107 | @_cdecl("__sanitizer_cov_trace_switch") 108 | func trace_switch(val: UInt64, cases: UnsafePointer) { 109 | guard CodeCoverageSensor.shared.isRecording else { return } 110 | 111 | // TODO: this was copied from libFuzzer and I never bothered understanding it 112 | // - why is `!(vals[Int(n - 1)] < 256 && val < 256)` “the most common and boring case”? 113 | // - why do I handle switches the same way as comparison operations? 114 | 115 | let n = cases[0] 116 | let valSizeInBits = cases[1] 117 | let vals = cases.advanced(by: 2) 118 | // Skip the most common and the most boring case. 119 | guard !(vals[Int(n - 1)] < 256 && val < 256) else { return } 120 | 121 | let pc = PC(bitPattern: __return_address()) 122 | 123 | var i: Int = 0 124 | var token: UInt64 = 0 125 | while i < n { 126 | defer { i += 1 } 127 | token = val ^ vals[i] 128 | guard val >= vals[i] else { break } 129 | } 130 | 131 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc + UInt(i)), arg1: token, arg2: 0) 132 | } 133 | 134 | @_cdecl("__sanitizer_cov_trace_div4") 135 | func trace_div4(val: UInt32) { 136 | guard CodeCoverageSensor.shared.isRecording else { return } 137 | 138 | let pc = PC(bitPattern: __return_address()) 139 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: val, arg2: 0) 140 | } 141 | 142 | @_cdecl("__sanitizer_cov_trace_div8") 143 | func trace_div8(val: UInt64) { 144 | guard CodeCoverageSensor.shared.isRecording else { return } 145 | 146 | let pc = PC(bitPattern: __return_address()) 147 | CodeCoverageSensor.shared.handleTraceCmp(pc: NormalizedPC(pc), arg1: val, arg2: 0) 148 | } 149 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/SignalHandlers.swift: -------------------------------------------------------------------------------- 1 | 2 | 3 | import Foundation 4 | import Basic 5 | 6 | typealias Thread = Basic.Thread 7 | 8 | /// Interrupt signal handling global variables 9 | private var receivedSignal: Signal? = nil 10 | 11 | private var signalSemaphore = DispatchSemaphore(value: 0) 12 | private var writeSignalSemaphore = DispatchSemaphore(value: 1) 13 | 14 | private var oldActions = Array.init(repeating: sigaction(), count: 32) 15 | 16 | /// This class can be used by command line tools to install a handler which 17 | /// should be called when a interrupt signal is delivered to the process. 18 | public final class SignalsHandler { 19 | 20 | /// The thread which waits to be notified when a signal is received. 21 | let thread: Thread 22 | 23 | let signals: [Signal] 24 | 25 | /// Start watching for interrupt signal and call the handler whenever the signal is received. 26 | public init(signals: [Signal], handler: @escaping (Signal) -> Void) { 27 | self.signals = signals 28 | // Create a signal handler. 29 | let signalHandler: @convention(c)(Int32) -> Void = { sig in 30 | writeSignalSemaphore.wait() 31 | receivedSignal = Signal(rawValue: sig)! 32 | signalSemaphore.signal() 33 | } 34 | var action = sigaction() 35 | action.__sigaction_u.__sa_handler = signalHandler 36 | for signal in signals { 37 | // Install the new handler. 38 | let result = sigaction(signal.rawValue, &action, &oldActions[Int(signal.rawValue)]) 39 | precondition(result == 0) 40 | } 41 | 42 | // This thread waits to be notified via semaphore. 43 | thread = Thread { 44 | while true { 45 | signalSemaphore.wait() 46 | if let _receivedSignal = receivedSignal { 47 | handler(_receivedSignal) 48 | receivedSignal = nil 49 | } else { // if the signal semaphore was signaled but no received signal exists, then 50 | // it means the Thread should finish its execution 51 | return 52 | } 53 | writeSignalSemaphore.signal() 54 | } 55 | } 56 | thread.start() 57 | } 58 | 59 | deinit { 60 | for sig in signals { 61 | // Restore the old action and close the write end of pipe. 62 | sigaction(sig.rawValue, &oldActions[Int(sig.rawValue)], nil) 63 | } 64 | receivedSignal = nil 65 | signalSemaphore.signal() 66 | thread.join() 67 | } 68 | } 69 | 70 | public enum Signal: Int32 { 71 | case terminalLineHangup = 1 72 | case interrupt 73 | case quit 74 | case illegalInstruction 75 | case traceTrap 76 | case abort 77 | case emulateInstructionExecuted 78 | case floatingPointException 79 | case kill 80 | case busError 81 | case segmentationViolation 82 | case nonExistentSystemCallInvoked 83 | case writeOnPipeWithNoReader 84 | case realTimeTimerExpired 85 | case softwareTermination 86 | case urgentConditionOnSocket 87 | case uncatchableStop 88 | case keyboardStop 89 | case continueAfterStop 90 | case childStatusHasChanged 91 | case backgroundReadAttemptedFromControlTerminal 92 | case backgroundWriteAttemptedToControlTerminal 93 | case ioPossibleOnADescriptor 94 | case cpuTimeLimitExceeded 95 | case fileSizeLimitExceeded 96 | case virtualTimeAlarm 97 | case profilingTimerAlarm 98 | case windowSizeChange 99 | case statusRequestFromKeyboard 100 | case userDefined1 101 | case userDefined2 102 | 103 | /* 104 | 1 SIGHUP terminate process terminal line hangup 105 | 2 SIGINT terminate process interrupt program 106 | 3 SIGQUIT create core image quit program 107 | 4 SIGILL create core image illegal instruction 108 | 5 SIGTRAP create core image trace trap 109 | 6 SIGABRT create core image abort program (formerly SIGIOT) 110 | 7 SIGEMT create core image emulate instruction executed 111 | 8 SIGFPE create core image floating-point exception 112 | 9 SIGKILL terminate process kill program 113 | 10 SIGBUS create core image bus error 114 | 11 SIGSEGV create core image segmentation violation 115 | 12 SIGSYS create core image non-existent system call invoked 116 | 13 SIGPIPE terminate process write on a pipe with no reader 117 | 14 SIGALRM terminate process real-time timer expired 118 | 15 SIGTERM terminate process software termination signal 119 | 16 SIGURG discard signal urgent condition present on socket 120 | 17 SIGSTOP stop process stop (cannot be caught or ignored) 121 | 18 SIGTSTP stop process stop signal generated from keyboard 122 | 19 SIGCONT discard signal continue after stop 123 | 20 SIGCHLD discard signal child status has changed 124 | 21 SIGTTIN stop process background read attempted from control terminal 125 | 22 SIGTTOU stop process background write attempted to control terminal 126 | 23 SIGIO discard signal I/O is possible on a descriptor (see fcntl(2)) 127 | 24 SIGXCPU terminate process cpu time limit exceeded (see setrlimit(2)) 128 | 25 SIGXFSZ terminate process file size limit exceeded (see setrlimit(2)) 129 | 26 SIGVTALRM terminate process virtual time alarm (see setitimer(2)) 130 | 27 SIGPROF terminate process profiling timer alarm (see setitimer(2)) 131 | 28 SIGWINCH discard signal Window size change 132 | 29 SIGINFO discard signal status request from keyboard 133 | 30 SIGUSR1 terminate process User defined signal 1 134 | 31 SIGUSR2 terminate process User defined signal 2 135 | */ 136 | } 137 | 138 | -------------------------------------------------------------------------------- /Sources/FuzzCheck/World.swift: -------------------------------------------------------------------------------- 1 | // 2 | // World.swift 3 | // FuzzCheck 4 | // 5 | // Created by Loïc Lecrenier on 03/06/2018. 6 | // 7 | 8 | import Files 9 | import Foundation 10 | 11 | /// An fuzzer event to communicate with the world 12 | public enum FuzzerEvent { 13 | /// The fuzzing process started 14 | case start 15 | /// The fuzzing process ended 16 | case done 17 | /// A new interesting input was discovered 18 | case new 19 | /// The initial corpus has been processed 20 | case didReadCorpus 21 | /// The pool of input has been reset and re-processed 22 | case didResetPool 23 | /// A signal sent to the process was caught 24 | case caughtSignal(Signal) 25 | /// A test failure was found 26 | case testFailure 27 | } 28 | 29 | public protocol FuzzerWorld { 30 | associatedtype Input 31 | associatedtype Properties: FuzzerInputProperties where Properties.Input == Input 32 | associatedtype Feature: Codable 33 | 34 | mutating func getPeakMemoryUsage() -> UInt 35 | mutating func clock() -> UInt 36 | mutating func readInputCorpus() throws -> [Input] 37 | mutating func readInputFile() throws -> Input 38 | 39 | mutating func saveArtifact(input: Input, features: [Feature]?, score: Double?, kind: ArtifactKind) throws 40 | mutating func addToOutputCorpus(_ input: Input) throws 41 | mutating func removeFromOutputCorpus(_ input: Input) throws 42 | mutating func reportEvent(_ event: FuzzerEvent, stats: FuzzerStats) 43 | 44 | var rand: FuzzerPRNG { get set } 45 | } 46 | 47 | public struct FuzzerStats { 48 | public var totalNumberOfRuns: Int = 0 49 | public var score: Int = 0 50 | public var poolSize: Int = 0 51 | public var executionsPerSecond: Int = 0 52 | public var averageComplexity: Int = 0 53 | public var rss: Int = 0 54 | } 55 | 56 | public struct FuzzerSettings { 57 | 58 | public enum Command: String { 59 | case minimize 60 | case fuzz 61 | case read 62 | } 63 | 64 | public var command: Command 65 | public var maxNumberOfRuns: Int 66 | public var maxInputComplexity: Double 67 | public var mutateDepth: Int 68 | 69 | public init(command: Command = .fuzz, maxNumberOfRuns: Int = Int.max, maxInputComplexity: Double = 256.0, mutateDepth: Int = 3) { 70 | self.command = command 71 | self.maxNumberOfRuns = maxNumberOfRuns 72 | self.maxInputComplexity = maxInputComplexity 73 | self.mutateDepth = mutateDepth 74 | } 75 | } 76 | 77 | public struct CommandLineFuzzerWorldInfo { 78 | public var rand: FuzzerPRNG = FuzzerPRNG(seed: arc4random()) 79 | public var inputFile: File? = nil 80 | public var inputCorpora: [Folder] = [] 81 | public var outputCorpus: Folder? = nil 82 | public var artifactsFolder: Folder? = (try? Folder.current.subfolder(named: "artifacts")) ?? Folder.current 83 | public var artifactsNameSchema: ArtifactSchema.Name = ArtifactSchema.Name(components: [.kind, .literal("-"), .hash], ext: "json") 84 | public var artifactsContentSchema: ArtifactSchema.Content = ArtifactSchema.Content(features: false, score: false, hash: false, complexity: false, kind: false) 85 | public init() {} 86 | } 87 | 88 | public struct CommandLineFuzzerWorld : FuzzerWorld 89 | where 90 | Properties: FuzzerInputProperties, 91 | Properties.Input == Input 92 | { 93 | public typealias Feature = CodeCoverageSensor.Feature 94 | 95 | public var info: CommandLineFuzzerWorldInfo 96 | public var rand: FuzzerPRNG { 97 | get { return info.rand } 98 | set { info.rand = newValue } 99 | } 100 | 101 | public init(info: CommandLineFuzzerWorldInfo) { 102 | self.info = info 103 | } 104 | 105 | public func clock() -> UInt { 106 | return UInt(DispatchTime.now().rawValue / 1_000) 107 | } 108 | public func getPeakMemoryUsage() -> UInt { 109 | var r: rusage = rusage.init() 110 | if getrusage(RUSAGE_SELF, &r) != 0 { 111 | return 0 112 | } 113 | return UInt(r.ru_maxrss) >> 20 114 | } 115 | 116 | public func saveArtifact(input: Input, features: [Feature]?, score: Double?, kind: ArtifactKind) throws { 117 | guard let artifactsFolder = info.artifactsFolder else { 118 | return 119 | } 120 | let complexity = Properties.complexity(of: input) 121 | let hash = Properties.hashValue(of: input) 122 | let content = Artifact.Content.init(schema: info.artifactsContentSchema, input: Properties.convertToCodable(input), features: features, score: score, hash: hash, complexity: complexity, kind: kind) 123 | let encoder = JSONEncoder() 124 | encoder.outputFormatting = .prettyPrinted 125 | let data = try encoder.encode(content) 126 | let nameInfo = ArtifactNameInfo(hash: hash, complexity: complexity, kind: kind) 127 | let name = ArtifactNameWithoutIndex(schema: info.artifactsNameSchema, info: nameInfo).fillGapToBeUnique(from: readArtifactsFolderNames()) 128 | print("Saving \(kind) at \(artifactsFolder.path)\(name)") 129 | try artifactsFolder.createFileIfNeeded(withName: name, contents: data) 130 | } 131 | 132 | public func readArtifactsFolderNames() -> Set { 133 | guard let artifactsFolder = info.artifactsFolder else { 134 | return [] 135 | } 136 | return Set(artifactsFolder.files.map { $0.name }) 137 | } 138 | 139 | public func readInputFile() throws -> Input { 140 | let decoder = JSONDecoder() 141 | 142 | let data = try info.inputFile!.read() 143 | return try Properties.convertFromCodable(decoder.decode(Artifact.Content.self, from: data).input) 144 | } 145 | 146 | public func readInputCorpus() throws -> [Input] { 147 | let decoder = JSONDecoder() 148 | return try info.inputCorpora 149 | .flatMap { $0.files } 150 | .map { try Properties.convertFromCodable(decoder.decode(Artifact.Content.self, from: $0.read()).input) } 151 | } 152 | 153 | public func removeFromOutputCorpus(_ input: Input) throws { 154 | guard let outputCorpus = info.outputCorpus else { return } 155 | let nameInfo = ArtifactNameInfo(hash: Properties.hashValue(of: input), complexity: Properties.complexity(of: input), kind: .input) 156 | let name = ArtifactNameWithoutIndex(schema: info.artifactsNameSchema, info: nameInfo).fillGapToBeUnique(from: []) 157 | try outputCorpus.file(named: name).delete() 158 | } 159 | 160 | public mutating func addToOutputCorpus(_ input: Input) throws { 161 | guard let outputCorpus = info.outputCorpus else { return } 162 | let encoder = JSONEncoder() 163 | encoder.outputFormatting = .prettyPrinted 164 | let data = try encoder.encode(Properties.convertToCodable(input)) 165 | let nameInfo = ArtifactNameInfo(hash: Properties.hashValue(of: input), complexity: Properties.complexity(of: input), kind: .input) 166 | let name = ArtifactNameWithoutIndex(schema: info.artifactsNameSchema, info: nameInfo).fillGapToBeUnique(from: []) 167 | try outputCorpus.createFileIfNeeded(withName: name, contents: data) 168 | } 169 | 170 | public func reportEvent(_ event: FuzzerEvent, stats: FuzzerStats) { 171 | switch event { 172 | case .start: 173 | print("START") 174 | case .done: 175 | print("DONE") 176 | case .new: 177 | print("NEW\t", terminator: "") 178 | case .didReadCorpus: 179 | print("FINISHED READING CORPUS") 180 | case .caughtSignal(let signal): 181 | switch signal { 182 | case .illegalInstruction, .abort, .busError, .floatingPointException: 183 | print("\n================ CRASH DETECTED ================") 184 | case .interrupt: 185 | print("\n================ RUN INTERRUPTED ================") 186 | default: 187 | print("\n================ SIGNAL \(signal) ================") 188 | } 189 | case .testFailure: 190 | print("\n================ TEST FAILED ================") 191 | case .didResetPool: 192 | print("RESET POOL\t", terminator: "") 193 | } 194 | print("\(stats.totalNumberOfRuns)", terminator: "\t") 195 | print("score: \(stats.score)", terminator: "\t") 196 | print("corp: \(stats.poolSize)", terminator: "\t") 197 | print("exec/s: \(stats.executionsPerSecond)", terminator: "\t") 198 | print("cplx: \(stats.averageComplexity)", terminator: "\t") 199 | //print("rss: \(stats.rss)") 200 | print() 201 | } 202 | } 203 | 204 | /** 205 | Return the hexadecimal representation of the given integer 206 | */ 207 | func hexString(_ h: Int) -> String { 208 | let bits = UInt64(bitPattern: Int64(h)) 209 | return String(bits, radix: 16, uppercase: false) 210 | } 211 | -------------------------------------------------------------------------------- /Sources/FuzzCheckTool/ArgumentsParser.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ArgumentsParser.swift 3 | // FuzzCheckJobsManager 4 | // 5 | 6 | import Basic 7 | import Files 8 | import Foundation 9 | import FuzzCheck 10 | 11 | func parseArguments() throws -> (FuzzerManagerSettings, FuzzerSettings, CommandLineFuzzerWorldInfo) { 12 | 13 | let (parser, workerSettingsBinder, worldBinder, settingsBinder) = CommandLineFuzzerWorldInfo.argumentsParser() 14 | 15 | let res = try parser.parse(Array(CommandLine.arguments.dropFirst())) 16 | var workerSettings = FuzzerSettings() 17 | try workerSettingsBinder.fill(parseResult: res, into: &workerSettings) 18 | 19 | var settings = FuzzerManagerSettings() 20 | try settingsBinder.fill(parseResult: res, into: &settings) 21 | 22 | print(workerSettings) 23 | var world = CommandLineFuzzerWorldInfo() 24 | try worldBinder.fill(parseResult: res, into: &world) 25 | 26 | return (settings, workerSettings, world) 27 | } 28 | -------------------------------------------------------------------------------- /Sources/FuzzCheckTool/Commands.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Commands.swift 3 | // FuzzCheckJobsManager 4 | // 5 | 6 | import Basic 7 | import Files 8 | import Foundation 9 | import FuzzCheck 10 | import Utility 11 | 12 | /// A wrapper type with reference semantics 13 | class Ref { 14 | var data: T 15 | 16 | init(_ data: T) { self.data = data } 17 | } 18 | 19 | /// Lock used to synchronize the launches of the child process with the interrupt/timer 20 | /// signal handlers. We don't want to launch an additional child process while we are 21 | /// shutting down the program. 22 | /// documentation reference: #wupKfXxNqM8 23 | let childProcessLock = Lock() 24 | 25 | /// Return a reference to the child process as well as the properties needed to launch it. 26 | func childProcessInfo(settings: FuzzerManagerSettings) -> (ref: Ref, launchPath: String, env: [String: String]) { 27 | let exec = settings.testExecutable! 28 | var fuzzerJobEnvironment = ProcessInfo.processInfo.environment 29 | fuzzerJobEnvironment["SWIFT_DETERMINISTIC_HASHING"] = "1" 30 | let process = Ref(Process()) 31 | let launchPath = exec.path 32 | let environment = fuzzerJobEnvironment 33 | 34 | return (process, launchPath, environment) 35 | } 36 | 37 | func minimizeCommand(settings: FuzzerManagerSettings, workerSettings: FuzzerSettings, world: CommandLineFuzzerWorldInfo) throws -> Never { 38 | var (settings, workerSettings, world) = (settings, workerSettings, world) 39 | 40 | let fileToMinimize = world.inputFile! // TODO: put that requirement in the arguments parser 41 | let (process, launchPath, environment) = childProcessInfo(settings: settings) 42 | let (sh, timerSource) = signalHandlers(process: process, globalTimeout: settings.globalTimeout, preexit: { 43 | // libFuzzer says “Failed to minimize beyond...”, but I am not so humble, FuzzCheck probably did a 👌 job 44 | print("Could not minimize beyond: \(world.inputFile!.name)") 45 | }) 46 | 47 | // The child processes will create artifacts, we put them all under a folder named .minimized 48 | // We will launch the child process with the simplest input file in that folder as argument. 49 | // That folder might already exist (because of a previous minimization attempt). In that case we do nothing. 50 | let artifactsFolder = try fileToMinimize.parent!.createSubfolderIfNeeded(withName: fileToMinimize.nameExcludingExtension + ".minimized") 51 | 52 | world.artifactsFolder = artifactsFolder 53 | // we store the complexity in the artifacts file so that we can determine which artifact is the simplest one. 54 | world.artifactsContentSchema = .init(features: false, score: false, hash: false, complexity: true, kind: false) 55 | // the complexity in the artifact files is stored under the key “complexity” 56 | // we create this simple wrapper type to decode it 57 | /** e.g. 58 | { 59 | input: {...}, 60 | complexity: 23.3 61 | } 62 | */ 63 | struct Complexity: Decodable { 64 | let complexity: Double 65 | } 66 | 67 | /// Return the artifact file containing the simplest input, or nil if the artifacts folder is empty 68 | func simplestInputFile() -> File? { 69 | let filesWithComplexity = artifactsFolder.files.map { f -> (File, Double) in 70 | (f, try! JSONDecoder().decode(Complexity.self, from: f.read()).complexity) 71 | } 72 | return filesWithComplexity.min { $0.1 < $1.1 }?.0 73 | } 74 | 75 | world.inputFile = simplestInputFile() ?? fileToMinimize 76 | 77 | workerSettings.command = .read 78 | world.artifactsFolder = artifactsFolder.containsFile(named: world.inputFile!.name) ? nil : artifactsFolder 79 | 80 | try run(process: process, launchPath: launchPath, arguments: workerSettings.commandLineArguments + world.commandLineArguments, env: environment) 81 | 82 | precondition(process.data.terminationStatus == FuzzerTerminationStatus.crash.rawValue || process.data.terminationStatus == FuzzerTerminationStatus.testFailure.rawValue, "The input to minimize didn't cause a crash") 83 | 84 | workerSettings.command = .minimize 85 | world.artifactsFolder = artifactsFolder 86 | 87 | // TODO: max number of runs 88 | while true { 89 | // By now we have added at least one file to the artifacts folder, so simplestInputFile() cannot be nil 90 | world.inputFile = simplestInputFile()! 91 | 92 | try run(process: process, launchPath: launchPath, arguments: workerSettings.commandLineArguments + world.commandLineArguments, env: environment) 93 | 94 | withExtendedLifetime(sh) { } 95 | withExtendedLifetime(timerSource) { } 96 | } 97 | } 98 | 99 | func fuzzCommand(settings: FuzzerManagerSettings, workerSettings: FuzzerSettings, world: CommandLineFuzzerWorldInfo) throws -> Never { 100 | 101 | let (process, launchPath, environment) = childProcessInfo(settings: settings) 102 | let (sh, timerSource) = signalHandlers(process: process, globalTimeout: settings.globalTimeout, preexit: {}) 103 | 104 | try run(process: process, launchPath: launchPath, arguments: workerSettings.commandLineArguments + world.commandLineArguments, env: environment) 105 | 106 | withExtendedLifetime(sh) { } 107 | withExtendedLifetime(timerSource) { } 108 | 109 | exit(0) 110 | } 111 | 112 | func run(process: Ref, launchPath: String, arguments: [String], env: [String: String]) throws { 113 | // see: #wupKfXxNqM8 114 | childProcessLock.withLock { 115 | process.data = Process() 116 | } 117 | let process = process.data 118 | process.launchPath = launchPath 119 | process.environment = env 120 | process.arguments = arguments 121 | if #available(OSX 10.13, *) { 122 | try process.run() 123 | } else { 124 | process.launch() 125 | } 126 | process.waitUntilExit() 127 | } 128 | 129 | /// Shut down the given process. First send an interrupt signal, then 130 | /// force-suspend it if it didn't handle the interrupt quickly enough. 131 | func interrupt(_ process: Ref) { 132 | // can't interrupt a process that is not running 133 | guard process.data.isRunning else { return } 134 | // Send interrupt signal 135 | process.data.interrupt() 136 | // Give the child process 0.1 seconds to exit 137 | Foundation.Thread.sleep(forTimeInterval: 0.1) 138 | 139 | // If child process has not exited yet 140 | if process.data.isRunning { 141 | // Give the child process an additional two seconds to exit 142 | Foundation.Thread.sleep(forTimeInterval: 2.0) 143 | // If it is *still* running, then 144 | if process.data.isRunning { 145 | // force-suspend it 146 | _ = process.data.suspend() 147 | } 148 | } 149 | } 150 | 151 | /** 152 | Create the interrupt signal handler and the Dispatch source timer. 153 | These two objects are responsible for stopping the child processes and 154 | the program itself. 155 | */ 156 | func signalHandlers(process: Ref, globalTimeout: UInt?, preexit: @escaping () -> Void) -> (SignalsHandler, DispatchSourceTimer) { 157 | 158 | let signals: [Signal] = [.segmentationViolation, .busError, .abort, .illegalInstruction, .floatingPointException, .interrupt, .softwareTermination, .fileSizeLimitExceeded] 159 | 160 | let sh = SignalsHandler(signals: signals) { signal in 161 | // Another part of the program might want to relaunch the process as soon as we 162 | // interrupt it, so we guard process creation/interrupt operations under a common lock. 163 | // see: #wupKfXxNqM8 164 | childProcessLock.withLock { 165 | interrupt(process) 166 | preexit() 167 | exit(0) 168 | } 169 | } 170 | 171 | let timerSource = DispatchSource.makeTimerSource(flags: .strict, queue: DispatchQueue.global()) 172 | if let globalTimeout = globalTimeout { 173 | let time: DispatchTime = DispatchTime(uptimeNanoseconds: DispatchTime.now().uptimeNanoseconds + UInt64(globalTimeout) * 1_000_000_000) 174 | timerSource.schedule(deadline: time) 175 | timerSource.setEventHandler { 176 | // Another part of the program might want to relaunch the process as soon as we 177 | // interrupt it, so we guard process creation/interrupt operations under a common lock. 178 | // see: #wupKfXxNqM8 179 | childProcessLock.withLock { 180 | interrupt(process) 181 | preexit() 182 | exit(0) 183 | } 184 | } 185 | if #available(OSX 10.12, *) { 186 | timerSource.activate() 187 | } else { 188 | timerSource.resume() // TODO: is this correct? 189 | } 190 | } 191 | 192 | return (sh, timerSource) 193 | } 194 | -------------------------------------------------------------------------------- /Sources/FuzzCheckTool/main.swift: -------------------------------------------------------------------------------- 1 | 2 | import Basic 3 | import Files 4 | import Foundation 5 | import FuzzCheck 6 | import Utility 7 | 8 | typealias Process = Foundation.Process 9 | typealias URL = Foundation.URL 10 | 11 | var (settings, workerSettings, world) = try parseArguments() 12 | 13 | if case .minimize = workerSettings.command { 14 | try minimizeCommand(settings: settings, workerSettings: workerSettings, world: world) 15 | } else if case .fuzz = workerSettings.command { 16 | try fuzzCommand(settings: settings, workerSettings: workerSettings, world: world) 17 | } else { 18 | fatalError("Unsupported command \(workerSettings.command)") 19 | } 20 | 21 | -------------------------------------------------------------------------------- /Tests/FuzzerTests/ArtifactsTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ArtifactsTest.swift 3 | // FuzzCheck 4 | // 5 | 6 | 7 | import XCTest 8 | @testable import FuzzCheck 9 | 10 | class ArtifactsTests: XCTestCase { 11 | /* 12 | func testParse() { 13 | let s = "??kind-?complexity?-?hash-?index" 14 | let ext = "" 15 | 16 | let existing: Set = [ 17 | "43.7fffffffffffffff.0.json", 18 | "43.7fffffffffffffff.1.json", 19 | "43.7fffffffffffffff.2.json", 20 | "43.7fffffffffffffff.3.json", 21 | "43.7fffffffffffffff.4.json", 22 | "43.7fffffffffffffff.5.json", 23 | ] 24 | 25 | let schemaAtoms = ArtifactSchema.Name.Atom.read(from: s) 26 | let schema = ArtifactSchema.Name(components: schemaAtoms, ext: ext) 27 | let artNameInfo = ArtifactNameInfo(hash: Int.max, complexity: .init(43.327), kind: .crash) 28 | let artName = artNameInfo.name(following: schema) 29 | //print(artName) 30 | print(artName.fillGapToBeUnique(from: existing)) 31 | }*/ 32 | } 33 | -------------------------------------------------------------------------------- /Tests/FuzzerTests/InputPoolTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // InputPoolTests.swift 3 | // FuzzerTests 4 | // 5 | 6 | import XCTest 7 | @testable import FuzzCheck 8 | 9 | /// A list of 100 unique features, each with a score of 1.0 10 | let f: [MockFeature] = { 11 | var f: [MockFeature] = [] 12 | for _ in 0 ..< 100 { 13 | f.append(MockFeature()) 14 | } 15 | return f 16 | }() 17 | 18 | class InputPoolTests: XCTestCase { 19 | 20 | func testCoverageScore0() { 21 | let u1 = MockInputPool.Element( 22 | input: (), 23 | complexity: 10.0, 24 | features: [f[0], f[1], f[2]] 25 | ) 26 | let u2 = MockInputPool.Element( 27 | input: (), 28 | complexity: 5.0, 29 | features: [f[1], f[2]] 30 | ) 31 | let u3 = MockInputPool.Element( 32 | input: (), 33 | complexity: 5.0, 34 | features: [f[1], f[3]] 35 | ) 36 | 37 | let pool = MockInputPool() 38 | _ = pool.add(u1) 39 | _ = pool.add(u2) 40 | _ = pool.add(u3) 41 | 42 | print(pool.inputs.map { $0.score }) 43 | } 44 | 45 | func testCoverageScore() { 46 | 47 | let u1 = MockInputPool.Element( 48 | input: (), 49 | complexity: 10.0, 50 | features: [f[0], f[1], f[2], f[3]] 51 | ) 52 | let u2 = MockInputPool.Element( 53 | input: (), 54 | complexity: 5.0, 55 | features: [f[4]] 56 | ) 57 | let u3 = MockInputPool.Element( 58 | input: (), 59 | complexity: 5.0, 60 | features: [f[5]] 61 | ) 62 | let u4 = MockInputPool.Element( 63 | input: (), 64 | complexity: 2.0, 65 | features: [f[6], f[7]] 66 | ) 67 | let u5 = MockInputPool.Element( 68 | input: (), 69 | complexity: 1.0, 70 | features: [f[6]] 71 | ) 72 | 73 | let pool = MockInputPool() 74 | _ = pool.add(u1) 75 | _ = pool.add(u2) 76 | _ = pool.add(u3) 77 | _ = pool.add(u4) 78 | _ = pool.add(u5) 79 | 80 | XCTAssertEqual(pool.inputs.count, 5) 81 | 82 | XCTAssertGreaterThan(pool.inputs[0].score, pool.inputs[1].score) 83 | XCTAssertGreaterThan(pool.inputs[0].score, pool.inputs[2].score) 84 | XCTAssertGreaterThan(pool.inputs[0].score, pool.inputs[3].score) 85 | XCTAssertGreaterThan(pool.inputs[0].score, pool.inputs[4].score) 86 | 87 | XCTAssertEqual(pool.inputs[1].score, pool.inputs[2].score) 88 | 89 | XCTAssertEqual(pool.inputs[3].score, f[6].score / 5 + f[7].score) 90 | XCTAssertEqual(pool.inputs[4].score, f[6].score * 4 / 5) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /Tests/FuzzerTests/Mock.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Mock.swift 3 | // FuzzerTests 4 | // 5 | 6 | @testable import FuzzCheck 7 | 8 | typealias MockFuzzerState = FuzzerState, MockWorld>, MockSensor> 9 | typealias MockInputPool = MockFuzzerState.InputPool 10 | 11 | struct MockInputProperties : FuzzerInputProperties { 12 | static func hash(of input: Input) -> Int { 13 | return 1 14 | } 15 | static func complexity(of input: Input) -> Double { 16 | return 1.0 17 | } 18 | } 19 | 20 | final class MockFeature: FuzzerSensorFeature, Hashable, Codable { 21 | let score: Double 22 | 23 | /// Initialize the MockFeature with a score of 1.0 24 | init() { 25 | self.score = 1.0 26 | } 27 | 28 | init(score: Double) { 29 | self.score = score 30 | } 31 | 32 | static func == (lhs: MockFeature, rhs: MockFeature) -> Bool { 33 | return lhs === rhs 34 | } 35 | func hash(into hasher: inout Hasher) { 36 | hasher.combine(ObjectIdentifier(self)) 37 | } 38 | } 39 | 40 | struct MockSensor: FuzzerSensor { 41 | typealias Feature = MockFeature 42 | 43 | init(features: AnyIterator<[MockFeature]>) { 44 | self.currentFeatures = features.next() ?? [] 45 | self.nextFeatures = features 46 | } 47 | 48 | private let nextFeatures: AnyIterator<[MockFeature]> 49 | private var currentFeatures: [MockFeature] 50 | 51 | var isRecording: Bool = false 52 | 53 | mutating func resetCollectedFeatures() { 54 | currentFeatures = nextFeatures.next() ?? [] 55 | } 56 | 57 | func iterateOverCollectedFeatures(_ handle: (MockFeature) -> Void) { 58 | currentFeatures.forEach(handle) 59 | } 60 | } 61 | 62 | struct MockWorld : FuzzerWorld { 63 | typealias Input = P.Input 64 | typealias Properties = P 65 | typealias Feature = MockFeature 66 | 67 | var _clock: UInt = 0 68 | var rand: FuzzerPRNG 69 | 70 | mutating func getPeakMemoryUsage() -> UInt { 71 | return 1 72 | } 73 | 74 | mutating func clock() -> UInt { 75 | _clock += 1 76 | return _clock 77 | } 78 | 79 | mutating func readInputCorpus() throws -> [P.Input] { 80 | return [] 81 | } 82 | 83 | mutating func readInputFile() throws -> P.Input { 84 | fatalError() 85 | } 86 | 87 | mutating func saveArtifact(input: P.Input, features: [Feature]?, score: Double?, kind: ArtifactKind) throws { 88 | return 89 | } 90 | 91 | mutating func addToOutputCorpus(_ input: P.Input) throws { 92 | return 93 | } 94 | 95 | mutating func removeFromOutputCorpus(_ input: P.Input) throws { 96 | return 97 | } 98 | 99 | mutating func reportEvent(_ event: FuzzerEvent, stats: FuzzerStats) { 100 | return 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Tests/FuzzerTests/RandomTests.swift: -------------------------------------------------------------------------------- 1 | 2 | import XCTest 3 | @testable import FuzzCheck 4 | 5 | class FuzzerTests: XCTestCase { 6 | func testWeightedPick() { 7 | var r = FuzzerPRNG.init(seed: 0) 8 | var weights: [UInt64] = Array.init() 9 | for i in 0 ..< 10 { 10 | weights.append(UInt64(i)) 11 | } 12 | let cumulativeWeights = [1, 6, 7, 10, 11, 12, 13, 14, 17, 18, 19, 20].enumerated().map { ($0.0, $0.1) } 13 | // print(weights) 14 | print(cumulativeWeights.map { $0.1 }) 15 | var timesChosen = cumulativeWeights.map { _ in 0 } 16 | for _ in 0 ..< 100_000 { 17 | 18 | timesChosen[r.weightedRandomElement(from: cumulativeWeights, minimum: 0)] += 1 19 | } 20 | print(timesChosen) 21 | } 22 | 23 | func testRandom() { 24 | var r = FuzzerPRNG(seed: 2) 25 | var timesChosen = Array.init(repeating: 0, count: 128) 26 | for _ in 0 ..< 1_000_000 { 27 | let i = Int.random(in: 0 ..< timesChosen.count, using: &r)// timesChosen.indices.randomElement(using: &r)! 28 | timesChosen[i] += 1 29 | } 30 | print(timesChosen) 31 | } 32 | 33 | func testBoolWithOdds() { 34 | var r = FuzzerPRNG.init(seed: 2) 35 | var timesTrue = 0 36 | for _ in 0 ..< 1_000_000 { 37 | if r.bool(odds: 0.27) { 38 | timesTrue += 1 39 | } 40 | } 41 | print(timesTrue) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Tests/GraphGeneratorTests/MutationTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // GraphGeneratorTests.swift 3 | // FuzzerTests 4 | // 5 | // Created by Loïc Lecrenier on 30/06/2018. 6 | // 7 | 8 | import XCTest 9 | import Fuzzer 10 | import ModuleToTest 11 | @testable import ModuleToTestMutators 12 | 13 | class MutationTests: XCTestCase { 14 | func testMut1() { 15 | let gen = GraphGenerator.init() 16 | var g = Graph.init() 17 | _ = g.addVertex(0) 18 | _ = g.addVertex(1) 19 | _ = g.addVertex(2) 20 | _ = g.addVertex(3) 21 | var r = Rand(seed: 0) 22 | print(g.dotDescription()) 23 | for i in 0 ..< 1000 { 24 | print("iter:", i) 25 | print("size:", g.totalSize) 26 | print() 27 | _ = gen.mutators.mutate(&g, &r) 28 | } 29 | print(g.dotDescription()) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # FuzzCheck 2 | 3 | (Note: I talked about FuzzCheck at the Functional Swift Conference [here](https://www.youtube.com/watch?v=23_qZePMQjA). The talk explains the motivation behind it better than this readme.) 4 | 5 | FuzzCheck is an experimental coverage-guided fuzzing engine for Swift packages that works with typed values instead of raw binary buffers. 6 | 7 | The name “FuzzCheck” is a mix of “Fuzzer” and “QuickCheck”. The goal is to create a fuzzing engine that is convenient enough to use as the input generator for property-based tests. 8 | 9 | Given a test function `(Input) -> Bool`, it tries to find values of `Input` that will trigger edge cases in your code. It can also automatically minimize an input that fails a test. 10 | 11 | Because FuzzCheck is a Swift package itself, it is easier to modify than libFuzzer. If you would like to contribute to it, I am happy to guide you through the code. 12 | 13 | ## Installation 14 | 15 | FuzzCheck is not production ready. Using it requires both a development snapshot of the Swift compiler, and a custom build of the Swift Package Manager. That is because the compile flag `-sanitize=fuzzer` must be enabled for the tested targets. 16 | 17 | The good news is that once these tools are installed, FuzzCheck is just another dependency in your `Package.swift` file! 18 | 19 | - go to [swift.org/downloads](https://swift.org/download#snapshots) and download the Swift 4.2 Development Snapshot by clicking on the “Xcode” link and following the instructions. 20 | - Find the path of the `swiftc` executable you just installed and assign it to the `SWIFT_EXEC` environment variable. For example, if you installed the snapshot from the 3rd of July, you should run: 21 | ```bash 22 | export SWIFT_EXEC=/Library/Developer/Toolchains/swift-4.2-DEVELOPMENT-SNAPSHOT-2018-07-03-a.xctoolchain/usr/bin/swiftc 23 | ``` 24 | - Clone my fork of the Swift Package Manager. 25 | ```bash 26 | git clone https://github.com/loiclec/swift-package-manager 27 | ``` 28 | - Then build it. It should not take more than a few minutes. 29 | ```bash 30 | cd swift-package-manager 31 | Utilities/bootstrap 32 | ``` 33 | 34 | That's it! You now have everything you need to use FuzzCheck! 35 | The executables that you will need to use to compile your Swift packages are located inside `swift-package-manager/.build/x86_64-apple-macosx10.10/debug/` 36 | 37 | ```bash 38 | # swiftc: verify that its version contains `Apple Swift version 4.2-dev` 39 | .build/x86_64-apple-macosx10.10/debug/swiftc --version 40 | Apple Swift version 4.2-dev (LLVM 647959670b, Clang 8756d7b836, Swift 107e307eae) 41 | Target: x86_64-apple-darwin18.0.0 42 | 43 | # swift-build replaces `swift build` 44 | .build/x86_64-apple-macosx10.10/debug/swift-build --version 45 | Swift Package Manager - Swift 4.2.0 46 | 47 | # swift-package replaces `swift package` 48 | .build/x86_64-apple-macosx10.10/debug/swift-package --version 49 | 50 | Swift Package Manager - Swift 4.2.0 51 | ``` 52 | 53 | ## Using FuzzCheck 54 | 55 | I have created a sample project called `FuzzCheckExample` that you can use to get familiar with FuzzCheck. But before explaining how it works, let's try to launch it and finally see some results! 56 | 57 | ```bash 58 | git clone https://github.com/loiclec/FuzzCheckExample.git 59 | cd FuzzCheckExample 60 | # Use the swift-build executable from the modified SwiftPM and use the fuzz-release configuration 61 | ../swift-package-manager/.build/x86_64-apple-macosx10.10/debug/swift-build -c fuzz-release 62 | # launch FuzzCheckTool with the test target as argument 63 | .build/fuzz-release/FuzzCheckTool --target FuzzCheckExample 64 | ``` 65 | After a few seconds, the process will stop: 66 | ``` 67 | ... 68 | ... 69 | DELETE 1 70 | NEW 528502 score: 122.0 corp: 81 exec/s: 103402 rss: 8 71 | DELETE 1 72 | NEW 528742 score: 122.0 corp: 81 exec/s: 103374 rss: 8 73 | 74 | ================ TEST FAILED ================ 75 | 529434 score: 122.0 corp: 81 exec/s: 103374 rss: 8 76 | Saving testFailure at /Users/loic/Projects/fuzzcheck-example/artifacts/testFailure-78a1af7b1be086ca0.json 77 | ``` 78 | 79 | It detected a test failure after 529434 iterations, and it saved the JSON-encoded crashing input inside the file `testFailure-78a1af7b1be086ca0.json`. Half a million iterations might seem like a lot, but if the test used a simple exhaustive or random search, it would have *never* found that test failure, even after trillions of iterations. 80 | 81 | The `Package.swift` manifest of FuzzCheckExample is: 82 | 83 | ```swift 84 | // swift-tools-version:4.2 85 | import PackageDescription 86 | 87 | let package = Package( 88 | name: "FuzzCheckExample", 89 | products: [ 90 | .library(name: "Graph", targets: ["Graph"]), 91 | .executable(name: "FuzzCheckExample", targets: ["FuzzCheckExample"]) 92 | ], 93 | dependencies: [ 94 | .package(url: "https://github.com/loiclec/FuzzCheck.git", .revision("b4abbf661f4d187ec88bc2811893283d4c091260")) 95 | ], 96 | targets: [ 97 | .target(name: "FuzzCheckExample", dependencies: [ 98 | "FuzzCheck", 99 | "FuzzCheckTool", 100 | "GraphFuzzerInputGenerator", 101 | "Graph" 102 | ]), 103 | .target(name: "GraphFuzzerInputGenerator", dependencies: ["FuzzCheck", "Graph"]), 104 | .target(name: "Graph", dependencies: []) 105 | ], 106 | fuzzedTargets: [ 107 | "FuzzCheckExample", 108 | "Graph" 109 | ] 110 | ) 111 | ``` 112 | 113 | This manifest: 114 | - has a `FuzzCheck` dependency, pinned to a specific commit (no stable version has been released yet) 115 | - contains one fuzz-test executable called `FuzzCheckExample`. Its target depends on `FuzzCheck` and `FuzzCheckTool`. 116 | - has a `fuzzedTargets` argument containing the targets that needs to be compiled with the `fuzzer` sanitizer 117 | - has a `GraphFuzzerInputGenerator` target, which defines how to mutate values of type `Graph`. This is required by FuzzCheck. 118 | 119 | The test itself is located inside `Sources/FuzzCheckExample/main.swift`: 120 | ```swift 121 | import FuzzCheck 122 | import GraphFuzzerInputGenerator 123 | import Graph 124 | 125 | func test(_ g: Graph) -> Bool { 126 | if 127 | g.count == 8, 128 | g.vertices[0].data == 100, 129 | g.vertices[1].data == 89, 130 | g.vertices[2].data == 10, 131 | g.vertices[3].data == 210, 132 | g.vertices[4].data == 1, 133 | g.vertices[5].data == 210, 134 | g.vertices[6].data == 9, 135 | g.vertices[7].data == 17, 136 | g.vertices[0].edges.count == 2, 137 | g.vertices[0].edges[0] == 1, 138 | g.vertices[0].edges[1] == 2, 139 | g.vertices[1].edges.count == 2, 140 | g.vertices[1].edges[0] == 3, 141 | g.vertices[1].edges[1] == 4, 142 | g.vertices[2].edges.count == 2, 143 | g.vertices[2].edges[0] == 5, 144 | g.vertices[2].edges[1] == 6, 145 | g.vertices[3].edges.count == 1, 146 | g.vertices[3].edges[0] == 7, 147 | g.vertices[4].edges.count == 0, 148 | g.vertices[5].edges.count == 0, 149 | g.vertices[6].edges.count == 0, 150 | g.vertices[7].edges.count == 0 151 | { 152 | return false 153 | } 154 | return true 155 | } 156 | 157 | let generator = 158 | GraphFuzzerInputGenerator>( 159 | vertexGenerator: .init() 160 | ) 161 | 162 | try CommandLineFuzzer.launch( 163 | test: test, 164 | generator: generator 165 | ) 166 | 167 | ``` 168 | 169 | It is a silly test that only fails when the graph data structure given as input is equal to this: 170 | ``` 171 | ┌─────┐ 172 | │ 100 │ 173 | └─┬─┬─┘ 174 | ┌──────┘ └──────┐ 175 | ┌──▼──┐ ┌──▼──┐ 176 | │ 89 │ │ 10 │ 177 | └──┬──┘ └──┬──┘ 178 | ┌────┴───┐ ┌───┴───┐ 179 | ┌──▼──┐ ┌──▼──┐ ┌──▼──┐ ┌──▼──┐ 180 | │ 210 │ │ 1 │ │ 210 │ │ 9 │ 181 | └──┬──┘ └─────┘ └─────┘ └─────┘ 182 | │ 183 | ┌──▼──┐ 184 | │ 17 │ 185 | └─────┘ 186 | ``` 187 | 188 | Without passing on any special knowledge to the fuzzer about the test, it was able to find this graph in less than 1_000_000 iterations! This is impressive considering that merely finding the 8 values of its vertices would take an average of `256^7 ~= 70_000_000_000_000_000` iterations by a simple exhaustive or random search. 189 | 190 | ## Creating a fuzz test 191 | 192 | To test a function `(Input) -> Bool`, you need a `FuzzerInputGenerator` to generate values of `Input`. A `FuzzerInputGenerator` has three requirements: 193 | 1. a property `baseInput` containing the simplest possible value of `Input` (e.g. the empty array) 194 | 2. a function to slightly mutate values of `Input` (e.g. append an element to an array) 195 | 3. a function to generate random values of `Input` (there is a default implementation of that one based on the `mutate` function) 196 | 197 | ```swift 198 | public protocol FuzzerInputGenerator: FuzzerInputProperties { 199 | associatedtype Input 200 | 201 | var baseInput: Input { get } 202 | func initialInputs(maxComplexity: Double, _ rand: inout FuzzerPRNG) -> [Input] 203 | func mutate(_ input: inout Input, _ spareComplexity: Double, _ rand: inout FuzzerPRNG) -> Bool 204 | } 205 | ``` 206 | 207 | `FuzzerInputGenerator` also conforms to `FuzzerInputProperties`, which gives the complexity of an input and its hash: 208 | ```swift 209 | public protocol FuzzerInputProperties { 210 | associatedtype Input 211 | 212 | static func complexity(of input: Input) -> Double 213 | static func hash(_ input: Input, into hasher: inout Hasher) 214 | } 215 | ``` 216 | 217 | I hope to provide many default implementations of these two protocols out-of-the-box, and to have tools to create them automatically (e.g. using Sourcery). But for now, you will have to implement them yourself. 218 | 219 | ## Design 220 | 221 | FuzzCheck works by maintaining a pool of test inputs and ranking them using the complexity of the input and the uniqueness of the code coverage caused by `test(input)`. From that pool, it selects a high-ranking input, mutates it, and runs the test function again. If the new mutated input discovered new code paths in the binary, then it is added to the pool, otherwise, FuzzCheck tries again with a different input and mutation. 222 | 223 | In pseudocode (the actual implementation is a bit more nuanced): 224 | ``` 225 | while true { 226 | var input = pool.select() 227 | mutate(&input) 228 | 229 | let analysis = analyze { test(input) } 230 | 231 | switch analysis { 232 | case .crashed, .failed: 233 | return reportFailure(input) 234 | case .interesting(let score): 235 | pool.add(input, score) 236 | case .notInteresting: 237 | continue 238 | } 239 | } 240 | ``` 241 | 242 | ## Caveats 243 | 244 | - tests that use more than one thread are not supported (but should be in the future) 245 | - almost no FuzzerInputGenerator implementations are provided yet 246 | - large codebases will be slow to fuzz-test. Work needs to be done to make it faster 247 | - some of the fundamental design and implementation details might still change 248 | 249 | ## History 250 | 251 | FuzzCheck was originally a copy of LLVM’s libFuzzer, but it has changed significantly since then. The main differences are that FuzzCheck: 252 | - works with typed values instead of raw binary buffers 253 | - uses a different algorithm to rank the inputs in the pool 254 | - is designed such that a different measure than code coverage could be used to rank the inputs in the pool 255 | - does not support crossover mutations 256 | - does not work with dictionaries, manual or automatic 257 | - only works with Swift programs 258 | - is not integrated with the address sanitizer 259 | - is not battle-tested and has not proven itself yet 260 | --------------------------------------------------------------------------------