├── .gitignore ├── LICENSE.md ├── README.md ├── project ├── Build.scala ├── build.properties └── plugins.sbt ├── src └── main │ ├── aspectj │ └── WeaveActor.aj │ ├── python │ ├── __init__.py │ ├── lifecycle.py │ ├── minimization_stats │ │ ├── __init__.py │ │ ├── combine_graphs.py │ │ ├── generate_graph.py │ │ └── gpi_template.py │ ├── setup.py │ └── util.py │ └── scala │ └── verification │ ├── CheckpointCollector.scala │ ├── DepTracker.scala │ ├── EventTrace.scala │ ├── ExternalEvents.scala │ ├── FailureDetector.scala │ ├── Instrumenter.scala │ ├── MessageFingerprints.scala │ ├── RunnerUtils.scala │ ├── SchedulerConfig.scala │ ├── Serialization.scala │ ├── fuzzing │ └── Fuzzer.scala │ ├── minification │ ├── DeltaDebugging.scala │ ├── IncrementalDeltaDebugging.scala │ ├── Minimizer.scala │ ├── OneAtATime.scala │ ├── TestOracle.scala │ ├── Util.scala │ ├── WildcardTestOracle.scala │ ├── internal_minimization │ │ ├── OneAtATimeRemoval.scala │ │ ├── RemovalStrategy.scala │ │ ├── ScheduleCheckers.scala │ │ └── StateMachineRemoval.scala │ └── wildcard_minimization │ │ ├── AmbiguityResolutionStrategies.scala │ │ ├── ClockClusterizer.scala │ │ ├── Clusterizer.scala │ │ ├── DeltaDebuggingClusterizer.scala │ │ ├── OneAtATimeClusterizer.scala │ │ └── WildcardMinimizer.scala │ └── schedulers │ ├── AbstractScheduler.scala │ ├── AuxilaryTypes.scala │ ├── BacktrackOrdering.scala │ ├── BasicScheduler.scala │ ├── DPOR.scala │ ├── DPORwHeuristics.scala │ ├── EventOrchestrator.scala │ ├── ExternalEventInjector.scala │ ├── FairScheduler.scala │ ├── InteractiveScheduler.scala │ ├── IntervalPeekScheduler.scala │ ├── NullScheduler.scala │ ├── PeekScheduler.scala │ ├── RandomScheduler.scala │ ├── ReplayScheduler.scala │ ├── STSScheduler.scala │ ├── Scheduler.scala │ └── Util.scala └── tools ├── combine_graphs_for_experiment.rb ├── indent.rb ├── overwrite_uninteresting_fuzz_runs.rb ├── rerun_experiments.sh ├── sanity.rb ├── spacify.rb └── subtree_pull_all.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache 6 | .history 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | 15 | # Scala-IDE specific 16 | .scala_dependencies 17 | .worksheet 18 | *.iml 19 | .idea/* 20 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Colin Scott, Aurojit Panda, Vjekoslav Brajkovic. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | The views and conclusions contained in the software and documentation are those 25 | of the authors and should not be interpreted as representing official policies, 26 | either expressed or implied, of the FreeBSD Project. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEMi 2 | 3 | This is the home of Distributed Execution Minimizer (DEMi), a fuzzing and test 4 | case reducing tool for distributed systems. Currently we only support applications 5 | built on top of [Akka](http://akka.io/), but adapting another RPC library to 6 | call into DEMi shouldn't be too difficult. 7 | 8 | ## Useful resources: 9 | 10 | - [Blog post](http://colin-scott.github.io/blog/2015/10/07/fuzzing-raft-for-fun-and-profit) covering how DEMi does fuzz testing, and the bugs we found in a Raft implementation using DEMi. 11 | - Our NSDI 2016 [paper](http://eecs.berkeley.edu/~rcs/research/nsdi16.pdf) describing the system in detail. 12 | - Example applications tested with DEMi can be found [here](https://github.com/NetSys/demi-applications). 13 | 14 | ## Current status of this project 15 | 16 | Although DEMi's features are fairly well fleshed out (e.g., we've used it to test [Spark](http://spark.apache.org/)), so far it has only been used by us. 17 | That is to say, there isn't a whole lot of documentation. 18 | 19 | If you're interested in using DEMi, we'd be more than happy to write up 20 | documentation, and help you iron out any issues you run into. We'd love to see 21 | DEMi applied more broadly! 22 | 23 | ## Whirlwind tour of how to use DEMi 24 | 25 | For now, a whirwind tour: 26 | 27 | - this [repository](https://github.com/NetSys/demi) contains the tool itself 28 | - an application pulls in DEMi by having it as a git subtree, located at `interposition/`. See the various branches of this [repository](https://github.com/NetSys/demi-applications) for examples. 29 | - in the application's Build.scala (or build.sbt), we need to include AspectJ, through the sbt-aspectj plugin. Here is an example: https://github.com/NetSys/demi-applications/blob/raft-45/project/Build.scala#L3. When we run `sbt build` from the application's top-level directory, AspectJ interposition will be automatically weaved in. The AspectJ code itself is [here](https://github.com/NetSys/demi/blob/master/src/main/aspectj/WeaveActor.aj). It mostly weaves interfaces within Akka, and should hopefully work on most versions of higher than 2.0. 30 | - In the application's Main method, we need to configure DEMi . For now, here is very verbose [example](https://github.com/NetSys/demi-applications/blob/raft-45/src/main/scala/pl/project13/Runner.scala). That example contains a whole bunch of cruft that is needed for minimization (+ other auxiliary stuff like recording the execution, writing it to disk, and replaying it later), but is not needed for simple fuzz testing. For just fuzz testing, you should be able to do it in a few lines of code. I'd be glad to come up with an example if you're interested. 31 | 32 | -------------------------------------------------------------------------------- /project/Build.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | import com.typesafe.sbt.SbtAspectj.{ Aspectj, aspectjSettings, useInstrumentedClasses } 4 | import com.typesafe.sbt.SbtAspectj.AspectjKeys.inputs 5 | 6 | object InterpositionBuild extends Build { 7 | lazy val interposition = Project( 8 | id = "interposition", 9 | base = file("."), 10 | settings = Defaults.defaultSettings ++ aspectjSettings ++ Seq( 11 | organization := "com.typesafe.sbt.aspectj", 12 | version := "0.1", 13 | scalaVersion := "2.11.2", 14 | libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.3.6", 15 | libraryDependencies += "com.typesafe.akka" %% "akka-cluster" % "2.3.6", 16 | libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.2", 17 | libraryDependencies += "com.assembla.scala-incubator" %% "graph-core" % "1.9.0", 18 | libraryDependencies += "com.assembla.scala-incubator" %% "graph-dot" % "1.9.0", 19 | libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0", 20 | libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.1.2", 21 | 22 | 23 | // add akka-actor as an aspectj input (find it in the update report) 24 | inputs in Aspectj <++= update map { report => 25 | report.matching(moduleFilter(organization = "com.typesafe.akka", name = "akka-actor*")) 26 | }, 27 | 28 | // replace the original akka-actor jar with the instrumented classes in runtime 29 | fullClasspath in Runtime <<= useInstrumentedClasses(Runtime) 30 | ) 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-aspectj" % "0.10.0") 2 | -------------------------------------------------------------------------------- /src/main/python/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NetSys/demi/e3ad09efa2c193bb8caac79a3141539e6eea9a3e/src/main/python/__init__.py -------------------------------------------------------------------------------- /src/main/python/lifecycle.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2013 Andreas Wundsam 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at: 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import getpass 16 | import os 17 | import re 18 | import socket 19 | import sys 20 | import json 21 | import time 22 | from util import timestamp_string, find, backtick, system 23 | import subprocess 24 | 25 | 26 | 27 | def dump_metadata(metadata_file, additional_metadata=None): 28 | with open(metadata_file, "w") as t: 29 | metadata = { 'timestamp' : timestamp_string(), 30 | 'argv' : sys.argv, 31 | 'user' : getpass.getuser(), 32 | 'cwd' : os.getcwd(), 33 | 'host' : { 34 | 'name' : socket.gethostname(), 35 | 'uptime' : backtick("uptime"), 36 | 'free' : backtick("exec 2>/dev/null free"), 37 | 'num_cores' : backtick("cat 2>/dev/null /proc/cpuinfo | grep '^processor[[:space:]]' | wc -l"), 38 | 'cpu_info' : backtick("cat 2>/dev/null /proc/cpuinfo | grep 'model name[[:space:]]' | uniq | sed 's/.*://' | perl -pi -e 's/\s+/ /g'") 39 | }, 40 | 'sys' : { 41 | 'lsb_release' : backtick("exec 2>/dev/null lsb_release --description --short"), 42 | 'uname' : backtick("uname -a") 43 | }, 44 | 'modules' : { 45 | module : { 'commit' : backtick("git rev-parse HEAD", cwd=path), 46 | 'branch' : backtick("git rev-parse --abbrev-ref HEAD", cwd=path), 47 | 'status' : backtick("git status", cwd=path), 48 | 'diff' : backtick("git diff", cwd=path) 49 | } for module, path in [("sts2", "./")] 50 | }, 51 | 'additional_metadata': additional_metadata, 52 | } 53 | t.write(json.dumps(metadata, sort_keys=True, indent=2, separators=(',', ": ")) + "\n") 54 | 55 | def walk_dirs_up(path): 56 | while path != "" and path != "/": 57 | yield path 58 | path = os.path.dirname(path) 59 | 60 | def find_git_dir(results_dir): 61 | return find(lambda f: os.path.exists(os.path.join(f, ".git" )), walk_dirs_up(results_dir)) 62 | 63 | def git_has_uncommitted_files(d): 64 | return system("git diff-files --quiet --ignore-submodules --", cwd=d) > 0 \ 65 | or system("git diff-index --cached --quiet HEAD --ignore-submodules --", cwd=d) > 0 66 | 67 | def publish_prepare(exp_name, results_dir): 68 | for module, path in sts_modules: 69 | if git_has_uncommitted_files(path): 70 | raise Exception("Cannot publish: uncommitted changes in sts module %s" % module) 71 | 72 | res_git_dir = find_git_dir(results_dir) 73 | if not res_git_dir: 74 | raise Exception("Cannot publish - no git dir found in results tree") 75 | 76 | def publish_results(exp_name, results_dir): 77 | import logging 78 | log = logging.getLogger("sts.exp_lifecycle") 79 | res_git_dir = find_git_dir(results_dir) 80 | rel_results_dir = os.path.relpath(results_dir, res_git_dir) 81 | log.info("Publishing results to git dir "+res_git_dir) 82 | system("git add %s" % rel_results_dir, cwd=res_git_dir) 83 | system("git commit -m '%s'" % exp_name, cwd=res_git_dir) 84 | system("git pull --rebase", cwd=res_git_dir) 85 | system("git push", cwd=res_git_dir) 86 | -------------------------------------------------------------------------------- /src/main/python/minimization_stats/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NetSys/demi/e3ad09efa2c193bb8caac79a3141539e6eea9a3e/src/main/python/minimization_stats/__init__.py -------------------------------------------------------------------------------- /src/main/python/minimization_stats/combine_graphs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | # 3 | # Copyright 2011-2013 Colin Scott 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at: 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | from generate_graph import write_data_file, load_json, DataInfo, write_gpi_template, invoke_gnuplot 18 | import string 19 | import argparse 20 | import json 21 | import os 22 | import sys 23 | 24 | if __name__ == '__main__': 25 | parser = argparse.ArgumentParser(description=( 26 | '''Specify any number of minimization_stats.json files to be combined into ''' 27 | '''a single graph. Usage:\n''' 28 | ''' $0 <path to 2nd json> ''' 29 | ''' <title for 2nd json> ...''' 30 | )) 31 | # Need at least one input file to know where to infer where to put the output. 32 | parser.add_argument('input1', metavar="INPUT1", 33 | help='''The first input json file''') 34 | parser.add_argument('title1', metavar="TITLE1", 35 | help='''The title for input1's line''') 36 | args, unknown = parser.parse_known_args() 37 | if (len(unknown) % 2) != 0: 38 | print >> sys.stderr, "Uneven number of arguments. Need titles!" 39 | sys.exit(1) 40 | 41 | basename = os.path.basename(args.input1) 42 | dirname = os.path.dirname(os.path.dirname(args.input1)) 43 | gpi_filename = string.replace(dirname + "/combined_" + basename, ".json", ".gpi") 44 | output_filename = string.replace(dirname + "/combined_" + basename, ".json", ".pdf") 45 | 46 | # Turn each adjacent pair of elements into a tuple 47 | # http://stackoverflow.com/questions/4628290/pairs-from-single-list 48 | pairs = zip(unknown[::2], unknown[1::2]) 49 | 50 | graph_title = "" 51 | 52 | data_info_list = [] 53 | for input_json, line_title in [(args.input1, args.title1)] + pairs: 54 | dat_filename = string.replace(input_json, ".json", ".dat") 55 | stats = load_json(input_json) 56 | write_data_file(dat_filename, stats) 57 | data_info_list.append(DataInfo(title=line_title, filename=dat_filename)) 58 | 59 | write_gpi_template(gpi_filename, output_filename, data_info_list, graph_title) 60 | invoke_gnuplot(gpi_filename) 61 | 62 | print "Output placed in %s" % output_filename 63 | -------------------------------------------------------------------------------- /src/main/python/minimization_stats/generate_graph.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | # 3 | # Copyright 2011-2013 Colin Scott 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at: 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # TODO(cs): should use matplotlib instead of the template 18 | from gpi_template import template 19 | import string 20 | import argparse 21 | import json 22 | import os 23 | from collections import namedtuple 24 | 25 | # Returns: 26 | # hashmap: 27 | # { 28 | # { iteration -> iteration size } 29 | # { internal_iteration -> iteration size } 30 | # } 31 | def interpolate_datapoints(stats): 32 | # Two tasks: 33 | # - String all of the inner json objects into a single hashmap of datapoints 34 | # - If there is a gap in iteration sizes, fill it in with the previous 35 | # value 36 | # TODO(cs): visually delineate the different stages of minimization 37 | return { 38 | "iteration_size": get_external_progress(stats), 39 | "internal_iteration_size": get_internal_progress(stats) 40 | } 41 | 42 | def get_internal_progress(stats): 43 | ret = {} 44 | iteration = 0 45 | 46 | # Provenance isn't applied in all cases, so use original 47 | # provenance = stats[1] 48 | orig = stats[1] 49 | ret[iteration] = int(orig["minimized_deliveries"]) 50 | iteration += 1 51 | 52 | for obj in stats[2:]: 53 | sorted_keys = sorted(map(int, obj["internal_iteration_size"].keys())) 54 | 55 | if (len(sorted_keys) == 0): 56 | # This wasn't an internal minimization run. The only (monotonic) progress that would 57 | # have been made would be at the very end, where the external event MCS 58 | # was verified, and there were absent expected events. 59 | # TODO(cs): probably off by one on the x-axis 60 | for i in range(obj["total_replays"]-1): 61 | ret[iteration] = ret[iteration-1] 62 | iteration += 1 63 | ret[iteration] = obj["minimized_deliveries"] 64 | iteration += 1 65 | continue 66 | 67 | # Smallest index so far for this object: 68 | bottom = 1 69 | for k in sorted_keys: 70 | while bottom < k: 71 | ret[iteration] = ret[iteration-1] 72 | iteration += 1 73 | bottom += 1 74 | ret[iteration] = int(obj["internal_iteration_size"][str(k)]) 75 | iteration += 1 76 | bottom += 1 77 | 78 | return ret 79 | 80 | def get_external_progress(stats): 81 | ret = {} 82 | iteration = 0 83 | 84 | # Get initial iteration_size from first DDMin run. 85 | # ["minimized_externals" does not include Start events] 86 | lowest_key = str(map(int, sorted(stats[2]["iteration_size"].keys()))[0]) 87 | ret[iteration] = stats[2]["iteration_size"][lowest_key] 88 | iteration += 1 89 | 90 | for obj in stats[2:]: 91 | sorted_keys = sorted(map(int, obj["iteration_size"].keys())) 92 | 93 | if (len(sorted_keys) == 0): 94 | # This wasn't an external minimization run, so no external progress was 95 | # made 96 | for i in range(obj["total_replays"]): 97 | ret[iteration] = ret[iteration-1] 98 | iteration += 1 99 | continue 100 | 101 | # Smallest index so far for this object: 102 | bottom = 1 103 | for k in sorted_keys: 104 | while bottom < k: 105 | ret[iteration] = ret[iteration-1] 106 | iteration += 1 107 | bottom += 1 108 | ret[iteration] = int(obj["iteration_size"][str(k)]) 109 | iteration += 1 110 | bottom += 1 111 | return ret 112 | 113 | def write_data_file(dat_filename, stats): 114 | ''' Write out the datapoints. Return the maximum x-value ''' 115 | xmax = 0 116 | datapoints = interpolate_datapoints(stats) 117 | for t in ["internal_iteration_size", "iteration_size"]: 118 | sorted_keys = datapoints[t].keys() 119 | sorted_keys.sort(lambda a,b: -1 if int(a) < int(b) else 1 if int(a) > int(b) else 0) 120 | if int(sorted_keys[-1]) > xmax: 121 | xmax = int(sorted_keys[-1]) 122 | with open(dat_filename + "_" + t, "w") as dat: 123 | for key in sorted_keys: 124 | dat.write(str(key) + " " + str(datapoints[t][key]) + '\n') 125 | return xmax 126 | 127 | def load_json(json_input): 128 | with open(json_input) as json_input_file: 129 | return json.load(json_input_file) 130 | 131 | DataInfo = namedtuple('DataInfo', ['filename', 'title']) 132 | 133 | def write_gpi_template(gpi_filename, output_filename, data_info_list, xmax=None, title=""): 134 | with open(gpi_filename, "w") as gpi: 135 | # Finish off the rest of the template 136 | gpi.write(template) 137 | if title != "": 138 | gpi.write('''set title "%s"\n''' % title) 139 | gpi.write('''set output "%s"\n''' % output_filename) 140 | if xmax != None: 141 | gpi.write('''set xrange [0:%d]\n''' % xmax) 142 | gpi.write('''plot ''') 143 | line_type_counter = 1 144 | for i, data_info in enumerate(data_info_list): 145 | first_t = True 146 | for t,title in [("internal_iteration_size", "Deliveries"), 147 | ("iteration_size", "Externals")]: 148 | expanded_title = title if data_info.title == "" else data_info.title + "_" + title 149 | gpi.write('''"%s" index 0:1 title "%s" with steps ls %d''' % 150 | (data_info.filename + "_" + t, expanded_title, 151 | line_type_counter)) 152 | line_type_counter += 1 153 | if first_t: 154 | gpi.write(", \\\n") 155 | first_t = False 156 | elif i != len(data_info_list) - 1: 157 | gpi.write(", \\\n") 158 | else: 159 | gpi.write("\n") 160 | 161 | def invoke_gnuplot(gpi_filename): 162 | os.system("gnuplot %s" % gpi_filename) 163 | 164 | if __name__ == '__main__': 165 | parser = argparse.ArgumentParser(description="generate a plot") 166 | parser.add_argument('input', metavar="INPUT", 167 | help='''The input json file''') 168 | parser.add_argument('-x', '--xmax', type=int, 169 | help='''Truncate the x dimension''') 170 | 171 | args = parser.parse_args() 172 | 173 | stats = load_json(args.input) 174 | 175 | gpi_filename = string.replace(args.input, ".json", ".gpi") 176 | output_filename = string.replace(args.input, ".json", ".pdf") 177 | dat_filename = string.replace(args.input, ".json", ".dat") 178 | 179 | max_x_value = write_data_file(dat_filename, stats) 180 | data_info_list = [DataInfo(title="", filename=dat_filename)] 181 | xmax = args.xmax 182 | if (xmax == None): 183 | xmax = max_x_value 184 | write_gpi_template(gpi_filename, output_filename, data_info_list, xmax=xmax) 185 | invoke_gnuplot(gpi_filename) 186 | -------------------------------------------------------------------------------- /src/main/python/minimization_stats/gpi_template.py: -------------------------------------------------------------------------------- 1 | 2 | template = """ 3 | # Note you need gnuplot 4.4 for the pdfcairo terminal. 4 | 5 | set terminal pdf font "Helvetica, 12" linewidth 4 6 | 7 | # Line style for axes 8 | set style line 80 lt rgb "#808080" 9 | 10 | # Line style for grid 11 | set style line 81 lt 0 # dashed 12 | set style line 81 lt rgb "#808080" # grey 13 | 14 | set grid back linestyle 81 15 | set border 3 back linestyle 80 # Remove border on top and right. These 16 | # borders are useless and make it harder 17 | # to see plotted lines near the border. 18 | # Also, put it in grey; no need for so much emphasis on a border. 19 | set xtics nomirror 20 | set ytics nomirror 21 | 22 | #set log x 23 | #set mxtics 10 # Makes logscale look good. 24 | 25 | # Line styles: try to pick pleasing colors, rather 26 | # than strictly primary colors or hard-to-see colors 27 | # like gnuplot's default yellow. Make the lines thick 28 | # so they're easy to see in small plots in papers. 29 | set style line 3 lt rgb "#00A000" lw 2 pt 6 30 | set style line 2 lt rgb "#A00000" lw 2 pt 1 31 | set style line 1 lt rgb "#5060D0" lw 2 pt 2 32 | set style line 4 lt rgb "#F25900" lw 2 pt 9 33 | 34 | set key top right 35 | 36 | set ylabel "Number of Remaining Events" 37 | set xlabel "Number of Schedules Executed" 38 | 39 | set yrange [0:] 40 | 41 | # Note that we're leaving out output, title, and plot 42 | #set output "runtime_graph.pdf" 43 | #set title "runtime seconds=?, replay duration=?" 44 | #plot 'foo.dat' index 0:1 title "" w lp ls 1 45 | """ 46 | -------------------------------------------------------------------------------- /src/main/python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2011-2013 Andreas Wundsam 3 | # Copyright 2011-2013 Colin Scott 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at: 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import lifecycle as exp_lifecycle 18 | from util import timestamp_string, mkdir_p, rm_rf, create_clean_dir, find 19 | 20 | import os 21 | import shutil 22 | import re 23 | import logging 24 | import errno 25 | import sys 26 | import argparse 27 | 28 | def setup_experiment(args, config): 29 | # Grab parameters 30 | if args.exp_name: 31 | config.exp_name = args.exp_name 32 | # elif not hasattr(config, 'exp_name'): 33 | # config.exp_name = exp_lifecycle.guess_config_name(config) 34 | 35 | if not hasattr(config, 'results_dir'): 36 | config.results_dir = "experiments/%s" % config.exp_name 37 | 38 | if args.timestamp_results is not None: 39 | # Note that argparse returns a list 40 | config.timestamp_results = args.timestamp_results 41 | 42 | if hasattr(config, 'timestamp_results') and config.timestamp_results: 43 | now = timestamp_string() 44 | config.results_dir += "_" + str(now) 45 | 46 | # Set up results directory 47 | mkdir_p("./experiments") 48 | create_clean_dir(config.results_dir) 49 | 50 | # Make sure that there are no uncommited changes 51 | if args.publish: 52 | exp_lifecycle.publish_prepare(config.exp_name, config.results_dir) 53 | 54 | # Record machine information for this experiment 55 | additional_metadata = None 56 | if hasattr(config, "get_additional_metadata"): 57 | additional_metadata = config.get_additional_metadata() 58 | 59 | exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir, 60 | additional_metadata=additional_metadata) 61 | 62 | # # Copy over config file 63 | # config_file = re.sub(r'\.pyc$', '.py', config.__file__) 64 | # if os.path.exists(config_file): 65 | # canonical_config_file = config.results_dir + "/orig_config.py" 66 | # if os.path.abspath(config_file) != os.path.abspath(canonical_config_file): 67 | # shutil.copy(config_file, canonical_config_file) 68 | 69 | def top_level_prefix(): 70 | # TODO(cs): use a string builder 71 | prefix = "." 72 | 73 | def areWeThereYet(prefix): 74 | return "interposition" in os.listdir(prefix) 75 | 76 | while (not areWeThereYet(prefix)): 77 | prefix = prefix + "/.." 78 | 79 | return prefix + "/" 80 | 81 | 82 | if __name__ == '__main__': 83 | description = """ 84 | Create an experiment directory, and fill it with a metadata file. 85 | """ 86 | 87 | parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, 88 | description=description) 89 | 90 | parser.add_argument('-v', '--verbose', action="count", default=0, 91 | help='''increase verbosity''') 92 | 93 | parser.add_argument('-n', '--exp-name', dest="exp_name", 94 | default=None, required=True, 95 | help='''experiment name (determines result directory name)''') 96 | 97 | parser.add_argument('-t', '--timestamp-results', dest="timestamp_results", 98 | default=False, action="store_true", 99 | help='''whether to append a timestamp to the result directory name''') 100 | 101 | parser.add_argument('-p', '--publish', action="store_true", default=False, 102 | help='''automatically publish experiment results to git''') 103 | 104 | args = parser.parse_args() 105 | class Config(object): 106 | pass 107 | config = Config() 108 | 109 | os.chdir(top_level_prefix()) 110 | 111 | setup_experiment(args, config) 112 | print os.path.abspath(config.results_dir) 113 | -------------------------------------------------------------------------------- /src/main/python/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import re 4 | import logging 5 | import errno 6 | import sys 7 | import subprocess 8 | import time 9 | 10 | def timestamp_string(): 11 | return time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime()) 12 | 13 | def mkdir_p(dst): 14 | try: 15 | os.makedirs(dst) 16 | except OSError as exc: 17 | if exc.errno == errno.EEXIST and os.path.isdir(dst): 18 | pass 19 | else: 20 | raise 21 | 22 | def rm_rf(dst): 23 | try: 24 | if os.path.exists(dst): 25 | shutil.rmtree(dst) 26 | except OSError: 27 | pass 28 | 29 | def create_clean_dir(results_dir): 30 | if os.path.exists(results_dir): 31 | print >> sys.stderr, "Results dir %s already exists. Overwriting.." % results_dir 32 | rm_rf(results_dir) 33 | mkdir_p(results_dir) 34 | 35 | def find(f, seq): 36 | """Return first item in sequence where f(item) == True.""" 37 | for item in seq: 38 | if f(item): 39 | return item 40 | 41 | def backtick(cmd, *args, **kwargs): 42 | return subprocess.Popen(cmd, *args, shell=True, stdout=subprocess.PIPE, **kwargs).stdout.read().strip() 43 | 44 | def system(cmd, *args, **kwargs): 45 | return subprocess.call(cmd, *args, shell=True, **kwargs) 46 | -------------------------------------------------------------------------------- /src/main/scala/verification/CheckpointCollector.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{Actor, ActorRef, ActorSystem, Props} 4 | 5 | import scala.collection.mutable.HashMap 6 | 7 | // For checking invariants on application state: 8 | final case object CheckpointRequest 9 | final case class CheckpointReply(data: Any) 10 | 11 | object CheckpointSink { 12 | var name = "checkpoint_sink" 13 | } 14 | 15 | // A fake actor, used as a placeholder to which checkpoint responses can be sent. 16 | // The scheduler intercepts messages sent to this actor. 17 | class CheckpointSink() extends Actor { 18 | def receive = { 19 | // This should never be called 20 | case _ => assert(false) 21 | } 22 | } 23 | 24 | class CheckpointCollector { 25 | // If a node is crashed, the value will be None rather than 26 | // Some(CheckpointReply) 27 | var checkpoints = new HashMap[String, Option[CheckpointReply]] 28 | var expectedResponses = 0 29 | 30 | def startCheckpointCollector(actorSystem: ActorSystem) { 31 | actorSystem.actorOf(Props[CheckpointSink], CheckpointSink.name) 32 | } 33 | 34 | // Pre: prepareRequests was recently invoked. 35 | def done() : Boolean = { 36 | return checkpoints.size == expectedResponses 37 | } 38 | 39 | def prepareRequests(actorRefs: Seq[ActorRef]) : Seq[(String, Any)] = { 40 | checkpoints.clear() 41 | val crashedActors = Instrumenter().crashedActors ++ 42 | actorRefs.filter(ref => ref.isTerminated).map(r => r.path.name) 43 | for (crashed <- crashedActors) { 44 | checkpoints(crashed) = None 45 | } 46 | expectedResponses = actorRefs.length - crashedActors.size 47 | return actorRefs.filterNot(ref => crashedActors contains ref.path.name). 48 | map(ref => ((ref.path.name, CheckpointRequest))) 49 | } 50 | 51 | def handleCheckpointResponse(checkpoint: Any, snd: String) { 52 | checkpoint match { 53 | case CheckpointReply(_) => checkpoints(snd) = Some(checkpoint.asInstanceOf[CheckpointReply]) 54 | case _ => throw new IllegalArgumentException("not a CheckpointReply: " + checkpoint) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/verification/DepTracker.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.Queue 4 | import akka.actor.{Actor, ActorCell, ActorRef, ActorSystem, Props} 5 | import akka.dispatch.Envelope 6 | 7 | import scalax.collection.mutable.Graph, 8 | scalax.collection.GraphEdge.DiEdge, 9 | scalax.collection.edge.LDiEdge 10 | 11 | import org.slf4j.LoggerFactory, 12 | ch.qos.logback.classic.Level, 13 | ch.qos.logback.classic.Logger 14 | 15 | object DepTracker { 16 | val root = Unique(MsgEvent("null", "null", null), 0) 17 | } 18 | 19 | /** 20 | * For every message we deliver, track which messages become enabled as a 21 | * result of our delivering that message. This can be used later by DPOR. 22 | * 23 | * - noopWaitQuiescence: if true, always attach external events to the root. 24 | */ 25 | // TODO(cs): Should probably factor out the Graph management code from DPOR 26 | // and put it here. 27 | class DepTracker(schedulerConfig: SchedulerConfig, 28 | noopWaitQuiescence:Boolean=true) { 29 | val log = LoggerFactory.getLogger("DepTracker") 30 | val depGraph = schedulerConfig.originalDepGraph match { 31 | case Some(g) => g 32 | case None => Graph[Unique, DiEdge]() 33 | } 34 | val initialTrace = new Queue[Unique] 35 | 36 | def getRootEvent() : Unique = { 37 | require(DepTracker.root != null) 38 | addGraphNode(DepTracker.root) 39 | DepTracker.root 40 | } 41 | 42 | // TODO(cs): probably not necessary? Just do getRootEvent() 43 | // Special marker for the 0th event 44 | val currentRoot = depGraph.isEmpty match { 45 | case true => getRootEvent() 46 | case false => 47 | // Since the graph does reference equality, need to reset _root to the 48 | // root in the graph. 49 | def matchesRoot(n: Unique) : Boolean = { 50 | return n.event == MsgEvent("null", "null", null) 51 | } 52 | depGraph.nodes.toList.find(matchesRoot _).map(_.value).getOrElse( 53 | throw new IllegalArgumentException("No root in initialDepGraph")) 54 | } 55 | 56 | def getAllNodes(): List[Unique] = { 57 | return depGraph.nodes.toList.map(_.value) 58 | } 59 | 60 | initialTrace += currentRoot 61 | // Most recent message we delivered. 62 | var parentEvent : Unique = currentRoot 63 | // Most recent quiescence we saw 64 | var lastQuiescence : Unique = currentRoot 65 | 66 | private[this] def addGraphNode (event: Unique) = { 67 | require(event != null) 68 | depGraph.add(event) 69 | } 70 | 71 | def isUnknown(unique: Unique) = (depGraph find unique) == None 72 | 73 | /** 74 | * Given a message, figure out if we have already seen 75 | * it before. We achieve this by consulting the 76 | * dependency graph. 77 | * 78 | * * @param (cell, envelope): Original message context. 79 | * 80 | * * @return A unique event. 81 | */ 82 | def getMessage(snd: String, rcv: String, msg: Any, external:Boolean=false) : Unique = { 83 | if (external) parentEvent = lastQuiescence 84 | val fingerprinted = schedulerConfig.messageFingerprinter.fingerprint(msg) 85 | val msgEvent = new MsgEvent(snd, rcv, fingerprinted) 86 | 87 | // Who cares if the parentEvent is in fact a message, as long as it is a parent. 88 | val parent = parentEvent 89 | 90 | def matchMessage (event: Event) : Boolean = { 91 | event match { 92 | // Second fingerprint isn't strictly necessary, but just be cautious 93 | // anyway 94 | case MsgEvent(s,r,m) => (s == snd && r == rcv && 95 | schedulerConfig.messageFingerprinter.fingerprint(m) == fingerprinted) 96 | case e => throw new IllegalStateException("Not a MsgEvent: " + e) 97 | } 98 | } 99 | 100 | val inNeighs = depGraph.get(parent).inNeighbors 101 | inNeighs.find { x => matchMessage(x.value.event) } match { 102 | case Some(x) => 103 | return x.value 104 | case None => 105 | val newMsg = Unique(msgEvent) 106 | return newMsg 107 | case _ => throw new Exception("wrong type") 108 | } 109 | } 110 | 111 | def addNodeAndEdge(child: Unique) { 112 | if (isUnknown(child)) { 113 | depGraph.add(child) 114 | depGraph.addEdge(child, parentEvent)(DiEdge) 115 | } 116 | } 117 | 118 | // External messages 119 | def reportNewlyEnabledExternal(snd: String, rcv: String, msg: Any): Unique = { 120 | parentEvent = lastQuiescence 121 | return reportNewlyEnabled(snd, rcv, msg) 122 | } 123 | 124 | // Return an id for this message. Must return this same id upon 125 | // reportNewlyDelivered. 126 | def reportNewlyEnabled(snd: String, rcv: String, msg: Any): Unique = { 127 | val child = getMessage(snd, rcv, msg) 128 | addNodeAndEdge(child) 129 | return child 130 | } 131 | 132 | def reportNewlyDelivered(u: Unique) { 133 | parentEvent = u 134 | initialTrace += u 135 | } 136 | 137 | // Report when quiescence has been arrived at as a result of an external 138 | // WaitQuiescence event. 139 | def reportQuiescence(w: WaitQuiescence) { 140 | if (!noopWaitQuiescence) { 141 | parentEvent = lastQuiescence 142 | val quiescence = Unique(w, id=w._id) 143 | println("reportQuiescence: " + quiescence) 144 | depGraph.add(quiescence) 145 | depGraph.addEdge(quiescence, parentEvent)(DiEdge) 146 | lastQuiescence = quiescence 147 | initialTrace += quiescence 148 | parentEvent = quiescence 149 | } 150 | } 151 | 152 | def reportKill(name: String, allActors: Set[String], id: Int) { 153 | val unique = Unique(NetworkPartition(Set(name), allActors), id=id) 154 | depGraph.add(unique) 155 | initialTrace += unique 156 | } 157 | 158 | def reportPartition(a: String, b: String, id: Int) { 159 | val unique = Unique(NetworkPartition(Set(a), Set(b)), id=id) 160 | depGraph.add(unique) 161 | initialTrace += unique 162 | } 163 | 164 | def reportUnPartition(a: String, b: String, id: Int) { 165 | val unique = Unique(NetworkUnpartition(Set(a), Set(b)), id=id) 166 | depGraph.add(unique) 167 | initialTrace += unique 168 | } 169 | 170 | def getGraph(): Graph[Unique, DiEdge] = depGraph 171 | 172 | def getInitialTrace(): Queue[Unique] = initialTrace 173 | } 174 | -------------------------------------------------------------------------------- /src/main/scala/verification/ExternalEvents.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{ActorCell, ActorRef, ActorSystem, Props} 4 | import akka.dispatch.{ Envelope } 5 | 6 | // ---------- External Events ----------- 7 | 8 | // External events used to specify a trace 9 | abstract trait ExternalEvent { 10 | def label: String 11 | } 12 | 13 | // Attaches unique IDs to external events. 14 | trait UniqueExternalEvent { 15 | var _id : Int = IDGenerator.get() 16 | 17 | def label: String = "e"+_id 18 | def toStringWithId: String = label+":"+toString() 19 | 20 | override def equals(other: Any): Boolean = { 21 | if (other.isInstanceOf[UniqueExternalEvent]) { 22 | return _id == other.asInstanceOf[UniqueExternalEvent]._id 23 | } else { 24 | return false 25 | } 26 | } 27 | 28 | override def hashCode: Int = { 29 | return _id 30 | } 31 | } 32 | 33 | /** 34 | * ExternalMessageConstructors are instance variables of Send() events. 35 | * They serve two purposes: 36 | * - They allow the client to late-bind the construction of their message. 37 | * apply() is invoked after the ActorSystem and all actors have been 38 | * created. 39 | * - Optionally: they provide an interface for `shrinking` the contents of 40 | * the messages. This is achieved through `getComponents` and 41 | * `maskComponents`. 42 | */ 43 | abstract class ExternalMessageConstructor extends Serializable { 44 | // Construct the message 45 | def apply() : Any 46 | // Optional, for `shrinking`: 47 | // Get the components that make up the content of the message we construct 48 | // in apply(). For now, only relevant to cluster membership messages. 49 | def getComponents() : Seq[ActorRef] = List.empty 50 | // Given a sequence of indices (pointing to elements in `getComponents()`), 51 | // create a new ExternalMessageConstructor that does not include those 52 | // components upon apply(). 53 | // Default: no-op 54 | def maskComponents(indices: Set[Int]): ExternalMessageConstructor = this 55 | } 56 | 57 | case class BasicMessageConstructor(msg: Any) extends ExternalMessageConstructor { 58 | def apply(): Any = msg 59 | } 60 | 61 | // External Event types. 62 | final case class Start (propCtor: () => Props, name: String) extends 63 | ExternalEvent with Event with UniqueExternalEvent 64 | // Really: isolate the actor. 65 | final case class Kill (name: String) extends 66 | ExternalEvent with Event with UniqueExternalEvent 67 | // Actually kill the actor rather than just isolating it. 68 | // TODO(cs): support killing of actors that aren't direct children of /user/ 69 | final case class HardKill (name: String) extends 70 | ExternalEvent with Event with UniqueExternalEvent 71 | final case class Send (name: String, messageCtor: ExternalMessageConstructor) extends 72 | ExternalEvent with Event with UniqueExternalEvent 73 | final case class WaitQuiescence() extends 74 | ExternalEvent with Event with UniqueExternalEvent 75 | // Stronger than WaitQuiescence: schedule indefinitely until cond returns true. 76 | // if quiescence has been reached but cond does 77 | // not return true, wait indefinitely until scheduler.enqueue_message is 78 | // invoked, schedule it, and again wait for quiescence. Repeat until cond 79 | // returns true. (Useful for systems that use external threads to send 80 | // messages indefinitely. 81 | final case class WaitCondition(cond: () => Boolean) extends 82 | ExternalEvent with Event with UniqueExternalEvent 83 | // Bidirectional partitions. 84 | final case class Partition (a: String, b: String) extends 85 | ExternalEvent with Event with UniqueExternalEvent 86 | final case class UnPartition (a: String, b: String) extends 87 | ExternalEvent with Event with UniqueExternalEvent 88 | // Executed synchronously, i.e. by the scheduler itself. The code block must 89 | // terminate (quickly)! 90 | final case class CodeBlock (block: () => Any) extends 91 | ExternalEvent with Event with UniqueExternalEvent 92 | 93 | // ---------- Failure Detector messages ----------- 94 | 95 | // Base class for failure detector messages 96 | abstract class FDMessage 97 | 98 | // Notification telling a node that it can query a failure detector by sending messages to fdNode. 99 | case class FailureDetectorOnline(fdNode: String) extends FDMessage 100 | 101 | // A node is unreachable, either due to node failure or partition. 102 | case class NodeUnreachable(actor: String) extends FDMessage with Event 103 | case class NodesUnreachable(actors: Set[String]) extends FDMessage with Event 104 | 105 | // A new node is now reachable, either because a partition healed or an actor spawned. 106 | case class NodeReachable(actor: String) extends FDMessage with Event 107 | // 108 | // A new node is now reachable, either because a partition healed or an actor spawned. 109 | case class NodesReachable(actors: Set[String]) extends FDMessage with Event 110 | 111 | // Query the failure detector for currently reachable actors. 112 | case object QueryReachableGroup extends FDMessage 113 | 114 | // Response to failure detector queries. 115 | case class ReachableGroup(actors: Set[String]) extends FDMessage 116 | 117 | 118 | // --------------- Utility functions ---------------- 119 | 120 | object MessageTypes { 121 | // Messages that the failure detector sends to actors. 122 | // Assumes that actors don't relay fd messages to eachother. 123 | def fromFailureDetector(m: Any) : Boolean = { 124 | m match { 125 | case _: FailureDetectorOnline | _: NodeUnreachable | _: NodeReachable | 126 | _: ReachableGroup => return true 127 | case _ => return false 128 | } 129 | } 130 | 131 | def fromCheckpointCollector(m: Any) : Boolean = { 132 | m match { 133 | case CheckpointRequest => return true 134 | case _ => return false 135 | } 136 | } 137 | 138 | def sanityCheckTrace(trace: Seq[ExternalEvent]) { 139 | trace foreach { 140 | case Send(_, msgCtor) => 141 | val msg = msgCtor() 142 | if (MessageTypes.fromFailureDetector(msg) || 143 | MessageTypes.fromCheckpointCollector(msg)) { 144 | throw new IllegalArgumentException( 145 | "trace contains system message: " + msg) 146 | } 147 | case _ => None 148 | } 149 | } 150 | } 151 | 152 | object ActorTypes { 153 | def systemActor(name: String) : Boolean = { 154 | return name == FailureDetector.fdName || name == CheckpointSink.name 155 | } 156 | } 157 | 158 | object EventTypes { 159 | // Should return true if the given message is an external message 160 | var externalMessageFilterHasBeenSet = false 161 | var externalMessageFilter: (Any) => Boolean = (_) => false 162 | // Should be set by applications during initialization. 163 | def setExternalMessageFilter(filter: (Any) => Boolean) { 164 | externalMessageFilterHasBeenSet = true 165 | externalMessageFilter = filter 166 | } 167 | 168 | def isMessageType(e: Event) : Boolean = { 169 | e match { 170 | case MsgEvent(_, _, m) => 171 | return true 172 | case MsgSend(_, _, m) => 173 | return true 174 | case UniqueMsgEvent(MsgEvent(_, _, m), _) => 175 | return true 176 | case UniqueMsgSend(MsgSend(_, _, m), _) => 177 | return true 178 | case _ => 179 | return false 180 | } 181 | } 182 | 183 | // Internal events that correspond to ExternalEvents. 184 | def isExternal(e: Event) : Boolean = { 185 | if (e.isInstanceOf[ExternalEvent]) { 186 | return true 187 | } 188 | return e match { 189 | case _: KillEvent | _: SpawnEvent | _: PartitionEvent | _: UnPartitionEvent => 190 | return true 191 | case MsgEvent(_, _, m) => 192 | return externalMessageFilter(m) 193 | case MsgSend(_, _, m) => 194 | return externalMessageFilter(m) 195 | case UniqueMsgEvent(MsgEvent(_, _, m), _) => 196 | return externalMessageFilter(m) 197 | case UniqueMsgSend(MsgSend(_, _, m), _) => 198 | return externalMessageFilter(m) 199 | case _ => return false 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/main/scala/verification/FailureDetector.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{Actor, ActorRef, ActorSystem, Props} 4 | 5 | import scala.collection.mutable.Queue 6 | import scala.collection.mutable.HashMap 7 | import scala.collection.immutable.Set 8 | import scala.collection.mutable.HashSet 9 | 10 | /* 11 | 12 | A utility class that can be used by schedulers to control failure detection 13 | events. 14 | 15 | FDMessageOrchestrator allows schedulers to: 16 | - register actors with the failure detector upon spawn. 17 | 18 | A scheduler that makes use of a FDMessageOrchestrator allows actors to: 19 | - be informed of failure detection events. Actors simply need to accept 20 | NodeReachable and NodeUnreachable messages to be informed. 21 | - Actors can also query the failure detector for all reachable nodes by calling: 22 | actor.contextFor("../" + FailureDetector.fdName) ? QueryReachableGroup 23 | and then later accepting a ReachableGroup message. 24 | */ 25 | 26 | object FailureDetector { 27 | var fdName = "failure_detector" 28 | } 29 | 30 | // A fake actor, used as a placeholder to which failure detector requests can be sent. 31 | // The scheduler intercepts messages sent to this actor. 32 | class FailureDetector () extends Actor { 33 | def receive = { 34 | // This should never be called 35 | case _ => assert(false) 36 | } 37 | } 38 | 39 | object FDMessageOrchestrator { 40 | // Function that queues a message to be sent later. 41 | type EnqueueMessage = (Option[ActorRef], String, Any) => Unit 42 | } 43 | 44 | class FDMessageOrchestrator (var enqueue_message: FDMessageOrchestrator.EnqueueMessage) { 45 | 46 | // Failure detector information 47 | // For each actor, track the set of other actors who are reachable. 48 | val fdState = new HashMap[String, HashSet[String]] 49 | val activeActors = new HashSet[String] 50 | val pendingFDRequests = new Queue[String] 51 | 52 | def clear() { 53 | fdState.clear 54 | activeActors.clear 55 | pendingFDRequests.clear 56 | } 57 | 58 | def startFD(actorSystem: ActorSystem) { 59 | actorSystem.actorOf(Props[FailureDetector], FailureDetector.fdName) 60 | } 61 | 62 | def create_node(node: String) { 63 | fdState(node) = new HashSet[String] 64 | } 65 | 66 | def isolate_node(node: String) { 67 | fdState(node).clear() 68 | } 69 | 70 | def unisolate_node(node: String) { 71 | fdState(node) ++= activeActors 72 | } 73 | 74 | def handle_start_event(node: String) { 75 | // Send FD message before adding an actor 76 | informFailureDetectorLocation(node) 77 | informNodeReachable(node) 78 | activeActors += node 79 | } 80 | 81 | def handle_kill_event(node: String) { 82 | activeActors -= node 83 | // Send FD message after removing the actor 84 | informNodeUnreachable(node) 85 | } 86 | 87 | def handle_partition_event(a: String, b: String) { 88 | // Send FD information to each of the actors 89 | informNodeUnreachable(a, b) 90 | informNodeUnreachable(b, a) 91 | } 92 | 93 | def handle_unpartition_event(a: String, b: String) { 94 | // Send FD information to each of the actors 95 | informNodeReachable(a, b) 96 | informNodeReachable(b, a) 97 | } 98 | 99 | def handle_fd_message(msg: Any, snd: String) { 100 | msg match { 101 | case QueryReachableGroup => 102 | // Allow queries to be made during class initialization (more than one might be happening at 103 | // a time) 104 | pendingFDRequests += snd 105 | case _ => 106 | assert(false) 107 | } 108 | } 109 | 110 | def send_all_pending_responses() { 111 | for (receiver <- pendingFDRequests) { 112 | answerFdQuery(receiver) 113 | } 114 | pendingFDRequests.clear 115 | } 116 | 117 | private[this] def informFailureDetectorLocation(actor: String) { 118 | enqueue_message(None, actor, FailureDetectorOnline(FailureDetector.fdName)) 119 | } 120 | 121 | private[this] def informNodeReachable(actor: String, node: String) { 122 | enqueue_message(None, actor, NodeReachable(node)) 123 | fdState(actor) += node 124 | } 125 | 126 | private[this] def informNodeReachable(node: String) { 127 | for (actor <- activeActors) { 128 | informNodeReachable(actor, node) 129 | } 130 | } 131 | 132 | private[this] def informNodeUnreachable(actor: String, node: String) { 133 | enqueue_message(None, actor, NodeUnreachable(node)) 134 | fdState(actor) -= node 135 | } 136 | 137 | private[this] def informNodeUnreachable(node: String) { 138 | for (actor <- activeActors) { 139 | informNodeUnreachable(actor, node) 140 | } 141 | } 142 | 143 | private[this] def answerFdQuery(sender: String) { 144 | // Compute the message 145 | val msg = ReachableGroup(fdState(sender).toSet) 146 | // Send failure detector information 147 | enqueue_message(None, sender, msg) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/main/scala/verification/MessageFingerprints.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.FSM 4 | import akka.actor.FSM.Timer 5 | 6 | import scala.reflect.ClassTag 7 | import scala.collection.mutable.ListBuffer 8 | 9 | trait MessageFingerprint { 10 | def hashCode: Int 11 | def equals(other: Any) : Boolean 12 | } 13 | 14 | trait MessageFingerprinter { 15 | // Return Some(fingerprint) if this fingerprinter knows anything about 16 | // msg, otherwise return None and fallback to another fingerprinter 17 | def fingerprint(msg: Any) : Option[MessageFingerprint] = { 18 | // Account for the possibility of serialized messages, that have been 19 | // serialized as MessageFingerprints. Subclasses should probably call this. 20 | if (msg.isInstanceOf[MessageFingerprint]) { 21 | return Some(msg.asInstanceOf[MessageFingerprint]) 22 | } 23 | return None 24 | } 25 | 26 | // Does this message trigger a logical clock contained in subsequent 27 | // messages to be incremented? 28 | def causesClockIncrement(msg: Any) : Boolean = false 29 | 30 | // Extract a clock value from the contents of this message 31 | def getLogicalClock(msg: Any) : Option[Long] = None 32 | } 33 | 34 | // A simple fingerprint template for user-defined fingerprinters. Should 35 | // usually not be instantiated directly; invoke BasicFingerprint.fromMessage() 36 | case class BasicFingerprint(val str: String) extends MessageFingerprint 37 | 38 | // Static constructor for BasicFingeprrint 39 | object BasicFingerprint { 40 | val systemRegex = ".*(new-system-\\d+).*".r 41 | 42 | def fromMessage(msg: Any): BasicFingerprint = { 43 | // Lazy person's approach: for any message that we didn't explicitly match on, 44 | // their toString might contain a string referring to the ActorSystem, which 45 | // changes across runs. Run a simple regex over it to catch that. 46 | val str = msg.toString match { 47 | case systemRegex(system) => msg.toString.replace(system, "") 48 | case _ => msg.toString 49 | } 50 | return BasicFingerprint(str) 51 | } 52 | } 53 | 54 | // Singleton for TimeoutMaker 55 | case object TimeoutMarkerFingerprint extends MessageFingerprint 56 | 57 | class BaseFingerprinter(parent: FingerprintFactory) extends MessageFingerprinter { 58 | override def fingerprint(msg: Any) : Option[MessageFingerprint] = { 59 | val alreadyFingerprint = super.fingerprint(msg) 60 | if (alreadyFingerprint != None) { 61 | return alreadyFingerprint 62 | } 63 | if (ClassTag(msg.getClass).toString == "akka.actor.FSM$TimeoutMarker") { 64 | return Some(TimeoutMarkerFingerprint) 65 | } 66 | msg match { 67 | case Timer(name, message, repeat, generation) => 68 | return Some(TimerFingerprint(name, parent.fingerprint(message), repeat, generation)) 69 | case _ => 70 | // BaseFingerprinter is the most general fingerprinter, so it always returns Some. 71 | return Some(BasicFingerprint.fromMessage(msg)) 72 | } 73 | } 74 | } 75 | 76 | object BaseFingerprinter { 77 | def isFSMTimer(f: MessageFingerprint) : Boolean = { 78 | return (f == TimeoutMarkerFingerprint || 79 | f.isInstanceOf[TimerFingerprint]) 80 | } 81 | } 82 | 83 | class FingerprintFactory { 84 | val fingerprinters = new ListBuffer[MessageFingerprinter] ++ 85 | List(new BaseFingerprinter(this)) 86 | 87 | def registerFingerprinter(fingerprinter: MessageFingerprinter) { 88 | // Always prepend. BaseFingerprinter should be last 89 | fingerprinters.+=:(fingerprinter) 90 | } 91 | 92 | def fingerprint(msg: Any) : MessageFingerprint = { 93 | for (fingerprinter <- fingerprinters) { 94 | val fingerprint = fingerprinter.fingerprint(msg) 95 | fingerprint match { 96 | case None => None 97 | case Some(f) => return f 98 | } 99 | } 100 | throw new IllegalStateException("Should have matched") 101 | } 102 | 103 | // Does this message trigger a logical clock contained in subsequent 104 | // messages to be incremented? 105 | def causesClockIncrement(msg: Any) : Boolean = { 106 | for (fingerprinter <- fingerprinters) { 107 | if (fingerprinter.causesClockIncrement(msg)) { 108 | return true 109 | } 110 | } 111 | return false 112 | } 113 | 114 | // Extract a clock value from the contents of this message 115 | def getLogicalClock(msg: Any) : Option[Long] = { 116 | for (fingerprinter <- fingerprinters) { 117 | val opt = fingerprinter.getLogicalClock(msg) 118 | if (!opt.isEmpty) { 119 | return opt 120 | } 121 | } 122 | return None 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/main/scala/verification/SchedulerConfig.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scalax.collection.mutable.Graph, 4 | scalax.collection.GraphEdge.DiEdge, 5 | scalax.collection.edge.LDiEdge 6 | 7 | 8 | // A constructor parameter, passed in to all Scheduler objects. 9 | case class SchedulerConfig( 10 | messageFingerprinter : FingerprintFactory=null, 11 | enableFailureDetector : Boolean=false, 12 | enableCheckpointing : Boolean=false, 13 | shouldShutdownActorSystem : Boolean=true, 14 | filterKnownAbsents : Boolean=false, 15 | invariant_check : Option[TestOracle.Invariant]=None, 16 | ignoreTimers : Boolean=false, 17 | storeEventTraces : Boolean=false, 18 | /** 19 | * - abortUponDivergence: return "no violation" if we detect that we've 20 | * diverged from the original schedule, i.e. we arrive at message transitions 21 | * that have not yet been encountered. The purpose of this option is really 22 | * just to compare against prior work. 23 | */ 24 | abortUponDivergence : Boolean=false, 25 | // Spark exhibits non-determinism, so instead of detecting specific 26 | // previously unobserved transitions, just count up all the pending messages at the 27 | // end of the execution, and see if any were not sent at some point in the 28 | // original execution. 29 | abortUponDivergenceLax : Boolean=false, 30 | 31 | /** 32 | * - originalDepGraph: DepGraph from the original execution, to detect if we 33 | * have diverged. 34 | */ 35 | // TODO(cs): maybe shouldn't store this here. 36 | originalDepGraph : Option[Graph[Unique,DiEdge]]=None 37 | ) 38 | -------------------------------------------------------------------------------- /src/main/scala/verification/fuzzing/Fuzzer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.ListBuffer 4 | import scala.util.Random 5 | 6 | 7 | // Needs to be implemented by the application. 8 | trait MessageGenerator { 9 | def generateMessage(aliveActors: RandomizedHashSet[String]) : Send 10 | } 11 | 12 | object StableClasses { 13 | // Deal with type erasue... 14 | // See: 15 | // http://stackoverflow.com/questions/7157143/how-can-i-match-classes-in-a-scala-match-statement 16 | val ClassOfKill = classOf[Kill] 17 | val ClassOfSend = classOf[Send] 18 | val ClassOfPartition = classOf[Partition] 19 | val ClassOfUnPartition = classOf[UnPartition] 20 | } 21 | 22 | // Weights should be between 0 and 1.0. 23 | // At least one weight should be greater than 0. 24 | case class FuzzerWeights( 25 | kill: Double = 0.01, 26 | send: Double = 0.3, 27 | wait_quiescence: Double = 0.1, 28 | partition: Double = 0.1, 29 | unpartition: Double = 0.1) { 30 | 31 | val allWeights = List(kill, send, partition, unpartition) 32 | 33 | val totalMass = allWeights.sum + wait_quiescence 34 | 35 | val eventTypes = List(StableClasses.ClassOfKill, StableClasses.ClassOfSend, 36 | StableClasses.ClassOfPartition, StableClasses.ClassOfUnPartition) 37 | 38 | // Takes a number between [0, 1.0] 39 | // Returns the class of an ExternalEvent, or None if the chosen class type 40 | // is WaitQuiescence. 41 | // We treat WaitQuiescence specially, since it's a case object instead of 42 | // a case class, and I don't really file like wrestling with the scala type system 43 | // right now. 44 | def getNextEventType(r: Double) : Option[Class[_ <: ExternalEvent]] = { 45 | // First scale r to [0, totalMass] 46 | val scaled = r * totalMass 47 | // Now, a simple O(n) algorithm to decide which event type to return. 48 | // N=7, so whatever. 49 | var currentWeight = 0.0 50 | for ((weight, index) <- allWeights.zipWithIndex) { 51 | currentWeight = currentWeight + weight 52 | if (scaled < currentWeight) { 53 | return Some(eventTypes(index)) 54 | } 55 | } 56 | return None 57 | } 58 | } 59 | 60 | // prefix must at least contain start events 61 | class Fuzzer(num_events: Integer, 62 | weights: FuzzerWeights, 63 | message_gen: MessageGenerator, 64 | prefix: Seq[ExternalEvent], 65 | postfix: Seq[ExternalEvent] = Seq.empty) { 66 | 67 | var seed = System.currentTimeMillis 68 | var rand = new Random(seed) 69 | 70 | val nodes = prefix flatMap { 71 | case Start(_, name) => Some(name) 72 | case _ => None 73 | } 74 | 75 | // Note that we don't currently support node recoveries. 76 | var currentlyAlive = new RandomizedHashSet[String] 77 | 78 | // N.B., only store one direction of the partition 79 | var currentlyPartitioned = new RandomizedHashSet[(String, String)] 80 | var currentlyUnpartitioned = new RandomizedHashSet[(String, String)] 81 | 82 | // Return None if we have no choice, e.g. if we've killed all nodes, and 83 | // there's nothing interesting left to do in the execution. 84 | def generateNextEvent() : Option[ExternalEvent] = { 85 | val nextEventType = weights.getNextEventType(rand.nextDouble()) 86 | nextEventType match { 87 | case None => return Some(WaitQuiescence()) 88 | case Some(cls) => 89 | cls match { 90 | case StableClasses.ClassOfKill => 91 | if (currentlyAlive.isEmpty) 92 | return None 93 | val nextVictim = currentlyAlive.removeRandomElement() 94 | return Some(Kill(nextVictim)) 95 | 96 | case StableClasses.ClassOfSend => 97 | val send = message_gen.generateMessage(currentlyAlive) 98 | return Some(send) 99 | 100 | case StableClasses.ClassOfPartition => 101 | if (currentlyUnpartitioned.isEmpty) { 102 | // Try again... 103 | return generateNextEvent() 104 | } 105 | val pair = currentlyUnpartitioned.removeRandomElement() 106 | currentlyPartitioned.insert(pair) 107 | return Some(Partition(pair._1, pair._2)) 108 | 109 | case StableClasses.ClassOfUnPartition => 110 | if (currentlyPartitioned.isEmpty) { 111 | // Try again... 112 | return generateNextEvent() 113 | } 114 | val pair = currentlyPartitioned.removeRandomElement() 115 | currentlyUnpartitioned.insert(pair) 116 | return Some(UnPartition(pair._1, pair._2)) 117 | } 118 | } 119 | throw new IllegalStateException("Shouldn't get here") 120 | } 121 | 122 | def generateFuzzTest() : Seq[ExternalEvent] = { 123 | reset() 124 | val fuzzTest = new ListBuffer[ExternalEvent] ++ prefix 125 | 126 | def validateFuzzTest(_fuzzTest: Seq[ExternalEvent]) { 127 | for (i <- (0 until _fuzzTest.length - 1)) { 128 | if (_fuzzTest(i).getClass() == classOf[WaitQuiescence] && 129 | _fuzzTest(i+1).getClass() == classOf[WaitQuiescence]) { 130 | throw new AssertionError(i + " " + (i+1) + ". Seed was: " + seed) 131 | } 132 | } 133 | } 134 | 135 | // Ensure that we don't inject two WaitQuiescense's in a row. 136 | var justInjectedWaitQuiescence = if (fuzzTest.length > 0) 137 | fuzzTest(fuzzTest.length-1).getClass == classOf[WaitQuiescence] 138 | else false 139 | 140 | def okToInject(event: Option[ExternalEvent]) : Boolean = { 141 | event match { 142 | case Some(WaitQuiescence()) => 143 | return !justInjectedWaitQuiescence 144 | case _ => return true 145 | } 146 | } 147 | 148 | for (_ <- (1 to num_events)) { 149 | var nextEvent = generateNextEvent() 150 | while (!okToInject(nextEvent)) { 151 | nextEvent = generateNextEvent() 152 | } 153 | nextEvent match { 154 | case Some(event) => 155 | event match { 156 | case WaitQuiescence() => 157 | justInjectedWaitQuiescence = true 158 | case _ => 159 | justInjectedWaitQuiescence = false 160 | } 161 | fuzzTest += event 162 | case None => 163 | validateFuzzTest(fuzzTest) 164 | return fuzzTest 165 | } 166 | } 167 | 168 | validateFuzzTest(fuzzTest) 169 | fuzzTest ++= postfix 170 | if (fuzzTest.size > 0 && 171 | fuzzTest.last.getClass != classOf[WaitQuiescence]) { 172 | fuzzTest += WaitQuiescence() 173 | } 174 | return fuzzTest 175 | } 176 | 177 | def reset() { 178 | seed = System.currentTimeMillis 179 | rand = new Random(seed) 180 | 181 | currentlyAlive = new RandomizedHashSet[String] 182 | for (node <- nodes) { 183 | currentlyAlive.insert(node) 184 | } 185 | 186 | currentlyPartitioned = new RandomizedHashSet[(String, String)] 187 | currentlyUnpartitioned = new RandomizedHashSet[(String, String)] 188 | for (i <- (0 to nodes.length-1)) { 189 | for (j <- (i+1 to nodes.length-1)) { 190 | currentlyUnpartitioned.insert((nodes(i), nodes(j))) 191 | } 192 | } 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/DeltaDebugging.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import org.slf4j.LoggerFactory, 4 | ch.qos.logback.classic.Level, 5 | ch.qos.logback.classic.Logger 6 | 7 | class DDMin (oracle: TestOracle, checkUnmodifed:Boolean=false, 8 | stats: Option[MinimizationStats]=None) extends Minimizer { 9 | val logger = LoggerFactory.getLogger("DDMin") 10 | 11 | var violation_fingerprint : ViolationFingerprint = null 12 | val _stats = stats match { 13 | case Some(s) => s 14 | case None => new MinimizationStats 15 | } 16 | _stats.updateStrategy("DDMin", oracle.getName) 17 | var initializationRoutine : Option[() => Any] = None 18 | var original_num_events = 0 19 | var total_inputs_pruned = 0 20 | 21 | // Taken from the 1999 version of delta debugging: 22 | // https://www.st.cs.uni-saarland.de/publications/files/zeller-esec-1999.pdf 23 | // Section 4. 24 | // 25 | // Note that this differs from the 2001 version: 26 | // https://www.cs.purdue.edu/homes/xyzhang/fall07/Papers/delta-debugging.pdf 27 | def minimize(dag: EventDag, 28 | _violation_fingerprint: ViolationFingerprint, 29 | _initializationRoutine: Option[() => Any]) : EventDag = { 30 | MessageTypes.sanityCheckTrace(dag.events) 31 | violation_fingerprint = _violation_fingerprint 32 | initializationRoutine = _initializationRoutine 33 | 34 | if (logger.isTraceEnabled()) { 35 | logger.trace("Minimizing:---") 36 | dag.events foreach { case e => logger.trace(e.asInstanceOf[UniqueExternalEvent].toStringWithId) } 37 | logger.trace("---") 38 | } 39 | 40 | // First check if the initial trace violates the exception 41 | if (checkUnmodifed) { 42 | logger.info("Checking if unmodified trace triggers violation...") 43 | if (oracle.test(dag.events, violation_fingerprint, _stats, 44 | initializationRoutine=initializationRoutine) == None) { 45 | throw new IllegalArgumentException("Unmodified trace does not trigger violation") 46 | } 47 | } 48 | _stats.reset() 49 | 50 | original_num_events = dag.length 51 | var remainder : EventDag = new UnmodifiedEventDag(List[ExternalEvent]()) 52 | 53 | _stats.record_prune_start() 54 | val mcs_dag = ddmin2(dag, remainder) 55 | val mcs = mcs_dag.get_all_events 56 | _stats.record_prune_end() 57 | 58 | assert(original_num_events - total_inputs_pruned == mcs.length) 59 | // Record the final iteration (fencepost) 60 | _stats.record_iteration_size(original_num_events - total_inputs_pruned) 61 | return mcs_dag 62 | } 63 | 64 | def verify_mcs(mcs: EventDag, 65 | _violation_fingerprint: ViolationFingerprint, 66 | initializationRoutine: Option[() => Any]=None): Option[EventTrace] = { 67 | val nop_stats = new MinimizationStats 68 | nop_stats.updateStrategy("NOP", "NOP") 69 | return oracle.test(mcs.events, _violation_fingerprint, 70 | nop_stats, initializationRoutine=initializationRoutine) 71 | } 72 | 73 | def ddmin2(dag: EventDag, remainder: EventDag): EventDag = { 74 | if (dag.get_atomic_events.length <= 1) { 75 | logger.info("base case") 76 | return dag 77 | } 78 | 79 | // N.B. we reverse to ensure that we test the left half of events before 80 | // the right half of events. (b/c of our use of remove_events()) 81 | // Just a matter of convention. 82 | val splits : Seq[EventDag] = 83 | MinificationUtil.split_list(dag.get_atomic_events, 2). 84 | asInstanceOf[Seq[Seq[AtomicEvent]]]. 85 | map(split => dag.remove_events(split)).reverse 86 | 87 | // First, check both halves. 88 | for ((split, i) <- splits.zipWithIndex) { 89 | val union = split.union(remainder) 90 | logger.info("Checking split " + union.get_all_events.map(e => e.label).mkString(",")) 91 | val trace = oracle.test(union.get_all_events, violation_fingerprint, 92 | _stats, initializationRoutine=initializationRoutine) 93 | val passes = trace == None 94 | _stats.record_iteration_size(original_num_events - total_inputs_pruned) 95 | if (!passes) { 96 | logger.info("Split fails. Recursing") 97 | total_inputs_pruned += (dag.length - split.length) 98 | return ddmin2(split, remainder) 99 | } else { 100 | logger.info("Split passes.") 101 | } 102 | } 103 | 104 | // Interference: 105 | logger.info("Interference") 106 | val left = ddmin2(splits(0), splits(1).union(remainder)) 107 | val right = ddmin2(splits(1), splits(0).union(remainder)) 108 | return left.union(right) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/IncrementalDeltaDebugging.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.HashMap 4 | import scala.collection.mutable.Queue 5 | 6 | import org.slf4j.LoggerFactory, 7 | ch.qos.logback.classic.Level, 8 | ch.qos.logback.classic.Logger 9 | 10 | /** 11 | * Invoke DDMin with maxDistance=0, then again with maxDistance=2, ... up to 12 | * given maxMaxDistance. 13 | * 14 | * Depends crucially on the assumption that DPOR tracks its history, so that 15 | * it doesn't ever explore the same path twice. 16 | * 17 | * stopAtSize: if MCS size <= stopAtSize, stop. 18 | */ 19 | // TODO(cs): propogate stopAtSize to DDMin? 20 | class IncrementalDDMin (oracle: ResumableDPOR, maxMaxDistance:Int=256, 21 | stopAtSize:Int=1, checkUnmodifed:Boolean=false, 22 | stats: Option[MinimizationStats]=None) extends Minimizer { 23 | 24 | val logger = LoggerFactory.getLogger("IncrementalDDMin") 25 | var ddmin = new DDMin(oracle, checkUnmodifed=false) 26 | val _stats = stats match { 27 | case Some(s) => s 28 | case None => new MinimizationStats 29 | } 30 | _stats.updateStrategy("IncDDMin", oracle.getName) 31 | 32 | private[this] def mergeStats(otherStats: MinimizationStats) { 33 | // Increment keys in otherStats.iterationSize by my current iteration 34 | for ((k, v) <- otherStats.inner().iterationSize) { 35 | _stats.inner().iterationSize(k + _stats.inner().total_replays) = v 36 | } 37 | // Now increment my own iteration 38 | _stats.inner().total_replays += otherStats.inner().total_replays 39 | // Ignore otherStats.stats 40 | } 41 | 42 | def minimize(dag: EventDag, 43 | violation_fingerprint: ViolationFingerprint, 44 | initializationRoutine: Option[()=>Any]) : EventDag = { 45 | var currentDistance = 0 46 | oracle.setMaxDistance(currentDistance) 47 | 48 | // First check if the initial trace violates the exception 49 | if (checkUnmodifed) { 50 | logger.info("Checking if unmodified trace triggers violation...") 51 | if (oracle.test(dag.events, violation_fingerprint, _stats) == None) { 52 | throw new IllegalArgumentException("Unmodified trace does not trigger violation") 53 | } 54 | } 55 | _stats.reset() 56 | 57 | var currentMCS = dag 58 | 59 | _stats.record_prune_start() 60 | 61 | while (currentDistance < maxMaxDistance && currentMCS.events.size > stopAtSize) { 62 | logger.info("Trying currentDistance="+currentDistance) 63 | ddmin = new DDMin(oracle, checkUnmodifed=false) 64 | currentMCS = ddmin.minimize(currentMCS, violation_fingerprint, initializationRoutine) 65 | RunnerUtils.printMCS(currentMCS.events) 66 | mergeStats(ddmin._stats) 67 | currentDistance = if (currentDistance == 0) 2 else currentDistance << 1 68 | oracle.setMaxDistance(currentDistance) 69 | _stats.record_distance_increase(currentDistance) 70 | } 71 | 72 | _stats.record_prune_end() 73 | 74 | return currentMCS 75 | } 76 | 77 | def verify_mcs(mcs: EventDag, 78 | _violation_fingerprint: ViolationFingerprint, 79 | initializationRoutine: Option[()=>Any]=None): Option[EventTrace] = { 80 | val nopStats = new MinimizationStats 81 | nopStats.updateStrategy("NOP", "NOP") 82 | return oracle.test(mcs.events, _violation_fingerprint, nopStats) 83 | } 84 | } 85 | 86 | object ResumableDPOR { 87 | type DPORConstructor = () => DPORwHeuristics 88 | } 89 | 90 | /** 91 | * A wrapper class that keeps separate instances of DPOR for each external 92 | * event subsequence. 93 | */ 94 | class ResumableDPOR(ctor: ResumableDPOR.DPORConstructor) extends TestOracle { 95 | // External event subsequence -> DPOR instance 96 | val subseqToDPOR = new HashMap[Seq[ExternalEvent], DPORwHeuristics] 97 | var invariant : Invariant = null 98 | var currentMaxDistance : Int = 0 99 | 100 | def getName: String = "DPOR" 101 | 102 | def setInvariant(inv: Invariant) = { 103 | invariant = inv 104 | } 105 | 106 | def setMaxDistance(dist: Int) { 107 | currentMaxDistance = dist 108 | } 109 | 110 | def test(events: Seq[ExternalEvent], 111 | violation_fingerprint: ViolationFingerprint, 112 | stats: MinimizationStats, 113 | init:Option[()=>Any]=None) : Option[EventTrace] = { 114 | if (!(subseqToDPOR contains events)) { 115 | subseqToDPOR(events) = ctor() 116 | subseqToDPOR(events).setInvariant(invariant) 117 | } 118 | val dpor = subseqToDPOR(events) 119 | dpor.setMaxDistance(currentMaxDistance) 120 | return dpor.test(events, violation_fingerprint, stats, init) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/Minimizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.ListBuffer 4 | import scala.collection.mutable.HashMap 5 | import scala.util.parsing.json.JSONObject 6 | import scala.util.parsing.json.JSONArray 7 | import scala.util.parsing.json.JSON 8 | 9 | trait Minimizer { 10 | // Returns the MCS. 11 | def minimize(events: EventDag, 12 | violation_fingerprint: ViolationFingerprint, 13 | initializationRoutine: Option[()=>Any]) : EventDag 14 | 15 | def minimize(events: Seq[ExternalEvent], 16 | violation_fingerprint: ViolationFingerprint, 17 | initializationRoutine: Option[()=>Any]=None) : EventDag = { 18 | return minimize(new UnmodifiedEventDag(events), violation_fingerprint, 19 | initializationRoutine=initializationRoutine) 20 | } 21 | 22 | // Returns Some(execution) if the final MCS was able to be reproduced, 23 | // otherwise returns None. 24 | def verify_mcs(mcs: EventDag, 25 | violation_fingerprint: ViolationFingerprint, 26 | initializationRoutine: Option[()=>Any]=None) : Option[EventTrace] 27 | } 28 | 29 | // Statistics about how the minimization process worked. 30 | class MinimizationStats { 31 | // minimization_strategy: 32 | // One of: {"LeftToRightRemoval", "DDMin"} 33 | // test_oracle: 34 | // One of: {"RandomScheduler", "STSSchedNoPeek", "STSSched", 35 | // "DPOR", "FairScheduler", "FungibleClocks"} 36 | var minimization_strategy : String = "" 37 | var test_oracle : String = "" 38 | var stats = new ListBuffer[MinimizationStats.InnerStats] 39 | 40 | // Update which <strategy, oracle> pair we're recording stats for 41 | def updateStrategy(_minimization_strategy: String, _test_oracle: String) { 42 | minimization_strategy = _minimization_strategy 43 | test_oracle = _test_oracle 44 | val name = ((minimization_strategy,test_oracle)).toString 45 | stats += new MinimizationStats.InnerStats(name) 46 | } 47 | 48 | def inner(): MinimizationStats.InnerStats = { 49 | stats.last 50 | } 51 | 52 | def reset() { 53 | inner().reset 54 | } 55 | 56 | def increment_replays() { 57 | inner().increment_replays 58 | } 59 | 60 | def record_replay_start() { 61 | inner().record_replay_start 62 | } 63 | 64 | def record_replay_end() { 65 | inner().record_replay_end 66 | } 67 | 68 | def record_prune_start() { 69 | inner().record_prune_start 70 | } 71 | 72 | def record_prune_end() { 73 | inner().record_prune_end 74 | } 75 | 76 | // For external events (legacy name), record how many external events were 77 | // left at the given iteration (replay #) 78 | def record_iteration_size(iteration_size: Integer) { 79 | inner().record_iteration_size(iteration_size) 80 | } 81 | 82 | // For internal events, record how many internal events were left at the 83 | // given iteration (replay #) 84 | def record_internal_size(iteration_size: Integer) { 85 | inner().record_internal_size(iteration_size) 86 | } 87 | 88 | def record_distance_increase(newDistance: Integer) { 89 | inner().record_distance_increase(newDistance) 90 | } 91 | 92 | def recordDeliveryStats(minimized_deliveries: Int, minimized_externals: Int, 93 | minimized_timers: Int) { 94 | inner().recordDeliveryStats(minimized_deliveries, minimized_externals, 95 | minimized_timers) 96 | } 97 | 98 | def toJson(): String = { 99 | return JSONArray(stats.map(s => s.toJson).toList).toString 100 | } 101 | } 102 | 103 | object MinimizationStats { 104 | class InnerStats(name: String) { 105 | // For the ith replay attempt, how many external events were left? 106 | val iterationSize = new HashMap[Int, Int] 107 | 108 | // For the ith replay attempt, how many internal events were left? 109 | val internalIterationSize = new HashMap[Int, Int] 110 | 111 | // For each increase in maxDistance, what was the current iteration when the 112 | // increase occurred? (See IncrementalDeltaDebugging.scala) 113 | // { new maxDistance -> current iteration } 114 | val maxDistance = new HashMap[Int, Int] 115 | 116 | // Number of schedules attempted in order to find the MCS, not including the 117 | // initial replay to verify the bug is still triggered 118 | var total_replays = 0 119 | 120 | // Other stats 121 | val stats = new HashMap[String, Double] 122 | 123 | reset() 124 | 125 | def reset() { 126 | iterationSize.clear 127 | internalIterationSize.clear 128 | maxDistance.clear 129 | stats.clear 130 | stats ++= Seq( 131 | // Overall minimization time 132 | "prune_duration_seconds" -> -1.0, 133 | "prune_start_epoch" -> -1.0, 134 | "prune_end_epoch" -> -1.0, 135 | // Time for the initial replay to verify the bug is still triggered 136 | "replay_duration_seconds" -> -1.0, 137 | "replay_end_epoch" -> -1.0, 138 | "replay_start_epoch" -> -1.0, 139 | // How long the original fuzz run took 140 | // TODO(cs): should be a part of EventTrace? 141 | "original_duration_seconds" -> -1.0, 142 | // Number of external events in the unmodified execution 143 | // TODO(cs): should be a part of EventTrace? 144 | "total_inputs" -> 0.0, 145 | // Number of events (both internal and external) in the unmodified execution 146 | // TODO(cs): should be a part of EventTrace? 147 | "total_events" -> 0.0, 148 | // How many times we tried replaying unmodified execution before we were 149 | // able to reproduce the violation 150 | "initial_verification_runs_needed" -> 0.0, 151 | "minimized_deliveries" -> 0.0, 152 | "minimized_externals" -> 0.0, 153 | "minimized_timers" -> 0.0 154 | ) 155 | } 156 | 157 | def increment_replays() { 158 | total_replays += 1 159 | } 160 | 161 | def record_replay_start() { 162 | stats("replay_start_epoch") = System.currentTimeMillis() 163 | } 164 | 165 | def record_replay_end() { 166 | stats("replay_end_epoch") = System.currentTimeMillis() 167 | stats("replay_duration_seconds") = 168 | (stats("replay_end_epoch") * 1.0 - stats("replay_start_epoch")) / 1000 169 | } 170 | 171 | def record_prune_start() { 172 | stats("prune_start_epoch") = System.currentTimeMillis() 173 | } 174 | 175 | def record_prune_end() { 176 | stats("prune_end_epoch") = System.currentTimeMillis() 177 | stats("prune_duration_seconds") = 178 | (stats("prune_end_epoch") * 1.0 - stats("prune_start_epoch")) / 1000 179 | } 180 | 181 | // For external events (legacy name), record how many external events were 182 | // left at the given iteration (replay #) 183 | def record_iteration_size(iteration_size: Int) { 184 | iterationSize(total_replays) = iteration_size 185 | } 186 | 187 | // For internal events, record how many internal events were left at the 188 | // given iteration (replay #) 189 | def record_internal_size(iteration_size: Integer) { 190 | internalIterationSize(total_replays) = iteration_size 191 | } 192 | 193 | def recordDeliveryStats(minimized_deliveries: Int, minimized_externals: Int, 194 | minimized_timers: Int) { 195 | stats("minimized_deliveries") = minimized_deliveries / 1.0 196 | stats("minimized_externals") = minimized_externals / 1.0 197 | stats("minimized_timers") = minimized_timers / 1.0 198 | } 199 | 200 | def record_distance_increase(newDistance: Int) { 201 | maxDistance(newDistance) = total_replays 202 | } 203 | 204 | def toJson(): JSONObject = { 205 | val map = new HashMap[String, Any] 206 | map("name") = name 207 | map("iteration_size") = JSONObject(iterationSize.map( 208 | pair => pair._1.toString -> pair._2).toMap) 209 | map("internal_iteration_size") = JSONObject(internalIterationSize.map( 210 | pair => pair._1.toString -> pair._2).toMap) 211 | map("total_replays") = total_replays 212 | map("maxDistance") = JSONObject(maxDistance.map( 213 | pair => pair._1.toString -> pair._2).toMap) 214 | map ++= stats 215 | return JSONObject(map.toMap) 216 | } 217 | } 218 | 219 | def fromJson(json: String): MinimizationStats = { 220 | val outer = JSON.parseFull(json).get.asInstanceOf[List[Map[String,Any]]] 221 | val outerObj = new MinimizationStats 222 | outer.foreach { case inner => 223 | val innerObj = new InnerStats(inner("name").asInstanceOf[String]) 224 | innerObj.total_replays = inner("total_replays").asInstanceOf[Double].toInt 225 | innerObj.stats.keys.foreach { case k => 226 | innerObj.stats(k) = inner(k).asInstanceOf[Double] } 227 | inner("maxDistance").asInstanceOf[Map[String,Any]].foreach { case (k,v) => 228 | innerObj.maxDistance(k.toInt) = v.asInstanceOf[Double].toInt } 229 | inner("iteration_size").asInstanceOf[Map[String,Any]].foreach { case (k,v) => 230 | innerObj.iterationSize(k.toInt) = v.asInstanceOf[Double].toInt } 231 | inner("internal_iteration_size").asInstanceOf[Map[String,Any]].foreach { case (k,v) => 232 | innerObj.internalIterationSize(k.toInt) = v.asInstanceOf[Double].toInt } 233 | outerObj.stats += innerObj 234 | } 235 | return outerObj 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/OneAtATime.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.ListBuffer 4 | import scala.collection.mutable.HashSet 5 | 6 | import org.slf4j.LoggerFactory, 7 | ch.qos.logback.classic.Level, 8 | ch.qos.logback.classic.Logger 9 | 10 | class LeftToRightRemoval (oracle: TestOracle, checkUnmodifed:Boolean=false, 11 | stats: Option[MinimizationStats]=None) extends Minimizer { 12 | val _stats = stats match { 13 | case Some(s) => s 14 | case None => new MinimizationStats 15 | } 16 | _stats.updateStrategy("LeftToRightRemoval", oracle.getName) 17 | 18 | val logger = LoggerFactory.getLogger("LeftToRight") 19 | 20 | def minimize(_dag: EventDag, 21 | violation_fingerprint: ViolationFingerprint, 22 | initializationRoutine: Option[() => Any]) : EventDag = { 23 | MessageTypes.sanityCheckTrace(_dag.events) 24 | // First check if the initial trace violates the exception 25 | if (checkUnmodifed) { 26 | logger.info("Checking if unmodified trace triggers violation...") 27 | if (oracle.test(_dag.events, violation_fingerprint, _stats, 28 | initializationRoutine=initializationRoutine) == None) { 29 | throw new IllegalArgumentException("Unmodified trace does not trigger violation") 30 | } 31 | _stats.reset 32 | } 33 | 34 | var dag = _dag 35 | var events_to_test = dag.get_atomic_events 36 | var tested_events = new HashSet[AtomicEvent]() 37 | 38 | while (events_to_test.length > 0) { 39 | // Try removing the event 40 | val event = events_to_test(0) 41 | logger.info("Trying removal of event " + event.toString) 42 | tested_events += event 43 | val new_dag = dag.remove_events(List(event)) 44 | 45 | if (oracle.test(new_dag.get_all_events, violation_fingerprint, 46 | _stats, initializationRoutine=initializationRoutine) == None) { 47 | logger.info("passes") 48 | // Move on to the next event to test 49 | events_to_test = events_to_test.slice(1, events_to_test.length) 50 | } else { 51 | logger.info("fails. Pruning") 52 | dag = new_dag 53 | // The atomic events to test may have changed after removing 54 | // the event we just pruned, so recompute. 55 | events_to_test = dag.get_atomic_events.filterNot(e => tested_events.contains(e)) 56 | } 57 | } 58 | 59 | return dag 60 | } 61 | 62 | def verify_mcs(mcs: EventDag, 63 | violation_fingerprint: ViolationFingerprint, 64 | initializationRoutine: Option[() => Any]=None): Option[EventTrace] = { 65 | val nop_stats = new MinimizationStats 66 | nop_stats.updateStrategy("NOP", "NOP") 67 | return oracle.test(mcs.events, violation_fingerprint, 68 | nop_stats, initializationRoutine=initializationRoutine) 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/TestOracle.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.HashMap 4 | 5 | /** 6 | * User-defined fingerprint for uniquely describing how one or more safety 7 | * violations manifests. 8 | */ 9 | trait ViolationFingerprint { 10 | def matches(other: ViolationFingerprint) : Boolean 11 | // Return the names of all nodes whose internal state exhibited the invariant violation. 12 | def affectedNodes(): Seq[String] 13 | } 14 | 15 | case object NoViolation extends ViolationFingerprint { 16 | def affectedNodes(): Seq[String] = Seq.empty 17 | def matches(other: ViolationFingerprint): Boolean = false 18 | } 19 | 20 | object TestOracle { 21 | // An predicate that returns None if the safety condition is not violated, 22 | // i.e. the execution is correct. Otherwise, returns a 23 | // `fingerprint` that identifies how the safety violation manifests itself. 24 | // The first argument is the current external event sequence, and the second 25 | // argument is a checkpoint map from actor -> Some(checkpointReply), or 26 | // actor -> None if the actor has crashed. 27 | type Invariant = (Seq[ExternalEvent], HashMap[String,Option[CheckpointReply]]) => Option[ViolationFingerprint] 28 | } 29 | 30 | trait TestOracle { 31 | type Invariant = (Seq[ExternalEvent], HashMap[String,Option[CheckpointReply]]) => Option[ViolationFingerprint] 32 | 33 | // Return one of: 34 | // {"RandomScheduler", "STSSchedNoPeek", "STSSched", "DPOR", 35 | // "FairScheduler", "FungibleClocks"} 36 | def getName() : String 37 | 38 | def setInvariant(invariant: Invariant) 39 | 40 | /** 41 | * Returns Some(trace) if there exists any execution trace containing the given external 42 | * events that causes the given invariant violation to reappear. 43 | * Otherwise, returns None. 44 | * 45 | * At the end of the invocation, it is the responsibility of the TestOracle 46 | * to ensure that the ActorSystem is returned to a clean initial state. 47 | * Throws an IllegalArgumentException if setInvariant has not been invoked. 48 | */ 49 | // Note that return value of this function is the opposite of what we 50 | // describe in the paper... 51 | def test(events: Seq[ExternalEvent], 52 | violation_fingerprint: ViolationFingerprint, 53 | stats: MinimizationStats, 54 | initializationRoutine:Option[()=>Any]=None) : Option[EventTrace] 55 | } 56 | 57 | object StatelessTestOracle { 58 | type OracleConstructor = () => TestOracle 59 | } 60 | 61 | /* 62 | * A TestOracle that throws away all state between invocations of test(). 63 | * Useful for debugging or avoiding statefulness problems, e.g. deadlocks. 64 | */ 65 | // TODO(cs): there is currently a deadlock in PeekScheduler: if you invoke 66 | // PeekScheduler.test() multiple times, it runs the first execution just fine, 67 | // but the second execution never reaches Quiescence, and blocks infinitely 68 | // trying to acquire traceSem. Figure out why. 69 | class StatelessTestOracle(oracle_ctor: StatelessTestOracle.OracleConstructor) extends TestOracle { 70 | var invariant : Invariant = null 71 | 72 | def getName: String = oracle_ctor().getName 73 | 74 | def setInvariant(inv: Invariant) = { 75 | invariant = inv 76 | } 77 | 78 | def test(events: Seq[ExternalEvent], 79 | violation_fingerprint: ViolationFingerprint, 80 | stats: MinimizationStats, 81 | init:Option[()=>Any]=None) : Option[EventTrace] = { 82 | val oracle = oracle_ctor() 83 | try { 84 | Instrumenter().scheduler = oracle.asInstanceOf[Scheduler] 85 | } catch { 86 | case e: Exception => println("oracle not a scheduler?") 87 | } 88 | oracle.setInvariant(invariant) 89 | val result = oracle.test(events, violation_fingerprint, stats, init) 90 | Instrumenter().restart_system 91 | return result 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/Util.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.ListBuffer 4 | import scala.collection.mutable.Map 5 | import scala.collection.mutable.HashSet 6 | import scala.collection.mutable.HashMap 7 | 8 | object MinificationUtil { 9 | def split_list(l: Seq[Any], split_ways: Int) : Seq[Seq[Any]] = { 10 | // Split our inputs into split_ways separate lists 11 | if (split_ways < 1) { 12 | throw new IllegalArgumentException("Split ways must be greater than 0") 13 | } 14 | 15 | var splits = List[Seq[Any]]() 16 | var split_interval = l.length / split_ways // integer division = floor 17 | var remainder = l.length % split_ways // remainder is guaranteed to be less than splitways 18 | 19 | var start_idx = 0 20 | while (splits.length < split_ways) { 21 | var split_idx = start_idx + split_interval 22 | /* 23 | * The first 'remainder' chunks are made one element larger to chew 24 | * up the remaining elements (remainder < splitways). 25 | * note: len(l) = split_ways * split_interval + remainder 26 | */ 27 | if (remainder > 0) { 28 | split_idx += 1 29 | remainder -= 1 30 | } 31 | 32 | val slice = l.slice(start_idx, split_idx) 33 | splits = splits :+ slice 34 | start_idx = split_idx 35 | } 36 | return splits 37 | } 38 | } 39 | 40 | /** 41 | * A collection of events that should be treated as an atomic unit, i.e. one 42 | * should never be removed without removing the others. 43 | * 44 | * Example: a Failure event and its subsequent Recovery event. 45 | */ 46 | class AtomicEvent(varargs: ExternalEvent*) { 47 | var events = List(varargs: _*) 48 | 49 | override def toString() : String = { 50 | return events.map(e => e.toString).mkString("-") 51 | } 52 | 53 | override def equals(other: Any) : Boolean = { 54 | other match { 55 | case a: AtomicEvent => return a.events == events 56 | case _ => return false 57 | } 58 | } 59 | 60 | override def hashCode() : Int = { 61 | return events.hashCode() 62 | } 63 | } 64 | 65 | /** 66 | * An ordered collection of ExternalEvent objects. EventDags are primarily used to present 67 | * a view of the underlying events with some subset of the input events pruned 68 | */ 69 | trait EventDag { 70 | /** 71 | * Ensure that: 72 | * - if a failure or partition is removed, its following recovery is also removed 73 | * - if a recovery is removed, its preceding failure or partition is also removed 74 | * ? if a Start event is removed, all subsequent Send events destined for 75 | * that node are removed. BROKEN, see TODO below. 76 | */ 77 | def remove_events(events: Seq[AtomicEvent]) : EventDag 78 | 79 | /** 80 | * Return the union of two EventDags. 81 | */ 82 | def union(other: EventDag) : EventDag 83 | 84 | /** 85 | * Return all ExternalEvents that can be removed. 86 | */ 87 | def get_atomic_events() : Seq[AtomicEvent] 88 | 89 | /** 90 | * Return all ExternalEvents, with AtomicEvents expanded. 91 | */ 92 | def get_all_events() : Seq[ExternalEvent] 93 | 94 | // Alias for get_all_events 95 | def events = get_all_events 96 | 97 | /** 98 | * Return get_all_events().length 99 | */ 100 | def length: Int 101 | def size = length 102 | } 103 | 104 | // Internal utility methods 105 | object EventDag { 106 | def remove_events(to_remove: Seq[AtomicEvent], events: Seq[ExternalEvent]) : Seq[ExternalEvent] = { 107 | val flattened = to_remove.map(atomic => atomic.events).flatten 108 | var all_removed = Set(flattened: _*) 109 | assume(flattened.length == all_removed.size) 110 | return events.filter(e => !(all_removed contains e)) 111 | 112 | // TODO(cs): figure out why the below optimization screws Delta Debugging 113 | // up... 114 | /* 115 | // The invariant that failures and recoveries are removed atomically is 116 | // already ensured by the way AtomicEvents are constructed. 117 | // 118 | // All that is left for us to handle is dependencies between Start and Send 119 | // events. 120 | 121 | // Collect up all remaining events. 122 | var remaining : Seq[ExternalEvent] = ListBuffer() 123 | // While we collect remaining events, also ensure that for all removed Start events, we 124 | // remove all subsequent Sends before the next Start event. 125 | // We accomplish this by tracking whether we recently passed a removed Start event 126 | // as we iterate through the list. 127 | val pending_starts = new HashSet[String]() 128 | for (event <- events) { 129 | event match { 130 | case Start(_, node) => { 131 | if (all_removed.contains(event)) { 132 | pending_starts += node 133 | } else if (pending_starts.contains(node)) { 134 | // The node has started up again, so stop filtering subsequent 135 | // Send events. 136 | pending_starts -= node 137 | } 138 | } 139 | case Send(node, _) => { 140 | if (pending_starts.contains(node)) { 141 | all_removed += event 142 | } 143 | } 144 | case _ => None 145 | } 146 | 147 | if (!all_removed.contains(event)) { 148 | remaining = remaining :+ event 149 | } 150 | } 151 | 152 | assume(remaining.length + all_removed.size == events.length) 153 | return remaining 154 | */ 155 | } 156 | } 157 | 158 | /** 159 | * An unmodified EventDag. 160 | */ 161 | class UnmodifiedEventDag(events: Seq[ExternalEvent]) extends EventDag { 162 | val event_to_idx : Map[ExternalEvent, Int] = Util.map_from_iterable(events.zipWithIndex) 163 | // Index of event -> index of pair 164 | // Should be symmetric 165 | val _conjoinedAtoms = new HashMap[Int, Int] 166 | 167 | def conjoinAtoms(e1: ExternalEvent, e2: ExternalEvent) { 168 | if (!(event_to_idx contains e1)) { 169 | throw new IllegalArgumentException("No such external event:" + e1) 170 | } 171 | if (!(event_to_idx contains e2)) { 172 | throw new IllegalArgumentException("No such external event:" + e2) 173 | } 174 | assert(!(_conjoinedAtoms contains event_to_idx(e1))) 175 | assert(!(_conjoinedAtoms contains event_to_idx(e2))) 176 | _conjoinedAtoms(event_to_idx(e1)) = event_to_idx(e2) 177 | _conjoinedAtoms(event_to_idx(e2)) = event_to_idx(e1) 178 | } 179 | 180 | def remove_events(to_remove: Seq[AtomicEvent]) : EventDag = { 181 | val remaining = EventDag.remove_events(to_remove, events) 182 | return new EventDagView(this, remaining) 183 | } 184 | 185 | def union(other: EventDag) : EventDag = { 186 | if (other.get_all_events.length != 0) { 187 | throw new IllegalArgumentException("Unknown events") 188 | } 189 | return this 190 | } 191 | 192 | def get_atomic_events() : Seq[AtomicEvent] = { 193 | return get_atomic_events(events) 194 | } 195 | 196 | // Internal API 197 | def get_atomic_events(given_events: Seq[ExternalEvent]) : Seq[AtomicEvent] = { 198 | // TODO(cs): memoize this computation? 199 | // As we iterate through events, check whether the "fingerprint" of the 200 | // current recovery event matches a "fingerprint" of a preceding failure 201 | // event. If so, group them together as an Atomic pair. 202 | // 203 | // For Partition/UnPartition, the Partition always comes first. For 204 | // Kill/Start, the Start always comes first. 205 | var fingerprint_2_previous_dual : Map[String, ExternalEvent] = Map() 206 | var atomics = new ListBuffer[AtomicEvent]() 207 | 208 | // First deal with explicitly conjoined atoms. 209 | val conjoined = new HashSet[ExternalEvent] ++ given_events.filter(e => _conjoinedAtoms contains event_to_idx(e)) 210 | assert(conjoined.forall(e => conjoined contains events(_conjoinedAtoms(event_to_idx(e))))) 211 | while (!conjoined.isEmpty) { 212 | val head = conjoined.head 213 | val tail = events(_conjoinedAtoms(event_to_idx(head))) 214 | atomics += new AtomicEvent(head, tail) 215 | conjoined -= head 216 | conjoined -= tail 217 | } 218 | 219 | // Now deal with domain knowledge. 220 | for (event <- given_events.filterNot(e => _conjoinedAtoms contains event_to_idx(e))) { 221 | event match { 222 | case Kill(name) => { 223 | fingerprint_2_previous_dual get name match { 224 | case Some(start) => { 225 | atomics = atomics :+ new AtomicEvent(start, event) 226 | fingerprint_2_previous_dual -= name 227 | } 228 | case None => { 229 | throw new RuntimeException("Kill without preceding Start") 230 | } 231 | } 232 | } 233 | case Partition(a,b) => { 234 | val key = a + "->" + b 235 | fingerprint_2_previous_dual(key) = event 236 | } 237 | case Start(_, name) => { 238 | fingerprint_2_previous_dual(name) = event 239 | } 240 | case UnPartition(a,b) => { 241 | val key = a + "->" + b 242 | fingerprint_2_previous_dual get key match { 243 | case Some(partition) => { 244 | atomics = atomics :+ new AtomicEvent(partition, event) 245 | fingerprint_2_previous_dual -= key 246 | } 247 | case None => { 248 | throw new RuntimeException("UnPartition without preceding Partition") 249 | } 250 | } 251 | } 252 | case _ => atomics = atomics :+ new AtomicEvent(event) 253 | } 254 | } 255 | 256 | // Make sure to include any remaining Start or Partition events that never had 257 | // corresponding Kill/UnPartition events. 258 | for (e <- fingerprint_2_previous_dual.values) { 259 | atomics = atomics :+ new AtomicEvent(e) 260 | } 261 | 262 | assume(atomics.map(atomic => atomic.events).flatten.length == given_events.length) 263 | // Sort by the original index of first element in the event list. 264 | return atomics.sortBy[Int](a => event_to_idx(a.events(0))) 265 | } 266 | 267 | def get_all_events() : Seq[ExternalEvent] = { 268 | return events 269 | } 270 | 271 | def length: Int = events.length 272 | } 273 | 274 | /** 275 | * A subsequence of an EventDag. 276 | */ 277 | // All EventDagViews have a pointer to the original UnmodifiedEventDag, so 278 | // that we can compute the proper ordering of events in a union() of two distinct 279 | // EventDagViews. 280 | class EventDagView(parent: UnmodifiedEventDag, events: Seq[ExternalEvent]) extends EventDag { 281 | 282 | def remove_events(to_remove: Seq[AtomicEvent]) : EventDag = { 283 | val remaining = EventDag.remove_events(to_remove, events) 284 | return new EventDagView(parent, remaining) 285 | } 286 | 287 | def union(other: EventDag) : EventDag = { 288 | // Set isn't strictly necessary, just a sanity check. 289 | val union = Set((events ++ other.get_all_events): _*). 290 | toList.sortBy[Int](e => parent.event_to_idx(e)) 291 | assume(events.length + other.get_all_events.length == union.length) 292 | return new EventDagView(parent, union) 293 | } 294 | 295 | def get_atomic_events() : Seq[AtomicEvent] = { 296 | return parent.get_atomic_events(events) 297 | } 298 | 299 | def get_all_events() : Seq[ExternalEvent] = { 300 | return events 301 | } 302 | 303 | def length: Int = events.length 304 | } 305 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/WildcardTestOracle.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.Props 4 | import scalax.collection.mutable.Graph, 5 | scalax.collection.GraphEdge.DiEdge, 6 | scalax.collection.edge.LDiEdge 7 | 8 | // Used in conjuction with DDMin for external events. 9 | // - timeBudgetSeconds: max time to give to *each* invocation of test 10 | class WildcardTestOracle( 11 | schedulerConfig: SchedulerConfig, 12 | originalTrace: EventTrace, 13 | actorNameProps: Seq[Tuple2[Props, String]], 14 | resolutionStrategy: AmbiguityResolutionStrategy=null, // if null, use BackTrackStrategy 15 | testScheduler:TestScheduler.TestScheduler=TestScheduler.STSSched, 16 | depGraph: Option[Graph[Unique,DiEdge]]=None, 17 | preTest: Option[STSScheduler.PreTestCallback]=None, 18 | timeBudgetSeconds:Long=Long.MaxValue, 19 | postTest: Option[STSScheduler.PostTestCallback]=None) extends TestOracle { 20 | 21 | def getName = "WildcardTestOracle" 22 | 23 | // Should already be specific in schedulerConfig 24 | def setInvariant(invariant: Invariant) {} 25 | 26 | var minTrace = originalTrace 27 | var externalsForMinTrace : Seq[ExternalEvent] = Seq.empty 28 | 29 | // TODO(cs): possible optimization: if we ever find a smaller trace that 30 | // triggers the bug, pass in that smaller trace from then on, rather than 31 | // originalTrace. 32 | 33 | def test(events: Seq[ExternalEvent], 34 | violation_fingerprint: ViolationFingerprint, 35 | stats: MinimizationStats, 36 | initializationRoutine:Option[()=>Any]=None) : Option[EventTrace] = { 37 | val minimizer = new WildcardMinimizer( 38 | schedulerConfig, 39 | events, 40 | originalTrace, 41 | actorNameProps, 42 | violation_fingerprint, 43 | skipClockClusters=true, 44 | stats=Some(stats), 45 | resolutionStrategy=resolutionStrategy, 46 | testScheduler=testScheduler, 47 | depGraph=depGraph, 48 | initializationRoutine=initializationRoutine, 49 | preTest=preTest, 50 | postTest=postTest, 51 | timeBudgetSeconds=timeBudgetSeconds) 52 | 53 | val (_, trace) = minimizer.minimize() 54 | if (trace != originalTrace) { 55 | if (trace.size < minTrace.size) { 56 | minTrace = trace 57 | externalsForMinTrace = events 58 | } 59 | return Some(trace) 60 | } 61 | return None 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/internal_minimization/OneAtATimeRemoval.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.SynchronizedQueue 4 | import scala.collection.mutable.Queue 5 | import scala.collection.mutable.HashMap 6 | import scala.collection.mutable.HashSet 7 | import scala.collection.mutable.ArrayBuffer 8 | import akka.actor.Props 9 | 10 | import org.slf4j.LoggerFactory, 11 | ch.qos.logback.classic.Level, 12 | ch.qos.logback.classic.Logger 13 | 14 | // Superclass for RemovalStrategy's that only ever remove events one at a time from the 15 | // original execution, but never add events. 16 | // Inheritors must implement the "choiceFilter" method. 17 | abstract class OneAtATimeStrategy( 18 | verified_mcs: EventTrace, 19 | messageFingerprinter: FingerprintFactory) extends RemovalStrategy { 20 | // MsgEvents we've tried ignoring so far. MultiSet to account for duplicate MsgEvent's 21 | val triedIgnoring = new MultiSet[(String, String, MessageFingerprint)] 22 | var _unignorable = 0 23 | val logger = LoggerFactory.getLogger("RemovalStrategy") 24 | 25 | // Populate triedIgnoring with all events that lie between an 26 | // UnignorableEvents block. Also, external messages. 27 | private def init() { 28 | var inUnignorableBlock = false 29 | 30 | verified_mcs.events.foreach { 31 | case BeginUnignorableEvents => 32 | inUnignorableBlock = true 33 | case EndUnignorableEvents => 34 | inUnignorableBlock = false 35 | case m @ UniqueMsgEvent(MsgEvent(snd, rcv, msg), id) => 36 | if (EventTypes.isExternal(m) || inUnignorableBlock) { 37 | triedIgnoring += ((snd, rcv, messageFingerprinter.fingerprint(msg))) 38 | } 39 | case t @ UniqueTimerDelivery(TimerDelivery(snd, rcv, msg), id) => 40 | if (inUnignorableBlock) { 41 | triedIgnoring += ((snd, rcv, messageFingerprinter.fingerprint(msg))) 42 | } 43 | case _ => 44 | } 45 | 46 | _unignorable = triedIgnoring.size 47 | } 48 | 49 | init() 50 | 51 | def unignorable: Int = _unignorable 52 | 53 | // Filter out the next MsgEvent, and return the resulting EventTrace. 54 | // If we've tried filtering out all MsgEvents, return None. 55 | def getNextTrace(trace: EventTrace, 56 | alreadyRemoved: MultiSet[(String,String,MessageFingerprint)], 57 | violationTriggered: Boolean) 58 | : Option[EventTrace] = { 59 | // Track what events we've kept so far because we 60 | // already tried ignoring them previously. MultiSet to account for 61 | // duplicate MsgEvent's. TODO(cs): this may lead to some ambiguous cases. 62 | val keysThisIteration = new MultiSet[(String, String, MessageFingerprint)] 63 | // We already tried 'keeping' prior events that were successfully ignored 64 | // but no longer show up in this trace. 65 | keysThisIteration ++= alreadyRemoved 66 | // Whether we've found the event we're going to try ignoring next. 67 | var foundIgnoredEvent = false 68 | 69 | // Return whether we should keep this event 70 | def checkDelivery(snd: String, rcv: String, msg: Any): Boolean = { 71 | val key = (snd, rcv, messageFingerprinter.fingerprint(msg)) 72 | keysThisIteration += key 73 | if (foundIgnoredEvent) { 74 | // We already chose our event to ignore. Keep all other events. 75 | return true 76 | } else { 77 | // Check if we should ignore or keep this one. 78 | if (keysThisIteration.count(key) > triedIgnoring.count(key) && 79 | choiceFilter(key._1, key._2, key._3)) { 80 | // We found something to ignore 81 | logger.info("Ignoring next: " + key) 82 | foundIgnoredEvent = true 83 | triedIgnoring += key 84 | return false 85 | } else { 86 | // Keep this one; we already tried ignoring it, but it was 87 | // not prunable. 88 | return true 89 | } 90 | } 91 | } 92 | 93 | // We accomplish two tasks as we iterate through trace: 94 | // - Finding the next event we want to ignore 95 | // - Filtering (keeping) everything that we don't want to ignore 96 | val modified = trace.events.flatMap { 97 | case m @ UniqueMsgEvent(MsgEvent(snd, rcv, msg), id) => 98 | if (checkDelivery(snd, rcv, msg)) { 99 | Some(m) 100 | } else { 101 | None 102 | } 103 | case t @ UniqueTimerDelivery(TimerDelivery(snd, rcv, msg), id) => 104 | if (checkDelivery(snd, rcv, msg)) { 105 | Some(t) 106 | } else { 107 | None 108 | } 109 | case _: MsgEvent => 110 | throw new IllegalArgumentException("Must be UniqueMsgEvent") 111 | case e => 112 | Some(e) 113 | } 114 | if (foundIgnoredEvent) { 115 | val queue = new SynchronizedQueue[Event] 116 | queue ++= modified 117 | return Some(new EventTrace(queue, 118 | verified_mcs.original_externals)) 119 | } 120 | // We didn't find anything else to ignore, so we're done 121 | return None 122 | } 123 | 124 | // choiceFilter: if the given (snd,rcv,fingerprint,UniqueMsgEvent.id) is eligible to be 125 | // picked next, return true or false if it should be picked. 126 | // Guarenteed to be invoked in left-to-right order 127 | def choiceFilter(snd: String, rcv: String, 128 | fingerprint: MessageFingerprint) : Boolean 129 | } 130 | 131 | // Removes events one at a time, from left to right order. 132 | class LeftToRightOneAtATime( 133 | verified_mcs: EventTrace, messageFingerprinter: FingerprintFactory) 134 | extends OneAtATimeStrategy(verified_mcs, messageFingerprinter) { 135 | 136 | override def choiceFilter(s: String, r: String, f: MessageFingerprint) = true 137 | } 138 | 139 | // For any <src, dst> pair, maintains FIFO delivery (i.e. assumes TCP as the 140 | // underlying transport medium), and only tries removing the last message 141 | // (iteratively) from each FIFO queue. 142 | // 143 | // Assumes that the original trace was generated using a src,dst FIFO delivery 144 | // discipline. 145 | class SrcDstFIFORemoval( 146 | verified_mcs: EventTrace, messageFingerprinter: FingerprintFactory) 147 | extends OneAtATimeStrategy(verified_mcs, messageFingerprinter) { 148 | 149 | // N.B. doesn't actually contain TimerDeliveries; only real messages. Timers 150 | // are harder to reason about, and this is just an optimization anyway, so 151 | // just try removing them in random order. 152 | // Queue is: ids of UniqueMsgEvents. 153 | val srcDstToMessages = new HashMap[(String, String), Vector[MessageFingerprint]] 154 | 155 | verified_mcs.events.foreach { 156 | case UniqueMsgEvent(MsgEvent("deadLetters", rcv, msg), id) => 157 | case m @ UniqueMsgEvent(MsgEvent(snd, rcv, msg), id) => 158 | val vec = srcDstToMessages.getOrElse((snd,rcv), Vector[MessageFingerprint]()) 159 | val newVec = vec :+ messageFingerprinter.fingerprint(msg) 160 | srcDstToMessages((snd,rcv)) = newVec 161 | case _ => 162 | } 163 | 164 | // The src,dst pair we chose last time getNextTrace was invoked. 165 | var previouslyChosenSrcDst : Option[(String,String)] = None 166 | // To deal with the possibility of multiple indistinguishable pending 167 | // messages at different points in the execution: 168 | // As choiceFilter is invoked, mark off where we are in the ArrayList for a 169 | // given src,dst pair. Only return true if we're at the end of the 170 | // ArrayList. 171 | val srcDstToCurrentIdx = new HashMap[(String, String), Int] 172 | def resetSrcDstToCurrentIdx() { 173 | srcDstToMessages.keys.foreach { 174 | case (src,dst) => 175 | srcDstToCurrentIdx((src,dst)) = -1 176 | } 177 | } 178 | resetSrcDstToCurrentIdx() 179 | 180 | def choiceFilter(snd: String, rcv: String, 181 | fingerprint: MessageFingerprint) : Boolean = { 182 | if (srcDstToMessages contains ((snd,rcv))) { 183 | srcDstToCurrentIdx((snd,rcv)) += 1 184 | val idx = srcDstToCurrentIdx((snd,rcv)) 185 | val lst = srcDstToMessages((snd,rcv)) 186 | if (idx == lst.length - 1) { 187 | // assert(lst(idx) == fingerprint) 188 | if (lst(idx) != fingerprint) { 189 | logger.error(s"lst(idx) ${lst(idx)} != fingerprint ${fingerprint}") 190 | } 191 | srcDstToMessages((snd,rcv)) = srcDstToMessages((snd,rcv)).dropRight(1) 192 | if (srcDstToMessages((snd,rcv)).isEmpty) { 193 | logger.info("src,dst is done!: " + ((snd,rcv))) 194 | srcDstToMessages -= ((snd,rcv)) 195 | } 196 | previouslyChosenSrcDst = Some((snd,rcv)) 197 | return true 198 | } 199 | } 200 | 201 | previouslyChosenSrcDst = None 202 | 203 | if (snd == "deadLetters") { // Timer 204 | return true 205 | } 206 | 207 | return false 208 | } 209 | 210 | // Filter out the next MsgEvent, and return the resulting EventTrace. 211 | // If we've tried filtering out all MsgEvents, return None. 212 | // TODO(cs): account for Kills & HardKills, which would reset the FIFO queues 213 | // involving that node. 214 | override def getNextTrace(trace: EventTrace, 215 | alreadyRemoved: MultiSet[(String,String,MessageFingerprint)], 216 | violationTriggeredLastRun: Boolean): Option[EventTrace] = { 217 | if (!violationTriggeredLastRun && !previouslyChosenSrcDst.isEmpty) { 218 | // Ignoring didn't work, so this src,dst is done. 219 | logger.info("src,dst is done: " + previouslyChosenSrcDst.get) 220 | srcDstToMessages -= previouslyChosenSrcDst.get 221 | } 222 | 223 | if (violationTriggeredLastRun) { 224 | // Some of the messages in srcDstToMessages may have been pruned as 225 | // "freebies" -- i.e. they may have been absent -- in the last run. 226 | // -> Recompute srcDstToMessages (in reverse order, in case there are 227 | // multiple indistinguishable events) 228 | srcDstToMessages.clear 229 | val alreadyRemovedCopy = new MultiSet[(String,String,MessageFingerprint)] 230 | alreadyRemovedCopy ++= alreadyRemoved 231 | verified_mcs.events.reverse.foreach { 232 | case UniqueMsgEvent(MsgEvent("deadLetters", rcv, msg), id) => 233 | case m @ UniqueMsgEvent(MsgEvent(snd, rcv, msg), id) => 234 | val tuple = ((snd,rcv,messageFingerprinter.fingerprint(msg))) 235 | if (alreadyRemovedCopy contains tuple) { 236 | alreadyRemovedCopy -= tuple 237 | } else { 238 | val vec = srcDstToMessages.getOrElse((snd,rcv), Vector[MessageFingerprint]()) 239 | // N.B. prepend, not append, so that it comes out in the same order 240 | // as the original trace 241 | val newVec = vec.+:(messageFingerprinter.fingerprint(msg)) 242 | srcDstToMessages((snd,rcv)) = newVec 243 | } 244 | case _ => 245 | } 246 | } 247 | resetSrcDstToCurrentIdx() 248 | 249 | return super.getNextTrace(trace, alreadyRemoved, violationTriggeredLastRun) 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/internal_minimization/RemovalStrategy.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | // Iteratively choose schedules to explore. 4 | trait RemovalStrategy { 5 | // Return how many events we were unwilling to ignore, e.g. because they've 6 | // been marked by the application as unignorable. 7 | def unignorable: Int 8 | 9 | // Return the next schedule to explore. 10 | // If there aren't any more schedules to check, return None. 11 | // Args: 12 | // - lastFailingTrace: the most recent schedule we've explored that has successfuly 13 | // resulted in the invariant violation. 14 | // - alreadyRemoved: any (src,dst,message fingerprint) pairs from the 15 | // original schedule that we've already successfully decided aren't 16 | // necessary 17 | // - violationTriggered: whether the last schedule we returned 18 | // successfully triggered the invariant violation, i.e. whether 19 | // lastFailingTrace == the most recent trace we returned from getNextTrace. 20 | def getNextTrace(lastFailingTrace: EventTrace, 21 | alreadyRemoved: MultiSet[(String,String,MessageFingerprint)], 22 | violationTriggered: Boolean) 23 | : Option[EventTrace] 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/internal_minimization/ScheduleCheckers.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.Props 4 | 5 | import org.slf4j.LoggerFactory, 6 | ch.qos.logback.classic.Level, 7 | ch.qos.logback.classic.Logger 8 | 9 | // Minimizes internal events. One-time-use -- you shouldn't invoke minimize() more 10 | // than once. 11 | trait InternalEventMinimizer { 12 | def minimize(): Tuple2[MinimizationStats, EventTrace] 13 | } 14 | 15 | // TODO(cs): we should try supporting DPOR removal of internals. 16 | 17 | // Given a RemovalStrategy, checks each schedule returned by the 18 | // RemovalStrategy with the STSSched scheduler. 19 | class STSSchedMinimizer( 20 | mcs: Seq[ExternalEvent], 21 | verified_mcs: EventTrace, 22 | violation: ViolationFingerprint, 23 | removalStrategy: RemovalStrategy, 24 | schedulerConfig: SchedulerConfig, 25 | actorNameProps: Seq[Tuple2[Props, String]], 26 | initializationRoutine: Option[() => Any]=None, 27 | preTest: Option[STSScheduler.PreTestCallback]=None, 28 | postTest: Option[STSScheduler.PostTestCallback]=None, 29 | stats: Option[MinimizationStats]=None) 30 | extends InternalEventMinimizer { 31 | 32 | val logger = LoggerFactory.getLogger("IntMin") 33 | 34 | def minimize(): Tuple2[MinimizationStats, EventTrace] = { 35 | val _stats = stats match { 36 | case Some(s) => s 37 | case None => new MinimizationStats 38 | } 39 | _stats.updateStrategy("InternalMin", "STSSched") 40 | 41 | val origTrace = verified_mcs.filterCheckpointMessages.filterFailureDetectorMessages 42 | var lastFailingTrace = origTrace 43 | var lastFailingSize = RunnerUtils.countMsgEvents(lastFailingTrace.filterCheckpointMessages.filterFailureDetectorMessages) 44 | // { (snd,rcv,fingerprint) } 45 | val prunedOverall = new MultiSet[(String,String,MessageFingerprint)] 46 | // TODO(cs): make this more efficient? Currently O(n^2) overall. 47 | var violationTriggered = false 48 | var nextTrace = removalStrategy.getNextTrace(lastFailingTrace, 49 | prunedOverall, violationTriggered) 50 | 51 | _stats.record_prune_start 52 | while (!nextTrace.isEmpty) { 53 | RunnerUtils.testWithStsSched(schedulerConfig, mcs, nextTrace.get, actorNameProps, 54 | violation, _stats, initializationRoutine=initializationRoutine, 55 | preTest=preTest, postTest=postTest) match { 56 | case Some(trace) => 57 | // Some other events may have been pruned by virtue of being absent. So 58 | // we reassign lastFailingTrace, then pick then next trace based on 59 | // it. 60 | violationTriggered = true 61 | val filteredTrace = trace.filterCheckpointMessages.filterFailureDetectorMessages 62 | val origSize = RunnerUtils.countMsgEvents(lastFailingTrace.filterCheckpointMessages.filterFailureDetectorMessages) 63 | val newSize = RunnerUtils.countMsgEvents(filteredTrace) 64 | val diff = origSize - newSize 65 | logger.info("Ignoring worked! Pruned " + diff + "/" + origSize + " deliveries") 66 | 67 | val priorDeliveries = new MultiSet[(String,String,MessageFingerprint)] 68 | priorDeliveries ++= RunnerUtils.getFingerprintedDeliveries( 69 | lastFailingTrace.filterCheckpointMessages.filterFailureDetectorMessages, 70 | schedulerConfig.messageFingerprinter 71 | ) 72 | 73 | val newDeliveries = new MultiSet[(String,String,MessageFingerprint)] 74 | newDeliveries ++= RunnerUtils.getFingerprintedDeliveries( 75 | filteredTrace, 76 | schedulerConfig.messageFingerprinter 77 | ) 78 | 79 | val prunedThisRun = priorDeliveries.setDifference(newDeliveries) 80 | logger.debug("Pruned: [ignore TimerDeliveries] ") 81 | prunedThisRun.foreach { case e => logger.debug("" + e) } 82 | logger.debug("---") 83 | 84 | prunedOverall ++= prunedThisRun 85 | 86 | lastFailingTrace = filteredTrace 87 | lastFailingTrace.setOriginalExternalEvents(mcs) 88 | lastFailingSize = newSize 89 | _stats.record_internal_size(lastFailingSize) 90 | case None => 91 | // We didn't trigger the violation. 92 | violationTriggered = false 93 | _stats.record_internal_size(lastFailingSize) 94 | logger.info("Ignoring didn't work.") 95 | None 96 | } 97 | nextTrace = removalStrategy.getNextTrace(lastFailingTrace, 98 | prunedOverall, violationTriggered) 99 | } 100 | _stats.record_prune_end 101 | val origSize = RunnerUtils.countMsgEvents(origTrace) 102 | val newSize = RunnerUtils.countMsgEvents(lastFailingTrace.filterCheckpointMessages.filterFailureDetectorMessages) 103 | val diff = origSize - newSize 104 | logger.info("Pruned " + diff + "/" + origSize + " deliveries (" + removalStrategy.unignorable + " unignorable)" + 105 | " in " + _stats.inner().total_replays + " replays") 106 | return (_stats, lastFailingTrace.filterCheckpointMessages.filterFailureDetectorMessages) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/internal_minimization/StateMachineRemoval.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.ListBuffer 4 | 5 | // A RemovalStrategy that maintains a model of the program's state 6 | // machine, and uses the model to decide which schedules to explore next. 7 | // We currently use Synoptic 8 | // (http://homes.cs.washington.edu/~mernst/pubs/synoptic-fse2011.pdf) 9 | // to build the model from console output of each execution we've tried so far. 10 | class StateMachineRemoval(originalTrace: EventTrace, messageFingerprinter: FingerprintFactory) extends RemovalStrategy { 11 | // Return how many events we were unwilling to ignore, e.g. because they've 12 | // been marked by the application as unignorable. 13 | def unignorable: Int = 0 14 | 15 | // Return the next schedule to explore. 16 | // If there aren't any more schedules to check, return None. 17 | // Args: 18 | // - lastFailingTrace: the most recent schedule we've explored that has successfuly 19 | // resulted in the invariant violation. 20 | // - alreadyRemoved: any (src,dst,message fingerprint) pairs from the 21 | // original schedule that we've already successfully decided aren't 22 | // necessary 23 | // - violationTriggered: whether the last schedule we returned 24 | // successfully triggered the invariant violation, i.e. whether 25 | // lastFailingTrace == the most recent trace we returned from getNextTrace. 26 | override def getNextTrace(lastFailingTrace: EventTrace, 27 | alreadyRemoved: MultiSet[(String,String,MessageFingerprint)], 28 | violationTriggered: Boolean): Option[EventTrace] = { 29 | return None 30 | } 31 | } 32 | 33 | // Stores all (Meta)EventTraces that have been executed in the past 34 | object HistoricalEventTraces { 35 | def current: MetaEventTrace = traces.last 36 | def isEmpty = traces.isEmpty 37 | 38 | // In order of least recent to most recent 39 | val traces = new ListBuffer[MetaEventTrace] 40 | 41 | // If you want fast lookup of EventTraces, you could populate a HashMap here: 42 | // { EventTrace -> MetaEventTrace } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/AmbiguityResolutionStrategies.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.HashSet 4 | 5 | // When there are multiple pending messages between a given src,dst pair, 6 | // choose how the WildCardMatch's message selector should be applied to the 7 | // pending messages. 8 | trait AmbiguityResolutionStrategy { 9 | type MessageSelector = (Any) => Boolean 10 | 11 | // Return the index of the selected message, if any. 12 | def resolve(msgSelector: MessageSelector, pending: Seq[Any], 13 | backtrackSetter: (Int) => Unit) : Option[Int] 14 | } 15 | 16 | // TODO(cs): SrcDstFIFOOnly technically isn't enough to ensure FIFO removal -- 17 | // currently ClockClusterizer can violate the FIFO scheduling discipline. 18 | class SrcDstFIFOOnly extends AmbiguityResolutionStrategy { 19 | // If the first pending message doesn't match, give up. 20 | // For Timers though, allow any of them to matched. 21 | def resolve(msgSelector: MessageSelector, pending: Seq[Any], 22 | backtrackSetter: (Int) => Unit) : Option[Int] = { 23 | pending.headOption match { 24 | case Some(msg) => 25 | if (msgSelector(msg)) 26 | Some(0) 27 | else 28 | None 29 | case None => 30 | None 31 | } 32 | } 33 | } 34 | 35 | // TODO(cs): implement SrcDstFIFO strategy that adds backtrack point for 36 | // playing unexpected events at the front of the FIFO queue whenever the WildCard 37 | // doesn't match the head (but a pending event later does?). 38 | // Key constraint: need to make sure that we're always making progress, i.e. 39 | // what we're exploring would be shorter if it pans out. 40 | 41 | // For UDP bugs: set a backtrack point every time there are multiple pending 42 | // messages of the same type, and have DPORwHeuristics go back and explore 43 | // those. 44 | class BackTrackStrategy(messageFingerprinter: FingerprintFactory) extends AmbiguityResolutionStrategy { 45 | def resolve(msgSelector: MessageSelector, pending: Seq[Any], 46 | backtrackSetter: (Int) => Unit) : Option[Int] = { 47 | val matching = pending.zipWithIndex.filter( 48 | { case (msg,i) => msgSelector(msg) }) 49 | 50 | if (!matching.isEmpty) { 51 | // Set backtrack points, if any, for any messages that are of the same 52 | // type, but not exactly the same as eachother. 53 | val alreadyTried = new HashSet[MessageFingerprint] 54 | alreadyTried += messageFingerprinter.fingerprint(matching.head._1) 55 | // Heuristic: reverse the remaining backtrack points, on the assumption 56 | // that the last one is more fruitful than the middle ones 57 | matching.tail.reverse.filter { 58 | case (msg,i) => 59 | val fingerprinted = messageFingerprinter.fingerprint(msg) 60 | if (!(alreadyTried contains fingerprinted)) { 61 | alreadyTried += fingerprinted 62 | true 63 | } else { 64 | false 65 | } 66 | }.foreach { 67 | case (msg,i) => backtrackSetter(i) 68 | } 69 | 70 | return matching.headOption.map(t => t._2) 71 | } 72 | 73 | return None 74 | } 75 | } 76 | 77 | // Same as above, but only focus on the first and last matching message. 78 | class FirstAndLastBacktrack(messageFingerprinter: FingerprintFactory) extends AmbiguityResolutionStrategy { 79 | def resolve(msgSelector: MessageSelector, pending: Seq[Any], 80 | backtrackSetter: (Int) => Unit) : Option[Int] = { 81 | val matching = pending.zipWithIndex.filter( 82 | { case (msg,i) => msgSelector(msg) }) 83 | 84 | if (!matching.isEmpty) { 85 | // Set backtrack points, if any, for any messages that are of the same 86 | // type, but not exactly the same as eachother. 87 | val alreadyTried = new HashSet[MessageFingerprint] 88 | alreadyTried += messageFingerprinter.fingerprint(matching.head._1) 89 | matching.tail.reverse.find { 90 | case (msg,i) => 91 | val fingerprinted = messageFingerprinter.fingerprint(msg) 92 | if (!(alreadyTried contains fingerprinted)) { 93 | alreadyTried += fingerprinted 94 | true 95 | } else { 96 | false 97 | } 98 | }.foreach { 99 | case (msg,i) => backtrackSetter(i) 100 | } 101 | 102 | return matching.headOption.map(t => t._2) 103 | } 104 | 105 | return None 106 | } 107 | } 108 | 109 | class LastOnlyStrategy extends AmbiguityResolutionStrategy { 110 | def resolve(msgSelector: MessageSelector, pending: Seq[Any], 111 | backtrackSetter: (Int) => Unit) : Option[Int] = { 112 | val matching = pending.zipWithIndex.filter( 113 | { case (msg,i) => msgSelector(msg) }) 114 | 115 | return matching.lastOption.map(t => t._2) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/ClockClusterizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.SynchronizedQueue 4 | import scala.util.Sorting 5 | import scala.reflect.ClassTag 6 | 7 | 8 | import org.slf4j.LoggerFactory, 9 | ch.qos.logback.classic.Level, 10 | ch.qos.logback.classic.Logger 11 | 12 | object Aggressiveness extends Enumeration { 13 | type Level = Value 14 | // Try all timers for all clusters. 15 | val NONE = Value 16 | // Try all timers for the first cluster iteration, then stop 17 | // as soon as a violation is found for all subsequent clusters. 18 | val ALL_TIMERS_FIRST_ITR = Value 19 | // Always stop as soon as a violation is found. 20 | val STOP_IMMEDIATELY = Value 21 | } 22 | 23 | // Cluster events as follows: 24 | // - Cluster all message deliveries according to their Term number. 25 | // - Finding which TimerDeliveries to include is hard [need to reason about 26 | // actor's current clock values], so instead have a nested for loop: 27 | // for each cluster we're going to try: 28 | // for each TimerDelivery: 29 | // try removing TimerDelivery from list of all TimerDeliveries 30 | // 31 | // See design doc for more info: 32 | // https://docs.google.com/document/d/1_EqOkSehZVC7oE2hjV1FxY8jw4mXAlM5ESigQYt4ojg/edit 33 | // 34 | // Args: 35 | // - aggressive: whether to stop trying to remove timers as soon as we've 36 | // found at least one failing violation for the current cluster. 37 | // - skipClockClusters: whether to only explore timers, and just play all 38 | // clock clusters. 39 | class ClockClusterizer( 40 | originalTrace: EventTrace, 41 | fingerprinter: FingerprintFactory, 42 | resolutionStrategy: AmbiguityResolutionStrategy, 43 | aggressiveness: Aggressiveness.Level=Aggressiveness.ALL_TIMERS_FIRST_ITR, 44 | skipClockClusters: Boolean=false) extends Clusterizer { 45 | 46 | val log = LoggerFactory.getLogger("ClockClusterizer") 47 | 48 | // Iteration: 49 | // - Pick a cluster to remove [starting with removing an empty set] 50 | // - Wildcard all subsequent clusters 51 | // - Always include all other events that do not have a Term number 52 | 53 | // Which UniqueMsgEvent ids (other than TimerDeliveries) to include next 54 | val clusterIterator = new ClockClusterIterator(originalTrace, fingerprinter) 55 | assert(clusterIterator.hasNext) 56 | // current set of messages we're *including* 57 | var currentCluster = clusterIterator.next // Start by not removing any clusters 58 | var tryingFirstCluster = true 59 | 60 | // Which ids of ElectionTimers we've already tried adding for the current 61 | // cluster. clear()'ed whenever we update the next cluster to remove. 62 | // TODO(cs): could optimize this by adding elements that we 63 | // know must be included. 64 | var timerIterator = OneAtATimeIterator.fromTrace(originalTrace, fingerprinter) 65 | // current set of timer ids we're *including* 66 | var currentTimers = Set[Int]() 67 | 68 | def approximateIterations: Int = { 69 | if (skipClockClusters) return timerIterator.toRemove.size 70 | return clusterIterator.clocks.size * timerIterator.toRemove.size 71 | } 72 | 73 | def getNextTrace(violationReproducedLastRun: Boolean, ignoredAbsentIds: Set[Int]) 74 | : Option[EventTrace] = { 75 | if (violationReproducedLastRun) { 76 | timerIterator.producedViolation(currentTimers, ignoredAbsentIds) 77 | clusterIterator.producedViolation(currentCluster, ignoredAbsentIds) 78 | } 79 | 80 | if (!timerIterator.hasNext || 81 | (aggressiveness == Aggressiveness.ALL_TIMERS_FIRST_ITR && 82 | violationReproducedLastRun && !tryingFirstCluster) || 83 | (aggressiveness == Aggressiveness.STOP_IMMEDIATELY && 84 | violationReproducedLastRun)) { 85 | tryingFirstCluster = false 86 | if (!clusterIterator.hasNext || skipClockClusters) { 87 | return None 88 | } 89 | 90 | timerIterator.reset 91 | currentCluster = clusterIterator.next 92 | log.info("Trying to remove clock cluster: " + 93 | clusterIterator.inverse(currentCluster).toSeq.sorted) 94 | } 95 | 96 | assert(timerIterator.hasNext) 97 | currentTimers = timerIterator.next 98 | log.info("Trying to remove timers: " + timerIterator.inverse(currentTimers).toSeq.sorted) 99 | 100 | val events = new SynchronizedQueue[Event] 101 | events ++= originalTrace.events.flatMap { 102 | case u @ UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) if EventTypes.isExternal(u) => 103 | Some(u) 104 | case UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 105 | if (currentTimers contains id) { 106 | Some(UniqueMsgEvent(MsgEvent(snd,rcv, 107 | WildCardMatch((lst, backtrackSetter) => { 108 | // Timers get to bypass the resolutionStrategy: allow any match 109 | val idx = lst.indexWhere(fingerprinter.causesClockIncrement) 110 | if (idx == -1) None else Some(idx) 111 | }, 112 | name="CausesClockIncrement")), 113 | id)) 114 | } else if (currentCluster contains id) { 115 | val classTag = ClassTag(msg.getClass) 116 | def messageFilter(pendingMsg: Any): Boolean = { 117 | ClassTag(pendingMsg.getClass) == classTag 118 | } 119 | // Choose the least recently sent message for now. 120 | Some(UniqueMsgEvent(MsgEvent(snd,rcv, 121 | WildCardMatch((lst, backtrackSetter) => 122 | resolutionStrategy.resolve(messageFilter, lst, backtrackSetter), 123 | name=classTag.toString)), id)) 124 | } else { 125 | None 126 | } 127 | case _: MsgEvent => 128 | throw new IllegalArgumentException("Must be UniqueMsgEvent") 129 | case t @ UniqueTimerDelivery(_, _) => 130 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 131 | case e => Some(e) 132 | } 133 | return Some(new EventTrace(events, originalTrace.original_externals)) 134 | } 135 | } 136 | 137 | // First iteration always includes all events 138 | class ClockClusterIterator(originalTrace: EventTrace, fingerprinter: FingerprintFactory) { 139 | val allIds = originalTrace.events.flatMap { 140 | case m @ UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 141 | if (!fingerprinter.causesClockIncrement(msg) && 142 | !fingerprinter.getLogicalClock(msg).isEmpty) { 143 | Some(id) 144 | } else { 145 | None 146 | } 147 | case t @ UniqueTimerDelivery(_, _) => 148 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 149 | case e => None 150 | }.toSet 151 | 152 | // Start by not removing any of the clock clusters, only trying to minimize 153 | // Timers. 154 | var firstClusterRemoval = true 155 | // Then: start with the first non-empty cluster 156 | var nextClockToRemove : Long = -1 157 | // Which clock values are safe to remove, i.e. we know that it's possible to 158 | // trigger the violation if they are removed. 159 | var blacklist = Set[Int]() 160 | 161 | var clocks : Seq[Long] = Seq.empty 162 | def computeRemainingClocks(): Seq[Long] = { 163 | val lowest = clocks.headOption.getOrElse(0:Long) 164 | return originalTrace.events.flatMap { 165 | case UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 166 | if (!(blacklist contains id)) { 167 | fingerprinter.getLogicalClock(msg) 168 | } else { 169 | None 170 | } 171 | case _ => None 172 | }.toSet.toSeq.sorted.dropWhile(c => c < lowest) 173 | } 174 | clocks = computeRemainingClocks 175 | 176 | private def current : Set[Int] = { 177 | val currentClockToRemove : Long = if (firstClusterRemoval) -1 else nextClockToRemove 178 | 179 | originalTrace.events.flatMap { 180 | case m @ UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 181 | if (fingerprinter.causesClockIncrement(msg)) { 182 | // Handled by OneAtATimeIterator 183 | None 184 | } else { 185 | val clock = fingerprinter.getLogicalClock(msg) 186 | if (clock.isEmpty) { 187 | Some(id) 188 | } else { 189 | val clockVal = clock.get 190 | if (clockVal == currentClockToRemove || (blacklist contains id)) { 191 | None 192 | } else { 193 | Some(id) 194 | } 195 | } 196 | } 197 | case t @ UniqueTimerDelivery(_, _) => 198 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 199 | case e => None 200 | }.toSet 201 | } 202 | 203 | // pre: hasNext 204 | def next : Set[Int] = { 205 | if (firstClusterRemoval) { 206 | val ret = current 207 | firstClusterRemoval = false 208 | ret 209 | } else { 210 | nextClockToRemove = clocks.head 211 | val ret = current 212 | clocks = clocks.tail 213 | ret 214 | } 215 | } 216 | 217 | def hasNext = firstClusterRemoval || !clocks.isEmpty 218 | 219 | def producedViolation(previouslyIncluded: Set[Int], ignoredAbsents: Set[Int]) { 220 | blacklist = blacklist ++ inverse(previouslyIncluded) 221 | if (!ignoredAbsents.isEmpty) { 222 | blacklist = blacklist ++ allIds.intersect(ignoredAbsents) 223 | clocks = computeRemainingClocks 224 | } 225 | } 226 | 227 | def inverse(toInclude: Set[Int]) = allIds -- toInclude 228 | } 229 | 230 | object OneAtATimeIterator { 231 | def fromTrace(originalTrace: EventTrace, fingerprinter: FingerprintFactory): OneAtATimeIterator = { 232 | var allTimerIds = Set[Int]() ++ originalTrace.events.flatMap { 233 | case UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 234 | if (fingerprinter.causesClockIncrement(msg)) { 235 | Some(id) 236 | } else { 237 | None 238 | } 239 | case _ => None 240 | } 241 | return new OneAtATimeIterator(allTimerIds) 242 | } 243 | } 244 | 245 | // First iteration always includes all timers 246 | // Second iteration includes all timers except the first, 247 | // etc. 248 | class OneAtATimeIterator(all: Set[Int]) { 249 | var toRemove = all.toSeq.sorted 250 | var first = true // if first, don't remove any elements 251 | // Timers that allow the violation to be triggered after having been removed 252 | var blacklist = Set[Int]() 253 | 254 | private def current = { 255 | if (first) { 256 | all -- blacklist 257 | } else { 258 | (all - toRemove.head) -- blacklist 259 | } 260 | } 261 | 262 | // pre: hasNext 263 | def next = { 264 | if (first) { 265 | val ret = current 266 | first = false 267 | ret 268 | } else { 269 | val ret = current 270 | toRemove = toRemove.tail 271 | ret 272 | } 273 | } 274 | 275 | def hasNext = { 276 | first || !toRemove.isEmpty 277 | } 278 | 279 | def producedViolation(previouslyIncluded: Set[Int], ignoredAbsents: Set[Int]) { 280 | blacklist = blacklist ++ inverse(previouslyIncluded) ++ 281 | all.intersect(ignoredAbsents) 282 | } 283 | 284 | def reset { 285 | toRemove = (all -- blacklist).toSeq.sorted 286 | first = true 287 | } 288 | 289 | def inverse(toInclude: Set[Int]) = all -- toInclude 290 | } 291 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/Clusterizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | // Iteratively choose schedules to explore. 4 | // Invoked by WildcardMinimizer. 5 | trait Clusterizer { 6 | // Approximately how many (possibly wildcarded) schedules are we going to 7 | // try before we are done? 8 | def approximateIterations: Int 9 | 10 | // Return the next schedule to explore. 11 | // If there aren't any more schedules to check, return None. 12 | // - violationReproducedLastRun: whether the last schedule we returned 13 | // successfully triggered the invariant violation 14 | // - ignoredAbsentIds: this set contains 15 | // the Unique.id's of all internal events that we tried to include last 16 | // time getNextTrace was invoked, but which did not appear in the 17 | // execution. These are "freebies" which can be removed without our trying 18 | // to remove them, whenever violationReproducedLastRun is true. 19 | def getNextTrace(violationReproducedLastRun: Boolean, ignoredAbsentIds: Set[Int]) 20 | : Option[EventTrace] 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/DeltaDebuggingClusterizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | /* 4 | import scala.collection.mutable.Stack 5 | 6 | class DDMinClusterizer( 7 | originalTrace: EventTrace, 8 | fingerprinter: FingerprintFactory, 9 | resolutionStrategy: AmbiguityResolutionStrategy) { 10 | 11 | val log = LoggerFactory.getLogger("DDMinClusterizer") 12 | 13 | val sortedIds = getIdsToRemove 14 | val allIds = idsToRemove.toSet 15 | 16 | def getIdsToRemove(): Seq[Int] = { 17 | var inUnignorableBlock = false 18 | originalTrace.flatMap { 19 | case BeginUnignorableEvents => 20 | inUnignorableBlock = true 21 | None 22 | case EndUnignorableEvents => 23 | inUnignorableBlock = false 24 | None 25 | case UniqueMsgEvent(m, id) 26 | if (!EventTypes.isExternal(m) && !inUnignorableBlock) => 27 | Some(id) 28 | case t @ UniqueTimerDelivery(_, _) => 29 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 30 | case e => None 31 | }.toSeq.sorted // Should already be sorted? 32 | } 33 | 34 | // ids to keep next, remainder 35 | val callStack = new Stack[(Seq[Int], Set[Int])] 36 | callStack.push((sortedIds, Set.empty)) 37 | 38 | // Iteration: 39 | // - Pick a subsequence to remove 40 | // - Wildcard all subsequent events 41 | 42 | def approximateIterations: Int = { 43 | return idsToRemove.size 44 | } 45 | 46 | def getNextTrace(violationReproducedLastRun: Boolean, ignoredAbsentIds: Set[Int]) 47 | : Option[EventTrace] = { 48 | if (callStack.isEmpty) { 49 | return None 50 | } 51 | 52 | var (idsToKeep, remainder) = callStack.pop 53 | 54 | // TODO(cs): not correct probably. 55 | // If base case: roll up the stack! 56 | while (idsToKeep.size <= 1 && !callStack.isEmpty) { 57 | var (idsToKeep, remainder) = callStack.pop 58 | } 59 | 60 | if (callStack.isEmpty) { 61 | return None 62 | } 63 | 64 | if (violationReproducedLastRun) { 65 | // TODO(cs): do something with ignoredAbsentIds 66 | } else { 67 | // Interference. 68 | } 69 | 70 | // TODO(cs): wildly redundant with FungibleClockClustering.scala 71 | val events = new SynchronizedQueue[Event] 72 | events ++= originalTrace.events.flatMap { 73 | case u @ UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) if EventTypes.isExternal(u) => 74 | Some(u) 75 | case UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 76 | if (currentCluster contains id) { 77 | val classTag = ClassTag(msg.getClass) 78 | def messageFilter(pendingMsg: Any): Boolean = { 79 | ClassTag(pendingMsg.getClass) == classTag 80 | } 81 | // Choose the least recently sent message for now. 82 | Some(UniqueMsgEvent(MsgEvent(snd,rcv, 83 | WildCardMatch((lst, backtrackSetter) => 84 | resolutionStrategy.resolve(messageFilter, lst, backtrackSetter), 85 | name=classTag.toString)), id)) 86 | } else { 87 | None 88 | } 89 | case _: MsgEvent => 90 | throw new IllegalArgumentException("Must be UniqueMsgEvent") 91 | case t @ UniqueTimerDelivery(_, _) => 92 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 93 | case e => Some(e) 94 | } 95 | return Some(new EventTrace(events, originalTrace.original_externals)) 96 | } 97 | } 98 | 99 | if (dag.get_atomic_events.length <= 1) { 100 | logger.info("base case") 101 | return dag 102 | } 103 | 104 | // N.B. we reverse to ensure that we test the left half of events before 105 | // the right half of events. (b/c of our use of remove_events()) 106 | // Just a matter of convention. 107 | val splits : Seq[EventDag] = 108 | MinificationUtil.split_list(dag.get_atomic_events, 2). 109 | asInstanceOf[Seq[Seq[AtomicEvent]]]. 110 | map(split => dag.remove_events(split)).reverse 111 | 112 | // First, check both halves. 113 | for ((split, i) <- splits.zipWithIndex) { 114 | val union = split.union(remainder) 115 | logger.info("Checking split " + union.get_all_events.map(e => e.label).mkString(",")) 116 | val trace = oracle.test(union.get_all_events, violation_fingerprint, 117 | _stats, initializationRoutine=initializationRoutine) 118 | val passes = trace == None 119 | _stats.record_iteration_size(original_num_events - total_inputs_pruned) 120 | if (!passes) { 121 | logger.info("Split fails. Recursing") 122 | total_inputs_pruned += (dag.length - split.length) 123 | return ddmin2(split, remainder) 124 | } else { 125 | logger.info("Split passes.") 126 | } 127 | } 128 | 129 | // Interference: 130 | logger.info("Interference") 131 | val left = ddmin2(splits(0), splits(1).union(remainder)) 132 | val right = ddmin2(splits(1), splits(0).union(remainder)) 133 | return left.union(right) 134 | 135 | */ 136 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/OneAtATimeClusterizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.SynchronizedQueue 4 | 5 | import scala.reflect.ClassTag 6 | 7 | import org.slf4j.LoggerFactory, 8 | ch.qos.logback.classic.Level, 9 | ch.qos.logback.classic.Logger 10 | 11 | class SingletonClusterizer( 12 | originalTrace: EventTrace, 13 | fingerprinter: FingerprintFactory, 14 | resolutionStrategy: AmbiguityResolutionStrategy) extends Clusterizer { 15 | 16 | val log = LoggerFactory.getLogger("SingletonClusterizer") 17 | 18 | var sortedIds = getIdsToRemove() 19 | val allIds = sortedIds.toSet | getUnignorableIds() 20 | var successfullyRemoved = Set[Int]() 21 | var ignoredLastRun = -1 22 | var firstRun = true 23 | 24 | def getIdsToRemove(): Seq[Int] = { 25 | var inUnignorableBlock = false 26 | originalTrace.events.flatMap { 27 | case BeginUnignorableEvents => 28 | inUnignorableBlock = true 29 | None 30 | case EndUnignorableEvents => 31 | inUnignorableBlock = false 32 | None 33 | case UniqueMsgEvent(m, id) 34 | if (!EventTypes.isExternal(m) && !inUnignorableBlock) => 35 | Some(id) 36 | case t @ UniqueTimerDelivery(_, _) => 37 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 38 | case e => None 39 | }.toSeq.sorted // Should already be sorted? 40 | } 41 | 42 | def getUnignorableIds(): Set[Int] = { 43 | var inUnignorableBlock = false 44 | originalTrace.events.flatMap { 45 | case BeginUnignorableEvents => 46 | inUnignorableBlock = true 47 | None 48 | case EndUnignorableEvents => 49 | inUnignorableBlock = false 50 | None 51 | case UniqueMsgEvent(m, id) 52 | if (EventTypes.isExternal(m) || inUnignorableBlock) => 53 | Some(id) 54 | case t @ UniqueTimerDelivery(_, _) => 55 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 56 | case e => None 57 | }.toSet 58 | } 59 | 60 | // Iteration: 61 | // - Pick an event to remove 62 | // - Wildcard all subsequent events 63 | 64 | def approximateIterations: Int = { 65 | return allIds.size 66 | } 67 | 68 | def getNextTrace(violationReproducedLastRun: Boolean, ignoredAbsentIds: Set[Int]) 69 | : Option[EventTrace] = { 70 | if (sortedIds.isEmpty) { 71 | return None 72 | } 73 | 74 | if (violationReproducedLastRun) { 75 | successfullyRemoved = (successfullyRemoved | ignoredAbsentIds) + ignoredLastRun 76 | } 77 | 78 | if (!firstRun) { 79 | ignoredLastRun = sortedIds.head 80 | log.info(s"Trying to ignore $ignoredLastRun") 81 | sortedIds = sortedIds.tail 82 | } else { 83 | firstRun = false 84 | } 85 | 86 | val currentCluster = allIds -- (successfullyRemoved - ignoredLastRun) 87 | 88 | // TODO(cs): wildly redundant with FungibleClockClustering.scala 89 | val events = new SynchronizedQueue[Event] 90 | events ++= originalTrace.events.flatMap { 91 | case u @ UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) if EventTypes.isExternal(u) => 92 | Some(u) 93 | case UniqueMsgEvent(MsgEvent(snd,rcv,msg), id) => 94 | if (currentCluster contains id) { 95 | assert(!msg.isInstanceOf[MessageFingerprint]) 96 | val classTag = ClassTag(msg.getClass) 97 | def messageFilter(pendingMsg: Any): Boolean = { 98 | ClassTag(pendingMsg.getClass) == classTag 99 | } 100 | // Choose the least recently sent message for now. 101 | Some(UniqueMsgEvent(MsgEvent(snd,rcv, 102 | WildCardMatch((lst, backtrackSetter) => 103 | resolutionStrategy.resolve(messageFilter, lst, backtrackSetter), 104 | name=classTag.toString)), id)) 105 | } else { 106 | None 107 | } 108 | case _: MsgEvent => 109 | throw new IllegalArgumentException("Must be UniqueMsgEvent") 110 | case t @ UniqueTimerDelivery(_, _) => 111 | throw new IllegalArgumentException("TimerDelivery not supported. Replay first") 112 | case e => Some(e) 113 | } 114 | return Some(new EventTrace(events, originalTrace.original_externals)) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/scala/verification/minification/wildcard_minimization/WildcardMinimizer.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import scala.collection.mutable.Queue 4 | import scala.collection.mutable.HashMap 5 | import scala.collection.mutable.HashSet 6 | import scala.collection.mutable.SynchronizedQueue 7 | import akka.actor.Props 8 | 9 | import scala.util.Sorting 10 | import scala.reflect.ClassTag 11 | 12 | import scalax.collection.mutable.Graph, 13 | scalax.collection.GraphEdge.DiEdge, 14 | scalax.collection.edge.LDiEdge 15 | 16 | import org.slf4j.LoggerFactory, 17 | ch.qos.logback.classic.Level, 18 | ch.qos.logback.classic.Logger 19 | 20 | // Which Scheduler to check each schedule with. 21 | object TestScheduler extends Enumeration { 22 | type TestScheduler = Value 23 | // Backtracks treated as no-ops. 24 | val STSSched = Value 25 | // Supports backtracks 26 | val DPORwHeuristics = Value 27 | } 28 | 29 | // Which RemovalStrategy (Clusterizer) to use to choose which schedules we'll 30 | // try exploring. 31 | object ClusteringStrategy extends Enumeration { 32 | type ClusteringStrategy = Value 33 | val ClockClusterizer = Value 34 | val SingletonClusterizer = Value 35 | val ClockThenSingleton = Value 36 | } 37 | 38 | // Minimizes internal events, by iteratively (i) invoking a Clusterizer to get 39 | // the next schedule to check, and (ii) checking each schedule with the given 40 | // TestScheduler. 41 | // 42 | // The Wildcarding isn't actually specified here -- wildcarding is specified 43 | // by the Clusterizer. 44 | // 45 | // See design doc: 46 | // https://docs.google.com/document/d/1_EqOkSehZVC7oE2hjV1FxY8jw4mXAlM5ESigQYt4ojg/edit 47 | class WildcardMinimizer( 48 | schedulerConfig: SchedulerConfig, 49 | mcs: Seq[ExternalEvent], 50 | trace: EventTrace, 51 | actorNameProps: Seq[Tuple2[Props, String]], 52 | violation: ViolationFingerprint, 53 | skipClockClusters:Boolean=false, // if true, only explore timers 54 | resolutionStrategy: AmbiguityResolutionStrategy=null, // if null, use BackTrackStrategy 55 | testScheduler:TestScheduler.TestScheduler=TestScheduler.STSSched, 56 | depGraph: Option[Graph[Unique,DiEdge]]=None, 57 | initializationRoutine: Option[() => Any]=None, 58 | preTest: Option[STSScheduler.PreTestCallback]=None, 59 | postTest: Option[STSScheduler.PostTestCallback]=None, 60 | stats: Option[MinimizationStats]=None, 61 | clusteringStrategy:ClusteringStrategy.ClusteringStrategy=ClusteringStrategy.ClockClusterizer, 62 | timeBudgetSeconds:Long=Long.MaxValue) 63 | extends InternalEventMinimizer { 64 | 65 | val log = LoggerFactory.getLogger("WildCardMin") 66 | 67 | def testWithDpor(nextTrace: EventTrace, stats: MinimizationStats, 68 | absentIgnored: STSScheduler.IgnoreAbsentCallback, 69 | resetCallback: DPORwHeuristics.ResetCallback, 70 | dporBudgetSeconds: Long): Option[EventTrace] = { 71 | val uniques = new Queue[Unique] ++ nextTrace.events.flatMap { 72 | case UniqueMsgEvent(m @ MsgEvent(snd,rcv,msg), id) => 73 | Some(Unique(m,id=(-1))) 74 | case s: SpawnEvent => None // DPOR ignores SpawnEvents 75 | case m: UniqueMsgSend => None // DPOR ignores MsgEvents 76 | case BeginWaitQuiescence => None // DPOR ignores BeginWaitQuiescence 77 | case Quiescence => None // forget about it for now? 78 | case c: ChangeContext => None 79 | // TODO(cs): deal with Partitions, etc. 80 | } 81 | 82 | // TODO(cs): keep the same dpor instance across runs, so that we don't 83 | // explore reduant executions. 84 | val dpor = new DPORwHeuristics(schedulerConfig, 85 | prioritizePendingUponDivergence=true, 86 | stopIfViolationFound=false, 87 | startFromBackTrackPoints=false, 88 | skipBacktrackComputation=true, 89 | stopAfterNextTrace=true, 90 | budgetSeconds=dporBudgetSeconds) 91 | dpor.setIgnoreAbsentCallback(absentIgnored) 92 | dpor.setResetCallback(resetCallback) 93 | depGraph match { 94 | case Some(g) => dpor.setInitialDepGraph(g) 95 | case None => 96 | } 97 | // dpor.setMaxDistance(0) x backtrack points are only set explicitly by us 98 | 99 | dpor.setMaxMessagesToSchedule(uniques.size) 100 | dpor.setActorNameProps(actorNameProps) 101 | val filtered_externals = DPORwHeuristicsUtil.convertToDPORTrace(mcs, 102 | actorNameProps.map(t => t._2), false) 103 | 104 | dpor.setInitialTrace(uniques) 105 | val ret = dpor.test(filtered_externals, violation, stats) 106 | // DPORwHeuristics doesn't play nicely with other schedulers, so we need 107 | // to clean up after it. 108 | // Counterintuitively, use a dummy Replayer to do this, since 109 | // DPORwHeuristics doesn't have shutdownSemaphore. 110 | val replayer = new ReplayScheduler(schedulerConfig, false) 111 | Instrumenter().scheduler = replayer 112 | replayer.shutdown() 113 | return ret 114 | } 115 | 116 | def testWithSTSSched(nextTrace: EventTrace, stats: MinimizationStats, 117 | absentIgnored: STSScheduler.IgnoreAbsentCallback): Option[EventTrace] = { 118 | return RunnerUtils.testWithStsSched( 119 | schedulerConfig, 120 | mcs, 121 | nextTrace, 122 | actorNameProps, 123 | violation, 124 | stats, 125 | initializationRoutine=initializationRoutine, 126 | preTest=preTest, 127 | postTest=postTest, 128 | absentIgnored=Some(absentIgnored)) 129 | } 130 | 131 | // N.B. for best results, run RunnerUtils.minimizeInternals on the result if 132 | // we managed to remove anything here. 133 | def minimize(): Tuple2[MinimizationStats, EventTrace] = { 134 | val _stats = stats match { 135 | case None => new MinimizationStats 136 | case Some(s) => s 137 | } 138 | val oracleName = if (testScheduler == TestScheduler.STSSched) 139 | "STSSched" else "DPOR" 140 | // don't updateStrategy if we're being used as a TestOracle 141 | if (!skipClockClusters) _stats.updateStrategy("FungibleClockMinimizer", oracleName) 142 | 143 | val aggressiveness = if (skipClockClusters) 144 | Aggressiveness.STOP_IMMEDIATELY 145 | else Aggressiveness.ALL_TIMERS_FIRST_ITR 146 | 147 | val _resolutionStrategy = if (resolutionStrategy != null) 148 | resolutionStrategy 149 | else new BackTrackStrategy(schedulerConfig.messageFingerprinter) 150 | 151 | val clusterizer = clusteringStrategy match { 152 | case ClusteringStrategy.ClockClusterizer | ClusteringStrategy.ClockThenSingleton => 153 | new ClockClusterizer(trace, 154 | schedulerConfig.messageFingerprinter, 155 | _resolutionStrategy, 156 | skipClockClusters=skipClockClusters, 157 | aggressiveness=aggressiveness) 158 | case ClusteringStrategy.SingletonClusterizer => 159 | new SingletonClusterizer(trace, 160 | schedulerConfig.messageFingerprinter, _resolutionStrategy) 161 | } 162 | 163 | // Each cluster gets timeBudgetSeconds / numberOfClusters seconds 164 | val dporBudgetSeconds = timeBudgetSeconds / (List(clusterizer.approximateIterations,1).max) 165 | val tStartSeconds = System.currentTimeMillis / 1000 166 | 167 | var minTrace = doMinimize(clusterizer, dporBudgetSeconds, trace, _stats) 168 | 169 | val timeElapsedSeconds = (System.currentTimeMillis / 1000) - tStartSeconds 170 | val remainingTimeSeconds = timeBudgetSeconds - timeElapsedSeconds 171 | 172 | if (clusteringStrategy == ClusteringStrategy.ClockThenSingleton && 173 | remainingTimeSeconds > 0) { 174 | log.info("Switching to SingletonClusterizer") 175 | val singletonClusterizer = new SingletonClusterizer(minTrace, 176 | schedulerConfig.messageFingerprinter, _resolutionStrategy) 177 | minTrace = doMinimize(singletonClusterizer, remainingTimeSeconds, minTrace, _stats) 178 | } 179 | 180 | // don't overwrite prune end if we're being used as a TestOracle 181 | if (!skipClockClusters) { 182 | _stats.record_prune_end 183 | // Fencepost 184 | _stats.record_internal_size(RunnerUtils.countMsgEvents(minTrace)) 185 | } 186 | 187 | return (_stats, minTrace) 188 | } 189 | 190 | def doMinimize(clusterizer: Clusterizer, budgetSeconds: Long, 191 | startTrace: EventTrace, _stats: MinimizationStats): EventTrace = { 192 | var minTrace = startTrace 193 | var nextTrace = clusterizer.getNextTrace(false, Set[Int]()) 194 | // don't overwrite prune start if we're being used as a TestOracle 195 | if (!skipClockClusters) _stats.record_prune_start 196 | while (!nextTrace.isEmpty) { 197 | val ignoredAbsentIndices = new HashSet[Int] 198 | def ignoreAbsentCallback(idx: Int) { 199 | ignoredAbsentIndices += idx 200 | } 201 | def resetCallback() { 202 | ignoredAbsentIndices.clear 203 | } 204 | val ret = if (testScheduler == TestScheduler.DPORwHeuristics) 205 | testWithDpor(nextTrace.get, _stats, ignoreAbsentCallback, 206 | resetCallback, budgetSeconds) 207 | else testWithSTSSched(nextTrace.get, _stats, ignoreAbsentCallback) 208 | 209 | // Record interation size if we're being used for internal minimization 210 | if (!skipClockClusters) { 211 | _stats.record_internal_size(RunnerUtils.countMsgEvents(minTrace)) 212 | } 213 | 214 | var ignoredAbsentIds = Set[Int]() 215 | if (!ret.isEmpty) { 216 | log.info("Pruning was successful.") 217 | if (ret.get.size <= minTrace.size) { 218 | minTrace = ret.get 219 | } 220 | val adjustedTrace = if (testScheduler == TestScheduler.STSSched) 221 | nextTrace.get. 222 | filterFailureDetectorMessages. 223 | filterCheckpointMessages. 224 | subsequenceIntersection(mcs, 225 | filterKnownAbsents=schedulerConfig.filterKnownAbsents). 226 | events 227 | else nextTrace.get.events.flatMap { // DPORwHeuristics 228 | case u: UniqueMsgEvent => Some(u) 229 | case _ => None 230 | } 231 | 232 | ignoredAbsentIndices.foreach { case i => 233 | ignoredAbsentIds = ignoredAbsentIds + 234 | adjustedTrace.get(i).get.asInstanceOf[UniqueMsgEvent].id 235 | } 236 | } 237 | nextTrace = clusterizer.getNextTrace(!ret.isEmpty, ignoredAbsentIds) 238 | } 239 | 240 | return minTrace 241 | } 242 | } 243 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/AbstractScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{Cell, ActorRef} 4 | 5 | import akka.dispatch.Envelope 6 | 7 | import scala.collection.mutable.Queue 8 | import scala.collection.mutable.HashMap 9 | import scala.collection.mutable.HashSet 10 | 11 | import org.slf4j.LoggerFactory, 12 | ch.qos.logback.classic.Level, 13 | ch.qos.logback.classic.Logger 14 | 15 | // Basically just keeps track of actor names. 16 | // 17 | // Subclasses need to at least implement: 18 | // - schedule_new_message 19 | // - enqueue_message 20 | // - event_produced(cell: Cell, envelope: Envelope) 21 | abstract class AbstractScheduler extends Scheduler { 22 | 23 | val logger = LoggerFactory.getLogger("AbstractScheduler") 24 | 25 | var instrumenter = Instrumenter() 26 | var currentTime = 0 27 | 28 | 29 | var actorNames = new HashSet[String] 30 | 31 | 32 | // Is this message a system message 33 | def isSystemCommunication(sender: ActorRef, receiver: ActorRef): Boolean = { 34 | if (sender == null && receiver == null) return true 35 | val senderPath = if (sender != null) sender.path.name else "deadLetters" 36 | val receiverPath = if (receiver != null) receiver.path.name else "deadLetters" 37 | return isSystemMessage(senderPath, receiverPath) 38 | } 39 | 40 | def isSystemMessage(src: String, dst: String): Boolean = { 41 | if ((actorNames contains src) || (actorNames contains dst)) 42 | return dst == "deadLetters" 43 | 44 | return true 45 | } 46 | 47 | // Notification that the system has been reset 48 | def start_trace() : Unit = { 49 | } 50 | 51 | 52 | // Record that an event was consumed 53 | def event_consumed(event: Event) = { 54 | } 55 | 56 | def event_consumed(cell: Cell, envelope: Envelope) = { 57 | } 58 | 59 | // Record that an event was produced 60 | def event_produced(event: Event) = { 61 | event match { 62 | case event : SpawnEvent => 63 | if (!(actorNames contains event.name)) { 64 | logger.debug("Sched knows about actor " + event.name) 65 | actorNames += event.name 66 | } 67 | } 68 | } 69 | 70 | 71 | // Called before we start processing a newly received event 72 | def before_receive(cell: Cell) { 73 | currentTime += 1 74 | } 75 | 76 | // Called after receive is done being processed 77 | def after_receive(cell: Cell) { 78 | } 79 | 80 | def notify_quiescence () { 81 | } 82 | 83 | def shutdown() { 84 | instrumenter.restart_system 85 | } 86 | 87 | // Get next event 88 | def next_event() : Event = { 89 | throw new Exception("NYI") 90 | } 91 | 92 | def reset_all_state () = { 93 | actorNames = new HashSet[String] 94 | currentTime = 0 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/AuxilaryTypes.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{ActorCell, ActorRef, ActorSystem, Props} 4 | import akka.dispatch.{Envelope} 5 | 6 | import java.util.concurrent.atomic.AtomicInteger 7 | import java.util.concurrent.Semaphore, 8 | scala.collection.mutable.HashMap, 9 | scala.collection.mutable.HashSet 10 | 11 | // ----------- Internal event types ------------- 12 | abstract trait Event 13 | 14 | // Metadata events, not actually events. 15 | // MsgEvents appearing between `BeginUnignorableEvents' and `EndUnigorableEvents' 16 | // will never be skipped over during replay. 17 | final case object BeginUnignorableEvents extends Event 18 | final case object EndUnignorableEvents extends Event 19 | // An external thread has just started an `atomic block`, where it will now 20 | // send some number of messages. Upon replay, wait until the end of the 21 | // atomic block before deciding whether those messages are or are not going 22 | // to show up. 23 | final case class BeginExternalAtomicBlock(taskId: Long) extends Event 24 | final case class EndExternalAtomicBlock(taskId: Long) extends Event 25 | 26 | // Internal events. 27 | // MsgSend is the initial send, not the delivery 28 | // N.B., if an event trace was serialized, it's possible that msg is of type 29 | // MessageFingerprint rather than a whole message! 30 | 31 | // Message delivery -- (not the initial send) 32 | // N.B., if an event trace was serialized, it's possible that msg is of type 33 | // MessageFingerprint rather than a whole message! 34 | case class MsgEvent( 35 | sender: String, receiver: String, msg: Any) extends Event 36 | case class SpawnEvent( 37 | parent: String, props: Props, name: String, actor: ActorRef) extends Event 38 | 39 | // (Used by DPOR) 40 | // TODO(cs): consolidate with redundant types below. 41 | case class NetworkPartition( 42 | first: Set[String], 43 | second: Set[String]) extends 44 | UniqueExternalEvent with ExternalEvent with Event 45 | // (Used by DPOR) 46 | case class NetworkUnpartition( 47 | first: Set[String], 48 | second: Set[String]) extends 49 | UniqueExternalEvent with ExternalEvent with Event 50 | 51 | // (More general than DPOR) 52 | final case class MsgSend (sender: String, 53 | receiver: String, msg: Any) extends Event 54 | final case class KillEvent (actor: String) extends Event 55 | final case class PartitionEvent (endpoints: (String, String)) extends Event 56 | final case class UnPartitionEvent (endpoints: (String, String)) extends Event 57 | // Marks when WaitQuiescence was first processed. 58 | final case object BeginWaitQuiescence extends Event 59 | // Marks when Quiescence was actually reached. 60 | final case object Quiescence extends Event 61 | final case class ChangeContext (actor: String) extends Event 62 | 63 | // Recording/Replaying Akka.FSM.Timer's (which aren't serializable! hence this madness) 64 | // N.B. these aren't explicitly recorded. We use them only when we want to serialize event 65 | // traces. 66 | final case class TimerFingerprint(name: String, 67 | msgFingerprint: MessageFingerprint, repeat: Boolean, generation: Int) extends MessageFingerprint 68 | // Corresponds to MsgEvent. 69 | final case class TimerDelivery(sender: String, receiver: String, fingerprint: TimerFingerprint) extends Event 70 | 71 | // Keep this as a static class rather than a trait for backwards-compatibility 72 | object MetaEvents { 73 | def isMetaEvent(e: Event): Boolean = { 74 | e match { 75 | case BeginUnignorableEvents | EndUnignorableEvents | _: BeginExternalAtomicBlock | 76 | _: EndExternalAtomicBlock | _: MsgSend | BeginWaitQuiescence | 77 | Quiescence | _: ChangeContext => return true 78 | case _ => return false 79 | } 80 | } 81 | } 82 | 83 | object IDGenerator { 84 | var uniqueId = new AtomicInteger // DPOR root event is assumed to be ID 0, incrementAndGet ensures starting at 1 85 | 86 | def get() : Integer = { 87 | val id = uniqueId.incrementAndGet() 88 | if (id == Int.MaxValue) { 89 | throw new RuntimeException("Deal with overflow..") 90 | } 91 | return id 92 | } 93 | } 94 | 95 | case class Unique( 96 | val event : Event, 97 | var id : Int = IDGenerator.get() 98 | ) extends ExternalEvent { 99 | def label: String = "e"+id 100 | } 101 | 102 | case class Uniq[E]( 103 | val element : E, 104 | var id : Int = IDGenerator.get() 105 | ) 106 | 107 | case object RootEvent extends Event 108 | 109 | case class WildCardMatch( 110 | // Given: 111 | // - a list of pending messages, sorted from least recently to most recently sent 112 | // - a "backtrack setter" function: given an index of the pending 113 | // messages, sets a backtrack point for that pending message, to be 114 | // replayed in the future. 115 | // return the index of the chosen one, or None 116 | msgSelector: (Seq[Any], (Int) => Unit) => Option[Int], 117 | name:String="" 118 | ) 119 | 120 | /** 121 | * TellEnqueue is a semaphore that ensures a linearizable execution, and protects 122 | * schedulers' data structures during akka's concurrent processing of `tell` 123 | * (the `!` operator). 124 | * 125 | * Instrumenter()'s control flow is as follows: 126 | * - Invoke scheduler.schedule_new_message to find a new message to deliver 127 | * - Call `dispatcher.dispatch` to deliver the message. Note that 128 | * `dispatcher.dispatch` hands off work to a separate thread and returns 129 | * immediately. 130 | * - The actor `receive()`ing the message now becomes active 131 | * - Every time that actor invokes `tell` to send a message to a known actor, 132 | * a ticket is taken from TellEnqueue via TellEnqueue.tell() 133 | * - Concurrently, akka will process the `tell`s by enqueuing the message in 134 | * the receiver's mailbox. 135 | * - Every time akka finishes enqueueing a message to the recevier's mailbox, 136 | * we first call scheduler.event_produced, and then replaces a ticket to 137 | * TellEnqueue via TellEnqueue.enqueue() 138 | * - When the actor returns from `receive`, we wait for all tickets to be 139 | * returned (via TellEnqueue.await()) before scheduling the next message. 140 | * 141 | * The `known actor` part is crucial. If the receiver is not an actor (e.g. 142 | * the main thread) or we do not interpose on the receiving actor, we will not 143 | * be able to return the ticket via TellEnqueue.enqueue(), and the system will 144 | * block forever on TellEnqueue.await(). 145 | */ 146 | trait TellEnqueue { 147 | def tell() 148 | def enqueue() 149 | def reset() 150 | def await () 151 | } 152 | 153 | class TellEnqueueBusyWait extends TellEnqueue { 154 | 155 | var enqueue_count = new AtomicInteger 156 | var tell_count = new AtomicInteger 157 | 158 | def tell() { 159 | tell_count.incrementAndGet() 160 | } 161 | 162 | def enqueue() { 163 | enqueue_count.incrementAndGet() 164 | } 165 | 166 | def reset() { 167 | tell_count.set(0) 168 | enqueue_count.set(0) 169 | } 170 | 171 | def await () { 172 | while (tell_count.get != enqueue_count.get) {} 173 | } 174 | 175 | } 176 | 177 | class TellEnqueueSemaphore extends Semaphore(1) with TellEnqueue { 178 | 179 | var enqueue_count = new AtomicInteger 180 | var tell_count = new AtomicInteger 181 | 182 | def tell() { 183 | tell_count.incrementAndGet() 184 | reducePermits(1) 185 | require(availablePermits() <= 0) 186 | } 187 | 188 | def enqueue() { 189 | enqueue_count.incrementAndGet() 190 | require(availablePermits() <= 0) 191 | release() 192 | } 193 | 194 | def reset() { 195 | tell_count.set(0) 196 | enqueue_count.set(0) 197 | // Set available permits to 0 198 | drainPermits() 199 | // Add a permit 200 | release() 201 | } 202 | 203 | def await() { 204 | acquire 205 | release 206 | } 207 | } 208 | 209 | class ExploredTacker { 210 | var exploredStack = new HashMap[Int, HashSet[(Unique, Unique)] ] 211 | 212 | def setExplored(index: Int, pair: (Unique, Unique)) = 213 | exploredStack.get(index) match { 214 | case Some(set) => set += pair 215 | case None => 216 | val newElem = new HashSet[(Unique, Unique)] + pair 217 | exploredStack(index) = newElem 218 | } 219 | 220 | def isExplored(pair: (Unique, Unique)): Boolean = { 221 | 222 | for ((index, set) <- exploredStack) set.contains(pair) match { 223 | case true => return true 224 | case false => 225 | } 226 | 227 | return false 228 | } 229 | 230 | def trimExplored(index: Int) = { 231 | exploredStack = exploredStack.filter { other => other._1 <= index } 232 | } 233 | 234 | 235 | def printExplored() = { 236 | for ((index, set) <- exploredStack.toList.sortBy(t => (t._1))) { 237 | println(index + ": " + set.size) 238 | //val content = set.map(x => (x._1.id, x._2.id)) 239 | //println(index + ": " + set.size + ": " + content)) 240 | } 241 | } 242 | 243 | def clear() = { 244 | exploredStack.clear() 245 | } 246 | } 247 | 248 | // Shared instance 249 | object ExploredTacker { 250 | var obj:ExploredTacker = null 251 | def apply() = { 252 | if (obj == null) { 253 | obj = new ExploredTacker 254 | } 255 | obj 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/BacktrackOrdering.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | // A subcomponent of DPOR schedulers. 4 | 5 | import akka.actor.Cell, 6 | akka.actor.ActorSystem, 7 | akka.actor.ActorRef, 8 | akka.actor.LocalActorRef, 9 | akka.actor.ActorRefWithCell, 10 | akka.actor.Actor, 11 | akka.actor.PoisonPill, 12 | akka.actor.Props 13 | 14 | import akka.dispatch.Envelope, 15 | akka.dispatch.MessageQueue, 16 | akka.dispatch.MessageDispatcher 17 | 18 | import scala.collection.concurrent.TrieMap, 19 | scala.collection.mutable.Queue, 20 | scala.collection.mutable.HashMap, 21 | scala.collection.mutable.HashSet, 22 | scala.collection.mutable.ArrayBuffer, 23 | scala.collection.mutable.ArraySeq, 24 | scala.collection.mutable.Stack, 25 | scala.collection.mutable.PriorityQueue, 26 | scala.math.Ordering, 27 | scala.reflect.ClassTag 28 | 29 | import java.util.concurrent.Semaphore 30 | import java.util.concurrent.atomic.AtomicBoolean 31 | 32 | import Function.tupled 33 | 34 | import scalax.collection.mutable.Graph, 35 | scalax.collection.GraphEdge.DiEdge, 36 | scalax.collection.edge.LDiEdge 37 | 38 | 39 | import org.slf4j.LoggerFactory, 40 | ch.qos.logback.classic.Level, 41 | ch.qos.logback.classic.Logger 42 | 43 | trait BacktrackOrdering { 44 | // The priority function we feed into the priority queue. 45 | // Higher return values get higher priority. 46 | def getOrdered(key: DPORwHeuristics.BacktrackKey) : Ordered[DPORwHeuristics.BacktrackKey] 47 | 48 | // If DPORwHeuristics is configured to ignore certain backtrack points, this 49 | // the integer we return here decides how we decide to ignore. Higher numbers 50 | // are ignored more quickly than lower numbers. 51 | def getDistance(key: DPORwHeuristics.BacktrackKey) : Int 52 | 53 | // If a key is no longer needed, give the BacktrackOrdering an opportunity 54 | // to clear state associated with the key. 55 | def clearKey(key: DPORwHeuristics.BacktrackKey) {} 56 | } 57 | 58 | class DefaultBacktrackOrdering extends BacktrackOrdering { 59 | // Default: order by depth. Longer depth is given higher priority. 60 | def getOrdered(tuple: DPORwHeuristics.BacktrackKey) = new Ordered[DPORwHeuristics.BacktrackKey] { 61 | def compare(other: DPORwHeuristics.BacktrackKey) = tuple._1.compare(other._1) 62 | } 63 | 64 | // TODO(cs): awkward interface, since distance doesn't really make sense 65 | // here... 66 | def getDistance(tuple: DPORwHeuristics.BacktrackKey) : Int = { 67 | return 0 68 | } 69 | } 70 | 71 | // Combine with: DPORwHeuristics.setMaxDistance(0) 72 | class StopImmediatelyOrdering extends BacktrackOrdering { 73 | // Default: order by depth. Longer depth is given higher priority. 74 | def getOrdered(tuple: DPORwHeuristics.BacktrackKey) = new Ordered[DPORwHeuristics.BacktrackKey] { 75 | def compare(other: DPORwHeuristics.BacktrackKey) = tuple._1.compare(other._1) 76 | } 77 | 78 | def getDistance(tuple: DPORwHeuristics.BacktrackKey) : Int = { 79 | return Int.MaxValue 80 | } 81 | } 82 | 83 | // Unlike traditional edit distance: 84 | // - do not consider any changes to word1, i.e. keep word1 fixed. 85 | // - do not penalize deletions. 86 | // 87 | // This strategy is simply (i) the number of misordered pairs of events from 88 | // the original sequence, plus (ii) any events in the new sequence that aren't 89 | // in the original. 90 | // 91 | // This has a few nice properties: 92 | // - It accounts for partial orderings (i.e. concurrent events), in the 93 | // following sense: rather than simply penalizing all events not in the 94 | // longest common subsequence (original proposal), it also adds extra 95 | // penalization to reorderings of events not in the longest common 96 | // subsequence. 97 | // - It can be computed online 98 | // - It's simpler than computing longest common subsequence 99 | class ArvindDistanceOrdering extends BacktrackOrdering { 100 | var sched : DPORwHeuristics = null 101 | var originalTrace : Queue[Unique] = null 102 | var originalIndices = new HashMap[Unique, Int] 103 | // Store all distances globally, to avoid redundant computation. 104 | var distances = new HashMap[DPORwHeuristics.BacktrackKey, Int] 105 | 106 | // Need a separate constructor b/c of circular dependence between 107 | // DPORwHeuristics. Might think about changing the interface. 108 | def init(_sched: DPORwHeuristics, _originalTrace: Queue[Unique]) { 109 | sched = _sched 110 | originalTrace = _originalTrace 111 | for ((e,i) <- originalTrace.zipWithIndex) { 112 | originalIndices(e) = i 113 | } 114 | } 115 | 116 | private[this] def arvindDistance(tuple: DPORwHeuristics.BacktrackKey) : Int = { 117 | def getPath() : Seq[Unique] = { 118 | val commonPrefix = sched.getCommonPrefix(tuple._2._1, tuple._2._1). 119 | map(node => node.value) 120 | val postfix = tuple._3 121 | return commonPrefix ++ postfix ++ Seq(tuple._2._1, tuple._2._2) 122 | } 123 | 124 | val path = getPath() 125 | // TODO(cs): make this computation online rather than offline, i.e. look 126 | // up the distance for the longest known prefix of this path, then compute 127 | // the remaining distance for the suffix. 128 | var distance = 0 129 | for ((e, i) <- path.zipWithIndex) { 130 | if (!(originalIndices contains e)) { 131 | // println "Not in original:", e 132 | distance += 1 133 | } else { 134 | for (pred <- path.slice(0,i)) { 135 | if ((originalIndices contains pred) && 136 | originalIndices(pred) > originalIndices(e)) { 137 | // println "Reordered: ", pred, e 138 | distance += 1 139 | } 140 | } 141 | } 142 | } 143 | return distance 144 | } 145 | 146 | private[this] def storeArvindDistance(tuple: DPORwHeuristics.BacktrackKey) : Int = { 147 | if (distances contains tuple) { 148 | return distances(tuple) 149 | } 150 | distances(tuple) = arvindDistance(tuple) 151 | return distances(tuple) 152 | } 153 | 154 | def getOrdered(tuple: DPORwHeuristics.BacktrackKey) = new Ordered[DPORwHeuristics.BacktrackKey] { 155 | def compare(other: DPORwHeuristics.BacktrackKey) : Int = { 156 | val myDistance = storeArvindDistance(tuple) 157 | val otherDistance = storeArvindDistance(other) 158 | if (otherDistance != myDistance) { 159 | return myDistance.compare(otherDistance) 160 | } 161 | // Break ties based on depth 162 | return tuple._1.compare(other._1) 163 | } 164 | } 165 | 166 | def getDistance(tuple: DPORwHeuristics.BacktrackKey) : Int = { 167 | return storeArvindDistance(tuple) 168 | } 169 | 170 | override def clearKey(tuple: DPORwHeuristics.BacktrackKey) = { 171 | distances -= tuple 172 | } 173 | } 174 | 175 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/BasicScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.Cell 4 | import akka.actor.ActorSystem 5 | import akka.actor.ActorRef 6 | import akka.actor.Actor 7 | import akka.actor.PoisonPill 8 | import akka.actor.Props; 9 | 10 | import akka.dispatch.Envelope 11 | import akka.dispatch.MessageQueue 12 | import akka.dispatch.MessageDispatcher 13 | 14 | import scala.collection.concurrent.TrieMap 15 | import scala.collection.mutable.Queue 16 | import scala.collection.mutable.HashMap 17 | import scala.collection.mutable.HashSet 18 | import scala.collection.Iterator 19 | 20 | import scala.collection.generic.GenericTraversableTemplate 21 | 22 | // A basic scheduler. Schedules events in the order they arrive. 23 | class BasicScheduler extends Scheduler { 24 | 25 | var instrumenter = Instrumenter() 26 | var currentTime = 0 27 | var index = 0 28 | 29 | type CurrentTimeQueueT = Queue[Event] 30 | 31 | var currentlyProduced = new CurrentTimeQueueT 32 | var currentlyConsumed = new CurrentTimeQueueT 33 | 34 | var producedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 35 | var consumedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 36 | 37 | var prevProducedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 38 | var prevConsumedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 39 | 40 | // Current set of enabled events. 41 | val pendingEvents = new HashMap[String, Queue[(Cell, Envelope)]] 42 | 43 | val actorNames = new HashSet[String] 44 | 45 | def isSystemCommunication(sender: ActorRef, receiver: ActorRef): Boolean = { 46 | if (sender == null || receiver == null) return true 47 | return isSystemMessage(sender.path.name, receiver.path.name) 48 | } 49 | 50 | // Is this message a system message 51 | def isSystemMessage(src: String, dst: String): Boolean = { 52 | if ((actorNames contains src) || (actorNames contains dst)) 53 | return dst == "deadLetters" 54 | 55 | return true 56 | } 57 | 58 | 59 | // Notification that the system has been reset 60 | def start_trace() : Unit = { 61 | prevProducedEvents = producedEvents 62 | prevConsumedEvents = consumedEvents 63 | producedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 64 | consumedEvents = new Queue[ (Integer, CurrentTimeQueueT) ] 65 | } 66 | 67 | 68 | // When executing a trace, find the next trace event. 69 | private[this] def mutable_trace_iterator( 70 | trace: Queue[ (Integer, CurrentTimeQueueT) ]) : Option[Event] = { 71 | 72 | if(trace.isEmpty) return None 73 | 74 | val (count, q) = trace.head 75 | q.isEmpty match { 76 | case true => 77 | trace.dequeue() 78 | mutable_trace_iterator(trace) 79 | case false => return Some(q.dequeue()) 80 | } 81 | } 82 | 83 | 84 | 85 | // Get next message event from the trace. 86 | private[this] def get_next_trace_message() : Option[MsgEvent] = { 87 | mutable_trace_iterator(prevConsumedEvents) match { 88 | case Some(v : MsgEvent) => Some(v) 89 | case Some(v : Event) => get_next_trace_message() 90 | case None => None 91 | } 92 | } 93 | 94 | 95 | 96 | // Figure out what is the next message to schedule. 97 | // TODO(cs): make sure not to send to blockedActors! 98 | def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] = { 99 | 100 | // Filter for messages belong to a particular actor. 101 | def is_the_same(e: MsgEvent, c: (Cell, Envelope)) : Boolean = { 102 | val (cell, env) = c 103 | e.receiver == cell.self.path.name 104 | } 105 | 106 | // Get from the current set of pending events. 107 | def get_pending_event() : Option[(Cell, Envelope)] = { 108 | // Do we have some pending events 109 | pendingEvents.headOption match { 110 | case Some((receiver, queue)) => 111 | if (queue.isEmpty == true) { 112 | 113 | pendingEvents.remove(receiver) match { 114 | case Some(key) => get_pending_event() 115 | case None => throw new Exception("internal error") 116 | } 117 | 118 | } else { 119 | Some(queue.dequeue()) 120 | } 121 | case None => None 122 | } 123 | } 124 | 125 | get_next_trace_message() match { 126 | // The trace says there is something to run. 127 | case Some(msg_event : MsgEvent) => 128 | pendingEvents.get(msg_event.receiver) match { 129 | case Some(queue) => queue.dequeueFirst(is_the_same(msg_event, _)) 130 | case None => None 131 | } 132 | // The trace says there is nothing to run so we have either exhausted our 133 | // trace or are running for the first time. Use any enabled transitions. 134 | case None => get_pending_event() 135 | 136 | } 137 | } 138 | 139 | 140 | // Get next event 141 | def next_event() : Event = { 142 | mutable_trace_iterator(prevConsumedEvents) match { 143 | case Some(v) => v 144 | case None => throw new Exception("no previously consumed events") 145 | } 146 | } 147 | 148 | 149 | // Record that an event was consumed 150 | def event_consumed(event: Event) = { 151 | currentlyConsumed.enqueue(event) 152 | } 153 | 154 | 155 | def event_consumed(cell: Cell, envelope: Envelope) = { 156 | currentlyConsumed.enqueue(new MsgEvent( 157 | envelope.sender.path.name, cell.self.path.name, envelope.message)) 158 | } 159 | 160 | // Record that an event was produced 161 | def event_produced(event: Event) = { 162 | event match { 163 | case event : SpawnEvent => actorNames += event.name 164 | } 165 | currentlyProduced.enqueue(event) 166 | } 167 | 168 | 169 | def event_produced(cell: Cell, envelope: Envelope) = { 170 | val snd = envelope.sender.path.name 171 | val rcv = cell.self.path.name 172 | val msgs = pendingEvents.getOrElse(rcv, new Queue[(Cell, Envelope)]) 173 | 174 | pendingEvents(rcv) = msgs += ((cell, envelope)) 175 | currentlyProduced.enqueue(new MsgEvent(snd, rcv, envelope.message)) 176 | if (!instrumenter.started.get) { 177 | instrumenter.start_dispatch() 178 | } 179 | } 180 | 181 | 182 | // Called before we start processing a newly received event 183 | def before_receive(cell: Cell) { 184 | producedEvents.enqueue( (currentTime, currentlyProduced) ) 185 | consumedEvents.enqueue( (currentTime, currentlyConsumed) ) 186 | currentlyProduced = new CurrentTimeQueueT 187 | currentlyConsumed = new CurrentTimeQueueT 188 | currentTime += 1 189 | println(Console.GREEN 190 | + " ↓↓↓↓↓↓↓↓↓ ⌚ " + currentTime + " | " + cell.self.path.name + " ↓↓↓↓↓↓↓↓↓ " + 191 | Console.RESET) 192 | } 193 | 194 | 195 | // Called after receive is done being processed 196 | def after_receive(cell: Cell) { 197 | println(Console.RED 198 | + " ↑↑↑↑↑↑↑↑↑ ⌚ " + currentTime + " | " + cell.self.path.name + " ↑↑↑↑↑↑↑↑↑ " 199 | + Console.RESET) 200 | 201 | } 202 | 203 | 204 | def notify_quiescence () { 205 | } 206 | 207 | def notify_timer_cancel(rcv: String, msg: Any) { 208 | pendingEvents(rcv).dequeueFirst(tuple => tuple._2.message == msg) 209 | if (pendingEvents(rcv).isEmpty) { 210 | pendingEvents -= rcv 211 | } 212 | } 213 | 214 | def enqueue_message(sender: Option[ActorRef], receiver: String, msg: Any) { 215 | throw new Exception("NYI") 216 | } 217 | 218 | def shutdown() { 219 | instrumenter.restart_system 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/EventOrchestrator.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import com.typesafe.config.ConfigFactory 4 | import akka.actor.{Actor, ActorCell, ActorRef, ActorSystem, Props, Terminated, ActorRefWithCell} 5 | import akka.actor.ActorDSL._ 6 | 7 | import akka.dispatch.Envelope 8 | 9 | import scala.collection.mutable.Queue 10 | import scala.collection.mutable.HashMap 11 | import scala.collection.mutable.SynchronizedQueue 12 | import scala.collection.immutable.Set 13 | import scala.collection.mutable.HashSet 14 | import scala.collection.mutable.ArrayBuffer 15 | 16 | import java.util.concurrent.Semaphore 17 | import java.util.concurrent.atomic.AtomicBoolean 18 | 19 | import org.slf4j.LoggerFactory, 20 | ch.qos.logback.classic.Level, 21 | ch.qos.logback.classic.Logger 22 | 23 | /** 24 | * Maintains the current state of the network, and provides 25 | * an interface for triggering events that affect that state, e.g. inducing 26 | * failures. 27 | * 28 | * Additionally records which events occured during the execution, possibly including internal events. 29 | * 30 | * Generic E specifies the type of given trace. Usually either Event or 31 | * ExternalEvent. 32 | */ 33 | class EventOrchestrator[E] { 34 | // Function that queues a message to be sent later. 35 | type EnqueueMessage = (Option[ActorRef], String, Any) => Unit 36 | 37 | // Callbacks 38 | type KillCallback = (String, Set[String], Int) => Unit 39 | type PartitionCallback = (String, String, Int) => Unit 40 | type UnPartitionCallback = (String, String, Int) => Unit 41 | var killCallback : KillCallback = (s: String, actors: Set[String], id: Int) => None 42 | def setKillCallback(c: KillCallback) = killCallback = c 43 | var partitionCallback : PartitionCallback = (a: String, b: String, id: Int) => None 44 | def setPartitionCallback(c: PartitionCallback) = partitionCallback = c 45 | var unPartitionCallback : UnPartitionCallback = (a: String, b: String, id: Int) => None 46 | def setUnPartitionCallback(c: UnPartitionCallback) = unPartitionCallback = c 47 | 48 | val log = LoggerFactory.getLogger("EventOrchestrator") 49 | 50 | // Pairs of actors that cannot communicate 51 | val partitioned = new HashSet[(String, String)] 52 | 53 | // Actors that are unreachable 54 | val inaccessible = new HashSet[String] 55 | 56 | // Actors that have been explicitly killed. 57 | // (Same as inaccessble, except that inaccessible also contains actors that 58 | // have been created but not Start()'ed) 59 | val killed = new HashSet[String] 60 | 61 | // Actors to actor ref 62 | // TODO(cs): make getter/setters for these 63 | val actorToActorRef = new HashMap[String, ActorRef] 64 | val actorToSpawnEvent = new HashMap[String, SpawnEvent] 65 | 66 | var trace: ArrayBuffer[E] = new ArrayBuffer[E] 67 | var traceIdx: Int = 0 68 | var events = new EventTrace 69 | 70 | var fd : FDMessageOrchestrator = null 71 | 72 | def set_trace(_trace: Seq[E]) { 73 | //println("Setting trace: " + _trace.size) 74 | trace = new ArrayBuffer() ++ _trace 75 | traceIdx = 0 76 | } 77 | 78 | def reset_events() { 79 | events = new EventTrace(events.original_externals) 80 | } 81 | 82 | def getRemainingTrace() : Seq[E] = { 83 | return trace.slice(traceIdx, trace.length) 84 | } 85 | 86 | def prepend(prefix: Seq[E]) { 87 | trace.++=:(prefix) 88 | } 89 | 90 | def trace_advanced() = { 91 | traceIdx += 1 92 | } 93 | 94 | def previous_event(): E = { 95 | if (traceIdx - 0 < 0) { 96 | throw new IllegalStateException("No previous event..") 97 | } 98 | trace(traceIdx - 1) 99 | } 100 | 101 | def trace_finished() : Boolean = { 102 | return traceIdx >= trace.size 103 | } 104 | 105 | def finish_early() = { 106 | traceIdx = trace.size 107 | } 108 | 109 | // A bit of a misnomer: current *trace* event, not current recorded event. 110 | def current_event() : E = { 111 | if (traceIdx >= trace.length) { 112 | throw new IllegalStateException("No current event..") 113 | } 114 | trace(traceIdx) 115 | } 116 | 117 | /** 118 | * Inform the given failure detector of any injected events in the future. 119 | */ 120 | def set_failure_detector(_fd: FDMessageOrchestrator) = { 121 | fd = _fd 122 | } 123 | 124 | /** 125 | * Injects ExternalEvents in trace until Quiescence it's time for the 126 | * scheduler to wait for quiescence. 127 | * 128 | * Should not be invoked if E != ExternalEvent. 129 | * 130 | * Return whether start_dispatch should be invoked after returning. 131 | */ 132 | def inject_until_quiescence(enqueue_message: EnqueueMessage): Boolean = { 133 | var loop = true 134 | while (loop && !trace_finished) { 135 | log.trace("Injecting " + traceIdx + "/" + trace.length + " " + current_event) 136 | current_event.asInstanceOf[ExternalEvent] match { 137 | case Start (_, name) => 138 | trigger_start(name) 139 | case k @ Kill (name) => 140 | killCallback(name, actorToActorRef.keys.toSet, k._id) 141 | trigger_kill(name) 142 | case k @ HardKill (name) => 143 | // If we just delivered a message to the actor we're about to kill, 144 | // the current thread is still in the process of 145 | // handling the Mailbox for that actor. In that 146 | // case we need to wait for the Mailbox to be set to "Idle" before 147 | // we can kill the actor, since otherwise the Mailbox will not be 148 | // able to process the akka-internal "Terminated" messages, i.e. 149 | // killing it now will result in a deadlock. 150 | if (Instrumenter().previousActor == name) { 151 | // N.B. because *this* thread is the one that will eventually set 152 | // the mailbox state to idle, we should be guarenteed that we will 153 | // not end up with start_dispatch being called concurrently with 154 | // what we're currently doing here. 155 | Instrumenter().dispatchAfterMailboxIdle(name) 156 | return false 157 | } 158 | killCallback(name, actorToActorRef.keys.toSet, k._id) 159 | trigger_hard_kill(k) 160 | case Send (name, messageCtor) => 161 | enqueue_message(None, name, messageCtor()) 162 | case p @ Partition (a, b) => 163 | partitionCallback(a,b,p._id) 164 | trigger_partition(a,b) 165 | case u @ UnPartition (a, b) => 166 | unPartitionCallback(a,b,u._id) 167 | trigger_unpartition(a,b) 168 | case WaitCondition(cond) => 169 | // Don't let trace advance here. Only let it advance when the 170 | // condition holds. 171 | return true 172 | case c @ CodeBlock(block) => 173 | events += c.asInstanceOf[Event] // keep the id the same 174 | // Since the block might send messages, make sure that we treat the 175 | // message sends as if they are being triggered by an external 176 | // thread, i.e. we enqueue them rather than letting them be sent 177 | // immediately. 178 | Instrumenter.overrideInternalThreadRule 179 | // This better terminate! 180 | block() 181 | Instrumenter.unsetInternalThreadRuleOverride 182 | case WaitQuiescence() => 183 | events += BeginWaitQuiescence 184 | loop = false // Start waiting for quiescence 185 | } 186 | trace_advanced() 187 | } 188 | return true 189 | } 190 | 191 | // Mark a couple of nodes as partitioned (so they cannot communicate) 192 | def add_to_partition (newly_partitioned: (String, String)) { 193 | partitioned += newly_partitioned 194 | } 195 | 196 | // Mark a couple of node as unpartitioned, i.e. they can communicate 197 | def remove_partition (newly_partitioned: (String, String)) { 198 | partitioned -= newly_partitioned 199 | } 200 | 201 | // Mark a node as unreachable, used to kill a node. 202 | // TODO(cs): to be implemented later: actually kill the node so that its state is cleared? 203 | def isolate_node (node: String) { 204 | inaccessible += node 205 | if (fd != null) { 206 | fd.isolate_node(node) 207 | } 208 | } 209 | 210 | // Mark a node as reachable, also used to start a node 211 | def unisolate_node (actor: String) { 212 | inaccessible -= actor 213 | killed -= actor 214 | if (fd != null) { 215 | fd.unisolate_node(actor) 216 | } 217 | } 218 | 219 | def trigger_start (name: String) = { 220 | events += actorToSpawnEvent(name) 221 | Util.logger.log(name, "God spawned me") 222 | unisolate_node(name) 223 | // If actor was previously killed, allow scheduler to send messages to it 224 | // again. 225 | if (Instrumenter().blockedActors contains name) { 226 | Instrumenter().blockedActors = Instrumenter().blockedActors - name 227 | } 228 | if (fd != null) { 229 | fd.handle_start_event(name) 230 | } 231 | } 232 | 233 | def trigger_kill (name: String) = { 234 | events += KillEvent(name) 235 | Util.logger.log(name, "God killed me") 236 | killed += name 237 | isolate_node(name) 238 | if (fd != null) { 239 | fd.handle_kill_event(name) 240 | } 241 | } 242 | 243 | def trigger_hard_kill (hardKill: HardKill) = { 244 | events += hardKill 245 | val name = hardKill.name 246 | Util.logger.log(name, "God is about to hard kill me") 247 | Instrumenter().preStartCalled.synchronized { 248 | // Actor should already have been fully initialized 249 | // TODO(cs): should probably synchronize over this whole method.. 250 | assert(Instrumenter().actorMappings contains name) 251 | val ref = Instrumenter().actorMappings(name) 252 | assert(!(Instrumenter().preStartCalled contains ref)) 253 | } 254 | 255 | // Get all the information we need before killing 256 | // TODO(cs): this may need to be recursive. 257 | val children = actorToActorRef(name).asInstanceOf[ActorRefWithCell].underlying. 258 | childrenRefs.children.map(ref => ref.path.name).toSeq 259 | val ref = actorToActorRef(name) 260 | 261 | // Clean up our data before killing 262 | log.debug("Cleaning up before hard kill...") 263 | for (name <- Seq(name) ++ children) { 264 | actorToActorRef -= name 265 | actorToSpawnEvent -= name 266 | // TODO(cs): move this into Instrumenter 267 | Instrumenter().actorMappings.synchronized { 268 | Instrumenter().dispatchers -= Instrumenter().actorMappings(name) 269 | Instrumenter().actorMappings -= name 270 | } 271 | // Unfortunately, all of the scheduler's pointers to this ActorCell may now become invalid, 272 | // i.e. the ActorCell's fields may now change dynamically and point to 273 | // another actor or be nulled out. See: 274 | // https://github.com/akka/akka/blob/release-2.2/akka-actor/src/main/scala/akka/actor/ActorCell.scala#L613 275 | // So we have scheduler remove all references to the actor cells. 276 | // TODO(cs): resend the pending messages for this actor, i.e. simulate a situation where 277 | // packets in the network destined for the old actor arrive at the newly 278 | // restarted actor. Tricky to deal with depGraph.. 279 | val sndMsgPairs = Instrumenter().scheduler.actorTerminated(name) 280 | Instrumenter().blockedActors = Instrumenter().blockedActors - name 281 | Instrumenter().timerToCancellable.keys.foreach { 282 | case (rcv,msg) => 283 | if (rcv == name) { 284 | val cancellable = Instrumenter().timerToCancellable((rcv,msg)) 285 | Instrumenter().removeCancellable(cancellable) 286 | } 287 | } 288 | } 289 | 290 | // stop()'ing is asynchronous. So, block until it completes. 291 | implicit val system = Instrumenter()._actorSystem 292 | val i = inbox() 293 | i watch ref 294 | // Kill it 295 | log.debug("Calling stop() and blocking... " + ref) 296 | Instrumenter()._actorSystem.stop(ref) 297 | // Now block 298 | def isTerminated(msg: Any) : Boolean = { 299 | msg match { 300 | case Terminated(_) => true 301 | case _ => false 302 | } 303 | } 304 | var msg = i.receive() 305 | while (!isTerminated(msg)) { 306 | msg = i.receive() 307 | } 308 | 309 | if (fd != null) { 310 | fd.handle_kill_event(name) 311 | } 312 | } 313 | 314 | def trigger_partition (a: String, b: String) = { 315 | events += PartitionEvent((a, b)) 316 | add_to_partition((a, b)) 317 | Util.logger.log(a, "God partitioned me from " + b) 318 | Util.logger.log(b, "God partitioned me from " + a) 319 | if (fd != null) { 320 | fd.handle_partition_event(a,b) 321 | } 322 | } 323 | 324 | def trigger_unpartition (a: String, b: String) = { 325 | events += UnPartitionEvent((a, b)) 326 | remove_partition((a, b)) 327 | Util.logger.log(a, "God reconnected me to " + b) 328 | Util.logger.log(b, "God reconnected me to " + a) 329 | if (fd != null) { 330 | fd.handle_unpartition_event(a,b) 331 | } 332 | } 333 | 334 | def handle_spawn_produced (event: SpawnEvent) = { 335 | actorToActorRef(event.name) = event.actor 336 | if (fd != null) { 337 | fd.create_node(event.name) 338 | } 339 | } 340 | 341 | def handle_spawn_consumed (event: SpawnEvent) = { 342 | actorToSpawnEvent(event.name) = event 343 | } 344 | 345 | def crosses_partition (snd: String, rcv: String) : Boolean = { 346 | if (snd == rcv && !(killed contains snd)) return false 347 | return ((partitioned contains (snd, rcv)) 348 | || (partitioned contains (rcv, snd)) 349 | || (inaccessible contains rcv) 350 | || (inaccessible contains snd)) 351 | } 352 | } 353 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/FairScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{Cell, ActorRef} 4 | 5 | import akka.dispatch.Envelope 6 | 7 | import scala.collection.mutable.Queue 8 | import scala.collection.mutable.HashMap 9 | import scala.collection.mutable.HashSet 10 | 11 | /** 12 | * Schedules events in a round-robin fashion. 13 | */ 14 | class FairScheduler extends AbstractScheduler { 15 | var actorQueue = new Queue[String] 16 | var index = 0 17 | 18 | // Current set of enabled events. 19 | val pendingEvents = new HashMap[String, Queue[Uniq[(Cell, Envelope)]]] 20 | 21 | def event_produced(cell: Cell, envelope: Envelope) = { 22 | val snd = envelope.sender.path.name 23 | val rcv = cell.self.path.name 24 | val msgs = pendingEvents.getOrElse(rcv, new Queue[Uniq[(Cell, Envelope)]]) 25 | 26 | val uniq = Uniq[(Cell, Envelope)]((cell, envelope)) 27 | pendingEvents(rcv) = msgs += uniq 28 | // Start dispatching events 29 | if (!instrumenter.started.get) { 30 | instrumenter.start_dispatch() 31 | } 32 | } 33 | 34 | def nextActor (blockedActors: Set[String]) : String = { 35 | if (actorQueue.size == 0) { 36 | actorQueue = actorQueue ++ actorNames.toList 37 | } 38 | var next = actorQueue(index) 39 | index = (index + 1) % actorQueue.size 40 | // Assume: never the case that all actors are blocked at the same time. 41 | while (!(blockedActors contains next)) { 42 | next = actorQueue(index) 43 | index = (index + 1) % actorQueue.size 44 | } 45 | next 46 | } 47 | 48 | def find_message_to_schedule(blockedActors: Set[String]) : Option[Uniq[(Cell, Envelope)]] = { 49 | val receiver = nextActor(blockedActors) 50 | // Do we have some pending events 51 | if (pendingEvents.isEmpty) { 52 | None 53 | } else { 54 | pendingEvents.get(receiver) match { 55 | case Some(queue) => 56 | if (queue.isEmpty == true) { 57 | pendingEvents.remove(receiver) match { 58 | case Some(key) => find_message_to_schedule(blockedActors) 59 | case None => throw new Exception("Internal error") // Really this is saying pendingEvents does not have 60 | // receiver as a key 61 | } 62 | } else { 63 | val uniq = queue.dequeue() 64 | Some(uniq) 65 | } 66 | case None => 67 | find_message_to_schedule(blockedActors) 68 | } 69 | } 70 | } 71 | 72 | // Figure out what is the next message to schedule. 73 | def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] = { 74 | find_message_to_schedule(blockedActors) match { 75 | case Some(uniq) => 76 | return Some(uniq.element) 77 | case None => 78 | return None 79 | } 80 | } 81 | 82 | def enqueue_message(sender: Option[ActorRef], receiver: String, msg: Any) { 83 | throw new Exception("NYI") 84 | } 85 | 86 | override def notify_quiescence () { 87 | println("No more messages to process " + pendingEvents) 88 | } 89 | 90 | def notify_timer_cancel(rcv: String, msg: Any) { 91 | pendingEvents(rcv).dequeueFirst(tuple => tuple.element._2.message == msg) 92 | if (pendingEvents(rcv).isEmpty) { 93 | pendingEvents -= rcv 94 | } 95 | } 96 | 97 | override def reset_all_state () = { 98 | super.reset_all_state 99 | index = 0 100 | pendingEvents.clear 101 | actorQueue.clear 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/IntervalPeekScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.{Actor, Cell, ActorRef, ActorSystem, Props} 4 | import akka.dispatch.Envelope 5 | 6 | import scala.collection.mutable.Queue 7 | import scala.collection.mutable.HashMap 8 | import scala.collection.mutable.SynchronizedQueue 9 | import scala.collection.immutable.Set 10 | import scala.collection.mutable.HashSet 11 | 12 | import java.util.concurrent.atomic.AtomicBoolean 13 | import java.util.concurrent.atomic.AtomicInteger 14 | import java.util.concurrent.Semaphore 15 | 16 | // TODO(cs): try placing newly scheduled timer events at the front of 17 | // pendingUnepxectedEvents, rather than at the back like other messages. 18 | 19 | object IntervalPeekScheduler { 20 | // N.B. enabled should contain non-fingerprinted messages, whereas _expected 21 | // should contain fingerprinted messages 22 | def unexpected(enabled: Seq[MsgEvent], _expected: MultiSet[MsgEvent], 23 | messageFingerprinter: FingerprintFactory) : Seq[MsgEvent] = { 24 | // TODO(cs): consider ordering of expected, rather than treating it as a Set? 25 | val expected = new MultiSet[MsgEvent] ++ _expected 26 | def fingerprintAndMatch(e: MsgEvent): Boolean = { 27 | val fingerprinted = MsgEvent(e.sender, e.receiver, 28 | messageFingerprinter.fingerprint(e.msg)) 29 | expected.contains(fingerprinted) match { 30 | case true => 31 | expected.remove(fingerprinted) 32 | return false 33 | case false => 34 | // Unexpected 35 | return true 36 | } 37 | } 38 | return enabled.filter(fingerprintAndMatch) 39 | } 40 | 41 | // Flatten all enabled events into a sorted list of (raw, non-fingerprinted) MsgEvents 42 | def flattenedEnabledNested(pendingEvents: 43 | HashMap[(String, String), HashMap[MessageFingerprint, 44 | Queue[Uniq[(Cell, Envelope)]]]]) 45 | : Seq[MsgEvent] = { 46 | // Last element of queue's tuple is the unique id, which is assumed to be 47 | // monotonically increasing by time of arrival. 48 | val unsorted = new Queue[(String, String, Any, Int)] 49 | pendingEvents.foreach { 50 | case (key, hash) => 51 | hash.foreach { 52 | case (msg, queue) => 53 | for (uniq <- queue) { 54 | if (key._2 != FailureDetector.fdName) { 55 | unsorted += ((key._1, key._2, uniq.element._2.message, uniq.id)) 56 | } 57 | } 58 | } 59 | } 60 | return unsorted.sortBy[Int](tuple => tuple._4). 61 | map(tuple => MsgEvent(tuple._1, tuple._2, tuple._3)) 62 | } 63 | 64 | // Flatten all enabled events into a sorted list of (raw, non-fingerprinted) MsgEvents 65 | def flattenedEnabled(pendingEvents: HashMap[ 66 | (String, String, MessageFingerprint), Queue[Uniq[(Cell, Envelope)]]]) : Seq[MsgEvent] = { 67 | // Last element of queue's tuple is the unique id, which is assumed to be 68 | // monotonically increasing by time of arrival. 69 | val unsorted = new Queue[(String, String, Any, Int)] 70 | pendingEvents.foreach { 71 | case (key, queue) => 72 | for (uniq <- queue) { 73 | if (key._2 != FailureDetector.fdName) { 74 | unsorted += ((key._1, key._2, uniq.element._2.message, uniq.id)) 75 | } 76 | } 77 | } 78 | return unsorted.sortBy[Int](tuple => tuple._4). 79 | map(tuple => MsgEvent(tuple._1, tuple._2, tuple._3)) 80 | } 81 | } 82 | 83 | /** 84 | * Similar to PeekScheduler(), except that: 85 | * a. we start from mid-way in the execution (or rather, we restore a checkpoint of 86 | * the state mid-way through the execution, by replaying all events that led up 87 | * to that point), 88 | * b. we only Peek() for a small interval (up to the next external event), 89 | * c. and we schedule in FIFO order rather than RR. 90 | * 91 | * Formal contract: 92 | * Schedules a fixed number of unexpected messages in FIFO order 93 | * to guess whether a particular message i is going to 94 | * become enabled or not. If i does become enabled, 95 | * we return the unexpected messages that lead up 96 | * to its being enabled. Otherwise, return null. 97 | */ 98 | // N.B. both expected and lookingFor should have their msg fields as a MessageFingerprint instead 99 | // of the raw message. 100 | class IntervalPeekScheduler(schedulerConfig: SchedulerConfig, 101 | expected: MultiSet[MsgEvent], lookingFor: MsgEvent, 102 | max_peek_messages:Int=10) extends 103 | ReplayScheduler(schedulerConfig, false) { 104 | // Whether we are currently restoring the checkpoint (by replaying a prefix 105 | // of events), or have moved on to peeking. 106 | val doneReplayingPrefix = new AtomicBoolean(false) 107 | 108 | // Semaphore to wait for peek to be done. We initialize the 109 | // semaphore to 0 rather than 1, so that the main thread blocks upon 110 | // invoking acquire() until another thread release()'s it. 111 | var donePeeking = new Semaphore(0) 112 | 113 | // Whether we ended up finding lookingFor 114 | val foundLookingFor = new AtomicBoolean(false) 115 | 116 | // Unexpected messages we have scheduled so far in search of lookingFor 117 | val postfix = new Queue[MsgEvent] 118 | 119 | // FIFO queue of unexpected events. 120 | var pendingUnexpectedEvents = new Queue[(Cell, Envelope)] 121 | 122 | /* 123 | * peek() schedules a fixed number of unexpected messages in FIFO order 124 | * to guess whether a particular message event m is going to become enabled or not. 125 | * 126 | * If msg does become enabled, return a prefix of messages that lead up to its being enabled. 127 | * 128 | * Otherwise, return None. 129 | */ 130 | def peek(prefix: EventTrace) : Option[Seq[MsgEvent]] = { 131 | if (!(Instrumenter().scheduler eq this)) { 132 | throw new IllegalStateException("Instrumenter().scheduler not set!") 133 | } 134 | doneReplayingPrefix.set(false) 135 | replay(prefix) 136 | println("Done replaying prefix. Proceeding with peek") 137 | started.set(true) 138 | doneReplayingPrefix.set(true) 139 | 140 | // Feed the unexpected events present at the end of replay into 141 | // pendingUnexpectedEvents. 142 | val unexpected = IntervalPeekScheduler.unexpected( 143 | IntervalPeekScheduler.flattenedEnabled(pendingEvents), expected, 144 | messageFingerprinter) 145 | for (msgEvent <- unexpected) { 146 | val key = (msgEvent.sender, msgEvent.receiver, 147 | messageFingerprinter.fingerprint(msgEvent.msg)) 148 | val nextMessage = pendingEvents.get(key) match { 149 | case Some(queue) => 150 | val willRet = queue.dequeue() 151 | if (queue.isEmpty) { 152 | pendingEvents.remove(key) 153 | } 154 | Some(willRet) 155 | case None => 156 | // Message not enabled 157 | None 158 | } 159 | nextMessage match { 160 | case Some(uniq) => pendingUnexpectedEvents += uniq.element 161 | case None => throw new RuntimeException("Shouldn't happen") 162 | } 163 | } 164 | 165 | Instrumenter().start_dispatch 166 | 167 | // Wait for peek scheduling to finish: 168 | donePeeking.acquire 169 | if (foundLookingFor.get) { 170 | return Some(postfix) 171 | } 172 | return None 173 | } 174 | 175 | override def event_produced(cell: Cell, envelope: Envelope) : Unit = { 176 | if (!doneReplayingPrefix.get) { 177 | return super.event_produced(cell, envelope) 178 | } 179 | 180 | val snd = envelope.sender.path.name 181 | val rcv = cell.self.path.name 182 | val msg = envelope.message 183 | val event = MsgEvent(snd, rcv, msg) 184 | val fingerprintedEvent = MsgEvent(snd, rcv, 185 | messageFingerprinter.fingerprint(msg)) 186 | if (expected.contains(fingerprintedEvent)) { 187 | expected -= fingerprintedEvent 188 | } else if (fingerprintedEvent == lookingFor) { 189 | foundLookingFor.set(true) 190 | } else if (!event_orchestrator.crosses_partition(snd, rcv)) { 191 | pendingUnexpectedEvents += ((cell, envelope)) 192 | } 193 | } 194 | 195 | // TODO(cs): make sure not to send to blockedActors! 196 | override def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] = { 197 | if (!doneReplayingPrefix.get) { 198 | return super.schedule_new_message(blockedActors) 199 | } 200 | 201 | if (foundLookingFor.get) { 202 | return None 203 | } 204 | 205 | // Send any pending timers. 206 | send_external_messages() 207 | 208 | if (pendingUnexpectedEvents.isEmpty) { 209 | println("No more events to schedule..") 210 | return None 211 | } 212 | 213 | val next = pendingUnexpectedEvents.dequeue() 214 | val snd = next._2.sender.path.name 215 | val rcv = next._1.self.path.name 216 | val msg = next._2.message 217 | val event = MsgEvent(snd, rcv, msg) 218 | postfix += event 219 | if (postfix.length > max_peek_messages) { 220 | println("Reached maximum unexpected messages") 221 | return None 222 | } 223 | 224 | return Some(next) 225 | } 226 | 227 | override def notify_quiescence() : Unit = { 228 | if (!doneReplayingPrefix.get) { 229 | return super.notify_quiescence 230 | } 231 | // Wake up the main thread. 232 | donePeeking.release 233 | } 234 | 235 | override def shutdown () = { 236 | // Don't restart the system (as in the other schedulers), just shut it 237 | // down. 238 | Instrumenter().shutdown_system(false) 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/NullScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.Cell 4 | import akka.actor.ActorSystem 5 | import akka.actor.ActorRef 6 | import akka.actor.Actor 7 | import akka.actor.PoisonPill 8 | import akka.actor.Props; 9 | 10 | import akka.dispatch.Envelope 11 | import akka.dispatch.MessageQueue 12 | import akka.dispatch.MessageDispatcher 13 | 14 | import scala.collection.concurrent.TrieMap 15 | import scala.collection.mutable.Queue 16 | import scala.collection.mutable.HashMap 17 | import scala.collection.mutable.HashSet 18 | import scala.collection.Iterator 19 | 20 | import scala.collection.generic.GenericTraversableTemplate 21 | 22 | // A scheduler that always forwards messages immediately. 23 | class NullScheduler extends Scheduler { 24 | 25 | // Mechanism to forward messages immediately: mark everything as a "system message". 26 | def isSystemCommunication(sender: ActorRef, receiver: ActorRef): Boolean = { 27 | return true 28 | } 29 | 30 | def isSystemMessage(src: String, dst: String): Boolean = { 31 | return true 32 | } 33 | 34 | def start_trace() : Unit = {} 35 | 36 | def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] = { 37 | return None 38 | } 39 | 40 | def next_event() : Event = { 41 | throw new Exception("no previously consumed events") 42 | } 43 | 44 | def event_consumed(event: Event) = {} 45 | def event_consumed(cell: Cell, envelope: Envelope) = {} 46 | def event_produced(event: Event) = {} 47 | def event_produced(cell: Cell, envelope: Envelope) = {} 48 | def before_receive(cell: Cell) {} 49 | def after_receive(cell: Cell) {} 50 | def notify_quiescence () {} 51 | def notify_timer_cancel(receiver: String, msg: Any) {} 52 | def enqueue_message(sender: Option[ActorRef], receiver: String, msg: Any) { 53 | throw new Exception("NYI") 54 | } 55 | def shutdown() {} 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/PeekScheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import com.typesafe.config.ConfigFactory 4 | import akka.actor.{Actor, Cell, ActorRef, ActorSystem, Props} 5 | 6 | import akka.dispatch.Envelope 7 | 8 | import scala.collection.mutable.Queue 9 | import scala.collection.mutable.HashMap 10 | import scala.collection.mutable.SynchronizedQueue 11 | import scala.collection.immutable.Set 12 | import scala.collection.mutable.HashSet 13 | 14 | import java.util.concurrent.Semaphore 15 | import java.util.concurrent.atomic.AtomicBoolean 16 | 17 | // TODO(cs): PeekScheduler should really be parameterized to allow us to try 18 | // different scheduling strategies (FIFO, round-robin) during Peek. 19 | 20 | /** 21 | * Takes a sequence of ExternalEvents as input, and plays the execution 22 | * forward in the same way as FairScheduler. While playing forward, 23 | * PeekScheduler records all internal events that occur, e.g. Message Sends. 24 | * PeekScheduler finally returns all events observed during the execution, including 25 | * external and internal events. 26 | */ 27 | class PeekScheduler(val schedulerConfig: SchedulerConfig) 28 | extends FairScheduler with ExternalEventInjector[ExternalEvent] with TestOracle { 29 | 30 | def getName: String = "FairScheduler" 31 | 32 | // Allow the user to place a bound on how many messages are delivered. 33 | // Useful for dealing with non-terminating systems. 34 | var maxMessages = Int.MaxValue 35 | def setMaxMessages(_maxMessages: Int) = { 36 | maxMessages = _maxMessages 37 | } 38 | 39 | var messagesScheduledSoFar = 0 40 | 41 | var test_invariant : Invariant = schedulerConfig.invariant_check match { 42 | case Some(i) => i 43 | case None => null 44 | } 45 | 46 | def peek (_trace: Seq[ExternalEvent]) : EventTrace = { 47 | if (!(Instrumenter().scheduler eq this)) { 48 | throw new IllegalStateException("Instrumenter().scheduler not set!") 49 | } 50 | event_orchestrator.events.setOriginalExternalEvents(_trace) 51 | return execute_trace(_trace) 52 | } 53 | 54 | override def event_produced(cell: Cell, envelope: Envelope) = { 55 | var snd = envelope.sender.path.name 56 | val rcv = cell.self.path.name 57 | val msgs = pendingEvents.getOrElse(rcv, new Queue[Uniq[(Cell, Envelope)]]) 58 | val uniq = Uniq[(Cell, Envelope)]((cell, envelope)) 59 | var isTimer = false 60 | handle_event_produced(snd, rcv, envelope) match { 61 | case FailureDetectorQuery => None 62 | case CheckpointReplyMessage => None 63 | case ExternalMessage => None 64 | case InternalMessage => { 65 | if (snd == "deadLetters") { 66 | isTimer = true 67 | } 68 | if (!crosses_partition(snd, rcv)) { 69 | pendingEvents(rcv) = msgs += uniq 70 | } 71 | } 72 | } 73 | // Record this MsgEvent as a special if it was sent from a timer. 74 | snd = if (isTimer) "Timer" else snd 75 | event_orchestrator.events.appendMsgSend(snd, rcv, envelope.message, uniq.id) 76 | } 77 | 78 | // Record a mapping from actor names to actor refs 79 | override def event_produced(event: Event) = { 80 | super.event_produced(event) 81 | handle_spawn_produced(event) 82 | } 83 | 84 | // Record that an event was consumed 85 | override def event_consumed(event: Event) = { 86 | handle_spawn_consumed(event) 87 | } 88 | 89 | // Record a message send event 90 | override def event_consumed(cell: Cell, envelope: Envelope) = { 91 | handle_event_consumed(cell, envelope) 92 | } 93 | 94 | // TODO(cs): make sure not to send to blockedActors! 95 | override def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] = { 96 | // Check if we've exceeded our message limit 97 | if (messagesScheduledSoFar > maxMessages) { 98 | println("Exceeded maxMessages") 99 | event_orchestrator.finish_early 100 | return None 101 | } 102 | 103 | send_external_messages() 104 | // FairScheduler gives us round-robin message dispatches. 105 | val uniq_option = find_message_to_schedule(blockedActors) 106 | uniq_option match { 107 | case Some(uniq) => 108 | event_orchestrator.events.appendMsgEvent(uniq.element, uniq.id) 109 | uniq.element._2.message match { 110 | case CheckpointRequest => None 111 | case _ => 112 | messagesScheduledSoFar += 1 113 | if (messagesScheduledSoFar == Int.MaxValue) { 114 | messagesScheduledSoFar = 1 115 | } 116 | } 117 | return Some(uniq.element) 118 | case None => 119 | return None 120 | } 121 | } 122 | 123 | override def notify_quiescence () { 124 | handle_quiescence 125 | } 126 | 127 | // Shutdown the scheduler, this ensures that the instrumenter is returned to its 128 | // original pristine form, so one can change schedulers 129 | override def shutdown () = { 130 | handle_shutdown 131 | } 132 | 133 | // Notification that the system has been reset 134 | override def start_trace() : Unit = { 135 | handle_start_trace 136 | } 137 | 138 | override def after_receive(cell: Cell) : Unit = { 139 | handle_after_receive(cell) 140 | } 141 | 142 | override def before_receive(cell: Cell) : Unit = { 143 | handle_before_receive(cell) 144 | } 145 | 146 | def setInvariant(invariant: Invariant) { 147 | test_invariant = invariant 148 | } 149 | 150 | override def enqueue_message(sender: Option[ActorRef], receiver: String, msg: Any) = { 151 | super[ExternalEventInjector].enqueue_message(sender, receiver, msg) 152 | } 153 | 154 | override def notify_timer_cancel(receiver: String, msg: Any): Unit = { 155 | if (handle_timer_cancel(receiver, msg)) { 156 | return 157 | } 158 | super.notify_timer_cancel(receiver, msg) 159 | } 160 | 161 | override def enqueue_timer(receiver: String, msg: Any) { handle_timer(receiver, msg) } 162 | 163 | override def reset_all_state() = { 164 | super.reset_all_state 165 | messagesScheduledSoFar = 0 166 | } 167 | 168 | def test(events: Seq[ExternalEvent], 169 | violation_fingerprint: ViolationFingerprint, 170 | stats: MinimizationStats, 171 | init:Option[()=>Any]=None) : Option[EventTrace] = { 172 | Instrumenter().scheduler = this 173 | peek(events) 174 | if (test_invariant == null) { 175 | throw new IllegalArgumentException("Must invoke setInvariant before test()") 176 | } 177 | val checkpoint = takeCheckpoint() 178 | val violation = test_invariant(events, checkpoint) 179 | var violation_found = false 180 | violation match { 181 | case Some(fingerprint) => 182 | violation_found = fingerprint.matches(violation_fingerprint) 183 | case None => None 184 | } 185 | val ret = violation_found match { 186 | case true => Some(event_orchestrator.events) 187 | case false => None 188 | } 189 | 190 | // reset ExternalEventInjector 191 | reset_state(true) 192 | // reset FairScheduler 193 | reset_all_state 194 | 195 | return ret 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/main/scala/verification/schedulers/Scheduler.scala: -------------------------------------------------------------------------------- 1 | package akka.dispatch.verification 2 | 3 | import akka.actor.Cell 4 | import akka.actor.ActorRef 5 | import akka.actor.Props; 6 | import akka.actor.AutoReceivedMessage; 7 | import akka.actor.ActorIdentity; 8 | import akka.actor.ReceiveTimeout; 9 | 10 | import akka.dispatch.Envelope 11 | 12 | // The interface for schedulers 13 | trait Scheduler { 14 | 15 | def isSystemCommunication(sender: ActorRef, receiver: ActorRef): Boolean 16 | def isSystemCommunication(sender: ActorRef, receiver: ActorRef, msg: Any): Boolean = { 17 | if (msg.isInstanceOf[AutoReceivedMessage] || 18 | msg.isInstanceOf[ActorIdentity] || 19 | msg.isInstanceOf[ReceiveTimeout]) { 20 | return true 21 | } 22 | return isSystemCommunication(sender, receiver) 23 | } 24 | 25 | 26 | // Is this message a system message 27 | def isSystemMessage(src: String, dst: String): Boolean 28 | def isSystemMessage(src: String, dst: String, msg: Any): Boolean = { 29 | if (msg.isInstanceOf[AutoReceivedMessage] || 30 | msg.isInstanceOf[ActorIdentity] || 31 | msg.isInstanceOf[ReceiveTimeout]) { 32 | return true 33 | } 34 | return isSystemMessage(src, dst) 35 | } 36 | 37 | // Notification that the system has been reset 38 | def start_trace() : Unit 39 | // Get the next message to schedule. Make sure not to return a message that 40 | // is destined for a blocked actor! Otherwise an exception will be thrown. 41 | def schedule_new_message(blockedActors: Set[String]) : Option[(Cell, Envelope)] 42 | // Get next event to schedule (used while restarting the system) 43 | def next_event() : Event 44 | // Notify that there are no more events to run 45 | def notify_quiescence () : Unit 46 | 47 | // Called before we start processing a newly received event 48 | def before_receive(cell: Cell) : Unit 49 | // Called after receive is done being processed 50 | def after_receive(cell: Cell) : Unit 51 | 52 | def before_receive(cell: Cell, msg: Any) : Unit = 53 | before_receive(cell) 54 | def after_receive(cell: Cell, msg: Any) : Unit = 55 | after_receive(cell) 56 | 57 | // Record that an event was produced 58 | def event_produced(event: Event) : Unit 59 | def event_produced(cell: Cell, envelope: Envelope) : Unit 60 | 61 | // Record that an event was consumed 62 | def event_consumed(event: Event) : Unit 63 | def event_consumed(cell: Cell, envelope: Envelope) 64 | // Tell the scheduler that it should eventually schedule the given message. 65 | // Used to feed messages from the external world into actor systems. 66 | 67 | // Called when timer is cancelled 68 | def notify_timer_cancel(receiver: String, msg: Any) 69 | 70 | // Interface for (safely) sending external messages 71 | def enqueue_message(sender: Option[ActorRef], receiver: String, msg: Any) 72 | 73 | // Interface for (safely) sending timers (akka.scheduler messages) 74 | def enqueue_timer(receiver: String, msg: Any) = enqueue_message(None, receiver, msg) 75 | 76 | // Interface for notifying the scheduler about a code block that has just 77 | // been scheduled by the application, through akka.scheduler.schedule(). 78 | // cell and envelope are fake, used as placeholders. 79 | def enqueue_code_block(cell: Cell, envelope: Envelope) { 80 | event_produced(cell, envelope) 81 | } 82 | 83 | // Shut down the actor system. 84 | def shutdown() 85 | 86 | // Invoked whenever the application logs to the console. Used mostly for 87 | // Synoptic integration. 88 | // [http://www.cs.ubc.ca/~bestchai/papers/esecfse2011-final.pdf] 89 | def notify_log_message(msg: String) = {} 90 | 91 | // When an actor has been terminated, the ActorCell references associated 92 | // with it are no longer valid. Remove all of them, and return all 93 | // (sender, message) pairs that used to be pending for this actor. These 94 | // may later be resent by Instrumenter. 95 | def actorTerminated(actor: String): Seq[(String, Any)] = { 96 | throw new RuntimeException("NYI") 97 | } 98 | 99 | // Invoked by Instrumenter after dispatchAfterMailboxIdle(name) has been 100 | // called and name's mailbox has been set to idle state 101 | def handleMailboxIdle() { 102 | Instrumenter().start_dispatch 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /tools/combine_graphs_for_experiment.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Must be invoked from the top-level sts2-applications directory. 4 | # 5 | # Given the directory containing an unmodified execution trace, find all 6 | # directories containing MCSes for that unmodified execution trace. 7 | # Combine their minimization_stats.json data into a single graph. 8 | 9 | if ARGV.length != 1 10 | $stderr.puts "Usage: #{$0} /path/to/experiment" 11 | exit(1) 12 | end 13 | 14 | basename = File.basename(ARGV[0]) 15 | dirname = File.dirname(ARGV[0]) 16 | 17 | args = [] 18 | Dir.chdir dirname do 19 | mcs_dirs = Dir.glob("#{basename}*").select do |name| 20 | name != basename and File.directory? name 21 | end 22 | 23 | mcs_dirs.each do |mcs_dir| 24 | title = mcs_dir.gsub(/#{basename}_/, '') 25 | path = File.absolute_path(mcs_dir) + "/minimization_stats.json" 26 | args << path 27 | args << title 28 | end 29 | end 30 | 31 | system("interposition/src/main/python/minimization_stats/combine_graphs.py #{args.join(' ')}") 32 | -------------------------------------------------------------------------------- /tools/indent.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | File.foreach(ARGV.shift) do |line| 4 | if line =~ /BEFORE/ 5 | puts " " + line 6 | elsif line =~ /AFTER/ 7 | puts " " + line 8 | elsif line =~ /schedule_new_message/ or line =~ /RAFT/ 9 | puts 10 | puts line 11 | puts 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /tools/overwrite_uninteresting_fuzz_runs.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/ruby 2 | 3 | outName = "console.out" 4 | out = File.new(outName, "w") 5 | 6 | ARGF.each do |line| 7 | out.puts line 8 | if line =~ /Trying random interleaving/ 9 | out.close 10 | File.delete(outName) 11 | out = File.new(outName, "w") 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /tools/rerun_experiments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | START_WITH=$1 4 | 5 | (cd experiments; git pull) 6 | 7 | for branch in raft-45 raft-46 raft-56 raft-58-initialization raft-58 raft-42 raft-66; do 8 | if [ "$START_WITH" != "" -a "$START_WITH" != $branch ]; then 9 | continue 10 | fi 11 | if [ "$START_WITH" != "" -a "$START_WITH" == $branch ]; then 12 | START_WITH="" 13 | fi 14 | 15 | echo "==================== Running $branch ==================" 16 | git checkout $branch 17 | git pull 18 | sbt assembly && java -d64 -Xmx15g -cp target/scala-2.11/randomSearch-assembly-0.1.jar akka.dispatch.verification.Main > console.out 19 | done 20 | -------------------------------------------------------------------------------- /tools/sanity.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'set' 4 | 5 | original = Set.new 6 | pruned = Set.new 7 | first_line = true 8 | 9 | File.foreach(ARGV.shift) do |line| 10 | events = line.chomp.split(", ") 11 | if first_line 12 | original += events 13 | first_line = false 14 | else 15 | pruned += events 16 | end 17 | end 18 | 19 | (original - pruned).each do |not_pruned| 20 | puts not_pruned 21 | end 22 | -------------------------------------------------------------------------------- /tools/spacify.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | File.foreach(ARGV.shift) do |line| 4 | puts line 5 | puts 6 | end 7 | -------------------------------------------------------------------------------- /tools/subtree_pull_all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUN_BASH=$1 4 | 5 | for branch in raft-45 raft-46 raft-56 raft-58 raft-58-initialization raft-42 raft-66 spark-2294 spark-2294-blocks spark-3150 spark-9256; do 6 | git checkout $branch 7 | git pull 8 | git subtree pull --prefix=interposition interposition master 9 | if [ "$RUN_BASH" != "" ]; then 10 | bash 11 | fi 12 | git push 13 | done 14 | --------------------------------------------------------------------------------