├── .github └── workflows │ └── test-runner.yml ├── LICENSE.txt ├── README.md ├── css ├── style.css └── style.scss ├── deploy_shiviz.py ├── doc-index.md ├── docgen.pl ├── fonts ├── icomoon.woff └── icons.woff ├── graph_viz ├── Readme.md ├── RpcClientServer.json ├── RpcClientServer.log └── log_graph_converter.py ├── images ├── cluster.png ├── diff.graffle ├── diff.png ├── event-expand-menu.png ├── event-selection.png ├── favico.ico ├── motifs.png ├── process-selection.png ├── search.png └── shiviz-example.png ├── index.html ├── js ├── builder │ ├── builderGraph.js │ ├── builderNode.js │ ├── graphBuilder.js │ ├── graphBuilderHost.js │ └── graphBuilderNode.js ├── clusterer.js ├── deployed.js ├── dev.js ├── graph │ ├── abstractGraph.js │ ├── abstractNode.js │ ├── dfsGraphTraversal.js │ ├── graphEvent.js │ └── graphTraversal.js ├── logEventMatcher │ ├── lemAST.js │ ├── lemInterpreter.js │ ├── lemParser.js │ ├── lemToken.js │ ├── lemTokenizer.js │ └── logEventMatcher.js ├── model │ ├── logEvent.js │ ├── modelGraph.js │ ├── modelNode.js │ ├── parser.js │ ├── vectorTimestamp.js │ └── vectorTimestampSerializer.js ├── motifFinder │ ├── broadcastGatherFinder.js │ ├── customMotifFinder.js │ ├── motif.js │ ├── motifDrawer.js │ ├── motifFinder.js │ ├── motifGroup.js │ ├── motifNavigator.js │ ├── requestResponseFinder.js │ └── textQueryMotifFinder.js ├── searchBar.js ├── shiviz.js ├── transform │ ├── collapseSequentialNodesTransformation.js │ ├── hideHostTransformation.js │ ├── highlightHostTransformation.js │ ├── highlightMotifTransformation.js │ ├── showDiffTransformation.js │ ├── transformation.js │ └── transformer.js ├── util │ ├── exception.js │ ├── regexp.js │ └── util.js └── visualization │ ├── abbreviation.js │ ├── controller.js │ ├── global.js │ ├── hostPermutation.js │ ├── layout.js │ ├── view.js │ ├── visualEdge.js │ ├── visualGraph.js │ └── visualNode.js ├── jsdoc.zip ├── local_scripts ├── d3.v4.min.js └── jquery-3.2.1.min.js ├── log ├── chord.log ├── ewd998.log ├── facebook-multiple-study.log ├── facebook-multiple.log ├── facebook-study.log ├── facebook.log ├── motifs.json ├── multiple-comparison.log ├── reliable-broadcast.log ├── simple-reliable-broadcast.log ├── simpledb.log ├── tsviz_fslock_24t_4sp.log ├── tsviz_shared_var_4_threads.log ├── voldemort-simple-threadnames.log └── voldemort.log └── test ├── index.html ├── logWithEscapeCharacters.js ├── test.js └── test.py /.github/workflows/test-runner.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Run tests 5 | 6 | on: 7 | push: 8 | branches: "**" 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python 3.9 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: 3.9 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install flake8 selenium 27 | - name: Run Tests 28 | run: | 29 | python test/test.py 30 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Unless otherwise noted, the ShiViz project is distributed under the 2 | MIT license, which appears below. Some files in the distribution are 3 | distributed under a different license. These exceptions are as 4 | follows: 5 | * ./jsdoc-master.zip 6 | * https://github.com/jsdoc3/jsdoc3.github.com 7 | * Creative Commons Attribution-ShareAlike 3.0 Unported license 8 | * D3.js (included from a remote site in index.html) 9 | * http://d3js.org/ 10 | * BSD license 11 | * JQuery (included from a remote site in index.html) 12 | * https://jquery.com/ 13 | * JQuery license 14 | 15 | 16 | The MIT License (MIT) 17 | 18 | Copyright (c) 2015 University of British Columbia 19 | 20 | Permission is hereby granted, free of charge, to any person obtaining a copy 21 | of this software and associated documentation files (the "Software"), to deal 22 | in the Software without restriction, including without limitation the rights 23 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 24 | copies of the Software, and to permit persons to whom the Software is 25 | furnished to do so, subject to the following conditions: 26 | 27 | The above copyright notice and this permission notice shall be included in all 28 | copies or substantial portions of the Software. 29 | 30 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 31 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 32 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 33 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 34 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 35 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 36 | SOFTWARE. 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ShiViz is a visualization tool to study executions of distributed systems. 2 | 3 | * [**Try the tool in your browser!**](https://bestchai.bitbucket.io/shiviz/) 4 | 5 | * ShiViz is described in several publications: 6 | 7 | * [Visualizing Distributed System Executions](https://homes.cs.washington.edu/~mernst/pubs/visualize-distributed-tosem2020.pdf), TOSEM 2020 8 | ``` 9 | @article{BeschastnikhLXWBE2020, 10 | author = {Ivan Beschastnikh and Perry Liu and Albert Xing 11 | and Patty Wang and Yuriy Brun and Michael D. Ernst}, 12 | title = {{Visualizing Distributed System Executions}}, 13 | journal = {ACM Transactions on Software Engineering and Methodology (TOSEM)}, 14 | volume = {29}, 15 | number = {2}, 16 | pages = {9:1--9:38}, 17 | month = Mar, 18 | year = {2020} 19 | } 20 | ``` 21 | * A previous version appeared as *[“Debugging distributed systems: Challenges and options for validation and debugging”](https://homes.cs.washington.edu/~mernst/pubs/debug-distributed-cacm2016.pdf)* by Ivan Beschastnikh, Patty Wang, Yuriy Brun, and Michael D. Ernst. Communications of the ACM, vol. 59, no. 8, Aug. 2016, pp. 32-37. 22 | 23 | 24 | * [Wiki](https://github.com/DistributedClocks/shiviz/wiki): explains ShiViz in more detail 25 | 26 | * [JS docs](http://bestchai.bitbucket.io/shiviz/docs/): documentation extracted from code 27 | 28 | * [DistributedClocks](https://distributedclocks.github.io/): ShiViz-compatible vector-clock instrumentation libraries for C/C++/Java/Go 29 | 30 | * [Sass](http://sass-lang.com/): a CSS preprocessor for easier and more intuitive styling 31 | 32 | - [Installation](http://sass-lang.com/install): instructions for setting up Sass 33 | - [Basics](http://sass-lang.com/guide): a quick guide to basic features in Sass 34 | - [Sassmeister](http://sassmeister.com/): test out Sass in real time 35 | - To compile a SCSS file into valid CSS from the command line, use: 36 | ```$ sass input.scss output.css --style expanded``` 37 | -------------------------------------------------------------------------------- /deploy_shiviz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ''' 4 | Purpose: 5 | ========= 6 | 7 | This script packages and deploys the shiviz project to: 8 | https://bestchai.bitbucket.io/shiviz/ 9 | 10 | 11 | Usage: 12 | ======= 13 | 14 | - This script must be run from within the top level shiviz source directory. 15 | 16 | - This script must be run from OSX :) 17 | 18 | - This script: 19 | 1. Generates the documentation for shiviz using jsdoc3 20 | 21 | 2. Removes the proxy hack that allows shiviz to access log files 22 | when shiviz is run locally. 23 | 24 | 3. Adds google analytics tracking. 25 | 26 | 4. Copies over the entire d3/ source tree over to a destination that 27 | is assumed to be the https://bitbucket.org/bestchai/shiviz/src/default/ repo. 28 | 29 | 5. Commits and pushes the https://bitbucket.org/bestchai/bestchai.bitbucket.org/src/default/ repo. 30 | ''' 31 | 32 | 33 | import sys 34 | import os 35 | import fileinput 36 | import subprocess 37 | import argparse 38 | import time 39 | 40 | def get_cmd_output(cmd, args): 41 | ''' 42 | Returns the standard output from running: 43 | $ cmd args[0] args[1] .. args[n] 44 | 45 | Where cmd is the command name (e.g., 'svn') and args is a list of 46 | arguments to the command (e.g., ['help', 'log']). 47 | ''' 48 | return subprocess.Popen([cmd] + args, 49 | stdout=subprocess.PIPE, 50 | stderr=subprocess.STDOUT).communicate()[0] 51 | 52 | 53 | def runcmd(s): 54 | ''' 55 | Logs and runs a shell command. 56 | ''' 57 | print "os.system: " + s 58 | return os.system(s) 59 | 60 | 61 | def minify(revid): 62 | ''' 63 | Minifies all of the js code under js/ using Google Closure Compiler and 64 | writes the minified resulting js code to js/min.js. 65 | ''' 66 | 67 | exit_code = runcmd("google-closure-compiler js/**.js !dev.js local_scripts/**.js --js_output_file js/min.js") 68 | if exit_code != 0: 69 | print("Minification failed!") 70 | sys.exit(-1) 71 | 72 | minified_file = fileinput.input("js/min.js", inplace=True) 73 | # Add hg revision id to the minified code. 74 | for line in minified_file: 75 | sys.stdout.write(line.replace("revision: ZZZ", "revision: %s" % revid)) 76 | 77 | 78 | def parse_args(): 79 | ''' 80 | Method to process the command line arguments. Expects only one of the two following options: 81 | -d or --dev to deploy to bestchai.bitbucket.org/shiviz-dev/ 82 | -p or --prod to deploy to bestchai.bitbucket.org/shiviz/ 83 | 84 | If no flag specified or the specified flag is invalid, outputs the help dialogue. 85 | ''' 86 | 87 | parser = argparse.ArgumentParser() 88 | argument_group = parser.add_mutually_exclusive_group(required=True) 89 | 90 | argument_group.add_argument("-d", "--dev", help="Deploys ShiViz to development environment.", action="store_true") 91 | argument_group.add_argument("-p", "--prod", help="Deploys ShiViz to production environment.", action="store_true") 92 | 93 | return parser.parse_args() 94 | 95 | 96 | def main(args): 97 | ''' 98 | Workhorse method to execute all the of the steps described in the file header. 99 | ''' 100 | 101 | src_dir = "./" 102 | 103 | if args.prod: 104 | dist_dir = "../bestchai.bitbucket.org/shiviz/" 105 | elif args.dev: 106 | dist_dir = "../bestchai.bitbucket.org/shiviz-dev/" 107 | 108 | 109 | print "Deploying to: " + dist_dir 110 | print "from: " + src_dir 111 | 112 | # TODO: add a confirmation yes/no dialog, before going ahead with rm. 113 | 114 | # Remove previously deployed version of shiviz. 115 | if (os.path.exists(dist_dir)): 116 | runcmd("rm -rf " + dist_dir + "*") 117 | else: 118 | print "Error: deployment dir is not where it is expected." 119 | sys.exit(-1) 120 | 121 | do_minification = raw_input("Do you want to minify the code? (Y/N) ") == "Y" 122 | if do_minification: 123 | # Check if google-closure-compiler is installed. 124 | if (runcmd("google-closure-compiler --version") != 0): 125 | print("You need to install google-closure-compiler for minification.") 126 | print("The easiest way to install the compiler is with NPM or Yarn.") 127 | 128 | continue_without_modification = raw_input("Do you want to continue without modification? (Y/N)") == "Y" 129 | if not continue_without_modification: 130 | sys.exit(-1) 131 | do_minification = False 132 | 133 | # Copy over the source. 134 | if (os.path.exists(src_dir)): 135 | runcmd("cp -R " + src_dir + "* " + dist_dir) 136 | if do_minification: 137 | # Remove js source code since we will be using a minified version (see below). 138 | runcmd("rm -rf " + dist_dir + "/js/*") 139 | else: 140 | print "Error: source dir is not where it is expected." 141 | sys.exit(-1) 142 | 143 | # Compile docs 144 | if (runcmd("perl docgen.pl " + dist_dir) != 0): 145 | sys.exit(-1) 146 | 147 | # Find out the current revision id: 148 | # hg: 149 | # revid = get_cmd_output('hg', ['id', '-i']); 150 | # git: 151 | revid = get_cmd_output('git', ['rev-parse', '--short', 'HEAD']); 152 | revid = revid.rstrip() 153 | 154 | # Find out the current branch: 155 | # hg: 156 | # branch = get_cmd_output('hg', ['branch']); 157 | # git: 158 | branch = get_cmd_output('git', ['rev-parse', '--abbrev-ref', 'HEAD']); 159 | branch = branch.rstrip(); 160 | 161 | print "Revid is : " + revid 162 | print "Branch is : " + branch 163 | 164 | # Remove any files containing '#' 165 | runcmd("cd " + dist_dir + " && find . | grep '#' | xargs rm") 166 | 167 | # Remove any files containing '~' 168 | runcmd("cd " + dist_dir + " && find . | grep '~' | xargs rm") 169 | 170 | # Remove any files containing '~' 171 | runcmd("cd " + dist_dir + " && find . | grep '.orig' | xargs rm") 172 | 173 | if do_minification: 174 | # Minify the code 175 | print "Minifying... please wait" 176 | minify(revid) 177 | 178 | minified_size = os.path.getsize('js/min.js') 179 | print "Minified size: %i" % minified_size 180 | 181 | if minified_size < 500: 182 | print "Minification failed!" 183 | return 184 | 185 | print "Minification successful!" 186 | 187 | # Replace reference to js files with minified js in deployed version 188 | # of index.html. 189 | runcmd("sed -i '' -e 's/]*><\/script>//g' " + dist_dir + "index.html") 190 | runcmd("sed -i '' -e 's/<\/body>/ 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.chrome.options import Options 3 | import os 4 | import unittest 5 | 6 | class TestRunner(unittest.TestCase): 7 | ''' 8 | Class to check the results of the tests in the test.js file. 9 | ''' 10 | 11 | chrome_options = Options() 12 | chrome_options.add_argument('--headless') 13 | driver = webdriver.Chrome(options=chrome_options) 14 | 15 | def test_js(self): 16 | ''' 17 | Workhorse method to check the results of the tests. 18 | 19 | Opens the index.html file in the test directory by using a headless Chrome browser. 20 | Then, parses the list elements from the page and checks its class attribute to determine 21 | if the test is failing or passing. If it is failing, treats it like a subtest and fails the 22 | subtest by using the test name given in the test.js file. 23 | ''' 24 | 25 | self.driver.get("file://" +os.getcwd()+"/test/index.html") 26 | output = self.driver.find_element_by_id("output") 27 | test_items = output.find_elements_by_tag_name("li") 28 | 29 | for test_item in test_items: 30 | is_passing = test_item.get_attribute("class") == "pass" 31 | 32 | if not is_passing: 33 | with self.subTest(): 34 | self.fail(test_item.text) 35 | 36 | self.driver.close() 37 | 38 | if __name__ == '__main__': 39 | unittest.main() --------------------------------------------------------------------------------