├── .bumpversion.cfg ├── .gitignore ├── .idea ├── .name ├── dictionaries │ └── sb.xml ├── emacs.xml ├── encodings.xml ├── fuzzing.iml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── jenkinsSettings.xml ├── libraries │ └── Buildout_Eggs.xml ├── misc.xml ├── modules.xml ├── mongoSettings.xml ├── scopes │ ├── code.xml │ ├── docs.xml │ ├── scope_settings.xml │ └── tests.xml ├── vagrant.xml ├── vcs.xml └── watcherTasks.xml ├── .pylintrc ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── builder.sh ├── docs ├── Makefile ├── make.bat └── source │ ├── api.rst │ ├── conf.py │ ├── index.rst │ ├── license.rst │ ├── release-notes.rst │ └── tutorial.rst ├── features ├── data │ ├── t1.pdf │ ├── t2.jpg │ └── t3.pdf ├── environment.py ├── fuzzer.feature ├── resources │ ├── test_config.yaml │ └── testfuzz.py ├── singleton.feature └── steps │ ├── ft_fuzzer.py │ └── ft_singleton.py ├── fuzzing.sublime-project ├── fuzzing ├── __init__.py ├── fuzzer.py ├── log.py └── resources │ └── log_config.yaml ├── gp_decorators ├── __init__.py └── singleton.py ├── pylint.ini ├── requirements.txt ├── resrc └── pylintrc ├── run_fuzzer.py ├── setup.py └── tests ├── __init__.py ├── run_config.yaml └── test_stat_counter_test.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.3.4 3 | commit = True 4 | files = setup.py -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /dist 3 | /src/fuzzing/pytest-2.6.4-py3.4.egg 4 | /*.egg-info 5 | /**/__pycache__/ 6 | *.pyc 7 | src/ 8 | fuzzing.log* 9 | .installed.cfg 10 | statistics 11 | .coverage 12 | .pytest_cache 13 | develop-eggs 14 | 15 | /reports/ 16 | 17 | fuzzing.sublime-project 18 | fuzzing.sublime-workspace 19 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | fuzzing -------------------------------------------------------------------------------- /.idea/dictionaries/sb.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | braun 5 | fuzzers 6 | testfuzz 7 | yaml 8 | 9 | 10 | -------------------------------------------------------------------------------- /.idea/emacs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/fuzzing.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 40 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /.idea/jenkinsSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/libraries/Buildout_Eggs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <<<<<<< HEAD 6 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/mongoSettings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 17 | 19 | -------------------------------------------------------------------------------- /.idea/scopes/code.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/scopes/docs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/scopes/scope_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/scopes/tests.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/vagrant.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.idea/watcherTasks.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 16 | 24 | 25 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Profiled execution. 11 | profile=no 12 | 13 | # Add files or directories to the blacklist. They should be base names, not 14 | # paths. 15 | ignore=CVS,reg_settings.py 16 | 17 | # Pickle collected data for later comparisons. 18 | persistent=yes 19 | 20 | # List of plugins (as comma separated values of python modules names) to load, 21 | # usually to register additional checkers. 22 | load-plugins= 23 | 24 | 25 | [MESSAGES CONTROL] 26 | 27 | # Enable the message, report, category or checker with the given id(s). You can 28 | # either give multiple identifier separated by comma (,) or put this option 29 | # multiple time. See also the "--disable" option for examples. 30 | #enable= 31 | 32 | # Disable the message, report, category or checker with the given id(s). You 33 | # can either give multiple identifiers separated by comma (,) or put this 34 | # option multiple times (only on the command line, not in the configuration 35 | # file where it should appear only once).You can also use "--disable=all" to 36 | # disable everything first and then reenable specific checks. For example, if 37 | # you want to run only the similarities checker, you can use "--disable=all 38 | # --enable=similarities". If you want to run only the classes checker, but have 39 | # no Warning level messages displayed, use"--disable=all --enable=classes 40 | # --disable=W" 41 | disable=RP0001,RP0003,RP0101,RP0801,RP0401,RP0701,I0011,C0325 42 | 43 | 44 | [REPORTS] 45 | 46 | # Set the output format. Available formats are text, parseable, colorized, msvs 47 | # (visual studio) and html. You can also give a reporter class, eg 48 | # mypackage.mymodule.MyReporterClass. 49 | output-format=text 50 | 51 | # Put messages in a separate file for each module / package specified on the 52 | # command line instead of printing them on stdout. Reports (if any) will be 53 | # written in a file name "pylint_global.[txt|html]". 54 | files-output=n 55 | 56 | # Tells whether to display a full report or only the messages 57 | reports=yes 58 | 59 | # Python expression which should return a note less than 10 (10 is the highest 60 | # note). You have access to the variables errors warning, statement which 61 | # respectively contain the number of errors / warnings messages and the total 62 | # number of statements analyzed. This is used by the global evaluation report 63 | # (RP0004). 64 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 65 | 66 | # Add a comment according to your evaluation note. This is used by the global 67 | # evaluation report (RP0004). 68 | comment=yes 69 | 70 | # Template used to display messages. This is a python new-style format string 71 | # used to format the message information. See doc for all details 72 | msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} 73 | 74 | 75 | [BASIC] 76 | 77 | # Required attributes for module, separated by a comma 78 | required-attributes= 79 | 80 | # List of builtins function names that should not be used, separated by a comma 81 | bad-functions=map,filter,apply,input 82 | 83 | # Regular expression which should only match correct module names 84 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 85 | 86 | # Regular expression which should only match correct module level names 87 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 88 | 89 | # Regular expression which should only match correct class names 90 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 91 | 92 | # Regular expression which should only match correct function names 93 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 94 | 95 | # Regular expression which should only match correct method names 96 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 97 | 98 | # Regular expression which should only match correct instance attribute names 99 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 100 | 101 | # Regular expression which should only match correct argument names 102 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 103 | 104 | # Regular expression which should only match correct variable names 105 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 106 | 107 | # Regular expression which should only match correct attribute names in class 108 | # bodies 109 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 110 | 111 | # Regular expression which should only match correct list comprehension / 112 | # generator expression variable names 113 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 114 | 115 | # Good variable names which should always be accepted, separated by a comma 116 | good-names=i,j,k,ex,Run,_ 117 | 118 | # Bad variable names which should always be refused, separated by a comma 119 | bad-names=foo,bar,baz,toto,tutu,tata 120 | 121 | # Regular expression which should only match function or class names that do 122 | # not require a docstring. 123 | no-docstring-rgx=__.*__ 124 | 125 | # Minimum line length for functions/classes that require docstrings, shorter 126 | # ones are exempt. 127 | docstring-min-length=-1 128 | 129 | 130 | [FORMAT] 131 | 132 | # Maximum number of characters on a single line. 133 | max-line-length=80 134 | 135 | # Regexp for a line that is allowed to be longer than the limit. 136 | ignore-long-lines=^\s*(# )??$ 137 | 138 | # Allow the body of an if to be on the same line as the test if there is no 139 | # else. 140 | single-line-if-stmt=no 141 | 142 | # List of optional constructs for which whitespace checking is disabled 143 | no-space-check=trailing-comma,dict-separator 144 | 145 | # Maximum number of lines in a module 146 | max-module-lines=1000 147 | 148 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 149 | # tab). 150 | indent-string=' ' 151 | 152 | 153 | [MISCELLANEOUS] 154 | 155 | # List of note tags to take in consideration, separated by a comma. 156 | notes=FIXME,TODO,TASK,XXX 157 | 158 | 159 | [SIMILARITIES] 160 | 161 | # Minimum lines number of a similarity. 162 | min-similarity-lines=4 163 | 164 | # Ignore comments when computing similarities. 165 | ignore-comments=yes 166 | 167 | # Ignore docstrings when computing similarities. 168 | ignore-docstrings=yes 169 | 170 | # Ignore imports when computing similarities. 171 | ignore-imports=no 172 | 173 | 174 | [TYPECHECK] 175 | 176 | # Tells whether missing members accessed in mixin class should be ignored. A 177 | # mixin class is detected if its name ends with "mixin" (case insensitive). 178 | ignore-mixin-members=yes 179 | 180 | # List of classes names for which member attributes should not be checked 181 | # (useful for classes with attributes dynamically set). 182 | ignored-classes=SQLObject 183 | 184 | # When zope mode is activated, add a predefined set of Zope acquired attributes 185 | # to generated-members. 186 | zope=no 187 | 188 | # List of members which are set dynamically and missed by pylint inference 189 | # system, and so shouldn't trigger E0201 when accessed. Python regular 190 | # expressions are accepted. 191 | generated-members=REQUEST,acl_users,aq_parent 192 | 193 | 194 | [VARIABLES] 195 | 196 | # Tells whether we should check for unused import in __init__ files. 197 | init-import=no 198 | 199 | # A regular expression matching the beginning of the name of dummy variables 200 | # (i.e. not used). 201 | dummy-variables-rgx=_$|dummy|^_ 202 | 203 | # List of additional names supposed to be defined in builtins. Remember that 204 | # you should avoid to define new builtins when possible. 205 | additional-builtins= 206 | 207 | 208 | [CLASSES] 209 | 210 | # List of interface methods to ignore, separated by a comma. This is used for 211 | # instance to not check methods defines in Zope's Interface base class. 212 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 213 | 214 | # List of method names used to declare (i.e. assign) instance attributes. 215 | defining-attr-methods=__init__,__new__,setUp 216 | 217 | # List of valid names for the first argument in a class method. 218 | valid-classmethod-first-arg=cls 219 | 220 | # List of valid names for the first argument in a metaclass class method. 221 | valid-metaclass-classmethod-first-arg=mcs 222 | 223 | 224 | [DESIGN] 225 | 226 | # Maximum number of arguments for function / method 227 | max-args=5 228 | 229 | # Argument names that match this expression will be ignored. Default to name 230 | # with leading underscore 231 | ignored-argument-names=_.* 232 | 233 | # Maximum number of locals for function / method body 234 | max-locals=15 235 | 236 | # Maximum number of return / yield for function / method body 237 | max-returns=6 238 | 239 | # Maximum number of branch for function / method body 240 | max-branches=12 241 | 242 | # Maximum number of statements in function / method body 243 | max-statements=50 244 | 245 | # Maximum number of parents for a class (see R0901). 246 | max-parents=7 247 | 248 | # Maximum number of attributes for a class (see R0902). 249 | max-attributes=7 250 | 251 | # Minimum number of public methods for a class (see R0903). 252 | min-public-methods=1 253 | 254 | # Maximum number of public methods for a class (see R0904). 255 | max-public-methods=20 256 | 257 | 258 | [IMPORTS] 259 | 260 | # Deprecated modules which should not be used, separated by a comma 261 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 262 | 263 | # Create a graph of every (i.e. internal and external) dependencies in the 264 | # given file (report RP0402 must not be disabled) 265 | import-graph= 266 | 267 | # Create a graph of external dependencies in the given file (report RP0402 must 268 | # not be disabled) 269 | ext-import-graph= 270 | 271 | # Create a graph of internal dependencies in the given file (report RP0402 must 272 | # not be disabled) 273 | int-import-graph= 274 | 275 | 276 | [EXCEPTIONS] 277 | 278 | # Exceptions that will emit a warning when being caught. Defaults to 279 | # "Exception" 280 | overgeneral-exceptions=Exception 281 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.7" 4 | install: 5 | - "pip install -r requirements.txt" 6 | - "pip install ." 7 | script: 8 | - behave --tags=-@slow features/ 9 | - py.test tests/ 10 | after_success: 11 | - true 12 | 13 | # TODO: 14 | # [ ] deploy package 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014,2015 Stefan Braun 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst CHANGELOG.txt LICENSE.txt fuzzing/resources/* docs/source/* docs/Makefile docs/make.bat 2 | include build.sh requirements.txt buildout.cfg -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ========================================================= 2 | fuzzing: tools for stress testing arbitrary applications. 3 | ========================================================= 4 | 5 | .. image:: https://travis-ci.org/stbraun/fuzzing.svg?branch=master 6 | 7 | .. image:: https://readthedocs.org/projects/fuzzing/badge/?version=master 8 | 9 | Stress testing of applications can be done in lots of different ways. 10 | This package provides an easy to use tool to stress test applications which take files 11 | as parameters. Editors, image viewers, compilers, and many more classes of apps come to mind. 12 | 13 | The stress test is based on a given set of files, binary or text. Those files are taken 14 | randomly and some bytes are modified also randomly (fuzzing). Then the application gets 15 | executed with the fuzzed file. Repeating this over and over again stresses the robustness 16 | for defective input data of the application. 17 | 18 | Tutorial and API documentation can be found on ReadTheDocs_. 19 | 20 | .. _ReadTheDocs: http://fuzzing.readthedocs.org/. 21 | 22 | What's new? 23 | ----------- 24 | 25 | Now you can run your tests in multiple processes. Test results are combined and printed. 26 | 27 | 28 | Installation 29 | ------------ 30 | 31 | The easiest way to install is via ``easy_install`` or ``pip`` :: 32 | 33 | $ pip install fuzzing 34 | 35 | There are feature related tests that can be run with ``behave`` and a 36 | couple of unit tests runnable with ``pytest`` or ``nosetest``. 37 | 38 | 39 | Example 40 | ------- 41 | 42 | :: 43 | 44 | from fuzzing.fuzzer import FuzzExecutor 45 | 46 | # Files to use as initial input seed. 47 | file_list = ["./features/data/t1.pdf", "./features/data/t3.pdf", "./features/data/t2.jpg"] 48 | 49 | # List of applications to test. 50 | apps_under_test = ["/Applications/Adobe Reader 9/Adobe Reader.app/Contents/MacOS/AdobeReader", 51 | "/Applications/PDFpen 6.app/Contents/MacOS/PDFpen 6", 52 | "/Applications/Preview.app/Contents/MacOS/Preview", 53 | ] 54 | 55 | number_of_runs = 13 56 | 57 | def test(): 58 | fuzz_executor = FuzzExecutor(apps_under_test, file_list) 59 | fuzz_executor.run_test(number_of_runs) 60 | return fuzz_executor.stats 61 | 62 | def main(): 63 | stats = test() 64 | print(stats) 65 | 66 | 67 | Using pre-built test runner and configuration 68 | --------------------------------------------- 69 | 70 | For convenience a test runner is provided which takes a test configuration. 71 | 72 | Example of a configuration YAML file: :: 73 | 74 | version: 1 75 | seed_files: ['requirements.txt', 'README.rst'] 76 | applications: ['MyFunnyApp', 'AdobeReader'] 77 | runs: 800 78 | processors: 4 79 | processes: 10 80 | 81 | Then call the test runner in a terminal session: :: 82 | 83 | $ run_fuzzer.py test.yaml 84 | 85 | This will execute the tests as configured and print the test result when done: :: 86 | 87 | $ run_fuzzer.py test.yaml 88 | Starting up ... 89 | ... finished 90 | 91 | Test Results: 92 | 93 | MyFunnyApp 94 | succeeded: 4021 95 | failed: 95 96 | AdobeReader 97 | succeeded: 3883 98 | failed: 1 99 | 100 | Copyright & License 101 | ------------------- 102 | 103 | * Copyright 2015, Stefan Braun 104 | * License: MIT 105 | -------------------------------------------------------------------------------- /builder.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export PATH=/usr/local/bin:/usr/local/sbin:$PATH 4 | 5 | # prepare folder for build reports 6 | if [ -d reports ]; then 7 | echo "report folder exists already." 8 | else 9 | mkdir reports 10 | fi 11 | 12 | if [ -z "$1" ]; then 13 | echo 'usage: builder.sh ' 14 | exit 1; 15 | fi 16 | 17 | 18 | mk_venv() { 19 | # setup virtual environment ... 20 | if python3 -m venv venv; then 21 | echo "==============================="; 22 | echo " virtual environment installed "; 23 | echo "==============================="; 24 | else 25 | exit 1; 26 | fi 27 | } 28 | 29 | activate_venv() { 30 | # activate virtual environment 31 | echo "activate virtual environment ..." 32 | source venv/bin/activate 33 | } 34 | 35 | install_requirements() { 36 | # install required packages 37 | pip install --upgrade pip 38 | if pip install -r requirements.txt; then 39 | echo "======================"; 40 | echo "requirements installed"; 41 | echo "======================"; 42 | else 43 | exit 1; 44 | fi 45 | } 46 | 47 | check_sources() { 48 | # run sanity checks 49 | if flake8 --output-file reports/flake8.txt --benchmark --count --statistics fuzzing gp_decorators run_fuzzer.py; then 50 | echo "====================="; 51 | echo " sanity tests passed "; 52 | echo "====================="; 53 | else 54 | echo "====================="; 55 | echo " sanity tests failed "; 56 | echo "====================="; 57 | 58 | exit 1; 59 | fi 60 | 61 | if pylint --rcfile=resrc/pylintrc fuzzing gp_decorators run_fuzzer.py | tee reports/pylint.txt; then 62 | echo "========================"; 63 | echo " static analysis passed"; 64 | echo "========================"; 65 | else 66 | echo "========================"; 67 | echo " static analysis failed "; 68 | echo "========================"; 69 | exit 1; 70 | fi 71 | } 72 | 73 | run_tests() { 74 | # run test and measure coverage 75 | if pytest --junit-xml=reports/unittests.xml --doctest-modules --cov=fuzzing --cov-branch --cov-report=html:reports/coverage --cov-report=xml:reports/coverage.xml tests/ ; then 76 | echo "==================="; 77 | echo " unit tests passed "; 78 | echo "==================="; 79 | else 80 | echo "==================="; 81 | echo " unit tests failed "; 82 | echo "==================="; 83 | exit 1; 84 | fi 85 | 86 | # run behave tests 87 | if behave --junit --junit-directory reports/ | tee reports/behave.txt; then 88 | echo "========================="; 89 | echo " behavioral tests passed "; 90 | echo "========================="; 91 | else 92 | echo "========================="; 93 | echo " behavioral tests failed "; 94 | echo "========================="; 95 | exit 1; 96 | fi 97 | } 98 | 99 | create_dist() { 100 | # build source distribution tarball 101 | python setup.py sdist 102 | 103 | # install package ... 104 | python setup.py install 105 | 106 | # ... and generate documentation 107 | pushd docs 108 | make html 109 | popd 110 | 111 | # package documentation 112 | echo "package documentation ..." 113 | pushd docs/build 114 | zip -r ../../dist/fuzzing-docs.zip html 115 | popd 116 | 117 | rm -rf fuzzing.egg-info 118 | rm dist/*.egg 119 | } 120 | 121 | case "$1" in 122 | venv ) 123 | mk_venv; 124 | ;; 125 | requ ) 126 | activate_venv; 127 | install_requirements; 128 | ;; 129 | checks ) 130 | activate_venv; 131 | check_sources; 132 | ;; 133 | tests ) 134 | activate_venv; 135 | run_tests; 136 | ;; 137 | dist ) 138 | activate_venv; 139 | create_dist; 140 | ;; 141 | all ) 142 | mk_venv; 143 | activate_venv; 144 | install_requirements; 145 | check_sources; 146 | run_tests; 147 | create_dist; 148 | ;; 149 | esac 150 | exit 0 151 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | @echo " coverage to run coverage check of the documentation (if enabled)" 49 | 50 | clean: 51 | rm -rf $(BUILDDIR)/* 52 | 53 | html: 54 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 55 | @echo 56 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 57 | 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | singlehtml: 64 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 65 | @echo 66 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 67 | 68 | pickle: 69 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 70 | @echo 71 | @echo "Build finished; now you can process the pickle files." 72 | 73 | json: 74 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 75 | @echo 76 | @echo "Build finished; now you can process the JSON files." 77 | 78 | htmlhelp: 79 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 80 | @echo 81 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 82 | ".hhp project file in $(BUILDDIR)/htmlhelp." 83 | 84 | qthelp: 85 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 86 | @echo 87 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 88 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 89 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/BasePackage.qhcp" 90 | @echo "To view the help file:" 91 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/BasePackage.qhc" 92 | 93 | devhelp: 94 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 95 | @echo 96 | @echo "Build finished." 97 | @echo "To view the help file:" 98 | @echo "# mkdir -p $$HOME/.local/share/devhelp/BasePackage" 99 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/BasePackage" 100 | @echo "# devhelp" 101 | 102 | epub: 103 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 104 | @echo 105 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 106 | 107 | latex: 108 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 109 | @echo 110 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 111 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 112 | "(use \`make latexpdf' here to do that automatically)." 113 | 114 | latexpdf: 115 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 116 | @echo "Running LaTeX files through pdflatex..." 117 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 118 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 119 | 120 | latexpdfja: 121 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 122 | @echo "Running LaTeX files through platex and dvipdfmx..." 123 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 124 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 125 | 126 | text: 127 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 128 | @echo 129 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 130 | 131 | man: 132 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 133 | @echo 134 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 135 | 136 | texinfo: 137 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 138 | @echo 139 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 140 | @echo "Run \`make' in that directory to run these through makeinfo" \ 141 | "(use \`make info' here to do that automatically)." 142 | 143 | info: 144 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 145 | @echo "Running Texinfo files through makeinfo..." 146 | make -C $(BUILDDIR)/texinfo info 147 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 148 | 149 | gettext: 150 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 151 | @echo 152 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 153 | 154 | changes: 155 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 156 | @echo 157 | @echo "The overview file is in $(BUILDDIR)/changes." 158 | 159 | linkcheck: 160 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 161 | @echo 162 | @echo "Link check complete; look for any errors in the above output " \ 163 | "or in $(BUILDDIR)/linkcheck/output.txt." 164 | 165 | doctest: 166 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 167 | @echo "Testing of doctests in the sources finished, look at the " \ 168 | "results in $(BUILDDIR)/doctest/output.txt." 169 | 170 | coverage: 171 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 172 | @echo "Testing of coverage in the sources finished, look at the " \ 173 | "results in $(BUILDDIR)/coverage/python.txt." 174 | 175 | xml: 176 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 177 | @echo 178 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 179 | 180 | pseudoxml: 181 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 182 | @echo 183 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 184 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\BasePackage.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\BasePackage.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | API Reference 3 | ============= 4 | 5 | 6 | Fuzzing 7 | ------- 8 | 9 | .. automodule:: fuzzing 10 | :members: 11 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Base Package documentation build configuration file, created by 5 | # sphinx-quickstart on Sat Nov 22 20:50:51 2014. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | #sys.path.insert(0, os.path.abspath('.')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | #needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = [ 33 | 'sphinx.ext.autodoc', 34 | 'sphinx.ext.doctest', 35 | 'sphinx.ext.todo', 36 | 'sphinx.ext.coverage', 37 | 'sphinx.ext.mathjax', 38 | 'sphinx.ext.viewcode', 39 | ] 40 | 41 | # Add any paths that contain templates here, relative to this directory. 42 | templates_path = ['_templates'] 43 | 44 | # The suffix of source filenames. 45 | source_suffix = '.rst' 46 | 47 | # The encoding of source files. 48 | #source_encoding = 'utf-8-sig' 49 | 50 | # The master toctree document. 51 | master_doc = 'index' 52 | 53 | # General information about the project. 54 | project = 'Fuzzing' 55 | copyright = '2014,2015 Stefan Braun' 56 | 57 | # The version info for the project you're documenting, acts as replacement for 58 | # |version| and |release|, also used in various other places throughout the 59 | # built documents. 60 | # 61 | # The short X.Y version. 62 | version = '0.3' 63 | # The full version, including alpha/beta/rc tags. 64 | release = '0.3.4' 65 | 66 | # The language for content autogenerated by Sphinx. Refer to documentation 67 | # for a list of supported languages. 68 | # 69 | # This is also used if you do content translation via gettext catalogs. 70 | # Usually you set "language" from the command line for these cases. 71 | language = None 72 | 73 | # There are two options for replacing |today|: either, you set today to some 74 | # non-false value, then it is used: 75 | #today = '' 76 | # Else, today_fmt is used as the format for a strftime call. 77 | #today_fmt = '%B %d, %Y' 78 | 79 | # List of patterns, relative to source directory, that match files and 80 | # directories to ignore when looking for source files. 81 | exclude_patterns = [] 82 | 83 | # The reST default role (used for this markup: `text`) to use for all 84 | # documents. 85 | #default_role = None 86 | 87 | # If true, '()' will be appended to :func: etc. cross-reference text. 88 | #add_function_parentheses = True 89 | 90 | # If true, the current module name will be prepended to all description 91 | # unit titles (such as .. function::). 92 | #add_module_names = True 93 | 94 | # If true, sectionauthor and moduleauthor directives will be shown in the 95 | # output. They are ignored by default. 96 | #show_authors = False 97 | 98 | # The name of the Pygments (syntax highlighting) style to use. 99 | pygments_style = 'sphinx' 100 | 101 | # A list of ignored prefixes for module index sorting. 102 | #modindex_common_prefix = [] 103 | 104 | # If true, keep warnings as "system message" paragraphs in the built documents. 105 | #keep_warnings = False 106 | 107 | 108 | # -- Options for HTML output ---------------------------------------------- 109 | 110 | # The theme to use for HTML and HTML Help pages. See the documentation for 111 | # a list of builtin themes. 112 | #html_theme = 'default' 113 | html_theme = 'haiku' 114 | 115 | # Theme options are theme-specific and customize the look and feel of a theme 116 | # further. For a list of options available for each theme, see the 117 | # documentation. 118 | #html_theme_options = {} 119 | 120 | # Add any paths that contain custom themes here, relative to this directory. 121 | #html_theme_path = [] 122 | 123 | # The name for this set of Sphinx documents. If None, it defaults to 124 | # " v documentation". 125 | #html_title = None 126 | 127 | # A shorter title for the navigation bar. Default is the same as html_title. 128 | #html_short_title = None 129 | 130 | # The name of an image file (relative to this directory) to place at the top 131 | # of the sidebar. 132 | #html_logo = None 133 | 134 | # The name of an image file (within the static path) to use as favicon of the 135 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 136 | # pixels large. 137 | #html_favicon = None 138 | 139 | # Add any paths that contain custom static files (such as style sheets) here, 140 | # relative to this directory. They are copied after the builtin static files, 141 | # so a file named "default.css" will overwrite the builtin "default.css". 142 | html_static_path = ['_static'] 143 | 144 | # Add any extra paths that contain custom files (such as robots.txt or 145 | # .htaccess) here, relative to this directory. These files are copied 146 | # directly to the root of the documentation. 147 | #html_extra_path = [] 148 | 149 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 150 | # using the given strftime format. 151 | #html_last_updated_fmt = '%b %d, %Y' 152 | 153 | # If true, SmartyPants will be used to convert quotes and dashes to 154 | # typographically correct entities. 155 | #html_use_smartypants = True 156 | 157 | # Custom sidebar templates, maps document names to template names. 158 | #html_sidebars = {} 159 | 160 | # Additional templates that should be rendered to pages, maps page names to 161 | # template names. 162 | #html_additional_pages = {} 163 | 164 | # If false, no module index is generated. 165 | #html_domain_indices = True 166 | 167 | # If false, no index is generated. 168 | #html_use_index = True 169 | 170 | # If true, the index is split into individual pages for each letter. 171 | #html_split_index = False 172 | 173 | # If true, links to the reST sources are added to the pages. 174 | #html_show_sourcelink = True 175 | 176 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 177 | #html_show_sphinx = True 178 | 179 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages will 183 | # contain a tag referring to it. The value of this option must be the 184 | # base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Language to be used for generating the HTML full-text search index. 191 | # Sphinx supports the following languages: 192 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 193 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' 194 | #html_search_language = 'en' 195 | 196 | # A dictionary with options for the search language support, empty by default. 197 | # Now only 'ja' uses this config value 198 | #html_search_options = {'type': 'default'} 199 | 200 | # The name of a javascript file (relative to the configuration directory) that 201 | # implements a search results scorer. If empty, the default will be used. 202 | #html_search_scorer = 'scorer.js' 203 | 204 | # Output file base name for HTML help builder. 205 | htmlhelp_basename = 'Fuzzingdoc' 206 | 207 | # -- Options for LaTeX output --------------------------------------------- 208 | 209 | latex_elements = { 210 | # The paper size ('letterpaper' or 'a4paper'). 211 | #'papersize': 'letterpaper', 212 | 213 | # The font size ('10pt', '11pt' or '12pt'). 214 | #'pointsize': '10pt', 215 | 216 | # Additional stuff for the LaTeX preamble. 217 | #'preamble': '', 218 | 219 | # Latex figure (float) alignment 220 | #'figure_align': 'htbp', 221 | } 222 | 223 | # Grouping the document tree into LaTeX files. List of tuples 224 | # (source start file, target name, title, 225 | # author, documentclass [howto, manual, or own class]). 226 | latex_documents = [ 227 | ('index', 'Fuzzing.tex', 'Fuzzing Documentation', 228 | 'Stefan Braun', 'manual'), 229 | ] 230 | 231 | # The name of an image file (relative to this directory) to place at the top of 232 | # the title page. 233 | #latex_logo = None 234 | 235 | # For "manual" documents, if this is true, then toplevel headings are parts, 236 | # not chapters. 237 | #latex_use_parts = False 238 | 239 | # If true, show page references after internal links. 240 | #latex_show_pagerefs = False 241 | 242 | # If true, show URL addresses after external links. 243 | #latex_show_urls = False 244 | 245 | # Documents to append as an appendix to all manuals. 246 | #latex_appendices = [] 247 | 248 | # If false, no module index is generated. 249 | #latex_domain_indices = True 250 | 251 | 252 | # -- Options for manual page output --------------------------------------- 253 | 254 | # One entry per manual page. List of tuples 255 | # (source start file, name, description, authors, manual section). 256 | man_pages = [ 257 | ('index', 'Fuzzing', 'Fuzzing Documentation', 258 | ['Stefan Braun'], 1) 259 | ] 260 | 261 | # If true, show URL addresses after external links. 262 | #man_show_urls = False 263 | 264 | 265 | # -- Options for Texinfo output ------------------------------------------- 266 | 267 | # Grouping the document tree into Texinfo files. List of tuples 268 | # (source start file, target name, title, author, 269 | # dir menu entry, description, category) 270 | texinfo_documents = [ 271 | ('index', 'Fuzzing', 'Fuzzing Documentation', 272 | 'Stefan Braun', 'Fuzzing', 'Stress testing.', 273 | 'Development'), 274 | ] 275 | 276 | # Documents to append as an appendix to all manuals. 277 | #texinfo_appendices = [] 278 | 279 | # If false, no module index is generated. 280 | #texinfo_domain_indices = True 281 | 282 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 283 | #texinfo_show_urls = 'footnote' 284 | 285 | # If true, do not generate a @detailmenu in the "Top" node's menu. 286 | #texinfo_no_detailmenu = False 287 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | Fuzzing - random stress testing 3 | =============================== 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | release-notes 11 | tutorial 12 | api 13 | license 14 | 15 | 16 | Indices and tables 17 | ================== 18 | 19 | * :ref:`genindex` 20 | * :ref:`modindex` 21 | * :ref:`search` 22 | 23 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | License 3 | ======= 4 | 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | 12 | Copyright (c) 2014,2015 Stefan Braun 13 | -------------------------------------------------------------------------------- /docs/source/release-notes.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | Release Notes 3 | ============= 4 | 5 | 6 | **Release 0.3.1** 7 | 8 | No functional changes. Only tested and released for Python 3.5. 9 | 10 | 11 | **Release 0.3.0** 12 | 13 | New features: 14 | 15 | * Run multiple tests in parallel on multiple processors. Number of processors and processes is configurable. 16 | * Test statistics of the processes are merged and printed. 17 | 18 | API changes: 19 | 20 | * ``FuzzExecutor.stats`` returns an instance of ``TestStatCounter``, not a simple dict anymore. 21 | 22 | You may want to look into ``TestStatCounter`` and ``Status``. 23 | See also ``run_fuzzer.py`` for intended usage. 24 | 25 | 26 | **Release 0.2.3** 27 | 28 | * Data structure for run statistics improved. 29 | * Tests can now be configured using a YAML file. 30 | * Test runner script added for improved user experience :: 31 | run_fuzzer.py config.yaml 32 | 33 | Reading the test runner script may help to get a clearer picture how to use the tool. 34 | 35 | 36 | **Release 0.2.3a1** 37 | 38 | Package structure simplified. 39 | 40 | 41 | **Release 0.2.2** 42 | 43 | Mainly cleanup. 44 | 45 | * Test uses pure Python test app. See ``features/resources/testfuzz.py``. 46 | 47 | 48 | **Release 0.2.1** 49 | 50 | * Class ``LoggerFactory``. Logger factory for configuration of the Python logging framework. 51 | 52 | * The ``fuzzer`` module uses logging. 53 | 54 | * Singleton decorator behaves much nicer since using ``wrapt``. 55 | See `Graham Dumpleton's talk `_ 56 | on the workings of wrapt. 57 | 58 | 59 | **Release 0.2.0** 60 | 61 | Improved fuzz testing. 62 | 63 | * Class ``FuzzExecutor`` makes fuzz testing of applications taking data files easy. 64 | 65 | 66 | **Release 0.1.0** 67 | 68 | First small step. 69 | 70 | * Basic functions for fuzz testing. 71 | * Decorator to declare a class as Singleton. 72 | -------------------------------------------------------------------------------- /docs/source/tutorial.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Tutorial 3 | ======== 4 | 5 | The following sections will show how to use the classes and functions of the package. 6 | 7 | * `Random testing`_ 8 | * `Logging`_ 9 | * `Singletons`_ 10 | 11 | 12 | 13 | 14 | .. index:: Random testing 15 | 16 | Random testing 17 | -------------- 18 | 19 | Systematic testing helps us to cover classes of equivalent test cases. 20 | Specifying those test classes largely reduces the effort for testing without sacrificing test coverage. 21 | 22 | One drawback of this approach is that we're testing only what we expect to break. This may allow defects 23 | caused by unexpected side effects or unexpected input data to pass the tests ... and show up in production systems. 24 | 25 | Random testing is an approach to increase the coverage of the domain of our software's inputs by automatically 26 | running large amounts of tests with randomized input data. This might be totally random 'byte noise', 27 | mostly valid data provided by a carefully crafted generator, or anything in between. 28 | 29 | .. index:: ! Charlie Miller 30 | 31 | Charlie Miller did some interesting work on fuzz testing. The function ``fuzzer()`` is essentially 32 | taken from *Babysitting an Army of Monkeys* (see references below). 33 | 34 | **References:** 35 | 36 | - http://fuzzinginfo.files.wordpress.com/2012/05/cmiller-csw-2010.pdf 37 | - https://cansecwest.com/csw08/csw08-miller.pdf 38 | 39 | 40 | How to do random testing on your own? 41 | +++++++++++++++++++++++++++++++++++++ 42 | 43 | Fuzz testing can be done on different levels: 44 | 45 | - unit (e.g. function, class, module), 46 | - integration (components built from units), 47 | - system (e.g. application). 48 | 49 | In each case you need to provide a source for test data, call your SUT, and check the result. 50 | Put this into a loop and start fuzzing. 51 | 52 | This is already good for robustness tests. In most cases you also want a kind 53 | of statistics and a documentation of the test cases resulting in an error. 54 | 55 | Generating test data 56 | ++++++++++++++++++++ 57 | 58 | .. index:: Charlie Miller 59 | 60 | In general random testing can be done with any kind of input data (I guess ;-). 61 | The code found in ``fuzzing.fuzzer.fuzzer()`` is working on a binary buffer. It is a copy of 62 | Charlie Miller's code mentioned above. 63 | 64 | The binary buffer may contain something 65 | like a pdf, an image, a presentation and so on. It also works fine for normal text, covering 66 | ASCII texts, HTML, XML, JSON and other text based formats. 67 | ``fuzzing.fuzzer.fuzz_string()`` is a wrapper simplifying such use cases a bit. 68 | 69 | Example of a simple generator: 70 | ++++++++++++++++++++++++++++++ 71 | Example 72 | :: 73 | 74 | import fuzzing 75 | seed = "This could be the content of a huge text file." 76 | number_of_fuzzed_variants_to_generate = 10 77 | fuzz_factor = 7 78 | fuzzed_data = fuzzing.fuzz_string(seed, number_of_fuzzed_variants_to_generate, fuzz_factor) 79 | print(fuzzed_data) 80 | 81 | Of course you can also create one fuzzed variant at a time and feed it directly into the SUT. 82 | 83 | 84 | Calling the SUT with the test data 85 | ++++++++++++++++++++++++++++++++++ 86 | 87 | How to call the SUT depends obviously on its type. A Python function can be called directly with the created 88 | data. It might make sense to enclose the call into a try / except block to catch errors. It is also easy to 89 | check the result value for failure. 90 | 91 | Testing software written in other languages works in the same way. You may want to write the fuzz generator in the 92 | target language, or just create the test data with Python and put it into a file for use by the target system. 93 | 94 | Applications reading files can be tested creating fuzzed files in the same manner as described above: 95 | Read a valid seed file into a buffer, fuzz it and write it back to a new file. Then run the application 96 | in a separate process for each fuzzed file. In this case it is not that easy to gather useful 97 | information about the success or failure of the run. At least crashes are easily recognized. 98 | 99 | 100 | The oracle - or: How to evaluate the test result? 101 | +++++++++++++++++++++++++++++++++++++++++++++++++ 102 | 103 | The function evaluating the result of a test run is called *oracle*. That's fine because the result 104 | is not always clear and understandable ;-). 105 | 106 | Running an application in a separate process as described above let us quite easily detect crashes. 107 | If we need more detailed information there is no general way to get at it. One of the most general 108 | information is a crash dump of the SUT. 109 | 110 | Detecting issues not leading to a crash depends largely on 111 | the application we are looking at. If it creates some accessible output, like a processed file 112 | or a log file, we may be able to write parsers that enable us to look for failures. 113 | 114 | 115 | Complete example: 116 | +++++++++++++++++ 117 | 118 | The following sample code runs 100 tests against the applications listed in ``apps_under_test``. 119 | Test data is generated using a simple fuzzer on a set of files defines in ``file_list``. 120 | 121 | After finishing the test runs a statistic is printed. 122 | 123 | Note that ``num_tests`` should be much bigger for real testing. But it makes sense to start with a small number 124 | to get the test harness working. Then increase this number to a couple of millions or so. 125 | 126 | Some of the code found in the ``fuzzer`` module is inlined for easier comprehension. 127 | 128 | :: 129 | 130 | import math 131 | import random 132 | import subprocess 133 | import time 134 | import os.path 135 | from tempfile import mkstemp 136 | from collections import Counter 137 | 138 | 139 | # Files to use as initial input seed. 140 | file_list = ["./data/pycse.pdf", "./data/PyOPC.pdf", "./data/003_overview.pdf", 141 | "./data/Clean-Code-V2.2.pdf", "./data/GraphDatabases.pdf", 142 | "./data/Intro_to_Linear_Algebra.pdf", "./data/zipser-1988.pdf", 143 | "./data/QR-denkenswert.JPG"] 144 | 145 | # List of applications to test. 146 | apps_under_test = ["/Applications/Adobe Reader 9/Adobe Reader.app/Contents/MacOS/AdobeReader", 147 | "/Applications/PDFpen 6.app/Contents/MacOS/PDFpen 6", 148 | "/Applications/Preview.app/Contents/MacOS/Preview", 149 | ] 150 | 151 | 152 | fuzz_factor = 50 # 250 153 | num_tests = 100 154 | 155 | # ##### End of configuration ##### 156 | 157 | def fuzzer(): 158 | """Fuzzing apps.""" 159 | stat_counter = Counter() 160 | for cnt in range(num_tests): 161 | file_choice = random.choice(file_list) 162 | app = random.choice(apps_under_test) 163 | app_name = app.split('/')[-1] 164 | file_name = file_choice.split('/')[-1] 165 | 166 | buf = bytearray(open(os.path.abspath(file_choice), 'rb').read()) 167 | 168 | # Charlie Miller's fuzzer code: 169 | num_writes = random.randrange(math.ceil((float(len(buf)) / fuzz_factor))) + 1 170 | 171 | for _ in range(num_writes): 172 | r_byte = random.randrange(256) 173 | rn = random.randrange(len(buf)) 174 | buf[rn] = r_byte 175 | # end of Charlie Miller's code 176 | 177 | fd, fuzz_output = mkstemp() 178 | open(fuzz_output, 'wb').write(buf) 179 | 180 | process = subprocess.Popen([app, fuzz_output]) 181 | 182 | time.sleep(1) 183 | crashed = process.poll() 184 | if crashed: 185 | logger.error("Process crashed ({} <- {})".format(app, file_choice)) 186 | stat_counter[(app_name, 'failed')] += 1 187 | else: 188 | process.terminate() 189 | stat_counter[(app_name, 'succeeded')] += 1 190 | return stat_counter 191 | 192 | if __name__ == '__main__': 193 | stats = fuzzer() 194 | print(stats) 195 | 196 | 197 | 198 | Using FuzzExecutor 199 | ++++++++++++++++++ 200 | 201 | Fuzz testing applications using files can be used often because it is quite generic. Therefore 202 | it makes sense to encapsulate this functionality and make it easy to apply. 203 | 204 | The example above can be written much faster using the class ``FuzzExecutor``: :: 205 | 206 | from fuzzing.fuzzer import FuzzExecutor 207 | 208 | # Files to use as initial input seed. 209 | file_list = ["./features/data/t1.pdf", "./features/data/t3.pdf", "./features/data/t2.jpg"] 210 | 211 | # List of applications to test. 212 | apps_under_test = ["/Applications/Adobe Reader 9/Adobe Reader.app/Contents/MacOS/AdobeReader", 213 | "/Applications/PDFpen 6.app/Contents/MacOS/PDFpen 6", 214 | "/Applications/Preview.app/Contents/MacOS/Preview", 215 | ] 216 | 217 | number_of_runs = 13 218 | 219 | def test(): 220 | fuzz_executor = FuzzExecutor(apps_under_test, file_list) 221 | fuzz_executor.run_test(number_of_runs) 222 | return fuzz_executor.stats 223 | 224 | def main(): 225 | stats = test() 226 | print(stats) 227 | 228 | 229 | 230 | Getting test statistics 231 | +++++++++++++++++++++++ 232 | 233 | The property ``FuzzExecutor.stat`` is an instance of ``TestStatCounter``. It provides the number 234 | of successful and failed runs for each application. 235 | 236 | To combine the statistics of multiple test runs ``TestStatCounter`` implements ``__add__``: :: 237 | 238 | // Run multiple tests yielding a set stats = set(c1, c2, c3) of stat counters 239 | ... 240 | // Then merge these counters to get a complete statistics of your test runs. 241 | // This operation does not modify c1 to c3. 242 | combined_stats = TestStatCounter(set()) 243 | for stat in stats: 244 | combined_stats += stat 245 | 246 | 247 | ``Status`` is an enum class providing the supported values for test status: :: 248 | 249 | @enum.unique 250 | class Status(enum.Enum): 251 | """Status values for test runs.""" 252 | FAILED = 0 253 | SUCCESS = 1 254 | 255 | 256 | Running tests without coding 257 | ++++++++++++++++++++++++++++ 258 | 259 | When running different sets of tests writing a script for each configuration is tedious. 260 | It would be nice to just write a configuration and feed it to a generic test runner. 261 | 262 | ``run_fuzzer.py`` now reads a test configuration written using YAML notation. Please note that each 263 | process will execute ``runs`` tests. Therefore the number of executed tests is the product of ``runs`` and ``processes``. 264 | :: 265 | 266 | version: 1 267 | seed_files: ['requirements.txt', 'README.rst'] 268 | applications: ['python & features/resources/testfuzz.py -p 0.3', 269 | '/Applications/Adobe Reader 9/Adobe Reader.app/Contents/MacOS/AdobeReader'] 270 | runs: 4 271 | processors: 3 272 | processes: 8 273 | 274 | If you want to run a couple of tests, just provide those configuration files and execute ``run_fuzzer.py``. 275 | For example: :: 276 | 277 | $ run_fuzzer.py test_config_one_processor.yaml 278 | $ run_fuzzer.py test_config_4_processors.yaml 279 | 280 | Each call to ``run_fuzzer.py`` will execute the tests as configured. It creates 281 | a ``ProcessPoolExecutor`` with pool size defined by the number of specified processors. 282 | The number of processes is (kind of) independent of the number processors; 283 | if there are more processes than processors, a new process will be started as soon as a processor is available. 284 | 285 | If for example 2 processors and 5 processes are specified, not more than 2 processes will run in parallel at each point in time. 286 | 287 | After executing all tests ``run_fuzzer.py`` merges the results of all processes and prints statistics like that: :: 288 | 289 | __________________________________________________ 290 | Test Results: 291 | __________________________________________________ 292 | Tests run/succeeded/failed: 32 / 25 / 7 293 | AdobeReader 294 | FAILED: 0 295 | SUCCESS: 14 296 | python 297 | FAILED: 7 298 | SUCCESS: 11 299 | 300 | __________________________________________________ 301 | 302 | 303 | Logging 304 | ------- 305 | 306 | The Python standard library provides good logging capabilities with the module ``logging``. 307 | Requirements on logging depend on the application and may change during the life cycle. Therefore 308 | a logging system must be flexible. The ``logging`` module is highly configurable and extendable. 309 | 310 | Class ``fuzzing.LoggerFactory`` reads a YAML configuration file and initializes the logging system. 311 | It is just a thin layer on top of ``logging`` abstracting from the details of initialization. 312 | Loggers can be used as usual; the use of ``LoggerFactory`` is transparent for the loggers. 313 | 314 | Configuration file 315 | ++++++++++++++++++ 316 | 317 | ``LoggerFactory`` expects to get a YAML file containing the configuration of the loggers. To deploy 318 | your configuration put it into a folder of your package, e.g.: :: 319 | 320 | 321 | 322 | 323 | log_config.yaml 324 | 325 | 326 | 327 | Then add it to your MANIFEST.in so that it will be packaged with your code: :: 328 | 329 | include /resources 330 | 331 | 332 | The documentation of the Python standard library describes how to write such a file: 333 | https://docs.python.org/3.4/library/logging.config.html. 334 | 335 | Example: :: 336 | 337 | version: 1 338 | formatters: 339 | concise: 340 | format: '%(asctime)s - %(levelname)s - %(message)s' 341 | detailed: 342 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() :: %(message)s' 343 | thread_info: 344 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() - %(thread)d:%(threadName)s :: %(message)s' 345 | handlers: 346 | console: 347 | class: logging.StreamHandler 348 | level: WARNING 349 | formatter: concise 350 | stream: ext://sys.stdout 351 | file: 352 | class: logging.handlers.RotatingFileHandler 353 | filename: 'fuzzer.log' 354 | maxBytes: 100000 355 | backupCount: 3 356 | level: DEBUG 357 | formatter: detailed 358 | loggers: 359 | fuzzing: 360 | level: DEBUG 361 | handlers: [console, file] 362 | propagate: no 363 | fuzzing.fuzzing: 364 | level: INFO 365 | handlers: [console, file] 366 | propagate: no 367 | fuzzing.fuzzing.FuzzExecutor: 368 | level: INFO 369 | handlers: [file, console] 370 | propagate: no 371 | root: 372 | level: WARNING 373 | handlers: [console] 374 | 375 | 376 | Initialization 377 | ++++++++++++++ 378 | 379 | The ``logging`` system must be initialized before the first use. So put something like the 380 | following into the startup code of your application: :: 381 | 382 | from gp_tools import LoggerFactory 383 | 384 | def my_main(): 385 | lf = LoggerFactory(package_name='my_package', config_file='resources/log_config.yaml') 386 | lf.initialize() 387 | 388 | 389 | Now you can log as you're used to it: :: 390 | 391 | import logging 392 | 393 | # 'my_logger' is the name as used in the configuration. 394 | logger = logging.getLogger('my_logger') 395 | 396 | logger.info('Happy Logging!') 397 | 398 | That's it :-) 399 | 400 | 401 | Singletons 402 | ---------- 403 | 404 | Singleton classes are characterized by the fact that there will be never more than a single instance. 405 | This may be useful for classes handling physical devices or any other stateful objects, e.g. caches, 406 | that need to be handled in a consistent way. 407 | 408 | Singletons should be used with care, because they may lead to high coupling if used in many places. 409 | So they may be comfortable first, but become a nightmare later on when extending or maintaining an application. 410 | 411 | Creating a singleton class using the singleton decorator is simple: :: 412 | 413 | from gp_decorators.singleton import singleton 414 | 415 | @singleton 416 | class SomeClass(object): 417 | """A singleton class.""" 418 | # 419 | 420 | -------------------------------------------------------------------------------- /features/data/t1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stbraun/fuzzing/bba3ad8a2b749e64a6e1f345ccd26a2675584d4e/features/data/t1.pdf -------------------------------------------------------------------------------- /features/data/t2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stbraun/fuzzing/bba3ad8a2b749e64a6e1f345ccd26a2675584d4e/features/data/t2.jpg -------------------------------------------------------------------------------- /features/data/t3.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stbraun/fuzzing/bba3ad8a2b749e64a6e1f345ccd26a2675584d4e/features/data/t3.pdf -------------------------------------------------------------------------------- /features/environment.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """General settings for feature tests.""" 3 | 4 | from fuzzing.log import LoggerFactory 5 | 6 | 7 | def before_all(context): 8 | """Set up before all tests. 9 | 10 | Initialize the logger framework. 11 | 12 | :param context: test context. 13 | """ 14 | lf = LoggerFactory(config_file='../features/resources/test_config.yaml') 15 | lf.initialize() 16 | ll = lf.get_instance('environment') 17 | ll.info('Logger initialized: %s', repr(lf.config)) 18 | ll.info('Initial test context: %s', repr(context)) 19 | -------------------------------------------------------------------------------- /features/fuzzer.feature: -------------------------------------------------------------------------------- 1 | # Created by sb at 06.12.14 2 | Feature: Provide a fuzz tester. 3 | A fuzz tester consists of a tests data generator, 4 | a driver feeding data to the SUT (Software Under Test), 5 | and an oracle detecting success or failure of a test run. 6 | 7 | | Test given applications with fuzzed files. Count successful and failed calls. 8 | | Call a given application with a file that was fuzzed before. 9 | | Example: Take a jpeg file, fuzz it and call Preview with the fuzzed file. 10 | 11 | # The exact number of modified bytes can't be guaranteed. 12 | # This is because a byte will be replaced with a random value which 13 | # may be the same as already there. 14 | # Therefore only the maximum number of modified bytes is known. 15 | Scenario: Fuzz a binary buffer with minimum number of modifications. 16 | Given a byte array of len 10 17 | When feeding it into the fuzzer, setting the fuzz_factor to 10 18 | Then it will return a buffer with up to two modified bytes. 19 | 20 | Scenario Outline: Fuzz a binary buffer with different numbers of modifications. 21 | Given a byte array of len 10 22 | When feeding it into the fuzzer, setting the fuzz_factor to 23 | Then it will return a buffer with up to modified bytes. 24 | 25 | Examples: 26 | | fuzz_factor | max_modified | 27 | | 1 | 10 | 28 | | 5 | 3 | 29 | | 11 | 2 | 30 | | 101 | 2 | 31 | 32 | 33 | Scenario Outline: Create a list of fuzzed variants of a given string. 34 | Given a string as seed. 35 | """ 36 | Create fuzzed variants of this string. 37 | A test seed for our fuzz tester. 38 | Multiple lines are fine. 39 | """ 40 | When feeding the seed into the fuzzer, providing a count of 41 | Then it will return a list of fuzzed variants of the seed. 42 | 43 | Examples: 44 | | count | len_list | 45 | | 0 | 0 | 46 | | 1 | 1 | 47 | | 11 | 11 | 48 | | 101 | 101 | 49 | 50 | 51 | Scenario Outline: File fuzzer - succeeding 52 | Given a list of file paths 53 | | file_path | 54 | | ./features/data/t1.pdf | 55 | | ./features/data/t2.jpg | 56 | | ./features/data/t3.pdf | 57 | And a list of applications 58 | | application | 59 | | python & features/resources/testfuzz.py | 60 | And a FuzzExecutor instance created with those lists. 61 | When running a test times 62 | Then results are recorded and succeeded. 63 | 64 | Examples: Test runs 65 | | runs | 66 | | 0 | 67 | | 1 | 68 | | 2 | 69 | 70 | 71 | Scenario Outline: File fuzzer - failing 72 | Given a list of file paths 73 | | file_path | 74 | | ./features/data/t1.pdf | 75 | | ./features/data/t2.jpg | 76 | | ./features/data/t3.pdf | 77 | And a list of applications 78 | | application | 79 | | python & features/resources/testfuzz.py -c | 80 | And a FuzzExecutor instance created with those lists. 81 | When running a test times 82 | Then results are recorded and failed. 83 | 84 | Examples: Test runs 85 | | runs | 86 | | 0 | 87 | | 1 | 88 | | 2 | 89 | 90 | 91 | @slow 92 | Scenario Outline: File fuzzer - randomly failing 93 | Given a list of file paths 94 | | file_path | 95 | | ./features/data/t1.pdf | 96 | | ./features/data/t2.jpg | 97 | | ./features/data/t3.pdf | 98 | And a list of applications 99 | | application | 100 | | python & features/resources/testfuzz.py -p 0.2 | 101 | And a FuzzExecutor instance created with those lists. 102 | When running a test times 103 | Then results are recorded. 104 | 105 | Examples: Test runs 106 | | runs | 107 | | 11 | 108 | -------------------------------------------------------------------------------- /features/resources/test_config.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | formatters: 3 | concise: 4 | format: 'TEST configuration - %(asctime)s - %(levelname)s - %(message)s' 5 | detailed: 6 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() :: %(message)s' 7 | thread_info: 8 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() - %(thread)d:%(threadName)s :: %(message)s' 9 | handlers: 10 | console: 11 | class: logging.StreamHandler 12 | level: WARNING 13 | formatter: concise 14 | stream: ext://sys.stdout 15 | file: 16 | class: logging.handlers.RotatingFileHandler 17 | filename: 'fuzzing.log' 18 | maxBytes: 100000 19 | backupCount: 3 20 | level: DEBUG 21 | formatter: detailed 22 | loggers: 23 | fuzzing: 24 | level: DEBUG 25 | handlers: [console, file] 26 | propagate: no 27 | fuzzing.fuzzing: 28 | level: INFO 29 | handlers: [console, file] 30 | propagate: no 31 | fuzzing.fuzzing.FuzzExecutor: 32 | level: INFO 33 | handlers: [file, console] 34 | propagate: no 35 | root: 36 | level: WARNING 37 | handlers: [console] 38 | -------------------------------------------------------------------------------- /features/resources/testfuzz.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding=utf-8 3 | """Test app for fuzzer.""" 4 | 5 | import sys 6 | import argparse 7 | from random import random 8 | import time 9 | 10 | 11 | def main(): 12 | """Test for fuzzer.""" 13 | description = "Simple app to test our fuzzer." 14 | parser = argparse.ArgumentParser(description=description) 15 | parser.add_argument('in_path', help='The name of a file to read.') 16 | parser.add_argument('-c', '--crash', help='Crash the app!', 17 | action="store_true") 18 | parser.add_argument('-p', '--probability', 19 | help='Crash the app with given probability (0.0-1.0)', 20 | type=float, 21 | default=0.0) 22 | args = parser.parse_args() 23 | if args.crash: 24 | return 1 / 0 25 | if random() < args.probability: 26 | return 2 / 0 27 | time.sleep(3) 28 | return 0 29 | 30 | 31 | if __name__ == '__main__': 32 | sys.exit(main()) 33 | -------------------------------------------------------------------------------- /features/singleton.feature: -------------------------------------------------------------------------------- 1 | # Created by sb at 07.12.14 2 | Feature: Provide support for implementation of singleton classes. 3 | Make it easy to write and identify singletons. 4 | 5 | Scenario: The singleton capability does not modify the functional behavior of a class. 6 | Given two functionally identical classes, one is a singleton. 7 | When creating an instance of the non-singleton class 8 | And creating an instance of the singleton class 9 | Then the functional behavior is identical. 10 | 11 | Scenario: All objects created from a singleton are identical. 12 | Given a singleton class. 13 | When creating multiple objects from the class 14 | Then all will be identical. 15 | 16 | Scenario: A change to a singleton object is visible to all references. 17 | Given a singleton class. 18 | When creating multiple objects from the class 19 | And modifying the value of an attribute in one of them 20 | Then this modification is visible to all. 21 | 22 | @wip 23 | Scenario: A singleton class shall report their own type not 'function' (basepkg-47). 24 | Given a singleton class. 25 | Then type() shall return the class. 26 | 27 | @wip 28 | Scenario: A singleton class shall report their own module (basepkg-47). 29 | Given a singleton class. 30 | Then __module__ shall be the module of the class. 31 | -------------------------------------------------------------------------------- /features/steps/ft_fuzzer.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Test steps for feature 'fuzzer'.""" 3 | 4 | from behave import given, when, then 5 | 6 | from fuzzing.fuzzer import fuzzer, fuzz_string, FuzzExecutor, Status 7 | 8 | 9 | @given("a byte array of len 10") 10 | def step_impl01(context): 11 | """Prepare a byte array. 12 | 13 | :param context: test context. 14 | """ 15 | context.buf = bytearray(10) 16 | 17 | 18 | @when("feeding it into the fuzzer, setting the fuzz_factor to 10") 19 | def step_impl02(context): 20 | """Execute fuzzer. 21 | 22 | :param context: test context. 23 | """ 24 | context.fuzzed_buf = fuzzer(context.buf, 10) 25 | 26 | 27 | @then("it will return a buffer with up to two modified bytes.") 28 | def step_impl03(context): 29 | """Check assertions. 30 | 31 | :param context: test context. 32 | """ 33 | assert len(context.buf) == len(context.fuzzed_buf) 34 | count = number_of_modified_bytes(context.buf, context.fuzzed_buf) 35 | assert count < 3 36 | assert count >= 0 37 | 38 | 39 | @when("feeding it into the fuzzer, setting the fuzz_factor to {fuzz_factor:d}") 40 | def step_impl04(context, fuzz_factor): 41 | """Execute fuzzer. 42 | 43 | :param fuzz_factor: specified fuzz_factor. 44 | :param context: test context. 45 | """ 46 | context.fuzzed_buf = fuzzer(context.buf, fuzz_factor) 47 | 48 | 49 | @then("it will return a buffer with up to {max_modified:d} modified bytes.") 50 | def step_impl(context, max_modified): 51 | """Check assertions. 52 | 53 | :param max_modified: maximum expected number of modifications. 54 | :param context: test context. 55 | """ 56 | assert len(context.buf) == len(context.fuzzed_buf) 57 | count = number_of_modified_bytes(context.buf, context.fuzzed_buf) 58 | assert count >= 0 59 | assert count <= max_modified 60 | 61 | 62 | @given("a string as seed.") 63 | def step_impl05(context): 64 | """Provide a string. 65 | 66 | :param context: test context. 67 | """ 68 | context.seed = context.text 69 | 70 | 71 | @when("feeding the seed into the fuzzer, providing a count of {count:d}") 72 | def step_impl06(context, count): 73 | """Execute fuzzer. 74 | 75 | :param count: number of string variants to generate. 76 | :param context: test context. 77 | """ 78 | fuzz_factor = 11 79 | context.fuzzed_string_list = fuzz_string(context.seed, count, fuzz_factor) 80 | 81 | 82 | @then("it will return a list of {len_list:d} fuzzed variants of the seed.") 83 | def step_impl07(context, len_list): 84 | """Check assertions. 85 | 86 | :param len_list: expected number of variants. 87 | :param context: test context. 88 | """ 89 | assert len(context.fuzzed_string_list) == len_list 90 | for fuzzed_string in context.fuzzed_string_list: 91 | assert len(context.seed) == len(fuzzed_string) 92 | count = number_of_modified_bytes(context.seed, fuzzed_string) 93 | assert count >= 0 94 | 95 | 96 | # ## file fuzzer 97 | 98 | 99 | @given("a list of file paths") 100 | def step_impl08(context): 101 | """Create file list. 102 | 103 | :param context: test context. 104 | """ 105 | assert context.table, "ENSURE: table is provided." 106 | context.file_list = [row['file_path'] for row in context.table.rows] 107 | 108 | 109 | @given("a list of applications") 110 | def step_impl09(context): 111 | """Create application list. 112 | 113 | :param context: test context. 114 | """ 115 | assert context.table, "ENSURE: table is provided." 116 | context.app_list = [row['application'] for row in context.table.rows] 117 | 118 | 119 | @given("a FuzzExecutor instance created with those lists.") 120 | def step_impl10(context): 121 | """Create application list. 122 | 123 | :param context: test context. 124 | """ 125 | app_list_valid = context.app_list is not None and context.app_list 126 | file_list_valid = context.file_list is not None and context.file_list 127 | assert app_list_valid, "ENSURE: app list is provided." 128 | assert file_list_valid, "ENSURE: file list is provided." 129 | context.fuzz_executor = FuzzExecutor(context.app_list, context.file_list) 130 | assert context.fuzz_executor, "VERIFY: fuzz executor created." 131 | 132 | 133 | @when("running a test {runs:d} times") 134 | def step_impl11(context, runs): 135 | """Execute multiple runs. 136 | 137 | :param runs: number of test runs to perform. 138 | :param context: test context. 139 | """ 140 | executor = context.fuzz_executor 141 | executor.run_test(runs) 142 | stats = executor.stats 143 | count = stats.cumulated_counts() 144 | assert count == runs, "VERIFY: stats available." 145 | 146 | 147 | @then("{runs:d} results are recorded.") 148 | def step_impl12(context, runs): 149 | """Check called apps / files. 150 | 151 | :param runs: expected number of records. 152 | :param context: test context. 153 | """ 154 | executor_ = context.fuzz_executor 155 | stats = executor_.stats 156 | count = stats.cumulated_counts() 157 | assert count == runs, "VERIFY: Number of recorded runs." 158 | 159 | 160 | @then("{runs:d} results are recorded and succeeded.") 161 | def step_impl13(context, runs): 162 | """Check called apps / files. 163 | 164 | :param runs: expected number of records. 165 | :param context: test context. 166 | """ 167 | executor_ = context.fuzz_executor 168 | stats = executor_.stats 169 | count = stats.cumulated_counts() 170 | assert count == runs, "VERIFY: Number of recorded runs." 171 | successful_runs = stats.cumulated_counts_for_status(Status.SUCCESS) 172 | assert successful_runs == runs 173 | 174 | 175 | @then("{runs:d} results are recorded and failed.") 176 | def step_impl14(context, runs): 177 | """Check called apps / files. 178 | 179 | :param runs: expected number of records. 180 | :param context: test context. 181 | """ 182 | executor_ = context.fuzz_executor 183 | stats = executor_.stats 184 | count = stats.cumulated_counts() 185 | assert count == runs, "VERIFY: Number of recorded runs." 186 | failed_runs = stats.cumulated_counts_for_status(Status.FAILED) 187 | assert failed_runs == runs 188 | 189 | 190 | # ##### helpers 191 | 192 | 193 | def number_of_modified_bytes(buf, fuzzed_buf): 194 | """Determine the number of differing bytes. 195 | 196 | :param buf: original buffer. 197 | :param fuzzed_buf: fuzzed buffer. 198 | :return: number of different bytes. 199 | :rtype: int 200 | """ 201 | count = 0 202 | for idx, byte in enumerate(buf): 203 | if byte != fuzzed_buf[idx]: 204 | count += 1 205 | return count 206 | -------------------------------------------------------------------------------- /features/steps/ft_singleton.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Test steps for feature 'singleton'.""" 3 | 4 | from behave import when, given, then 5 | 6 | from gp_decorators.singleton import singleton 7 | 8 | 9 | @given("two functionally identical classes, one is a singleton.") 10 | def step_impl01(context): 11 | """Create test classes. 12 | :param context: test context. 13 | """ 14 | context.GeneralStore = GeneralStore 15 | context.SingleStore = SingleStore 16 | 17 | 18 | @when("creating an instance of the non-singleton class") 19 | def step_impl02(context): 20 | """Put instance into context for later use. 21 | 22 | :param context: test context. 23 | """ 24 | context.generalStore = context.GeneralStore() 25 | 26 | 27 | @when("creating an instance of the singleton class") 28 | def step_impl03(context): 29 | """Put instance of singleton class into context. 30 | 31 | :param context: test context. 32 | """ 33 | context.singleStore = context.SingleStore() 34 | 35 | 36 | @then("the functional behavior is identical.") 37 | def step_impl04(context): 38 | """Compare behavior of singleton vs. non-singleton. 39 | 40 | :param context: test context. 41 | """ 42 | single = context.singleStore 43 | general = context.generalStore 44 | key = 13 45 | item = 42 46 | assert single.request(key) == general.request(key) 47 | single.add_item(key, item) 48 | general.add_item(key, item) 49 | assert single.request(key) == general.request(key) 50 | 51 | 52 | @given("a singleton class.") 53 | def step_impl05(context): 54 | """Just put singleton class into context. 55 | 56 | :param context: test context. 57 | """ 58 | context.SingleStore = SingleStore 59 | 60 | 61 | @when("creating multiple objects from the class") 62 | def step_impl06(context): 63 | """Prepare test for singleton property. 64 | 65 | :param context: test context. 66 | """ 67 | store = context.SingleStore 68 | context.st_1 = store() 69 | context.st_2 = store() 70 | context.st_3 = store() 71 | 72 | 73 | @then("all will be identical.") 74 | def step_impl07(context): 75 | """Test for singleton property. 76 | 77 | :param context: test context. 78 | """ 79 | assert context.st_1 is context.st_2 80 | assert context.st_2 is context.st_3 81 | 82 | 83 | @when("modifying the value of an attribute in one of them") 84 | def step_impl08(context): 85 | """ 86 | 87 | :param context: test context. 88 | """ 89 | context.key = 'the key' 90 | context.value = 'the value' 91 | context.st_1.add_item(context.key, context.value) 92 | 93 | 94 | @then("this modification is visible to all.") 95 | def step_impl09(context): 96 | """ 97 | 98 | :param context: test context. 99 | """ 100 | assert context.st_1.request(context.key) == context.value 101 | assert context.st_2.request(context.key) == context.value 102 | assert context.st_3.request(context.key) == context.value 103 | 104 | 105 | @then("type() shall return the class.") 106 | def step_impl10(context): 107 | """ 108 | 109 | :param context: test context. 110 | """ 111 | assert context.SingleStore.__class__ == SingleStore.__class__ 112 | assert issubclass(context.SingleStore, BaseStore) 113 | 114 | 115 | @then("__module__ shall be the module of the class.") 116 | def step_impl11(context): 117 | """Test for module. 118 | 119 | :param context: test context. 120 | """ 121 | assert context.SingleStore.__module__ == BaseStore.__module__ 122 | 123 | 124 | # ###### helpers 125 | 126 | 127 | class BaseStore(object): 128 | """Class for test purposes only.""" 129 | 130 | def __init__(self): 131 | self.cache_ = {} 132 | 133 | def add_item(self, k, v): 134 | """Add an item. 135 | :param k: key. 136 | :param v: value. 137 | """ 138 | self.cache_[k] = v 139 | 140 | def request(self, k): 141 | """Request an item by key. 142 | :param k: key. 143 | """ 144 | if k not in self.cache_: 145 | return None 146 | return self.cache_[k] 147 | 148 | 149 | class GeneralStore(BaseStore): 150 | """Non-singleton class.""" 151 | pass 152 | 153 | 154 | # noinspection PyArgumentList 155 | @singleton 156 | class SingleStore(BaseStore): 157 | """Singleton class.""" 158 | pass 159 | -------------------------------------------------------------------------------- /fuzzing.sublime-project: -------------------------------------------------------------------------------- 1 | { 2 | "folders": 3 | [ 4 | { 5 | "path": "." 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /fuzzing/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Tools for development. 3 | 4 | Logger. 5 | Random testing tool. 6 | 7 | Copyright (c) 2015 Stefan Braun 8 | 9 | Permission is hereby granted, free of charge, to any person 10 | obtaining a copy of this software and associated documentation 11 | files (the "Software"), to deal in the Software without restriction, 12 | including without limitation the rights to use, copy, modify, merge, 13 | publish, distribute, sublicense, and/or sell copies of the Software, 14 | and to permit persons to whom the Software is furnished to do so, 15 | subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be 18 | included in all copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 21 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 22 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 23 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 24 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 25 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 26 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 27 | DEALINGS IN THE SOFTWARE. 28 | """ 29 | 30 | import logging 31 | from .log import LoggerFactory 32 | from .fuzzer import fuzzer, fuzz_string, FuzzExecutor, TestStatCounter, Status 33 | 34 | # Symbols available when importing with *. 35 | __all__ = ['LoggerFactory', 'fuzzer', 'fuzz_string', 36 | 'FuzzExecutor', 'TestStatCounter', 'Status'] 37 | 38 | # Configure NullHandler to prevent warning in case logging is not configured. 39 | # See https://docs.python.org/2/howto/logging.html#library-config 40 | logging.getLogger('fuzzing').addHandler(logging.NullHandler()) 41 | -------------------------------------------------------------------------------- /fuzzing/fuzzer.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | Fuzz testing module. 4 | 5 | A Toolbox to create fuzzers for random testing of software. 6 | 7 | Copyright (c) 2015-2018 Stefan Braun 8 | """ 9 | # Permission is hereby granted, free of charge, to any person obtaining a copy 10 | # of this software and 11 | # associated documentation files (the "Software"), to deal in the Software 12 | # without restriction, 13 | # including without limitation the rights to use, copy, modify, merge, 14 | # publish, distribute, 15 | # sublicense, and/or sell copies of the Software, and to permit persons to 16 | # whom the Software is 17 | # furnished to do so, subject to the following conditions: 18 | # 19 | # The above copyright notice and this permission notice shall be included in 20 | # all copies or 21 | # substantial portions of the Software. 22 | # 23 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 | # IMPLIED, 25 | # INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR 26 | # A PARTICULAR PURPOSE 27 | # AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 28 | # LIABLE FOR ANY CLAIM, 29 | # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 30 | # OTHERWISE, ARISING FROM, 31 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 32 | # THE SOFTWARE. 33 | 34 | import random 35 | import time 36 | import math 37 | from copy import deepcopy 38 | from collections import Counter 39 | import os.path 40 | from tempfile import mkstemp 41 | import subprocess 42 | import logging 43 | import enum 44 | 45 | 46 | def logger(): 47 | """Provide logger. 48 | 49 | :return: local logger. 50 | :rtype: Logger 51 | """ 52 | return logging.getLogger('fuzzing.fuzzer') 53 | 54 | 55 | def fuzz_string(seed_str, runs=100, fuzz_factor=50): 56 | """Generate runs fuzzed strings from seed_str. 57 | 58 | A random fuzzer for a simulated text viewer application. 59 | It takes a string as seed and generates runs variant of it. 60 | 61 | :param seed_str: the string to use as seed for fuzzing. 62 | :param runs: number of fuzzed variants to supply. 63 | :param fuzz_factor: degree of fuzzing = 1 / fuzz_factor. 64 | :return: list of fuzzed variants of seed_str. 65 | :rtype: [str] 66 | """ 67 | buf = bytearray(seed_str, encoding="utf8") 68 | variants = [] 69 | for _ in range(runs): 70 | fuzzed = fuzzer(buf, fuzz_factor) 71 | variants.append(''.join([chr(b) for b in fuzzed])) 72 | logger().info('Fuzzed strings: %s', variants) 73 | return variants 74 | 75 | 76 | def fuzzer(buffer, fuzz_factor=101): 77 | """Fuzz given buffer. 78 | 79 | Take a buffer of bytes, create a copy, and replace some bytes 80 | with random values. Number of bytes to modify depends on fuzz_factor. 81 | This code is taken from Charlie Miller's fuzzer code. 82 | 83 | :param buffer: the data to fuzz. 84 | :type buffer: byte array 85 | :param fuzz_factor: degree of fuzzing. 86 | :type fuzz_factor: int 87 | :return: fuzzed buffer. 88 | :rtype: byte array 89 | """ 90 | buf = deepcopy(buffer) 91 | num_writes = number_of_bytes_to_modify(len(buf), fuzz_factor) 92 | for _ in range(num_writes): 93 | random_byte = random.randrange(256) 94 | random_position = random.randrange(len(buf)) 95 | buf[random_position] = random_byte 96 | return buf 97 | 98 | 99 | def number_of_bytes_to_modify(buf_len, fuzz_factor): 100 | """Calculate number of bytes to modify. 101 | 102 | :param buf_len: len of data buffer to fuzz. 103 | :param fuzz_factor: degree of fuzzing. 104 | :return: number of bytes to change. 105 | """ 106 | return random.randrange(math.ceil((float(buf_len) / fuzz_factor))) + 1 107 | 108 | 109 | @enum.unique 110 | class Status(enum.Enum): 111 | """Status values for test runs.""" 112 | 113 | FAILED = 0 114 | SUCCESS = 1 115 | 116 | 117 | class TestStatCounter(): 118 | """Hold a set of test results.""" 119 | 120 | def __init__(self, keys): 121 | """Prepare instance for test setup. 122 | 123 | :param keys: set of keys_. 124 | :type keys: [str] 125 | """ 126 | self.keys_ = set(keys) 127 | self.stats_ = {} 128 | for key in keys: 129 | self.stats_[key] = Counter() 130 | 131 | @property 132 | def keys(self): 133 | """Retrieve the set of keys. 134 | 135 | :return: set of keys. 136 | :rtype: set(str) 137 | """ 138 | return deepcopy(self.keys_) 139 | 140 | def add(self, key, status): 141 | """Add a new test result to the statistics. 142 | 143 | :param key: key of the test run. Must be in key set! 144 | :type key: str 145 | :param status: status of the test run. 146 | :type status: Status 147 | """ 148 | assert key in self.keys_, 'ENSURE: key is valid.' 149 | self.stats_[key][status] += 1 150 | 151 | def cumulated_counts(self): 152 | """Return sum over all counters. 153 | 154 | :return: The number of test runs; failed and successful. 155 | """ 156 | return sum([sum(v.values()) for v in self.stats_.values()]) 157 | 158 | def cumulated_counts_for_status(self, status): 159 | """Return sum over all counters for given status. 160 | 161 | :param status: the status to summarize. 162 | :type status: Status 163 | :return: number of tests resulting in status. 164 | """ 165 | return sum([v[status] for v in self.stats_.values()]) 166 | 167 | def retrieve_count(self, key, status): 168 | """Return count of key / status pair. 169 | 170 | :param key: key to retrieve count for. 171 | :type key: str 172 | :param status: status to retrieve count for. 173 | :type status: Status 174 | :return: count 175 | """ 176 | assert key in self.keys_, 'ENSURE: key is valid.' 177 | assert status in Status, 'ENSURE: status is valid.' 178 | return self.stats_[key][status] 179 | 180 | def __add__(self, other): 181 | """Merge test statistics. 182 | 183 | Does not modify the statistics, but creates and returns a new one. 184 | 185 | :param other: test statistics to merge with self. 186 | :type other: TestStatCounter 187 | :return: the merged statistics. 188 | :rtype: TestStatCounter 189 | """ 190 | combined_keys = self.keys_.union(other.keys_) 191 | tsc = TestStatCounter(combined_keys) 192 | for key in self.stats_: 193 | tsc.stats_[key].update(self.stats_[key]) 194 | for key in other.stats_: 195 | tsc.stats_[key].update(other.stats_[key]) 196 | return tsc 197 | 198 | def __repr__(self): 199 | """Create printable representation. 200 | 201 | :return: printable statistics. 202 | :rtype: str 203 | """ 204 | count_failed = self.cumulated_counts_for_status(Status.FAILED) 205 | count_succeeded = self.cumulated_counts_for_status(Status.SUCCESS) 206 | count_all = count_succeeded + count_failed 207 | tmpl = 'Tests run/succeeded/failed: {} / {} / {}\n' 208 | info = tmpl.format(count_all, count_succeeded, count_failed) 209 | for key in self.keys_: 210 | info += '{}\n'.format(key) 211 | for status in Status: 212 | count = self.retrieve_count(key, status) 213 | info += '\t{}: {}\n'.format(status.name, count) 214 | return info 215 | 216 | 217 | class FuzzExecutor(): 218 | """Run fuzz tests on applications.""" 219 | 220 | def __init__(self, app_list, file_list): 221 | """Take apps under test and test data. 222 | 223 | :param app_list: list of applications. 224 | :param file_list: list of files for testing. 225 | """ 226 | self.logger = logging.getLogger('fuzzing.fuzzer.FuzzExecutor') 227 | self.logger.info('Initializing FuzzExecutor ...') 228 | self.apps, self.args = FuzzExecutor.__parse_app_list(app_list) 229 | self.file_list = file_list 230 | self.fuzz_factor = 251 231 | keys = [os.path.basename(app) for app in self.apps] 232 | self.stats_ = TestStatCounter(keys) 233 | 234 | def run_test(self, runs): 235 | """Run tests and build up statistics. 236 | 237 | :param runs: number of tests to run. 238 | """ 239 | self.logger.info('Start fuzzing ...') 240 | for _ in range(runs): 241 | app = random.choice(self.apps) 242 | data_file = random.choice(self.file_list) 243 | fuzzed_file = self._fuzz_data_file(data_file) 244 | self._execute(app, fuzzed_file) 245 | self.logger.info('Fuzzing completed.') 246 | 247 | @property 248 | def stats(self): 249 | """Retrieve statistics of last run. 250 | 251 | :return: statistic counters. 252 | :rtype: TestStatCounter 253 | """ 254 | return self.stats_ 255 | 256 | def _fuzz_data_file(self, data_file): 257 | """Generate fuzzed variant of given file. 258 | 259 | :param data_file: path to file to fuzz. 260 | :type data_file: str 261 | :return: path to fuzzed file. 262 | :rtype: str 263 | """ 264 | buf = bytearray(open(os.path.abspath(data_file), 'rb').read()) 265 | fuzzed = fuzzer(buf, self.fuzz_factor) 266 | try: 267 | _, fuzz_output = mkstemp(prefix='fuzzed_') 268 | open(fuzz_output, 'wb').write(fuzzed) 269 | finally: 270 | pass 271 | return fuzz_output 272 | 273 | def _execute(self, app_, file_): 274 | """Run app with file as input. 275 | 276 | :param app_: application to run. 277 | :param file_: file to run app with. 278 | :return: success True, else False 279 | :rtype: bool 280 | """ 281 | app_name = os.path.basename(app_) 282 | args = [app_] 283 | args.extend(self.args[app_]) 284 | args.append(file_) 285 | process = subprocess.Popen(args) 286 | 287 | time.sleep(1) 288 | status = {True: Status.SUCCESS, False: Status.FAILED} 289 | crashed = process.poll() 290 | result = status[crashed is None] 291 | self.stats_.add(app_name, result) 292 | if result is Status.SUCCESS: 293 | # process did not crash, so just terminate it 294 | process.terminate() 295 | 296 | @staticmethod 297 | def __parse_app_list(app_list): 298 | """Parse list of apps for arguments. 299 | 300 | :param app_list: list of apps with optional arguments. 301 | :return: list of apps and assigned argument dict. 302 | :rtype: [String], {String: [String]} 303 | """ 304 | args = {} 305 | apps = [] 306 | for app_str in app_list: 307 | parts = app_str.split("&") 308 | app_path = parts[0].strip() 309 | apps.append(app_path) 310 | if len(parts) > 1: 311 | args[app_path] = [arg.strip() for arg in parts[1].split()] 312 | else: 313 | args[app_path] = [] 314 | return apps, args 315 | -------------------------------------------------------------------------------- /fuzzing/log.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Logging support. 3 | 4 | More convenient logging support built on top of logging package. 5 | """ 6 | 7 | import logging 8 | import logging.config 9 | import pkgutil 10 | import yaml 11 | 12 | from gp_decorators.singleton import singleton 13 | 14 | 15 | # noinspection PyArgumentList 16 | @singleton 17 | class LoggerFactory(): 18 | """Create and manage logger instances. 19 | 20 | It is important to initialize the logging framework before 21 | the first call to a logger. 22 | """ 23 | 24 | def __init__(self, package_name='fuzzing', 25 | config_file='resources/log_config.yaml'): 26 | """Create logger and set configuration.""" 27 | self.package_name = package_name 28 | self.config_file = config_file 29 | self.config = None 30 | 31 | def initialize(self): 32 | """Initialize logging framework. 33 | 34 | Must be called before first logging attempt! 35 | """ 36 | self.config = self.__read_configuration() 37 | logging.config.dictConfig(self.config) 38 | 39 | @staticmethod 40 | def get_instance(identifier): 41 | """Get a logger instance. 42 | 43 | :param identifier: identifier of logger as addressed in configuration. 44 | :return: the requested logger. 45 | """ 46 | logger = logging.getLogger(identifier) 47 | return logger 48 | 49 | def __read_configuration(self): 50 | """Read the logging configuration from file. 51 | 52 | :return: configuration dictionary 53 | :rtype: dict 54 | """ 55 | cfg = pkgutil.get_data(self.package_name, self.config_file) 56 | conf_dict = yaml.load(cfg) 57 | return conf_dict 58 | -------------------------------------------------------------------------------- /fuzzing/resources/log_config.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | formatters: 3 | concise: 4 | format: '%(asctime)s - %(levelname)s - %(message)s' 5 | detailed: 6 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() :: %(message)s' 7 | thread_info: 8 | format: '%(asctime)s - %(levelname)s - %(filename)s:%(lineno)3d - %(funcName)s() - %(thread)d:%(threadName)s :: %(message)s' 9 | handlers: 10 | console: 11 | class: logging.StreamHandler 12 | level: WARNING 13 | formatter: concise 14 | stream: ext://sys.stdout 15 | file: 16 | class: logging.handlers.RotatingFileHandler 17 | filename: 'fuzzing.log' 18 | maxBytes: 1000000 19 | backupCount: 3 20 | level: INFO 21 | formatter: detailed 22 | loggers: 23 | fuzzing: 24 | level: DEBUG 25 | handlers: [console, file] 26 | propagate: no 27 | fuzzing.fuzzing: 28 | level: INFO 29 | handlers: [console, file] 30 | propagate: no 31 | fuzzing.fuzzing.FuzzExecutor: 32 | level: WARNING 33 | handlers: [console, file] 34 | propagate: no 35 | root: 36 | level: WARNING 37 | handlers: [console] 38 | -------------------------------------------------------------------------------- /gp_decorators/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Decorators. 3 | 4 | Copyright 2014, Stefan Braun 5 | Licensed under MIT. 6 | """ 7 | 8 | import logging 9 | from gp_decorators import singleton 10 | 11 | # Symbols available when importing with *. 12 | __all__ = ['singleton'] 13 | 14 | # Configure NullHandler to prevent warning in case logging is not configured. 15 | # See https://docs.python.org/2/howto/logging.html#library-config 16 | logging.getLogger('gp_decorators').addHandler(logging.NullHandler()) 17 | -------------------------------------------------------------------------------- /gp_decorators/singleton.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """A decorator for Singleton classes.""" 3 | __author__ = 'sb' 4 | 5 | import wrapt 6 | 7 | _INSTANCES = {} 8 | 9 | 10 | @wrapt.decorator 11 | def singleton(wrapped, _, args, kwargs): 12 | """Return the single instance of wrapped. 13 | :param wrapped: the wrapped class. 14 | :param _: unused 15 | :param args: optional arguments for wrapped object. 16 | :param kwargs: optional arguments for wrapped object. 17 | """ 18 | if wrapped not in _INSTANCES: 19 | _INSTANCES[wrapped] = wrapped(*args, **kwargs) 20 | return _INSTANCES[wrapped] 21 | -------------------------------------------------------------------------------- /pylint.ini: -------------------------------------------------------------------------------- 1 | [pylint] 2 | testpaths=tests 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | wrapt 2 | PyYAML>=3.11 3 | argh>=0.26.1 4 | coverage>=3.7.1 5 | pathtools>=0.1.2 6 | Sphinx 7 | behave 8 | setuptools 9 | pylint 10 | pytest 11 | pytest-cover 12 | pytest-watch 13 | zc.buildout 14 | flake8 15 | bumpversion 16 | -------------------------------------------------------------------------------- /resrc/pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code 6 | extension-pkg-whitelist= 7 | 8 | # Add files or directories to the blacklist. They should be base names, not 9 | # paths. 10 | ignore=CVS,.git,model,venv,reports-hooks,resrc 11 | 12 | # Add files or directories matching the regex patterns to the blacklist. The 13 | # regex matches against base names, not paths. 14 | ignore-patterns= 15 | 16 | # Python code to execute, usually for sys.path manipulation such as 17 | # pygtk.require(). 18 | #init-hook= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # List of plugins (as comma separated values of python modules names) to load, 24 | # usually to register additional checkers. 25 | load-plugins= 26 | 27 | # Pickle collected data for later comparisons. 28 | persistent=yes 29 | 30 | # Specify a configuration file. 31 | #rcfile= 32 | 33 | # When enabled, pylint would attempt to guess common misconfiguration and emit 34 | # user-friendly hints instead of false-positive error messages 35 | suggestion-mode=yes 36 | 37 | # Allow loading of arbitrary C extensions. Extensions are imported into the 38 | # active Python interpreter and may run arbitrary code. 39 | unsafe-load-any-extension=no 40 | 41 | 42 | [MESSAGES CONTROL] 43 | 44 | # Only show warnings with the listed confidence levels. Leave empty to show 45 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 46 | confidence= 47 | 48 | # Disable the message, report, category or checker with the given id(s). You 49 | # can either give multiple identifiers separated by comma (,) or put this 50 | # option multiple times (only on the command line, not in the configuration 51 | # file where it should appear only once).You can also use "--disable=all" to 52 | # disable everything first and then reenable specific checks. For example, if 53 | # you want to run only the similarities checker, you can use "--disable=all 54 | # --enable=similarities". If you want to run only the classes checker, but have 55 | # no Warning level messages displayed, use"--disable=all --enable=classes 56 | # --disable=W" 57 | disable=print-statement, 58 | parameter-unpacking, 59 | unpacking-in-except, 60 | old-raise-syntax, 61 | backtick, 62 | long-suffix, 63 | old-ne-operator, 64 | old-octal-literal, 65 | import-star-module-level, 66 | non-ascii-bytes-literal, 67 | raw-checker-failed, 68 | bad-inline-option, 69 | locally-disabled, 70 | locally-enabled, 71 | file-ignored, 72 | suppressed-message, 73 | useless-suppression, 74 | deprecated-pragma, 75 | apply-builtin, 76 | basestring-builtin, 77 | buffer-builtin, 78 | cmp-builtin, 79 | coerce-builtin, 80 | execfile-builtin, 81 | file-builtin, 82 | long-builtin, 83 | raw_input-builtin, 84 | reduce-builtin, 85 | standarderror-builtin, 86 | unicode-builtin, 87 | xrange-builtin, 88 | coerce-method, 89 | delslice-method, 90 | getslice-method, 91 | setslice-method, 92 | no-absolute-import, 93 | old-division, 94 | dict-iter-method, 95 | dict-view-method, 96 | next-method-called, 97 | metaclass-assignment, 98 | indexing-exception, 99 | raising-string, 100 | reload-builtin, 101 | oct-method, 102 | hex-method, 103 | nonzero-method, 104 | cmp-method, 105 | input-builtin, 106 | round-builtin, 107 | intern-builtin, 108 | unichr-builtin, 109 | map-builtin-not-iterating, 110 | zip-builtin-not-iterating, 111 | range-builtin-not-iterating, 112 | filter-builtin-not-iterating, 113 | using-cmp-argument, 114 | eq-without-hash, 115 | div-method, 116 | idiv-method, 117 | rdiv-method, 118 | exception-message-attribute, 119 | invalid-str-codec, 120 | sys-max-int, 121 | bad-python3-import, 122 | deprecated-string-function, 123 | deprecated-str-translate-call, 124 | deprecated-itertools-function, 125 | deprecated-types-field, 126 | next-method-defined, 127 | dict-items-not-iterating, 128 | dict-keys-not-iterating, 129 | dict-values-not-iterating 130 | 131 | # Enable the message, report, category or checker with the given id(s). You can 132 | # either give multiple identifier separated by comma (,) or put this option 133 | # multiple time (only on the command line, not in the configuration file where 134 | # it should appear only once). See also the "--disable" option for examples. 135 | enable=c-extension-no-member 136 | 137 | 138 | [REPORTS] 139 | 140 | # Python expression which should return a note less than 10 (10 is the highest 141 | # note). You have access to the variables errors warning, statement which 142 | # respectively contain the number of errors / warnings messages and the total 143 | # number of statements analyzed. This is used by the global evaluation report 144 | # (RP0004). 145 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 146 | 147 | # Template used to display messages. This is a python new-style format string 148 | # used to format the message information. See doc for all details 149 | #msg-template= 150 | 151 | # Set the output format. Available formats are text, parseable, colorized, json 152 | # and msvs (visual studio).You can also give a reporter class, eg 153 | # mypackage.mymodule.MyReporterClass. 154 | output-format=text 155 | 156 | # Tells whether to display a full report or only the messages 157 | reports=no 158 | 159 | # Activate the evaluation score. 160 | score=yes 161 | 162 | 163 | [REFACTORING] 164 | 165 | # Maximum number of nested blocks for function / method body 166 | max-nested-blocks=5 167 | 168 | # Complete name of functions that never returns. When checking for 169 | # inconsistent-return-statements if a never returning function is called then 170 | # it will be considered as an explicit return statement and no message will be 171 | # printed. 172 | never-returning-functions=optparse.Values,sys.exit 173 | 174 | 175 | [BASIC] 176 | 177 | # Naming style matching correct argument names 178 | argument-naming-style=snake_case 179 | 180 | # Regular expression matching correct argument names. Overrides argument- 181 | # naming-style 182 | #argument-rgx= 183 | 184 | # Naming style matching correct attribute names 185 | attr-naming-style=snake_case 186 | 187 | # Regular expression matching correct attribute names. Overrides attr-naming- 188 | # style 189 | #attr-rgx= 190 | 191 | # Bad variable names which should always be refused, separated by a comma 192 | bad-names=foo, 193 | bar, 194 | baz, 195 | toto, 196 | tutu, 197 | tata 198 | 199 | # Naming style matching correct class attribute names 200 | class-attribute-naming-style=any 201 | 202 | # Regular expression matching correct class attribute names. Overrides class- 203 | # attribute-naming-style 204 | #class-attribute-rgx= 205 | 206 | # Naming style matching correct class names 207 | class-naming-style=PascalCase 208 | 209 | # Regular expression matching correct class names. Overrides class-naming-style 210 | #class-rgx= 211 | 212 | # Naming style matching correct constant names 213 | const-naming-style=UPPER_CASE 214 | 215 | # Regular expression matching correct constant names. Overrides const-naming- 216 | # style 217 | #const-rgx= 218 | 219 | # Minimum line length for functions/classes that require docstrings, shorter 220 | # ones are exempt. 221 | docstring-min-length=-1 222 | 223 | # Naming style matching correct function names 224 | function-naming-style=snake_case 225 | 226 | # Regular expression matching correct function names. Overrides function- 227 | # naming-style 228 | #function-rgx= 229 | 230 | # Good variable names which should always be accepted, separated by a comma 231 | good-names=i, 232 | j, 233 | k, 234 | ex, 235 | Run, 236 | _ 237 | 238 | # Include a hint for the correct naming format with invalid-name 239 | include-naming-hint=no 240 | 241 | # Naming style matching correct inline iteration names 242 | inlinevar-naming-style=any 243 | 244 | # Regular expression matching correct inline iteration names. Overrides 245 | # inlinevar-naming-style 246 | #inlinevar-rgx= 247 | 248 | # Naming style matching correct method names 249 | method-naming-style=snake_case 250 | 251 | # Regular expression matching correct method names. Overrides method-naming- 252 | # style 253 | #method-rgx= 254 | 255 | # Naming style matching correct module names 256 | module-naming-style=snake_case 257 | 258 | # Regular expression matching correct module names. Overrides module-naming- 259 | # style 260 | #module-rgx= 261 | 262 | # Colon-delimited sets of names that determine each other's naming style when 263 | # the name regexes allow several styles. 264 | name-group= 265 | 266 | # Regular expression which should only match function or class names that do 267 | # not require a docstring. 268 | no-docstring-rgx=^_ 269 | 270 | # List of decorators that produce properties, such as abc.abstractproperty. Add 271 | # to this list to register other decorators that produce valid properties. 272 | property-classes=abc.abstractproperty 273 | 274 | # Naming style matching correct variable names 275 | variable-naming-style=snake_case 276 | 277 | # Regular expression matching correct variable names. Overrides variable- 278 | # naming-style 279 | #variable-rgx= 280 | 281 | 282 | [FORMAT] 283 | 284 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 285 | expected-line-ending-format= 286 | 287 | # Regexp for a line that is allowed to be longer than the limit. 288 | ignore-long-lines=^\s*(# )??$ 289 | 290 | # Number of spaces of indent required inside a hanging or continued line. 291 | indent-after-paren=4 292 | 293 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 294 | # tab). 295 | indent-string=' ' 296 | 297 | # Maximum number of characters on a single line. 298 | max-line-length=100 299 | 300 | # Maximum number of lines in a module 301 | max-module-lines=1000 302 | 303 | # List of optional constructs for which whitespace checking is disabled. `dict- 304 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 305 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 306 | # `empty-line` allows space-only lines. 307 | no-space-check=trailing-comma, 308 | dict-separator 309 | 310 | # Allow the body of a class to be on the same line as the declaration if body 311 | # contains single statement. 312 | single-line-class-stmt=no 313 | 314 | # Allow the body of an if to be on the same line as the test if there is no 315 | # else. 316 | single-line-if-stmt=no 317 | 318 | 319 | [LOGGING] 320 | 321 | # Logging modules to check that the string format arguments are in logging 322 | # function parameter format 323 | logging-modules=logging 324 | 325 | 326 | [MISCELLANEOUS] 327 | 328 | # List of note tags to take in consideration, separated by a comma. 329 | notes=FIXME, 330 | XXX, 331 | TODO 332 | 333 | 334 | [SIMILARITIES] 335 | 336 | # Ignore comments when computing similarities. 337 | ignore-comments=yes 338 | 339 | # Ignore docstrings when computing similarities. 340 | ignore-docstrings=yes 341 | 342 | # Ignore imports when computing similarities. 343 | ignore-imports=no 344 | 345 | # Minimum lines number of a similarity. 346 | min-similarity-lines=4 347 | 348 | 349 | [SPELLING] 350 | 351 | # Limits count of emitted suggestions for spelling mistakes 352 | max-spelling-suggestions=4 353 | 354 | # Spelling dictionary name. Available dictionaries: none. To make it working 355 | # install python-enchant package. 356 | spelling-dict= 357 | 358 | # List of comma separated words that should not be checked. 359 | spelling-ignore-words= 360 | 361 | # A path to a file that contains private dictionary; one word per line. 362 | spelling-private-dict-file= 363 | 364 | # Tells whether to store unknown words to indicated private dictionary in 365 | # --spelling-private-dict-file option instead of raising a message. 366 | spelling-store-unknown-words=no 367 | 368 | 369 | [TYPECHECK] 370 | 371 | # List of decorators that produce context managers, such as 372 | # contextlib.contextmanager. Add to this list to register other decorators that 373 | # produce valid context managers. 374 | contextmanager-decorators=contextlib.contextmanager 375 | 376 | # List of members which are set dynamically and missed by pylint inference 377 | # system, and so shouldn't trigger E1101 when accessed. Python regular 378 | # expressions are accepted. 379 | generated-members= 380 | 381 | # Tells whether missing members accessed in mixin class should be ignored. A 382 | # mixin class is detected if its name ends with "mixin" (case insensitive). 383 | ignore-mixin-members=yes 384 | 385 | # This flag controls whether pylint should warn about no-member and similar 386 | # checks whenever an opaque object is returned when inferring. The inference 387 | # can return multiple potential results while evaluating a Python object, but 388 | # some branches might not be evaluated, which results in partial inference. In 389 | # that case, it might be useful to still emit no-member and other checks for 390 | # the rest of the inferred objects. 391 | ignore-on-opaque-inference=yes 392 | 393 | # List of class names for which member attributes should not be checked (useful 394 | # for classes with dynamically set attributes). This supports the use of 395 | # qualified names. 396 | ignored-classes=optparse.Values,thread._local,_thread._local 397 | 398 | # List of module names for which member attributes should not be checked 399 | # (useful for modules/projects where namespaces are manipulated during runtime 400 | # and thus existing member attributes cannot be deduced by static analysis. It 401 | # supports qualified module names, as well as Unix pattern matching. 402 | ignored-modules= 403 | 404 | # Show a hint with possible names when a member name was not found. The aspect 405 | # of finding the hint is based on edit distance. 406 | missing-member-hint=yes 407 | 408 | # The minimum edit distance a name should have in order to be considered a 409 | # similar match for a missing member name. 410 | missing-member-hint-distance=1 411 | 412 | # The total number of similar names that should be taken in consideration when 413 | # showing a hint for a missing member. 414 | missing-member-max-choices=1 415 | 416 | 417 | [VARIABLES] 418 | 419 | # List of additional names supposed to be defined in builtins. Remember that 420 | # you should avoid to define new builtins when possible. 421 | additional-builtins= 422 | 423 | # Tells whether unused global variables should be treated as a violation. 424 | allow-global-unused-variables=yes 425 | 426 | # List of strings which can identify a callback function by name. A callback 427 | # name must start or end with one of those strings. 428 | callbacks=cb_, 429 | _cb 430 | 431 | # A regular expression matching the name of dummy variables (i.e. expectedly 432 | # not used). 433 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 434 | 435 | # Argument names that match this expression will be ignored. Default to name 436 | # with leading underscore 437 | ignored-argument-names=_.*|^ignored_|^unused_ 438 | 439 | # Tells whether we should check for unused import in __init__ files. 440 | init-import=no 441 | 442 | # List of qualified module names which can have objects that can redefine 443 | # builtins. 444 | redefining-builtins-modules=six.moves,past.builtins,future.builtins 445 | 446 | 447 | [CLASSES] 448 | 449 | # List of method names used to declare (i.e. assign) instance attributes. 450 | defining-attr-methods=__init__, 451 | __new__, 452 | setUp 453 | 454 | # List of member names, which should be excluded from the protected access 455 | # warning. 456 | exclude-protected=_asdict, 457 | _fields, 458 | _replace, 459 | _source, 460 | _make 461 | 462 | # List of valid names for the first argument in a class method. 463 | valid-classmethod-first-arg=cls 464 | 465 | # List of valid names for the first argument in a metaclass class method. 466 | valid-metaclass-classmethod-first-arg=mcs 467 | 468 | 469 | [DESIGN] 470 | 471 | # Maximum number of arguments for function / method 472 | max-args=5 473 | 474 | # Maximum number of attributes for a class (see R0902). 475 | max-attributes=7 476 | 477 | # Maximum number of boolean expressions in a if statement 478 | max-bool-expr=5 479 | 480 | # Maximum number of branch for function / method body 481 | max-branches=12 482 | 483 | # Maximum number of locals for function / method body 484 | max-locals=15 485 | 486 | # Maximum number of parents for a class (see R0901). 487 | max-parents=7 488 | 489 | # Maximum number of public methods for a class (see R0904). 490 | max-public-methods=20 491 | 492 | # Maximum number of return / yield for function / method body 493 | max-returns=6 494 | 495 | # Maximum number of statements in function / method body 496 | max-statements=50 497 | 498 | # Minimum number of public methods for a class (see R0903). 499 | min-public-methods=2 500 | 501 | 502 | [IMPORTS] 503 | 504 | # Allow wildcard imports from modules that define __all__. 505 | allow-wildcard-with-all=no 506 | 507 | # Analyse import fallback blocks. This can be used to support both Python 2 and 508 | # 3 compatible code, which means that the block might have code that exists 509 | # only in one or another interpreter, leading to false positives when analysed. 510 | analyse-fallback-blocks=no 511 | 512 | # Deprecated modules which should not be used, separated by a comma 513 | deprecated-modules=optparse,tkinter.tix 514 | 515 | # Create a graph of external dependencies in the given file (report RP0402 must 516 | # not be disabled) 517 | ext-import-graph= 518 | 519 | # Create a graph of every (i.e. internal and external) dependencies in the 520 | # given file (report RP0402 must not be disabled) 521 | import-graph= 522 | 523 | # Create a graph of internal dependencies in the given file (report RP0402 must 524 | # not be disabled) 525 | int-import-graph= 526 | 527 | # Force import order to recognize a module as part of the standard 528 | # compatibility libraries. 529 | known-standard-library= 530 | 531 | # Force import order to recognize a module as part of a third party library. 532 | known-third-party=enchant 533 | 534 | 535 | [EXCEPTIONS] 536 | 537 | # Exceptions that will emit a warning when being caught. Defaults to 538 | # "Exception" 539 | overgeneral-exceptions=Exception 540 | -------------------------------------------------------------------------------- /run_fuzzer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | """A simple script to make fuzz-testing more convenient. 4 | 5 | Copyright (c) 2015 Stefan Braun 6 | 7 | Permission is hereby granted, free of charge, to any person 8 | obtaining a copy of this software and associated documentation 9 | files (the "Software"), to deal in the Software without restriction, 10 | including without limitation the rights to use, copy, modify, 11 | merge, publish, distribute, sublicense, and/or sell copies of 12 | the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall 16 | be included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 22 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | """ 27 | 28 | import sys 29 | import argparse 30 | from concurrent.futures import ProcessPoolExecutor 31 | import yaml 32 | 33 | from fuzzing import FuzzExecutor, TestStatCounter 34 | 35 | APPLICATIONS = 'applications' 36 | SEED_FILES = 'seed_files' 37 | PROCESSORS = 'processors' 38 | PROCESSES = 'processes' 39 | RUNS = 'runs' 40 | DEFAULT_RUNS = 10 41 | DEFAULT_PROCESSORS = 1 42 | DEFAULT_PROCESSES = 3 43 | 44 | HELP_CONFIGURATION = """ 45 | version: 1 46 | seed_files: ['requirements.txt', 'README.rst'] 47 | applications: ['python & features/resources/testfuzz.py -p 0.5'] 48 | runs: 15 49 | processors: 3 50 | processes: 8 51 | """ 52 | 53 | 54 | class InvalidConfigurationError(Exception): 55 | """Raised if test configuration is invalid.""" 56 | def __init__(self, info): 57 | """Take info describing cause of exception.""" 58 | super(InvalidConfigurationError, self).__init__() 59 | self.info = info 60 | 61 | 62 | def load_configuration(conf_path): 63 | """Load and validate test configuration. 64 | 65 | :param conf_path: path to YAML configuration file. 66 | :return: configuration as dict. 67 | """ 68 | with open(conf_path) as f_conf: 69 | conf_dict = yaml.load(f_conf) 70 | validate_config(conf_dict) 71 | return conf_dict 72 | 73 | 74 | def validate_config(conf_dict): 75 | """Validate configuration. 76 | 77 | :param conf_dict: test configuration. 78 | :type conf_dict: {} 79 | :raise InvalidConfigurationError: 80 | """ 81 | if APPLICATIONS not in conf_dict.keys(): 82 | raise InvalidConfigurationError('Missing application configuration.') 83 | if SEED_FILES not in conf_dict.keys(): 84 | raise InvalidConfigurationError('Missing seed file configuration.') 85 | if RUNS not in conf_dict.keys(): 86 | conf_dict[RUNS] = DEFAULT_RUNS 87 | if PROCESSES not in conf_dict.keys(): 88 | conf_dict[PROCESSES] = DEFAULT_PROCESSES 89 | if PROCESSORS not in conf_dict.keys(): 90 | conf_dict[PROCESSORS] = DEFAULT_PROCESSORS 91 | 92 | 93 | def execute_test(config): 94 | """Run tests. 95 | 96 | :param config: test configuration. 97 | :type config: {} 98 | """ 99 | import os 100 | print('Starting process: {}'.format(os.getpid())) 101 | executor = FuzzExecutor(config[APPLICATIONS], config[SEED_FILES]) 102 | executor.run_test(config[RUNS]) 103 | print('Process {} finishes.'.format(os.getpid())) 104 | return executor.stats 105 | 106 | 107 | def show_test_stats(test_stats): 108 | """Print test statistics. 109 | 110 | :param test_stats: result of test runs. 111 | :type test_stats: TestStatCounter 112 | """ 113 | print('\n{}'.format('_' * 50)) 114 | print('Test Results:') 115 | print('{}'.format('_' * 50)) 116 | print(test_stats) 117 | print('{}\n'.format('_' * 50)) 118 | 119 | 120 | def combine_test_stats(results): 121 | """Combine the test results. 122 | 123 | :param results: list of test results. 124 | :type results: [TestStatCounter] 125 | :return: combined statistics. 126 | :rtype: TestStatCounter 127 | """ 128 | combined = TestStatCounter(set()) 129 | for res in results: 130 | combined += res 131 | return combined 132 | 133 | 134 | def main(): 135 | """Read configuration and execute test runs.""" 136 | parser = argparse.ArgumentParser(description='Stress test applications.') 137 | parser.add_argument('config_path', help='Path to configuration file.') 138 | args = parser.parse_args() 139 | try: 140 | configuration = load_configuration(args.config_path) 141 | except InvalidConfigurationError: 142 | print("\nConfiguration is not valid.") 143 | print('Example:\n{}'.format(HELP_CONFIGURATION)) 144 | return 1 145 | print("Starting up ...") 146 | futures = [] 147 | with ProcessPoolExecutor(configuration[PROCESSORS]) as executor: 148 | for _ in range(configuration[PROCESSES]): 149 | futures.append(executor.submit(execute_test, configuration)) 150 | print("... finished") 151 | test_stats = combine_test_stats([f.result() for f in futures]) 152 | show_test_stats(test_stats) 153 | return 0 154 | 155 | 156 | if __name__ == '__main__': 157 | sys.exit(main()) 158 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | fuzzing: Some general meta classes. 4 | 5 | Note that "python setup.py test" invokes pytest on the package. 6 | With appropriately configured setup.cfg, this will check both 7 | xxx_test modules and docstrings. 8 | Currently all tests are written for behave! 9 | 10 | Copyright 2015, Stefan Braun. 11 | Licensed under MIT. 12 | """ 13 | import sys 14 | from setuptools import setup 15 | from setuptools.command.test import test 16 | 17 | 18 | class PyTest(test): 19 | """This is a plug-in for setuptools. 20 | 21 | It will invoke py.test when you run python setup.py test 22 | """ 23 | 24 | def finalize_options(self): 25 | """Configure.""" 26 | test.finalize_options(self) 27 | self.test_args = [] 28 | self.test_suite = True 29 | 30 | def run_tests(self): 31 | """Execute tests.""" 32 | # import here, because outside the required eggs aren't loaded yet 33 | import pytest 34 | sys.exit(pytest.main(self.test_args)) 35 | 36 | 37 | version = '0.3.4' 38 | 39 | setup(name="fuzzing", 40 | version=version, 41 | description="Tools for stress testing applications.", 42 | long_description=open("README.rst").read(), 43 | # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 44 | classifiers=[ 45 | 'Development Status :: 4 - Beta', 46 | 'Intended Audience :: Developers', 47 | 'License :: OSI Approved :: MIT License', 48 | 'Programming Language :: Python', 49 | 'Programming Language :: Python :: 3.4', 50 | 'Programming Language :: Python :: 3.5', 51 | 'Programming Language :: Python :: 3.6', 52 | 'Programming Language :: Python :: 3.7', 53 | 'Programming Language :: Python :: 3 :: Only', 54 | 'Programming Language :: Python :: Implementation :: CPython', 55 | 'Programming Language :: Python :: Implementation :: PyPy', 56 | 'Topic :: Software Development :: Libraries :: Python Modules', 57 | 'Topic :: Software Development :: Testing' 58 | ], 59 | keywords="development tools", # Separate with spaces 60 | author="Stefan Braun", 61 | author_email="sb@action.ms", 62 | url="https://github.com/stbraun/fuzzing", 63 | license="MIT", 64 | packages=['fuzzing'], 65 | include_package_data=True, 66 | zip_safe=False, 67 | tests_require=['pytest', 'behave>=1.2.4', 'pytest-cover', 'pytest-watch'], 68 | cmdclass={'test': PyTest}, 69 | scripts=['run_fuzzer.py', ], 70 | 71 | # List of packages that this one depends upon: 72 | install_requires=['sphinx', 'wrapt', 'PyYAML', 'argh', 'pathtools', 73 | 'setuptools', 'zc.buildout'], 74 | requires=['wrapt', 'PyYAML', 'argh', 'pathtools', 'setuptools'], 75 | provides=['fuzzing', 'gp_decorators'], 76 | ) 77 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | basepkg: tests module. 4 | 5 | Meant for use with py.test. 6 | Organize tests into files, each named xxx_test.py 7 | Read more here: http://pytest.org/ 8 | 9 | Copyright 2014, Stefan Braun 10 | Licensed under MIT 11 | """ -------------------------------------------------------------------------------- /tests/run_config.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | seed_files: ['requirements.txt', 'README.rst'] 3 | applications: ['python & features/resources/testfuzz.py -p 0.3', 4 | '/Applications/Adobe Reader 9/Adobe Reader.app/Contents/MacOS/AdobeReader'] 5 | runs: 4 6 | processors: 3 7 | processes: 8 -------------------------------------------------------------------------------- /tests/test_stat_counter_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Test cases for test result class.""" 3 | # Copyright (c) 2015-2018 Stefan Braun 4 | # 5 | # Permission is hereby granted, free of charge, to any person 6 | # obtaining a copy of this software and associated documentation 7 | # files (the "Software"), to deal in the Software without restriction, 8 | # including without limitation the rights to use, copy, modify, merge, 9 | # publish, distribute, sublicense, and/or sell copies of the Software, 10 | # and to permit persons to whom the Software is furnished to do so, 11 | # subject to the following conditions: 12 | # 13 | # The above copyright notice and this permission notice shall be 14 | # included in all copies or substantial portions of the Software. 15 | # 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | # DEALINGS IN THE SOFTWARE. 24 | 25 | 26 | from fuzzing.fuzzer import TestStatCounter, Status 27 | 28 | 29 | def test_create_empty_instance(): 30 | """Create instance of status counter without keys_.""" 31 | tsc = TestStatCounter([]) 32 | assert tsc is not None, 'Instance created.' 33 | 34 | 35 | def test_create_instance(): 36 | """Create instance of status counter.""" 37 | tsc = TestStatCounter(['a', 'b']) 38 | assert tsc is not None, 'Instance created.' 39 | 40 | 41 | def test_add_new_test_result(): 42 | """Add new test result and check update of stats.""" 43 | key = 'aaa' 44 | status = Status.SUCCESS 45 | tsc = TestStatCounter([key]) 46 | assert tsc.retrieve_count(key, status) == 0, 'Counter is zero yet.' 47 | tsc.add(key=key, status=status) 48 | assert tsc.retrieve_count(key, status) == 1 49 | tsc.add(key=key, status=status) 50 | tsc.add(key=key, status=status) 51 | assert tsc.retrieve_count(key, status) == 3 52 | 53 | 54 | def test_cumulated_count(): 55 | """Retrieve the cumulated count for the instance.""" 56 | keys = ['a', 'b', 'c'] 57 | counts = [2, 3, 4, 5, 6, 7] 58 | expected = sum(counts) 59 | tsc = TestStatCounter(keys=keys) 60 | assert tsc.cumulated_counts() == 0 61 | __increment_counter(tsc, keys, counts) 62 | assert tsc.cumulated_counts() == expected 63 | 64 | 65 | def test_cumulated_count_for_status(): 66 | """Retrieve the cumulated count for given status.""" 67 | keys = ['a', 'b', 'c'] 68 | counts = [2, 3, 4, 5, 6, 7] 69 | expected = sum([x for x in counts if x % 2 == 1]) 70 | tsc = TestStatCounter(keys=keys) 71 | assert tsc.cumulated_counts_for_status(Status.SUCCESS) == 0 72 | __increment_counter(tsc, keys, counts) 73 | actual = tsc.cumulated_counts_for_status(Status.SUCCESS) 74 | assert actual == expected 75 | 76 | 77 | def test_merge_matching_keys(): 78 | """Merge two test statistics. 79 | 80 | Both have same set of keys_. 81 | """ 82 | keys = ['a', 'b', 'c'] 83 | counts_1 = [2, 3, 4, 5, 6, 7] 84 | counts_2 = [3, 4, 5, 6, 7, 8] 85 | tsc_1 = TestStatCounter(keys=keys) 86 | __increment_counter(tsc_1, keys, counts_1) 87 | tsc_2 = TestStatCounter(keys=keys) 88 | __increment_counter(tsc_2, keys, counts_2) 89 | tsc = tsc_1 + tsc_2 90 | expected = tsc.cumulated_counts() 91 | actual = tsc_1.cumulated_counts() + tsc_2.cumulated_counts() 92 | assert actual == expected 93 | 94 | 95 | def test_merge_differing_keys(): 96 | """Merge two test statistics. 97 | 98 | The statistics have different sets of keys_. 99 | """ 100 | keys_1 = ['a', 'b', 'c'] 101 | keys_2 = ['b', 'c', 'd'] 102 | counts_1 = [2, 3, 4, 5, 6, 7] 103 | counts_2 = [3, 4, 5, 6, 7, 8] 104 | tsc_1 = TestStatCounter(keys=keys_1) 105 | __increment_counter(tsc_1, keys_1, counts_1) 106 | tsc_2 = TestStatCounter(keys=keys_2) 107 | __increment_counter(tsc_2, keys_2, counts_2) 108 | tsc = tsc_1 + tsc_2 109 | expected = tsc.cumulated_counts() 110 | actual = tsc_1.cumulated_counts() + tsc_2.cumulated_counts() 111 | assert actual == expected 112 | 113 | 114 | def test_retrieve_count(): 115 | """Retrieve count for specified key and status.""" 116 | keys = ['a', 'b'] 117 | counts = [2, 3, 4, 5] 118 | tsc = TestStatCounter(keys) 119 | __increment_counter(tsc, keys, counts) 120 | assert counts[0] == tsc.retrieve_count(keys[0], Status.FAILED) 121 | assert counts[1] == tsc.retrieve_count(keys[0], Status.SUCCESS) 122 | 123 | 124 | def __increment_counter(tsc, keys, counts): 125 | """Increment counters for each key/status pair. 126 | 127 | Iterate over keys_ / status / counts and increment the counters. 128 | 129 | :param tsc: the unit under test. 130 | :param keys: list of keys_ 131 | :param counts: list of counts to set. 132 | """ 133 | assert len(keys) * 2 <= len(counts) 134 | for i, key in enumerate(keys): 135 | for _ in range(counts[2 * i]): 136 | tsc.add(key, Status.FAILED) 137 | for _ in range(counts[2 * i + 1]): 138 | tsc.add(key, Status.SUCCESS) 139 | --------------------------------------------------------------------------------