├── reqs.txt
├── phuzzer
├── extensions
│ ├── __init__.py
│ ├── grease_callback.py
│ └── extender.py
├── errors.py
├── __init__.py
├── phuzzers
│ ├── afl_ijon.py
│ ├── afl_multicb.py
│ ├── afl_plusplus.py
│ ├── __init__.py
│ ├── witcherafl.py
│ └── afl.py
├── util.py
├── timer.py
├── minimizer.py
├── seed.py
├── showmap.py
├── hierarchy.py
├── reporter.py
└── __main__.py
├── .github
├── workflows
│ ├── ci.yml
│ └── nightly-ci.yml
└── ISSUE_TEMPLATE
│ ├── config.yml
│ ├── question.yml
│ ├── feature-request.yml
│ └── bug-report.yml
├── bin
├── kernel_config.sh
└── analyze_result.py
├── setup.py
├── .shellphuzz.ini
├── LICENSE
├── .gitignore
├── README.md
├── Dockerfile
└── tests
└── test_fuzzer.py
/reqs.txt:
--------------------------------------------------------------------------------
1 | tqdm
2 |
3 |
--------------------------------------------------------------------------------
/phuzzer/extensions/__init__.py:
--------------------------------------------------------------------------------
1 | from .extender import Extender
2 | from .grease_callback import GreaseCallback
3 |
--------------------------------------------------------------------------------
/phuzzer/errors.py:
--------------------------------------------------------------------------------
1 | class PhuzzerError(Exception):
2 | pass
3 |
4 | class InstallError(PhuzzerError):
5 | pass
6 |
7 | class AFLError(PhuzzerError):
8 | pass
9 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on:
4 | push:
5 | branches:
6 | - "**"
7 | pull_request:
8 | workflow_dispatch:
9 |
10 | jobs:
11 | ci:
12 | uses: angr/ci-settings/.github/workflows/angr-ci.yml@master
13 |
--------------------------------------------------------------------------------
/bin/kernel_config.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -x
2 |
3 | echo 1 | sudo tee /proc/sys/kernel/sched_child_runs_first
4 | echo core | sudo tee /proc/sys/kernel/core_pattern
5 | cd /sys/devices/system/cpu; echo performance | sudo tee cpu*/cpufreq/scaling_governor
6 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 | contact_links:
3 | - name: Join our Slack community
4 | url: https://angr.io/invite/
5 | about: For questions and help with angr, you are invited to join the angr Slack community
6 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 |
2 | from distutils.core import setup
3 |
4 | setup(
5 | name='phuzzer', version='8.19.4.30.pre3', description="Python wrapper for multiarch AFL",
6 | packages=['phuzzer', 'phuzzer.extensions', 'phuzzer.phuzzers'],
7 | install_requires=['tqdm','networkx']
8 | )
9 |
--------------------------------------------------------------------------------
/.github/workflows/nightly-ci.yml:
--------------------------------------------------------------------------------
1 | name: Nightly CI
2 |
3 | on:
4 | schedule:
5 | - cron: "0 0 * * *"
6 | workflow_dispatch:
7 |
8 | jobs:
9 | ci:
10 | uses: angr/ci-settings/.github/workflows/angr-ci.yml@master
11 | with:
12 | nightly: true
13 | afl: true
14 | secrets: inherit
15 |
--------------------------------------------------------------------------------
/phuzzer/__init__.py:
--------------------------------------------------------------------------------
1 | from .errors import *
2 | from .phuzzers import *
3 | from .minimizer import Minimizer
4 | from .showmap import Showmap
5 | from .extensions import *
6 | from .seed import Seed
7 | from .hierarchy import *
8 | from .phuzzers.afl import AFL
9 | from .phuzzers.afl_plusplus import AFLPlusPlus
10 | from .phuzzers.afl_multicb import AFLMultiCB
11 | from .phuzzers.afl_ijon import AFLIJON
12 |
--------------------------------------------------------------------------------
/.shellphuzz.ini:
--------------------------------------------------------------------------------
1 | [loggers]
2 | keys=root,fuzzer.fuzzer
3 |
4 | [logger_root]
5 | level=NOTSET
6 | handlers=hand01
7 |
8 | [logger_fuzzer.fuzzer]
9 | level=WARNING
10 | handlers=hand01
11 | qualname=fuzzer.fuzzer
12 |
13 | [handlers]
14 | keys=hand01
15 |
16 | [handler_hand01]
17 | class=StreamHandler
18 | level=NOTSET
19 | formatter=form01
20 | args=(sys.stdout,)
21 |
22 | [formatters]
23 | keys=form01
24 |
25 | [formatter_form01]
26 | format=F1 %(asctime)s %(levelname)s %(message)s
27 | datefmt=
28 | class=logging.Formatter
29 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/afl_ijon.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 |
4 | from .afl import AFL
5 |
6 | l = logging.getLogger(__name__)
7 |
8 |
9 | class AFLIJON(AFL):
10 | """ IJON port of AFL phuzzer.
11 | Paper found here:
12 | https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2020/02/27/IJON-Oakland20.pdf
13 | """
14 | def __init__(self, **kwargs):
15 | super().__init__(**kwargs)
16 |
17 | def choose_afl(self):
18 | self.afl_bin_dir = '/phuzzers/ijon/'
19 | afl_bin_path = os.path.join(self.afl_bin_dir, "afl-fuzz")
20 | return afl_bin_path
21 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/afl_multicb.py:
--------------------------------------------------------------------------------
1 | from . import Phuzzer
2 | from .afl import AFL
3 | import logging
4 | import os
5 |
6 | l = logging.getLogger("phuzzer.phuzzers.afl")
7 |
8 |
9 | class AFLMultiCB(AFL):
10 | '''This is a multi-CB AFL phuzzer (for CGC).'''
11 |
12 | def __init__(self, targets, **kwargs):
13 | self.targets = targets
14 | super().__init__(targets[0], **kwargs)
15 |
16 | self.timeout = 1000 * len(targets)
17 | self.target_opts = targets[1:]
18 |
19 | def choose_afl(self):
20 | self.afl_bin_dir, _ = Phuzzer.init_afl_config(self.targets[0], is_multicb=True)
21 | afl_bin_path = os.path.join(self.afl_bin_dir, "afl-fuzz")
22 | return afl_bin_path
23 |
--------------------------------------------------------------------------------
/phuzzer/util.py:
--------------------------------------------------------------------------------
1 | import string
2 | import os
3 |
4 | def hexescape(s):
5 | '''
6 | perform hex escaping on a raw string s
7 | '''
8 |
9 | out = []
10 | acceptable = (string.ascii_letters + string.digits + " .").encode()
11 | for c in s:
12 | if c not in acceptable:
13 | out.append("\\x%02x" % c)
14 | else:
15 | out.append(chr(c))
16 |
17 | return ''.join(out)
18 |
19 | def _get_bindir():
20 | base = os.path.dirname(__file__)
21 | while not "bin" in os.listdir(base) and os.path.abspath(base) != "/":
22 | base = os.path.join(base, "..")
23 | if os.path.abspath(base) == "/":
24 | raise InstallError("could not find afl install directory")
25 | return base
26 |
27 | from .errors import InstallError
28 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/afl_plusplus.py:
--------------------------------------------------------------------------------
1 | import subprocess
2 | import logging
3 | import os
4 |
5 | from .afl import AFL
6 |
7 | l = logging.getLogger(__name__)
8 |
9 |
10 | class AFLPlusPlus(AFL):
11 | """ AFL++ port of AFL phuzzer.
12 | Paper found here:
13 | https://aflplus.plus//papers/aflpp-woot2020.pdf
14 | """
15 | def __init__(self, *args, **kwargs):
16 | super().__init__(*args, **kwargs)
17 |
18 | def choose_afl(self):
19 | self.afl_bin_dir = '/phuzzers/AFLplusplus/' if 'AFL_PATH' not in os.environ else os.environ['AFL_PATH']
20 | afl_bin_path = os.path.join(self.afl_bin_dir, "afl-fuzz")
21 | return afl_bin_path
22 |
23 | def _start_afl_instance(self, instance_cnt=0):
24 |
25 | args, fuzzer_id = self.build_args()
26 | my_env = os.environ.copy()
27 |
28 | if "AFL_SET_AFFINITY" in my_env:
29 | core_num = int(my_env["AFL_SET_AFFINITY"])
30 | core_num += instance_cnt
31 | print(args)
32 | args = [args[0]] + [f"-b {core_num}"] + args[1:]
33 |
34 | logpath = os.path.join(self.work_dir, fuzzer_id + ".log")
35 | l.debug("execing: %s > %s", ' '.join(args), logpath)
36 |
37 | with open(logpath, "w") as fp:
38 | return subprocess.Popen(args, stdout=fp, stderr=fp, close_fds=True, env=my_env)
39 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2015, The Regents of the University of California
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 |
7 | * Redistributions of source code must retain the above copyright notice, this
8 | list of conditions and the following disclaimer.
9 |
10 | * Redistributions in binary form must reproduce the above copyright notice,
11 | this list of conditions and the following disclaimer in the documentation
12 | and/or other materials provided with the distribution.
13 |
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 |
25 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/question.yml:
--------------------------------------------------------------------------------
1 | name: Ask a question
2 | description: Ask a question about phuzzer
3 | labels: [question,needs-triage]
4 | body:
5 | - type: markdown
6 | attributes:
7 | value: |
8 | If you have a question about phuzzer, that is not a bug report or a feature request, you can ask it here. For more real-time help with phuzzer, from us and the community, join our [Slack](https://angr.io/invite/).
9 |
10 | Before submitting this question, please check the following, which may answer your question:
11 | * Have you checked the [documentation](https://docs.angr.io/)?
12 | * Have you checked the [FAQ](https://docs.angr.io/introductory-errata/faq)?
13 | * Have you checked our library of [examples](https://github.com/angr/angr-doc/tree/master/examples)?
14 | * Have you [searched existing issues](https://github.com/angr/phuzzer/issues?q=is%3Aissue+label%3Aquestion) to see if this question has been answered before?
15 | * Have you checked that you are running the latest versions of angr and its components. angr is rapidly-evolving!
16 |
17 | **Please note: This repo is effectively unmaintained. While we appreciate bug reports and feature requests, we cannot commit to a timely response.** For more real-time help with angr, from us and the community, join our [Slack](https://angr.io/invite/).
18 |
19 | - type: textarea
20 | attributes:
21 | label: Question
22 | description:
23 | validations:
24 | required: true
25 |
--------------------------------------------------------------------------------
/phuzzer/timer.py:
--------------------------------------------------------------------------------
1 | import threading
2 | import logging
3 |
4 | l = logging.getLogger("phuzzer.phuzzers")
5 |
6 | config = { }
7 |
8 | class InstallError(Exception):
9 | pass
10 |
11 |
12 | # http://stackoverflow.com/a/41450617
13 | class InfiniteTimer():
14 | """A Timer class that does not stop, unless you want it to."""
15 |
16 | def __init__(self, seconds, target):
17 | self._should_continue = False
18 | self.is_running = False
19 | self.seconds = seconds
20 | self.target = target
21 | self.thread = None
22 |
23 | def _handle_target(self):
24 | self.is_running = True
25 | self.target()
26 | self.is_running = False
27 | self._start_timer()
28 |
29 | def _start_timer(self):
30 | if self._should_continue: # Code could have been running when cancel was called.
31 | self.thread = threading.Timer(self.seconds, self._handle_target)
32 | self.thread.start()
33 |
34 | def start(self):
35 | if not self._should_continue and not self.is_running:
36 | self._should_continue = True
37 | self._start_timer()
38 | else:
39 | print("Timer already started or running, please wait if you're restarting.")
40 |
41 | def cancel(self):
42 | if self.thread is not None:
43 | self._should_continue = False # Just in case thread is running and cancel fails.
44 | self.thread.cancel()
45 | else:
46 | pass
47 | #print("Timer never started or failed to initialize.")
48 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature-request.yml:
--------------------------------------------------------------------------------
1 | name: Request a feature
2 | description: Request a new feature for phuzzer
3 | labels: [enhancement,needs-triage]
4 | body:
5 | - type: markdown
6 | attributes:
7 | value: |
8 | Thank you for taking the time to submit this feature request!
9 |
10 | Before submitting this feature request, please check the following:
11 | * Have you checked that you are running the latest versions of angr and its components? angr is rapidly-evolving!
12 | * Have you checked the [documentation](https://docs.angr.io/) to see if this feature exists already?
13 | * Have you [searched existing issues](https://github.com/angr/phuzzer/issues?q=is%3Aissue+label%3Aenhancement+) to see if this feature has been requested before?
14 |
15 | **Please note: This repo is effectively unmaintained. While we appreciate bug reports and feature requests, we cannot commit to a timely response.** For more real-time help with angr, from us and the community, join our [Slack](https://angr.io/invite/).
16 |
17 | - type: textarea
18 | attributes:
19 | label: Description
20 | description: |
21 | Brief description of the desired feature. If the feature is intended to solve some problem, please clearly describe the problem, including any relevant binaries, etc.
22 |
23 | **Tip:** You can attach files to the issue by first clicking on the textarea to select it, then dragging & dropping the file onto the textarea.
24 | validations:
25 | required: true
26 |
27 | - type: textarea
28 | attributes:
29 | label: Alternatives
30 | description: Possible alternative solutions or features that you have considered.
31 |
32 | - type: textarea
33 | attributes:
34 | label: Additional context
35 | description: Any other context or screenshots about the feature request.
36 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.ipch
2 | .vscode
3 |
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | # C extensions
10 | #*.so
11 |
12 | .vagrant/*
13 | .idea/*
14 |
15 | # Distribution / packaging
16 | .Python
17 | build/
18 | develop-eggs/
19 | dist/
20 | downloads/
21 | eggs/
22 | .eggs/
23 | lib/
24 | lib64/
25 | parts/
26 | sdist/
27 | var/
28 | wheels/
29 | *.egg-info/
30 | .installed.cfg
31 | *.egg
32 | MANIFEST
33 |
34 | # PyInstaller
35 | # Usually these files are written by a python script from a template
36 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
37 | *.manifest
38 | *.spec
39 |
40 | # Installer logs
41 | pip-log.txt
42 | pip-delete-this-directory.txt
43 |
44 | # Unit test / coverage reports
45 | htmlcov/
46 | .tox/
47 | .coverage
48 | .coverage.*
49 | .cache
50 | nosetests.xml
51 | coverage.xml
52 | *.cover
53 | .hypothesis/
54 | .pytest_cache/
55 |
56 | # Translations
57 | *.mo
58 | *.pot
59 |
60 | # Django stuff:
61 | *.log
62 | local_settings.py
63 | db.sqlite3
64 |
65 | # Flask stuff:
66 | instance/
67 | .webassets-cache
68 |
69 | # Scrapy stuff:
70 | .scrapy
71 |
72 | # Sphinx documentation
73 | docs/_build/
74 |
75 | # PyBuilder
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # pyenv
82 | .python-version
83 |
84 | # celery beat schedule file
85 | celerybeat-schedule
86 |
87 | # SageMath parsed files
88 | *.sage.py
89 |
90 | # Environments
91 | .env
92 | .venv
93 | env/
94 | venv/
95 | ENV/
96 | env.bak/
97 | venv.bak/
98 |
99 | # Spyder project settings
100 | .spyderproject
101 | .spyproject
102 |
103 | # Rope project settings
104 | .ropeproject
105 |
106 | # mkdocs documentation
107 | /site
108 |
109 | # mypy
110 | .mypy_cache/
111 |
112 | #AFL file names with :
113 | *:*
114 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Phuzzer
2 |
3 | This module provides a Python wrapper for interacting with fuzzers, such as AFL (American Fuzzy Lop: http://lcamtuf.coredump.cx/afl/).
4 | It supports starting an AFL instance, adding slave workers, injecting and retrieving testcases, and checking various performance metrics.
5 | It is based on the module that Shellphish used in Mechanical Phish (our CRS for the Cyber Grand Challenge) to interact with AFL.
6 |
7 | ## Installation
8 |
9 | /!\ We recommend installing our Python packages in a Python virtual environment. That is how we do it, and you'll likely run into problems if you do it otherwise.
10 |
11 | The fuzzer has some dependencies.
12 | First, here's a probably-incomplete list of debian packages that might be useful:
13 |
14 | sudo apt-get install build-essential gcc-multilib libtool automake autoconf bison debootstrap debian-archive-keyring libtool-bin
15 | sudo apt-get build-dep qemu
16 |
17 | Then, the fuzzer also depends on a few modules: `shellphish-afl`, `driller` and `tracer`.
18 |
19 | pip install git+https://github.com/shellphish/shellphish-afl
20 | pip install git+https://github.com/shellphish/driller
21 | pip install git+https://github.com/angr/tracer
22 |
23 | That'll pull a ton of other stuff, compile qemu about 4 times, and set everything up.
24 | Then, install this fuzzer wrapper:
25 |
26 | pip install git+https://github.com/angr/phuzzer
27 |
28 | ## Usage
29 |
30 | There are two ways of using this package.
31 | The easy way is to use the `shellphuzz` script, which allows you to specify various options, enable [driller](https://sites.cs.ucsb.edu/~vigna/publications/2016_NDSS_Driller.pdf), etc.
32 | The script has explanations about its usage with `--help`.
33 |
34 | A quick example:
35 |
36 | ```
37 | # fuzz with 4 AFL cores
38 | python -m phuzzer -i -c 4 /path/to/binary
39 |
40 | # perform symbolic-assisted fuzzing with 4 AFL cores and 2 symbolic tracing (drilling) cores.
41 | python -m phuzzer -i -c 4 -d 2 /path/to/binary
42 | ```
43 |
44 | You can also use it programmatically, but we have no documentation for that.
45 | For now, `import fuzzer` or look at the shellphuz script and figure it out ;-)
46 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM ubuntu:18.04
2 |
3 | # build essentials,
4 | RUN apt-get update && apt-get install -y software-properties-common && \
5 | apt-add-repository -y universe && \
6 | apt-get update && \
7 | apt-get install -y \
8 | build-essential \
9 | gcc-multilib \
10 | libtool \
11 | automake \
12 | autoconf \
13 | bison \
14 | git \
15 | gcc \
16 | debootstrap \
17 | debian-archive-keyring \
18 | libtool-bin \
19 | python3 \
20 | python3-dev \
21 | python3-pip
22 |
23 |
24 | # install QEMU
25 | RUN cp /etc/apt/sources.list /etc/apt/sources.list~ && sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list && \
26 | apt-get update && DEBIAN_FRONTEND=noninteractive apt-get build-dep -y qemu
27 |
28 | # Shellphish-AFL Deps
29 | RUN bash -c "pip3 install https://github.com/angr/wheels/blob/master/shellphish_afl-1.2.1-py2.py3-none-manylinux1_x86_64.whl?raw=true && \
30 | pip3 install git+https://github.com/shellphish/driller && \
31 | pip3 install git+https://github.com/angr/tracer"
32 |
33 | # Shellphish-AFL Symlinks
34 | RUN bash -c "ln -s /usr/local/bin/afl-cgc /usr/bin/afl-cgc && \
35 | ln -s /usr/local/bin/afl-multi-cgc /usr/bin/afl-multi-cgc && \
36 | ln -s /usr/local/bin/afl-unix /usr/bin/afl-unix && \
37 | ln -s /usr/local/bin/fuzzer-libs /usr/bin/fuzzer-libs && \
38 | ln -s /usr/local/bin/driller /usr/bin/driller"
39 |
40 | # Construct place for new phuzzers to live
41 | RUN mkdir /phuzzers
42 |
43 | # --- new fuzzer backends go here --- #
44 |
45 | # Install IJON Fuzzer
46 | RUN cd /phuzzers && \
47 | git clone https://github.com/RUB-SysSec/ijon && \
48 | apt-get install clang -y && \
49 | cd ijon && make && cd llvm_mode && LLVM_CONFIG=llvm-config-6.0 CC=clang-6.0 make
50 |
51 | # Install AFL++
52 | RUN cd /phuzzers/ && \
53 | bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" && \
54 | git clone https://github.com/AFLplusplus/AFLplusplus && \
55 | cd AFLplusplus && \
56 | apt install build-essential libtool-bin python3-dev automake flex bison ipython3 \
57 | libglib2.0-dev libpixman-1-dev clang python3-setuptools llvm -y && \
58 | LLVM_CONFIG=llvm-config-11 make distrib && \
59 | make install -j 8
60 |
61 | # Install the phuzzer framework
62 | COPY . ./phuzzer
63 | RUN pip3 install ./phuzzer
64 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug-report.yml:
--------------------------------------------------------------------------------
1 | name: Report a bug
2 | description: Report a bug in phuzzer
3 | labels: [bug,needs-triage]
4 | body:
5 | - type: markdown
6 | attributes:
7 | value: |
8 | Thank you for taking the time to submit this bug report!
9 |
10 | Before submitting this bug report, please check the following, which may resolve your issue:
11 | * Have you checked that you are running the latest versions of angr and its components? angr is rapidly-evolving!
12 | * Have you [searched existing issues](https://github.com/angr/phuzzer/issues?q=is%3Aopen+is%3Aissue+label%3Abug) to see if this bug has been reported before?
13 | * Have you checked the [documentation](https://docs.angr.io/)?
14 | * Have you checked the [FAQ](https://docs.angr.io/introductory-errata/faq)?
15 |
16 | **Important:** If this bug is a security vulnerability, please submit it privately. See our [security policy](https://github.com/angr/angr/blob/master/SECURITY.md) for more details.
17 |
18 | **Please note: This repo is effectively unmaintained. While we appreciate bug reports and feature requests, we cannot commit to a timely response.** For more real-time help with angr, from us and the community, join our [Slack](https://angr.io/invite/).
19 |
20 | - type: textarea
21 | attributes:
22 | label: Description
23 | description: Brief description of the bug, with any relevant log messages.
24 | validations:
25 | required: true
26 |
27 | - type: textarea
28 | attributes:
29 | label: Steps to reproduce the bug
30 | description: |
31 | If appropriate, include both a **script to reproduce the bug**, and if possible **attach the binary used**.
32 |
33 | **Tip:** You can attach files to the issue by first clicking on the textarea to select it, then dragging & dropping the file onto the textarea.
34 | - type: textarea
35 | attributes:
36 | label: Environment
37 | description: Many common issues are caused by problems with the local Python environment. Before submitting, double-check that your versions of all modules in the angr suite (angr, cle, pyvex, ...) are up to date and include the output of `python -m angr.misc.bug_report` here.
38 |
39 | - type: textarea
40 | attributes:
41 | label: Additional context
42 | description: Any additional context about the problem.
43 |
--------------------------------------------------------------------------------
/phuzzer/extensions/grease_callback.py:
--------------------------------------------------------------------------------
1 | import os
2 | import shutil
3 | import logging
4 | from .. import Showmap
5 |
6 | l = logging.getLogger("grease_callback")
7 |
8 | class GreaseCallback(object):
9 | def __init__(self, grease_dir, grease_filter=None, grease_sorter=None):
10 | self._grease_dir = grease_dir
11 | assert os.path.exists(grease_dir)
12 | self._grease_filter = grease_filter if grease_filter is not None else lambda x: True
13 | self._grease_sorter = grease_sorter if grease_sorter is not None else lambda x: x
14 |
15 | def grease_callback(self, fuzz):
16 | l.warning("we are stuck, trying to grease the wheels!")
17 |
18 | # find an unused input
19 | grease_inputs = [
20 | os.path.join(self._grease_dir, x) for x in os.listdir(self._grease_dir)
21 | if self._grease_filter(os.path.join(self._grease_dir, x))
22 | ]
23 |
24 | if len(grease_inputs) == 0:
25 | l.warning("no grease inputs remaining")
26 | return
27 |
28 | # iterate until we find one with a new bitmap
29 | bitmap = fuzz.bitmap()
30 | for a in self._grease_sorter(grease_inputs):
31 | if os.path.getsize(a) == 0:
32 | continue
33 | with open(a) as sf:
34 | seed_content = sf.read()
35 | smap = Showmap(fuzz.binary_path, seed_content)
36 | shownmap = smap.showmap()
37 | for k in shownmap:
38 | #print(shownmap[k], (ord(bitmap[k % len(bitmap)]) ^ 0xff))
39 | if shownmap[k] > (ord(bitmap[k % len(bitmap)]) ^ 0xff):
40 | l.warning("Found interesting, syncing to tests")
41 |
42 | fuzzer_out_dir = fuzz.out_dir
43 | grease_dir = os.path.join(fuzzer_out_dir, "grease")
44 | grease_queue_dir = os.path.join(grease_dir, "queue")
45 | try:
46 | os.mkdir(grease_dir)
47 | os.mkdir(grease_queue_dir)
48 | except OSError:
49 | pass
50 | id_num = len(os.listdir(grease_queue_dir))
51 | filepath = "id:" + ("%d" % id_num).rjust(6, "0") + ",grease"
52 | filepath = os.path.join(grease_queue_dir, filepath)
53 | shutil.copy(a, filepath)
54 | l.warning("copied grease input: %s", os.path.basename(a))
55 | return
56 |
57 | l.warning("No interesting inputs found")
58 | __call__ = grease_callback
59 |
--------------------------------------------------------------------------------
/phuzzer/minimizer.py:
--------------------------------------------------------------------------------
1 | import os
2 | import shutil
3 | import tempfile
4 | import subprocess
5 | from .phuzzers import Phuzzer
6 | from .phuzzers.afl import AFL
7 |
8 | import logging
9 | l = logging.getLogger("phuzzer.Minimizer")
10 |
11 | class Minimizer:
12 | """Testcase minimizer"""
13 |
14 | def __init__(self, binary_path, testcase):
15 | """
16 | :param binary_path: path to the binary which the testcase applies to
17 | :param testcase: string representing the contents of the testcase
18 | """
19 |
20 | self.binary_path = binary_path
21 | self.testcase = testcase
22 |
23 | AFL.check_environment()
24 |
25 | afl_dir, _ = AFL.init_afl_config(binary_path)
26 | self.tmin_path = os.path.join(afl_dir, "afl-tmin")
27 |
28 | # create temp
29 | self.work_dir = tempfile.mkdtemp(prefix='tmin-', dir='/tmp/')
30 |
31 | # flag for work directory removal
32 | self._removed = False
33 |
34 | self.input_testcase = os.path.join(self.work_dir, 'testcase')
35 | self.output_testcase = os.path.join(self.work_dir, 'minimized_result')
36 |
37 | l.debug("input_testcase: %s", self.input_testcase)
38 | l.debug("output_testcase: %s", self.output_testcase)
39 |
40 | # populate contents of input testcase
41 | with open(self.input_testcase, 'wb') as f:
42 | f.write(testcase)
43 |
44 | self.errlog = ""
45 |
46 | def __del__(self):
47 | if not self._removed:
48 | import traceback
49 | traceback.print_stack()
50 | shutil.rmtree(self.work_dir)
51 |
52 | def minimize(self):
53 | """Start minimizing"""
54 |
55 | self._start_minimizer().wait()
56 | if os.path.isfile(self.output_testcase):
57 | with open(self.output_testcase, 'rb') as f:
58 | result = f.read()
59 | else:
60 | print(open(self.errlog, "r").read())
61 | raise ValueError(f"minized version not created see error output above {self.output_testcase} ")
62 |
63 |
64 | shutil.rmtree(self.work_dir)
65 | self._removed = True
66 |
67 | return result
68 |
69 | def _start_minimizer(self, memory="8G"):
70 |
71 | args = [self.tmin_path]
72 |
73 | args += ["-i", self.input_testcase]
74 | args += ["-o", self.output_testcase]
75 | args += ["-m", memory]
76 | args += ["-Q"]
77 |
78 | args += ["--"]
79 | args += [self.binary_path]
80 |
81 | outfile = "minimizer.log"
82 |
83 | l.debug("execing: %s > %s", " ".join(args), outfile)
84 |
85 | self.errlog = os.path.join(self.work_dir, outfile)
86 |
87 | with open(self.errlog, "wb") as fp:
88 | return subprocess.Popen(args, stderr=fp)
89 |
--------------------------------------------------------------------------------
/bin/analyze_result.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import os
4 | import sys
5 | import tqdm
6 | import json
7 | import phuzzer
8 |
9 | DIR = sys.argv[1].rstrip('/')
10 | BIN = os.path.basename(DIR).split('-')[-1]
11 | print(DIR,BIN)
12 | f = phuzzer.Phuzzer('/results/bins/%s'%BIN, '', job_dir=DIR)
13 | h = phuzzer.InputHierarchy(fuzzer=f, load_crashes=True)
14 |
15 | def good(_i):
16 | return _i.instance not in ('fuzzer-1', 'fuzzer-2', 'fuzzer-3', 'fuzzer-4', 'fuzzer-5')
17 |
18 | all_blocks = set()
19 | all_transitions = set()
20 | all_inputs = [ i for i in h.inputs.values() if not i.crash and good(i) ]
21 | all_crashes = [ i for i in h.inputs.values() if i.crash ]
22 | min_timestamp = min(i.timestamp for i in all_inputs)
23 | if all_crashes:
24 | first_crash = min(all_crashes, key=lambda i: i.timestamp)
25 | time_to_crash = first_crash.timestamp - min_timestamp
26 | first_crash_techniques = first_crash.contributing_techniques
27 | if 'grease' in first_crash_techniques :
28 | # TODO: figure out how long that input took
29 | time_to_crash += 120
30 | else:
31 | first_crash = None
32 | time_to_crash = -1
33 | first_crash_techniques = set()
34 |
35 | for i in tqdm.tqdm(all_inputs):
36 | all_blocks.update(i.block_set)
37 | all_transitions.update(i.transition_set)
38 |
39 | fuzzer_only = { i for i in all_inputs if list(i.contributing_techniques) == ['fuzzer'] }
40 | grease_derived = { i for i in all_inputs if 'grease' in i.contributing_techniques }
41 | driller_derived = { i for i in all_inputs if 'driller' in i.contributing_techniques }
42 | hybrid_derived = grease_derived & driller_derived
43 | #tc = h.technique_contributions()
44 |
45 | tag = ''.join(DIR.split('/')[-1].split('-')[:-2])
46 |
47 | results = {
48 | 'bin': BIN,
49 | 'tag': tag,
50 | 'testcase_count': len(all_inputs),
51 | 'crash_count': len(all_crashes),
52 | 'crashed': len(all_crashes)>0,
53 | 'crash_time': time_to_crash,
54 | 'crash_techniques': tuple(first_crash_techniques),
55 | 'grease_assisted_crash': 'grease' in first_crash_techniques,
56 | 'driller_assisted_crash': 'driller' in first_crash_techniques,
57 | 'fuzzer_assisted_crash': 'fuzzer' in first_crash_techniques,
58 | 'fuzzer_only_testcases': len(fuzzer_only),
59 | 'greese_derived_testcases': len(grease_derived),
60 | 'driller_derived_testcases': len(driller_derived),
61 | 'hybrid_derived_testcases': len(hybrid_derived),
62 | 'blocks_triggered': len(all_blocks),
63 | 'transitions_triggered': len(all_transitions),
64 | }
65 |
66 | print("")
67 | for k,v in results.items():
68 | print("RESULT", results['tag'], results['bin'], k, v)
69 | print("")
70 | print("JSON", json.dumps(results))
71 |
72 | #print("RESULT",tag,BIN,": fuzzer blocks:",tc.get('fuzzer', (0,0))[0])
73 | #print("RESULT",tag,BIN,": driller blocks:",tc.get('driller', (0,0))[0])
74 | #print("RESULT",tag,BIN,": grease blocks:",tc.get('grease', (0,0))[0])
75 | #print("RESULT",tag,BIN,": fuzzer crashes:",tc.get('fuzzer', (0,0))[1])
76 | #print("RESULT",tag,BIN,": driller crashes:",tc.get('driller', (0,0))[1])
77 | #print("RESULT",tag,BIN,": grease crashes:",tc.get('grease', (0,0))[1])
78 |
--------------------------------------------------------------------------------
/phuzzer/seed.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 |
4 | l = logging.getLogger('fuzzer.seed')
5 |
6 | class Seed:
7 | def __init__(self, filepath):
8 | self.filepath = filepath
9 | self.filename = os.path.basename(filepath)
10 | self.worker = os.path.basename(os.path.dirname(os.path.dirname(filepath)))
11 | self.technique = self.worker.split("-")[0].split("_")[0].split(":")[0]
12 |
13 | self.id = None
14 | self.source_ids = [ ]
15 | self.cov = False
16 | self.op = None
17 | self.synced_from = None
18 | self.other_fields = { }
19 | self.val = None
20 | self.rep = None
21 | self.pos = None
22 | self.orig = None
23 | self.crash = False
24 | self.sig = None
25 | self._process_filename(self.filename)
26 |
27 | self.timestamp = os.stat(self.filepath).st_mtime
28 |
29 | # these are resolved by the hierarchy
30 | self.parents = None
31 | self.origins = None
32 | self.contributing_techniques = None
33 |
34 | def _process_filename(self, filename):
35 | # process the fields
36 | fields = filename.split(',')
37 | for f in fields:
38 | if f == "+cov":
39 | self.cov = True
40 | elif f == "grease":
41 | assert self.id
42 | self.orig = "greased_%s" % self.id
43 | else:
44 | n,v = f.split(':', 1)
45 | if n == 'id':
46 | assert not self.id
47 | self.id = v
48 | elif n == 'src':
49 | assert not self.source_ids
50 | self.source_ids = v.split('+')
51 | elif n == 'sync':
52 | assert not self.synced_from
53 | self.synced_from = v
54 | elif n == 'op':
55 | assert not self.op
56 | self.op = v
57 | elif n == 'rep':
58 | assert not self.rep
59 | self.rep = v
60 | elif n == 'orig':
61 | assert not self.orig
62 | self.orig = v
63 | elif n == 'pos':
64 | assert not self.pos
65 | self.pos = v
66 | elif n == 'val':
67 | assert not self.val
68 | self.val = v
69 | elif n == 'from': # driller uses this instead of synced/src
70 | instance, from_id = v[:-6], v[-6:]
71 | self.synced_from = instance
72 | self.source_ids.append(from_id)
73 | elif n == 'sig':
74 | assert not self.crash
75 | assert not self.sig
76 | assert self.id
77 | self.crash = True
78 | self.sig = v
79 | self.id = 'c'+self.id
80 | else:
81 | l.warning("Got unexpected field %s with value %s for file %s.", n, v, filename)
82 | self.other_fields[n] = v
83 |
84 | assert self.id is not None
85 | assert self.source_ids or self.orig
86 |
87 | def read(self):
88 | with open(self.filepath, 'rb') as f:
89 | return f.read()
90 |
91 | def __repr__(self):
92 | s = "" % (self.worker, self.filename)
93 | #if self.synced_from:
94 | # s += " sync:%s" % self.synced_from
95 | #s += "src:%s" % self.source_ids
96 | return s
97 |
--------------------------------------------------------------------------------
/phuzzer/showmap.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import os
3 | import shutil
4 | import subprocess
5 | import tempfile
6 |
7 | from phuzzer.phuzzers import Phuzzer
8 | from phuzzer.phuzzers.afl import AFL
9 |
10 | l = logging.getLogger("phuzzer.Showmap")
11 |
12 |
13 | class Showmap:
14 | """Show map"""
15 |
16 | def __init__(self, binary_path, testcase, timeout=None):
17 | """
18 | :param binary_path: path to the binary which the testcase applies to
19 | :param testcase: string representing the contents of the testcase
20 | :param timeout: millisecond timeout
21 | """
22 |
23 | self.binary_path = binary_path
24 | self.testcase = testcase
25 | self.timeout = None
26 |
27 | if isinstance(binary_path, str):
28 | self.is_multicb = False
29 | self.binaries = [binary_path]
30 | elif isinstance(binary_path, (list,tuple)):
31 | self.is_multicb = True
32 | self.binaries = binary_path
33 | else:
34 | raise ValueError("Was expecting either a string or a list/tuple for binary_path! "
35 | "It's {} instead.".format(type(binary_path)))
36 |
37 | if timeout is not None:
38 | if isinstance(timeout, int):
39 | self.timeout = str(timeout)
40 | elif isinstance(timeout, (str)):
41 | self.timeout = timeout
42 | elif isinstance(timeout, (bytes)):
43 | self.timeout = timeout.decode('utf-8')
44 | else:
45 | raise ValueError("timeout param must be of type int or str")
46 |
47 | # will be set by showmap's return code
48 | self.causes_crash = False
49 |
50 | AFL.check_environment()
51 | afl_dir, _ = Phuzzer.init_afl_config(self.binaries[0])
52 | self.showmap_path = os.path.join(afl_dir, "afl-showmap")
53 |
54 | l.debug("showmap_path: %s", self.showmap_path)
55 |
56 | # create temp
57 | self.work_dir = tempfile.mkdtemp(prefix='showmap-', dir='/tmp/')
58 |
59 | # flag for work directory removal
60 | self._removed = False
61 |
62 | self.input_testcase = os.path.join(self.work_dir, 'testcase')
63 | self.output = os.path.join(self.work_dir, 'out')
64 |
65 | l.debug("input_testcase: %s", self.input_testcase)
66 | l.debug("output: %s", self.output)
67 |
68 | # populate contents of input testcase
69 | with open(self.input_testcase, 'wb') as f:
70 | f.write(testcase)
71 |
72 | def __del__(self):
73 | if not self._removed:
74 | shutil.rmtree(self.work_dir)
75 |
76 | def showmap(self):
77 | """Create the map"""
78 |
79 | if self._start_showmap().wait() == 2:
80 | self.causes_crash = True
81 |
82 | with open(self.output) as f: result = f.read()
83 |
84 | shutil.rmtree(self.work_dir)
85 | self._removed = True
86 |
87 | shownmap = dict()
88 | for line in result.split("\n")[:-1]:
89 | key, h_count = map(int, line.split(":"))
90 | shownmap[key] = h_count
91 |
92 | return shownmap
93 |
94 | def _start_showmap(self, memory="8G"):
95 |
96 | args = [self.showmap_path]
97 |
98 | args += ["-o", self.output]
99 | if not self.is_multicb:
100 | args += ["-m", memory]
101 | args += ["-Q"]
102 |
103 | if self.timeout:
104 | args += ["-t", self.timeout]
105 | else:
106 | args += ["-t", "%d+" % (len(self.binaries) * 1000)]
107 |
108 | args += ["--"]
109 | args += self.binaries
110 |
111 | outfile = "minimizer.log"
112 |
113 | l.debug("execing: %s > %s", " ".join(args), outfile)
114 |
115 | outfile = os.path.join(self.work_dir, outfile)
116 | with open(outfile, "w") as fp, open(self.input_testcase, 'rb') as it, open("/dev/null", 'wb') as devnull:
117 | return subprocess.Popen(args, stdin=it, stdout=devnull, stderr=fp, close_fds=True)
118 |
--------------------------------------------------------------------------------
/tests/test_fuzzer.py:
--------------------------------------------------------------------------------
1 | from os.path import join
2 | import time
3 | import phuzzer
4 | import logging
5 | l = logging.getLogger("fuzzer.tests.test_fuzzer")
6 | l.setLevel(logging.WARNING)
7 |
8 | import os
9 | bin_location = None
10 |
11 | for x in range(0, 3):
12 | bin_location = str(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../'*x, '../../binaries'))
13 | if os.path.exists(bin_location):
14 | break
15 |
16 | if bin_location is None or not os.path.exists(bin_location):
17 | raise ValueError("Error cannot find binaries intallation path")
18 |
19 | def test_parallel_execution():
20 | """
21 | test parallel execution, summary stats, and the timed_out method of Phuzzer
22 | """
23 |
24 | timeout_value = 5
25 | binary = os.path.join(bin_location, "tests/cgc/ccf3d301_01")
26 | afl = phuzzer.AFL(binary, work_dir="/tmp/testwork", afl_count=2, create_dictionary=True, resume=False,
27 | timeout=timeout_value)
28 |
29 | afl.start()
30 |
31 | start_time = time.time()
32 | while not afl.timed_out():
33 | time.sleep(.75)
34 | elapsed_time = time.time() - start_time
35 | assert elapsed_time <= (timeout_value + 1)
36 |
37 | assert os.path.exists(join(afl.work_dir, "fuzzer-master", "queue"))
38 | assert os.path.exists(join(afl.work_dir, "fuzzer-1", "queue"))
39 | #assert os.path.exists(join(afl.work_dir, "fuzzer-2", "queue"))
40 |
41 | assert afl.summary_stats["execs_done"] > 0
42 | assert afl.summary_stats["execs_per_sec"] > 0
43 |
44 | afl.stop()
45 |
46 |
47 | def test_dictionary_creation_cgc():
48 | """
49 | test dictionary creation on a binary
50 | """
51 |
52 | binary = os.path.join(bin_location, "tests/cgc/ccf3d301_01")
53 | afl = phuzzer.AFL(binary, create_dictionary=True, resume=False)
54 | assert len(afl.dictionary) >= 60
55 | assert not os.path.exists(afl.dictionary_file)
56 | afl.start()
57 | assert os.path.exists(afl.dictionary_file)
58 | afl.stop()
59 |
60 |
61 | def test_minimizer():
62 | """
63 | Test minimization of an input
64 | """
65 |
66 | binary = os.path.join(bin_location, "tests/cgc/PIZZA_00001")
67 |
68 | crash = bytes.fromhex('66757fbeff10ff7f1c3131313131413131317110314301000080006980009fdce6fecc4c66747fbeffffff7f1c31313131314131313171793143cfcfcfcfcfcfcf017110314301000000003e3e3e3e3e413e3e2e3e3e383e317110000000003e3e3e3e3e413e3e2e3e3e383e31713631310031103c3b6900ff3e3131413131317110313100000000006900ff91dce6fecc7e6e000200fecc4c66747fbeffffff7f1c31313131314131313171793143cf003100000000006900ff91dcc3c3c3479fdcffff084c3131313133313141314c6f00003e3e3e3e30413e3e2e3e3e383e31712a000000003e3e3e3e3eedededededededededededededededededededededededededededededededededededededededededede0dadada4c4c4c4c333054c4c4c401000000fb6880009fdce6fecc4c66757fbeffffff7f1c31313131314131313171793143cfcfcfcfcfcfcf017110314301000000003e3e3e3e3e413e3e2e343e383e317110000000003e3e3e3e3e413e3e2e3e3e383e31713631310031103c3b6900ff3e3131413131317110313100000000006900ff91dce6fecc7e6e000200003100000000006900ff91dcc3c3c3479fdcffff084c0d0d0d0d0dfa1d7f')
69 |
70 | m = phuzzer.Minimizer(binary, crash)
71 |
72 | assert m.minimize() == b'100'
73 |
74 |
75 | def test_showmap():
76 | """
77 | Test the mapping of an input
78 | """
79 |
80 | true_dict = {7525: 1, 14981: 1, 25424: 1, 31473: 1, 33214: 1, 37711: 1, 64937: 1, 65353: 4, 66166: 1, 79477: 1, 86259: 1, 86387: 1, 96625: 1, 107932: 1, 116010: 1, 116490: 1, 117482: 4, 120443: 1}
81 |
82 | binary = os.path.join(bin_location, "tests/cgc/cfe_CADET_00003")
83 |
84 | testcase = b"hello"
85 |
86 | s = phuzzer.Showmap(binary, testcase)
87 | smap = s.showmap()
88 |
89 | for te in true_dict:
90 | assert true_dict[te] == smap[te]
91 |
92 |
93 | def test_fuzzer_spawn():
94 | """
95 | Test that the fuzzer spawns correctly
96 | """
97 |
98 | binary = os.path.join(bin_location, "tests/cgc/PIZZA_00001")
99 |
100 | f = phuzzer.AFL(binary, resume=False)
101 | f.start()
102 |
103 | for _ in range(15):
104 | if f.alive:
105 | break
106 | time.sleep(1)
107 |
108 | assert f.alive
109 | if f.alive:
110 | f.stop()
111 |
112 |
113 | def test_multicb_spawn():
114 | """
115 | Test that the fuzzer spins up for a multicb challenge.
116 | """
117 | binaries = [os.path.join(bin_location, "tests/cgc/251abc02_01"),
118 | os.path.join(bin_location, "tests/cgc/251abc02_02")]
119 |
120 | f = phuzzer.AFLMultiCB(binaries, create_dictionary=True)
121 | f.start()
122 |
123 | for _ in range(15):
124 | if f.alive:
125 | break
126 | time.sleep(1)
127 |
128 | assert f.alive
129 |
130 | dictionary_path = os.path.join(f.work_dir, "dict.txt")
131 | assert os.path.isfile(dictionary_path)
132 |
133 | if f.alive:
134 | f.stop()
135 |
136 |
137 | def test_pollenate():
138 | fauxware = os.path.join(bin_location, "tests/i386/fauxware")
139 | f = phuzzer.AFL(fauxware, resume=False)
140 | f.start()
141 |
142 | time.sleep(1)
143 |
144 | # this should get synchronized
145 | f.pollenate(b"A"*9+b"SOSNEAKY\0")
146 | for _ in range(30):
147 | if any(b"SOSNEAKY" in t for t in f.queue()):
148 | break
149 | time.sleep(1)
150 | else:
151 | assert False, "AFL failed to synchronize pollenated seed."
152 |
153 |
154 | def inprogress_dict():
155 | va = os.path.join(bin_location, "tests/x86_64/veritesting_a")
156 | f = phuzzer.AFL(va, resume=False, dictionary=[b"B"])
157 | f.start()
158 |
159 | time.sleep(1)
160 | assert f.alive
161 |
162 | # this should get synchronized
163 | for _ in range(30):
164 | if any(t.count(b"B") == 10 in t for t in f.queue()):
165 | break
166 | time.sleep(1)
167 | else:
168 | assert False, "AFL failed to find the easter egg given a dict."
169 |
170 |
171 | def run_all():
172 | functions = globals()
173 | all_functions = dict(filter((lambda kv: kv[0].startswith('test_')), functions.items()))
174 | for f in sorted(all_functions.keys()):
175 | if hasattr(all_functions[f], '__call__'):
176 | all_functions[f]()
177 |
178 |
179 | if __name__ == "__main__":
180 | logging.getLogger("phuzzer").setLevel("WARNING")
181 | run_all()
182 |
--------------------------------------------------------------------------------
/phuzzer/hierarchy.py:
--------------------------------------------------------------------------------
1 | import networkx
2 | import logging
3 | import tqdm
4 | import glob
5 | import os
6 |
7 | l = logging.getLogger('fuzzer.input_hierarchy')
8 |
9 | class InputHierarchy:
10 | """
11 | This class deals with the AFL input hierarchy and analyses done on it.
12 | """
13 |
14 | def __init__(self, fuzzer_dir, load_crashes=True):
15 | self._dir = fuzzer_dir
16 | self.inputs = { }
17 | self.worker_inputs = { }
18 | self.workers = [ ]
19 |
20 | self.reload(load_crashes)
21 |
22 | while self._remove_cycles():
23 | pass
24 |
25 | def _remove_cycles(self):
26 | """
27 | Really hacky way to remove cycles in hierarchies (wtf).
28 | """
29 |
30 | G = self.make_graph()
31 | cycles = list(networkx.simple_cycles(G))
32 | if not cycles:
33 | return False
34 | else:
35 | cycles[0][0].looped = True
36 | cycles[0][0].parents[:] = [ ]
37 | return True
38 |
39 | def triggered_blocks(self):
40 | """
41 | Gets the triggered blocks by all the testcases.
42 | """
43 | return set.union(*(i.block_set for i in tqdm.tqdm(self.inputs.values())))
44 |
45 | def crashes(self):
46 | """
47 | Returns the crashes, if they are loaded.
48 | """
49 | return [ i for i in self.inputs.values() if i.crash ]
50 |
51 | def technique_contributions(self):
52 | """
53 | Get coverage and crashes by technique.
54 | """
55 | results = { }
56 | for s,(b,c) in self.seed_contributions():
57 | results.setdefault(s.worker.split('-')[0], [0,0])[0] += b
58 | results.setdefault(s.worker.split('-')[0], [0,0])[1] += c
59 | return results
60 |
61 | def reload(self, load_crashes):
62 | self._load_workers()
63 | for i in self.workers:
64 | self._load_inputs(i)
65 | if load_crashes:
66 | self._load_inputs(i, input_type="crashes")
67 | self._resolve_all_parents()
68 | return self
69 |
70 | def _load_workers(self):
71 | self.workers = [
72 | os.path.basename(os.path.dirname(n))
73 | for n in glob.glob(os.path.join(self._dir, "*", "queue"))
74 | ]
75 | self.worker_inputs = { i: { } for i in self.workers }
76 | l.debug("Instances: %s", self.workers)
77 |
78 | def _load_inputs(self, worker, input_type="queue"):
79 | l.info("Loading inputs from worker %s", worker)
80 | for fp in glob.glob(os.path.join(self._dir, worker, input_type, "id*")):
81 | l.debug("Adding input %s", fp)
82 | i = Seed(fp)
83 | self.inputs[i.worker + ':' + i.id] = i
84 | self.worker_inputs[i.worker][i.id] = i
85 |
86 | def _resolve_all_parents(self):
87 | for i in self.inputs.values():
88 | self._resolve_parents(i)
89 |
90 | def _resolve_parents(self, seed):
91 | try:
92 | if seed.source_ids and seed.source_ids[0] == "pollenation":
93 | # this is pollenated in
94 | seed.parents = [ ]
95 | elif seed.synced_from:
96 | seed.parents = [ self.input_from_worker(seed.synced_from, seed.source_ids[0]) ]
97 | else:
98 | seed.parents = [ self.input_from_worker(seed.worker, i) for i in seed.source_ids ]
99 | except KeyError as e:
100 | l.warning("Unable to resolve source ID %s for %s", e, self)
101 | seed.parents = [ ]
102 |
103 |
104 | def input_from_worker(self, worker, id): #pylint:disable=redefined-builtin
105 | return self.worker_inputs[worker][id]
106 |
107 | def make_graph(self):
108 | G = networkx.DiGraph()
109 | for child in self.inputs.values():
110 | for parent in child.parents:
111 | G.add_edge(parent, child)
112 | return G
113 |
114 | def plot(self, output=None):
115 | import matplotlib.pyplot as plt #pylint:disable=import-error,import-outside-toplevel
116 | plt.close()
117 | networkx.draw(self.make_graph())
118 | if output:
119 | plt.savefig(output)
120 | else:
121 | plt.show()
122 |
123 | #
124 | # Lineage analysis
125 | #
126 |
127 | def seed_parents(self, seed):
128 | if seed.parents is None:
129 | self._resolve_parents(seed)
130 | return seed.parents
131 |
132 | def seed_lineage(self, seed):
133 | for p in self.seed_parents(seed):
134 | yield from self.seed_lineage(p)
135 | yield seed
136 |
137 | def print_lineage(self, seed, depth=0):
138 | if depth:
139 | print(' '*depth + str(seed))
140 | else:
141 | print(seed)
142 | for parent in self.seed_parents(seed):
143 | self.print_lineage(parent, depth=depth+1)
144 |
145 | def seed_origins(self, seed):
146 | """
147 | Return the origins of the given seed.
148 |
149 | :param seed: the seed
150 | """
151 | if seed.origins is not None:
152 | return seed.origins
153 |
154 | if not self.seed_parents(seed):
155 | o = { seed }
156 | else:
157 | o = set.union(*(self.seed_origins(s) for s in self.seed_parents(seed)))
158 | seed.origins = o
159 | return seed.origins
160 |
161 | def contributing_techniques(self, seed):
162 | if seed.contributing_techniques is None:
163 | # don't count this current technique if we synced it
164 | if seed.synced_from:
165 | new_technique = frozenset()
166 | else:
167 | new_technique = frozenset([seed.technique])
168 | seed.contributing_techniques = frozenset.union(
169 | new_technique, *(i.contributing_techniques for i in self.seed_parents(seed))
170 | )
171 | return seed.contributing_techniques
172 |
173 | def contributing_workers(self, seed):
174 | return set(i.worker for i in self.seed_lineage(seed))
175 |
176 | def seed_contributions(self):
177 | """
178 | Get the seeds (including inputs introduced by extensions) that
179 | resulted in coverage and crashes.
180 | """
181 | sorted_inputs = sorted((
182 | i for i in self.inputs.values() if i.worker.startswith('fuzzer-')
183 | ), key=lambda j: j.timestamp)
184 |
185 | found = set()
186 | contributions = { }
187 | for s in tqdm.tqdm(sorted_inputs):
188 | o = max(s.origins, key=lambda i: i.timestamp)
189 | if s.crash:
190 | contributions.setdefault(o, (set(),set()))[1].add(s)
191 | else:
192 | c = o.transition_set - found
193 | if not c:
194 | continue
195 | contributions.setdefault(o, (set(),set()))[0].update(c)
196 | found |= c
197 |
198 | return sorted(((k, list(map(len,v))) for k,v in contributions.items()), key=lambda x: x[0].timestamp)
199 |
200 | from .seed import Seed
201 |
--------------------------------------------------------------------------------
/phuzzer/reporter.py:
--------------------------------------------------------------------------------
1 | import os
2 | import time
3 | import glob
4 | import datetime
5 | import traceback
6 | from threading import Thread
7 | from collections import defaultdict
8 |
9 |
10 | class Reporter(Thread):
11 | DETAIL_FREQ = 1
12 |
13 | def __init__(self, binary, reportdir, afl_cores, first_crash, timeout, work_dir, testversion=""):
14 | Thread.__init__(self)
15 | self.binary = binary
16 | self.reportdir = reportdir
17 | self.afl_cores = afl_cores
18 | self.first_crash = first_crash
19 | self.timeout = timeout
20 | self.timeout_seen = False
21 | self.work_dir = work_dir
22 |
23 | self.details_fn = f"{reportdir}/run_details.txt"
24 | self.summary_fn = f"{reportdir}/run_summary.txt"
25 |
26 | if not os.path.exists(self.details_fn):
27 | open(self.details_fn, "w").write('Date\tTime\tBinary\tTarget\tElapsed\tCores\tExecs\tExec/sec\tCycles\tPaths\tCrashes\tReason\tTestVer\n')
28 |
29 | if not os.path.exists(self.summary_fn):
30 | open(self.summary_fn, "w").write('Date\tTime\tBinary\tTarget\tElapsed\tCores\tExecs\tExec/sec\tCycles\tPaths\tCrashes\tReason\tTestVer\n')
31 |
32 | self.start_time = time.time()
33 | self.statement_cnt = 0
34 | self.get_fuzzer_stats()
35 | self.keepgoing=True
36 | self.summary_stats = defaultdict(lambda: 0)
37 | self.last_printed_crashes = self.summary_stats["unique_crashes"]
38 | self.last_printed_paths_total = self.summary_stats["paths_total"]
39 | self._crash_seen=False
40 | self._timeout_reached=False
41 | self.statement_cnt = 0
42 | self.do_printing = False
43 | self.elapsed_time = 0
44 | self.testversion = testversion
45 | self.script_filename = ""
46 |
47 | def set_script_filename(self, script_fn):
48 | self.script_filename = script_fn
49 |
50 | def run(self):
51 | while self.keepgoing:
52 | self.generate_report_line()
53 | time.sleep(1)
54 |
55 | def enable_printing(self):
56 | self.do_printing = True
57 |
58 | def summarize_stats(self):
59 |
60 | summary_stats = defaultdict(lambda: 0)
61 | for _, fuzzstats in self.stats.items():
62 | for fstat, value in fuzzstats.items():
63 | try:
64 | fvalue = float(value)
65 | if fstat in ('paths_total', 'unique_crashes'):
66 | summary_stats[fstat] = max(summary_stats[fstat], int(fvalue))
67 | else:
68 | try:
69 | summary_stats[fstat] += int(fvalue)
70 | except Exception:
71 | summary_stats[fstat] += 0
72 | except ValueError:
73 | pass
74 |
75 | self.summary_stats = summary_stats
76 |
77 | def get_fuzzer_stats(self):
78 | self.stats = {}
79 | if os.path.isdir(self.work_dir):
80 | for fuzzer_dir in os.listdir(self.work_dir):
81 | if os.path.isdir(os.path.join(self.work_dir, fuzzer_dir)):
82 | stat_path = os.path.join(self.work_dir, fuzzer_dir, "fuzzer_stats")
83 | self.stats[fuzzer_dir] = {}
84 | if os.path.isfile(stat_path):
85 | with open(stat_path, "r") as f:
86 | stat_blob = f.read()
87 | stat_lines = stat_blob.split("\n")[:-1]
88 | for stat in stat_lines:
89 | if ":" in stat:
90 | try:
91 | key, val = stat.split(":")
92 | except Exception:
93 | index = stat.find(":")
94 | key = stat[:index]
95 | val = stat[index + 1:]
96 | else:
97 | print(f"Skipping stat '${stat}' in \n${stat_lines} because no split value")
98 | continue
99 | try:
100 | self.stats[fuzzer_dir][key.strip()] = val.strip()
101 | except KeyError as ke:
102 | print(ke)
103 | traceback.format_exc()
104 | print(self.stats.keys())
105 |
106 | try:
107 | fuzz_q_mask = os.path.join(self.work_dir, fuzzer_dir, "crashes*", "id*")
108 | self.stats[fuzzer_dir]["unique_crashes"] = len(glob.glob(fuzz_q_mask))
109 | fuzz_q_mask = os.path.join(self.work_dir, fuzzer_dir, "queue", "id*")
110 | self.stats[fuzzer_dir]["paths_total"] = len(glob.glob(fuzz_q_mask))
111 | except KeyError as ke:
112 | print(ke)
113 | traceback.format_exc()
114 | print(self.stats.keys())
115 |
116 | def print_details(self, mandatory_print=False):
117 | timeout_str = ""
118 | run_until_str = ""
119 | self.elapsed_time = time.time() - self.start_time
120 | if self.timeout:
121 | if self.first_crash:
122 | run_until_str = "until first crash or "
123 | run_until_str += "timeout "
124 | timeout_str = "for %d of %d seconds " % (self.elapsed_time, self.timeout)
125 | elif self.first_crash:
126 | run_until_str = "until first crash "
127 | else:
128 | run_until_str = "until stopped by you "
129 |
130 | outstr = f'[*] {self.afl_cores} fuzzers running {run_until_str}{timeout_str}completed '
131 | outstr += f'{self.summary_stats["execs_done"]} at {self.summary_stats["execs_per_sec"]} execs/sec '
132 | outstr += f'with {self.summary_stats["cycles_done"]} cycles finding {self.summary_stats["paths_total"]} paths and '
133 | outstr += f'\033[32;5;3m{self.summary_stats["unique_crashes"]} crashes \033[0m'
134 |
135 | if self.last_printed_crashes != self.summary_stats["unique_crashes"] or mandatory_print or (
136 | self.elapsed_time > 3600 and self.summary_stats["paths_total"] != self.last_printed_paths_total):
137 | print(outstr)
138 | else:
139 | print(outstr, end="\r")
140 | self.last_printed_crashes = self.summary_stats["unique_crashes"]
141 | self.last_printed_paths_total = self.summary_stats["paths_total"]
142 |
143 | def generate_report_line(self, mandatory_record=False):
144 | self.elapsed_time = time.time() - self.start_time
145 |
146 | self.get_fuzzer_stats()
147 | self.summarize_stats()
148 |
149 | self.build_report_stats()
150 | self.statement_cnt += 1
151 | if self.statement_cnt % Reporter.DETAIL_FREQ == 0 or mandatory_record:
152 | with open(self.details_fn, "a+") as fp :
153 | fp.write(self.build_report_stats() + "\n")
154 |
155 | if self.do_printing:
156 | self.print_details(mandatory_record)
157 |
158 | def build_report_stats(self, end_reason=""):
159 |
160 | dt = datetime.datetime.now()
161 | binary_version = self.binary.replace("/p/webcam/php/", "").replace("/sapi/cgi/php-cgi", "")
162 |
163 | return f'{dt:%Y-%m-%d}\t{dt:%H:%M:%S}\t{binary_version}\t{self.script_filename:<25}' \
164 | f'\t{self.elapsed_time:.0f}\t{self.afl_cores}\t{self.summary_stats["execs_done"]:.0f}' \
165 | f'\t{float(self.summary_stats["execs_per_sec"]):.0f}\t{self.summary_stats["cycles_done"]}' \
166 | f'\t{self.summary_stats["paths_total"]:.0f}\t{self.summary_stats["unique_crashes"]:.0f}' \
167 | f'\t{end_reason}\t{self.testversion}'
168 |
169 | def set_crash_seen(self):
170 | self._crash_seen = True
171 |
172 | def set_timeout_seen(self):
173 | self._timeout_seen = True
174 |
175 | def save_summary_line(self, end_reason):
176 | run_results = self.build_report_stats(end_reason)
177 | with open(self.summary_fn, "a+") as fp:
178 | fp.write(run_results + "\n")
179 |
180 | def stop(self):
181 | end_reason=""
182 | if self.first_crash:
183 | end_reason = "First Crash"
184 |
185 | if self._crash_seen:
186 | # print ("\n[*] Crash found!")
187 | end_reason = "Crash Found."
188 |
189 | if self._timeout_reached:
190 | end_reason = "Max Time Reached"
191 |
192 | self.keepgoing = False
193 |
194 | self.generate_report_line(mandatory_record=True)
195 |
196 | self.save_summary_line(end_reason)
197 |
--------------------------------------------------------------------------------
/phuzzer/__main__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from .reporter import Reporter
3 | from .phuzzers import Phuzzer
4 | import pkg_resources
5 | import logging.config
6 | import importlib
7 | import argparse
8 | import tarfile
9 | import shutil
10 | import socket
11 | import time
12 | import imp
13 | import os
14 |
15 | try:
16 | import driller
17 | DRILLER_EXISTS = True
18 | except ImportError:
19 | DRILLER_EXISTS=False
20 |
21 | from . import GreaseCallback
22 |
23 |
24 | def main():
25 | parser = argparse.ArgumentParser(description="Shellphish fuzzer interface")
26 | parser.add_argument('binary', help="the path to the target binary to fuzz")
27 | parser.add_argument('-g', '--grease-with', help="A directory of inputs to grease the fuzzer with when it gets stuck.")
28 | parser.add_argument('-d', '--driller_workers', help="When the fuzzer gets stuck, drill with N workers.", type=int)
29 | parser.add_argument('-f', '--force_interval', help="Force greaser/fuzzer assistance at a regular interval (in seconds).", type=float)
30 | parser.add_argument('-w', '--work-dir', help="The work directory for AFL.", default="/dev/shm/work/")
31 |
32 | parser.add_argument('-l', '--login-data', help="The json file from which to get the login information", default="")
33 | parser.add_argument('-c', '--afl-cores', help="Number of AFL workers to spin up.", default=1, type=int)
34 | parser.add_argument('-C', '--first-crash', help="Stop on the first crash.", action='store_true', default=False)
35 | parser.add_argument('-Q', '--use-qemu', help="Use qemu to trace binary.", action='store_true', default=False)
36 | parser.add_argument('-t', '--timeout', help="Timeout (in seconds).", type=float, default=None)
37 | parser.add_argument('-i', '--ipython', help="Drop into ipython after starting the fuzzer.", action='store_true')
38 | parser.add_argument('-T', '--tarball', help="Tarball the resulting AFL workdir for further analysis to this file -- '{}' is replaced with the hostname.")
39 | parser.add_argument('-m', '--helper-module',
40 | help="A module that includes some helper scripts for seed selection and such.")
41 | parser.add_argument('-D', '--dictionary', default=None,
42 | help="Load the dictionary from a file, with each on a single line ")
43 | parser.add_argument('--memory', help="Memory limit to pass to AFL (MB, or use k, M, G, T suffixes)", default="8G")
44 | parser.add_argument('--no-dictionary', help="Do not create a dictionary before fuzzing.", action='store_true', default=False)
45 | parser.add_argument('--logcfg', help="The logging configuration file.", default=".shellphuzz.ini")
46 | parser.add_argument('-s', '--seed-dir', action="append", help="Directory of files to seed fuzzer with")
47 | parser.add_argument('--run-timeout', help="Number of milliseconds permitted for each run of binary", type=int, default=None)
48 | parser.add_argument('--driller-timeout', help="Number of seconds to allow driller to run", type=int, default=10*60)
49 | parser.add_argument('--length-extension', help="Try extending inputs to driller by this many bytes", type=int)
50 | parser.add_argument('--target-opts', help="Options to pass to target.", default=None, nargs='+')
51 | parser.add_argument('-r', '--resume', help="Resume prior run if possible and do not destroy work directory.",
52 | action='store_true', default=False)
53 | parser.add_argument('--reportdir', help="The directory to use for the reports.", default=".")
54 | parser.add_argument('-p','--phuzzer-type', '--fuzzer-type', help="Which phuzzer are you using: AFL, AFL_IJON, AFL++, Witcher, AFL_MULTICB.", default=Phuzzer.AFL)
55 | args = parser.parse_args()
56 |
57 | if os.path.isfile(os.path.join(os.getcwd(), args.logcfg)):
58 | logging.config.fileConfig(os.path.join(os.getcwd(), args.logcfg))
59 |
60 | try: os.mkdir("/dev/shm/work/")
61 | except OSError: pass
62 |
63 | if args.helper_module:
64 | try:
65 | helper_module = importlib.import_module(args.helper_module)
66 | except (ImportError, TypeError):
67 | helper_module = imp.load_source('fuzzing_helper', args.helper_module)
68 | else:
69 | helper_module = None
70 |
71 | drill_extension = None
72 | grease_extension = None
73 |
74 | if args.grease_with:
75 | print ("[*] Greasing...")
76 | grease_extension = GreaseCallback(
77 | args.grease_with,
78 | grease_filter=helper_module.grease_filter if helper_module is not None else None,
79 | grease_sorter=helper_module.grease_sorter if helper_module is not None else None
80 | )
81 |
82 | if args.driller_workers and DRILLER_EXISTS:
83 | print ("[*] Drilling...")
84 | drill_extension = driller.LocalCallback(num_workers=args.driller_workers, worker_timeout=args.driller_timeout, length_extension=args.length_extension)
85 |
86 | stuck_callback = (
87 | (lambda f: (grease_extension(f), drill_extension(f))) if drill_extension and grease_extension
88 | else drill_extension or grease_extension
89 | )
90 |
91 | seeds = None
92 | if args.seed_dir:
93 | seeds = []
94 | print ("[*] Seeding...")
95 | for dirpath in args.seed_dir:
96 | for filename in os.listdir(dirpath):
97 | filepath = os.path.join(dirpath, filename)
98 | if not os.path.isfile(filepath):
99 | continue
100 | with open(filepath, 'rb') as seedfile:
101 | seeds.append(seedfile.read())
102 |
103 | if args.dictionary:
104 | built_dict = open(args.dictionary,"rb").read().split(b"\n")
105 | else:
106 | built_dict = None
107 |
108 | print ("[*] Creating fuzzer...")
109 | fuzzer = Phuzzer.phactory(phuzzer_type=args.phuzzer_type,
110 | target=args.binary, work_dir=args.work_dir, seeds=seeds, afl_count=args.afl_cores,
111 | create_dictionary=not args.no_dictionary, timeout=args.timeout,
112 | memory=args.memory, run_timeout=args.run_timeout, dictionary=built_dict, use_qemu=args.use_qemu,
113 | resume=args.resume, target_opts=args.target_opts
114 | )
115 |
116 | # start it!
117 | print ("[*] Starting fuzzer...")
118 | fuzzer.start()
119 | start_time = time.time()
120 |
121 | reporter = Reporter(args.binary, args.reportdir, args.afl_cores, args.first_crash, args.timeout, fuzzer.work_dir )
122 |
123 | reporter.start()
124 |
125 | if args.ipython:
126 | print ("[!]")
127 | print ("[!] Launching ipython shell. Relevant variables:")
128 | print ("[!]")
129 | print ("[!] fuzzer")
130 | if args.driller_workers and DRILLER_EXISTS:
131 | print ("[!] driller_extension")
132 | if args.grease_with:
133 | print ("[!] grease_extension")
134 | print ("[!]")
135 | import IPython; IPython.embed()
136 |
137 | try:
138 | loopcnt = 0
139 | #print ("[*] Waiting for fuzzer completion (timeout: %s, first_crash: %s)." % (args.timeout, args.first_crash))
140 | crash_seen = False
141 | reporter.enable_printing()
142 |
143 | while True:
144 |
145 | if not crash_seen and fuzzer.found_crash():
146 | # print ("\n[*] Crash found!")
147 | crash_seen = True
148 | reporter.set_crash_seen()
149 | if args.first_crash:
150 | break
151 | if fuzzer.timed_out():
152 | reporter.set_timeout_seen()
153 | print("\n[*] Timeout reached.")
154 | break
155 |
156 | time.sleep(1)
157 | loopcnt += 1
158 |
159 | except KeyboardInterrupt:
160 | end_reason = "Keyboard Interrupt"
161 | print ("\n[*] Aborting wait. Ctrl-C again for KeyboardInterrupt.")
162 | except Exception as e:
163 | end_reason = "Exception occurred"
164 | print ("\n[*] Unknown exception received (%s). Terminating fuzzer." % e)
165 | fuzzer.stop()
166 | if drill_extension:
167 | drill_extension.kill()
168 | raise
169 |
170 | print ("[*] Terminating fuzzer.")
171 | reporter.stop()
172 |
173 | fuzzer.stop()
174 | if drill_extension:
175 | drill_extension.kill()
176 |
177 | if args.tarball:
178 | print ("[*] Dumping results...")
179 | p = os.path.join("/tmp/", "afl_sync")
180 | try:
181 | shutil.rmtree(p)
182 | except (OSError, IOError):
183 | pass
184 | shutil.copytree(fuzzer.work_dir, p)
185 |
186 | tar_name = args.tarball.replace("{}", socket.gethostname())
187 |
188 | tar = tarfile.open("/tmp/afl_sync.tar.gz", "w:gz")
189 | tar.add(p, arcname=socket.gethostname()+'-'+os.path.basename(args.binary))
190 | tar.close()
191 | print ("[*] Copying out result tarball to %s" % tar_name)
192 | shutil.move("/tmp/afl_sync.tar.gz", tar_name)
193 |
194 |
195 | if __name__ == "__main__":
196 | main()
197 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/__init__.py:
--------------------------------------------------------------------------------
1 | import distutils.spawn #pylint:disable=no-name-in-module,import-error
2 | import subprocess
3 | import traceback
4 | import logging
5 | import signal
6 | import time
7 | import sys
8 | import os
9 | import re
10 | try:
11 | import angr
12 | ANGR_INSTALLED = True
13 | except ImportError:
14 | ANGR_INSTALLED = False
15 | try:
16 | import shellphish_afl
17 | SHELLPHISH_AFL_INSTALLED = True
18 | except ImportError:
19 | SHELLPHISH_AFL_INSTALLED = False
20 | try:
21 | import elftools
22 | ELFTOOLS_INSTALLED = True
23 | except ImportError:
24 | ELFTOOLS_INSTALLED = False
25 |
26 |
27 | l = logging.getLogger("phuzzer.phuzzers")
28 |
29 |
30 | class Phuzzer:
31 | """ Phuzzer object, spins up a fuzzing job on a binary """
32 |
33 | AFL_MULTICB = "AFLMULTICB"
34 | WITCHER_AFL = "WITCHERAFL"
35 | AFL = "AFL"
36 | AFL_IJON = "AFL_IJON"
37 | AFL_PLUSPLUS = "AFL++"
38 |
39 | qemu_arch_name = ""
40 | afl_bin_dir = None
41 |
42 | def __init__(self, target, seeds=None, dictionary=None, create_dictionary=False, timeout=None):
43 | """
44 | :param target: the target (i.e., path to the binary to fuzz, or a docker target)
45 | :param seeds: list of inputs to seed fuzzing with
46 | :param dictionary: a list of bytes objects to seed the dictionary with
47 | :param create_dictionary: create a dictionary from the string references in the binary
48 | :param timeout: duration to run fuzzing session
49 | """
50 |
51 | self.target = target
52 | self.target_os = ""
53 | self.target_qemu_arch = ""
54 | self.seeds = seeds or [ ]
55 |
56 | # processes spun up
57 | self.processes = [ ]
58 |
59 | self.start_time = None
60 | self.end_time = None
61 | self.timeout = timeout
62 |
63 | self.check_environment()
64 |
65 | # token dictionary
66 | self.dictionary = dictionary or (self.create_dictionary() if create_dictionary else [])
67 |
68 | @staticmethod
69 | def phactory(*args, **kwargs):
70 | if len(args) < 1 and 'phuzzer_type' not in kwargs:
71 | raise TypeError("The phactory() requires 'type' argument")
72 | if len(args) > 1:
73 | raise TypeError("The phactory() allows only 1 positional argument")
74 |
75 | if len(args) == 1:
76 | classtype = args[0]
77 | else:
78 | classtype = kwargs.get('phuzzer_type')
79 | del kwargs['phuzzer_type']
80 | classtype = classtype.upper()
81 |
82 | if classtype == Phuzzer.AFL:
83 | from .afl import AFL
84 | return AFL(**kwargs)
85 | elif classtype == Phuzzer.AFL_MULTICB:
86 | from .afl_multicb import AFLMultiCB
87 | return AFLMultiCB(**kwargs)
88 | elif classtype == Phuzzer.WITCHER_AFL:
89 | from .witcherafl import WitcherAFL
90 | return WitcherAFL(**kwargs)
91 | elif classtype == Phuzzer.AFL_IJON:
92 | from .afl_ijon import AFLIJON
93 | return AFLIJON(**kwargs)
94 | elif classtype == Phuzzer.AFL_PLUSPLUS:
95 | from .afl_plusplus import AFLPlusPlus
96 | return AFLPlusPlus(**kwargs)
97 | else:
98 | raise ValueError(f"Fuzzer type {classtype} is not found.")
99 |
100 | #
101 | # Some convenience functionality.
102 | #
103 |
104 | def found_crash(self):
105 | return len(self.crashes()) > 0
106 |
107 | def add_cores(self, n):
108 | for _ in range(n):
109 | self.add_core()
110 |
111 | def remove_cores(self, n):
112 | """
113 | remove multiple fuzzers
114 | """
115 | for _ in range(n):
116 | self.remove_core()
117 |
118 | def timed_out(self):
119 | if self.timeout is None:
120 | return False
121 |
122 | return time.time() - self.start_time > self.timeout
123 |
124 | def start(self):
125 | self.start_time = int(time.time())
126 | return self
127 | __enter__ = start
128 |
129 | def stop(self):
130 | self.end_time = int(time.time())
131 | if self.start_time is not None:
132 | l.info("Phuzzer %s shut down after %d seconds.", self, self.end_time - self.start_time)
133 | for p in self.processes:
134 | p.terminate()
135 | p.wait()
136 | __exit__ = stop
137 |
138 | @staticmethod
139 | def init_afl_config(binary_path, is_multicb=False):
140 | """
141 | Returns AFL_PATH and AFL_DIR, if AFL_PATH is set in os.environ it returns that, if not it attempts to auto-detect
142 | :param binary_path:
143 | :return: afl_path_var, afl_dir, qemu_arch_name: afl_path_var is location of qemu_trace to use, afl_dir is the location of the afl binaries, qemu_arch_name is the name of the binary's architecture
144 | """
145 |
146 | if Phuzzer.afl_bin_dir is not None:
147 | return Phuzzer.afl_bin_dir, Phuzzer.qemu_arch_name
148 |
149 | if "AFL_PATH" in os.environ:
150 | Phuzzer.afl_bin_dir = os.environ["AFL_PATH"]
151 | else:
152 |
153 | if not ANGR_INSTALLED:
154 | raise ModuleNotFoundError("AFL_PATH was found in enviornment variables and angr is not installed.")
155 | if not SHELLPHISH_AFL_INSTALLED:
156 | raise ModuleNotFoundError(
157 | "AFL_PATH was found in enviornment variables and either shellphish_afl is not installed.")
158 | try:
159 | p = angr.Project(binary_path)
160 | Phuzzer.qemu_arch_name = p.arch.qemu_name
161 | tracer_id = 'cgc' if p.loader.main_object.os == 'cgc' else p.arch.qemu_name
162 | if is_multicb:
163 | tracer_id = 'multi-{}'.format(tracer_id)
164 |
165 | afl_path_var = shellphish_afl.afl_path_var(tracer_id)
166 | os.environ['AFL_PATH'] = afl_path_var
167 |
168 | Phuzzer.afl_bin_dir = shellphish_afl.afl_dir(tracer_id)
169 | print(f"afl_dir {Phuzzer.afl_bin_dir}")
170 |
171 | except Exception:
172 |
173 | traceback.format_exc()
174 | raise ModuleNotFoundError("AFL_PATH was found in enviornment variables and "
175 | "either angr or shellphish_afl is not installed.")
176 |
177 | return Phuzzer.afl_bin_dir, Phuzzer.qemu_arch_name
178 |
179 | @classmethod
180 | def check_environment(cls):
181 | try:
182 | cls._check_environment()
183 | except InstallError as e:
184 | l.error("Your system is misconfigured for fuzzing! Please run the following commands to fix this issue:\n%s", e.args[0])
185 | raise
186 |
187 |
188 | #
189 | # Dictionary creation
190 | #
191 | def create_dictionary(self):
192 | if ANGR_INSTALLED:
193 | return self.create_dictionary_angr()
194 | elif ELFTOOLS_INSTALLED:
195 | return self.create_dictionary_elftools()
196 | else:
197 | raise ModuleNotFoundError("Cannot create a dictionary without angr or elftools being installed")
198 |
199 | def create_dictionary_elftools(self):
200 | from elftools.elf.elffile import ELFFile
201 | MAX = 120
202 | strings = set()
203 | with open(self.target, 'rb') as f:
204 | elf = ELFFile(f)
205 |
206 | for sec in elf.iter_sections():
207 | if sec.name not in {'.rodata'}:
208 | continue
209 | for match in re.findall(b"[a-zA-Z0-9_]{4}[a-zA-Z0-9_]*", sec.data()):
210 | t = match.decode()
211 | for i in range(0, len(t), MAX):
212 | strings.add(t[i:i + MAX])
213 | return strings
214 |
215 | def create_dictionary_angr(self):
216 |
217 | l.warning("creating a dictionary of string references within target \"%s\"", self.target)
218 |
219 | b = angr.Project(self.target, load_options={'auto_load_libs': False})
220 | cfg = b.analyses.CFG(resolve_indirect_jumps=True, collect_data_references=True)
221 | state = b.factory.blank_state()
222 |
223 | string_references = []
224 | for v in cfg._memory_data.values():
225 | if v.sort == "string" and v.size > 1:
226 | st = state.solver.eval(state.memory.load(v.address, v.size), cast_to=bytes)
227 | string_references.append((v.address, st))
228 |
229 | strings = [] if len(string_references) == 0 else list(list(zip(*string_references))[1])
230 | return strings
231 |
232 |
233 | #
234 | # Subclasses should override this.
235 | #
236 |
237 | @staticmethod
238 | def _check_environment():
239 | raise NotImplementedError()
240 |
241 | def crashes(self, signals=(signal.SIGSEGV, signal.SIGILL)):
242 | """
243 | Retrieve the crashes discovered by AFL. Since we are now detecting flag
244 | page leaks (via SIGUSR1) we will not return these leaks as crashes.
245 | Instead, these 'crashes' can be found with the leaks function.
246 |
247 | :param signals: list of valid kill signal numbers to override the default (SIGSEGV and SIGILL)
248 | :return: a list of strings which are crashing inputs
249 | """
250 | raise NotImplementedError()
251 |
252 | def queue(self, fuzzer='fuzzer-master'):
253 | """
254 | retrieve the current queue of inputs from a fuzzer
255 | :return: a list of strings which represent a fuzzer's queue
256 | """
257 | raise NotImplementedError()
258 |
259 | def pollenate(self, *testcases):
260 | """
261 | pollenate a fuzzing job with new testcases
262 |
263 | :param testcases: list of bytes objects representing new inputs to introduce
264 | """
265 | raise NotImplementedError()
266 |
267 | def add_core(self):
268 | raise NotImplementedError()
269 |
270 | def remove_core(self):
271 | raise NotImplementedError()
272 |
273 | def __del__(self):
274 | self.stop()
275 |
276 | from ..errors import InstallError
277 |
--------------------------------------------------------------------------------
/phuzzer/extensions/extender.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | import time
4 | import random
5 | import struct
6 | import tempfile
7 | import subprocess
8 | from ..showmap import Showmap
9 | import logging
10 |
11 | try:
12 | import shellphish_qemu
13 | SHELLPHISH_QEMU = True
14 | except ImportError:
15 | SHELLPHISH_QEMU = False
16 |
17 | l = logging.getLogger("fuzzer.extensions.Extender")
18 |
19 |
20 | class Extender:
21 |
22 | def __init__(self, binary, sync_dir):
23 |
24 | self.binary = binary
25 | self.sync_dir = sync_dir
26 |
27 | self.current_fuzzer = None
28 |
29 | self.crash_count = 0
30 | self.test_count = 0
31 |
32 | self.name = self.__class__.__name__.lower()
33 |
34 | directories = [os.path.join(self.sync_dir, self.name),
35 | os.path.join(self.sync_dir, self.name, "crashes"),
36 | os.path.join(self.sync_dir, self.name, "queue"),
37 | os.path.join(self.sync_dir, self.name, ".synced")]
38 |
39 | self.crash_bitmap = dict()
40 |
41 | for directory in directories:
42 | try:
43 | os.makedirs(directory)
44 | except OSError:
45 | continue
46 |
47 | l.debug("Fuzzer extension %s initialized", self.name)
48 |
49 | def _current_sync_count(self, fuzzer):
50 | """
51 | Get the current number of inputs belonging to `fuzzer` which we've already mutated.
52 | """
53 |
54 | sync_file = os.path.join(self.sync_dir, self.name, ".synced", fuzzer)
55 | if os.path.exists(sync_file):
56 | with open(sync_file, 'rb') as f:
57 | sc = struct.unpack(" self.crash_bitmap[i]:
177 | interesting = True
178 | self.crash_bitmap[i] = shownmap[i]
179 |
180 | return interesting
181 |
182 | @staticmethod
183 | def _interesting_test(shownmap, bitmap):
184 |
185 | for i in shownmap.keys():
186 | if shownmap[i] > (ord(bitmap[i]) ^ 0xff):
187 | return True
188 |
189 | return False
190 |
191 | def _submit_test(self, test_input, bitmap):
192 |
193 | sm = Showmap(self.binary, test_input)
194 | shownmap = sm.showmap()
195 |
196 | if sm.causes_crash and self._interesting_crash(shownmap):
197 | self._new_crash(test_input)
198 | l.info("Found a new crash (length %d)", len(test_input))
199 | elif not sm.causes_crash and self._interesting_test(shownmap, bitmap):
200 | self._new_test(test_input)
201 | l.info("Found an interesting new input (length %d)", len(test_input))
202 | else:
203 | l.debug("Found a dud")
204 |
205 | @staticmethod
206 | def _new_mutation(payload, extend_amount):
207 |
208 | def random_string(n):
209 | return bytes(random.choice(list(range(1, 9)) + list(range(11, 256))) for _ in range(n))
210 |
211 | np = payload + random_string(extend_amount + random.randint(0, 0x1000))
212 | l.debug("New mutation of length %d", len(np))
213 |
214 | return np
215 |
216 | def _mutate(self, r, bitmap):
217 |
218 | receive_counts = self._get_receive_counts(r)
219 |
220 | for numerator, denominator in receive_counts:
221 | if numerator != denominator:
222 | extend_by = denominator - numerator
223 |
224 | if extend_by > 1000000:
225 | l.warning("Amount to extend is greater than 1,000,000, refusing to perform extension")
226 | continue
227 |
228 | for _ in range(10):
229 | test_input = self._new_mutation(r, extend_by)
230 | self._submit_test(test_input, bitmap)
231 |
232 | def _do_round(self):
233 | """
234 | Single round of extending mutations.
235 | """
236 |
237 | def _extract_number(iname):
238 | attrs = dict(map(lambda x: (x[0], x[-1]), map(lambda y: y.split(":"), iname.split(","))))
239 | if "id" in attrs:
240 | return int(attrs["id"])
241 | return 0
242 |
243 | for fuzzer in os.listdir(self.sync_dir):
244 | if fuzzer == self.name:
245 | continue
246 | l.debug("Looking to extend inputs in fuzzer '%s'", fuzzer)
247 |
248 | self.current_fuzzer = fuzzer
249 | synced = self._current_sync_count(fuzzer)
250 | c_synced = self._current_crash_sync_count(fuzzer)
251 |
252 | l.debug("Already worked on %d inputs from fuzzer '%s'", synced, fuzzer)
253 |
254 | bitmap = self._current_bitmap(fuzzer)
255 |
256 | # no bitmap, fuzzer probably hasn't started
257 | if bitmap is None:
258 | l.warning("No bitmap for fuzzer '%s', skipping", fuzzer)
259 | continue
260 |
261 | queue_dir = os.path.join(self.sync_dir, fuzzer, "queue")
262 |
263 | queue_l = [n for n in os.listdir(queue_dir) if n != '.state']
264 | new_q = [i for i in queue_l if _extract_number(i) >= synced]
265 |
266 | crash_dir = os.path.join(self.sync_dir, fuzzer, "crashes")
267 | crash_l = [n for n in os.listdir(crash_dir) if n != 'README.txt']
268 | new_c = [i for i in crash_l if _extract_number(i) >= c_synced]
269 | new = new_q + new_c
270 | if len(new):
271 | l.info("Found %d new inputs to extend", len(new))
272 |
273 | for ninput in new_q:
274 | n_path = os.path.join(queue_dir, ninput)
275 | with open(n_path, "rb") as f:
276 | self._mutate(f.read(), bitmap)
277 |
278 | for ninput in new_c:
279 | n_path = os.path.join(crash_dir, ninput)
280 | with open(n_path, "rb") as f:
281 | self._mutate(f.read(), bitmap)
282 |
283 | self._update_sync_count(fuzzer, len(queue_l))
284 | self._update_crash_sync_count(fuzzer, len(crash_l))
285 |
286 | def run(self):
287 |
288 | while True:
289 | self._do_round()
290 | time.sleep(3)
291 |
292 | if __name__ == "__main__":
293 | l.setLevel("INFO")
294 |
295 | if len(sys.argv) > 2:
296 | b = sys.argv[1]
297 | s = sys.argv[2]
298 |
299 | e = Extender(b, s)
300 | e.run()
301 | else:
302 | sys.exit(1)
303 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/witcherafl.py:
--------------------------------------------------------------------------------
1 |
2 | from queue import Queue, Empty
3 | from threading import Thread
4 | from .afl import AFL
5 | import json
6 | import os
7 | import re
8 | import subprocess
9 | import shutil
10 | import time
11 | import stat
12 | import glob
13 | import logging
14 | import urllib.request
15 |
16 | l = logging.getLogger("phuzzer.phuzzers.wafl")
17 | l.setLevel(logging.INFO)
18 |
19 | class WitcherAFL(AFL):
20 | """ WitcherAFL launches the web fuzzer building on the AFL object """
21 |
22 | def __init__(
23 | self, target, seeds=None, dictionary=None, create_dictionary=None,
24 | work_dir=None, resume=False,
25 | afl_count=1, memory="8G", timeout=None,
26 | target_opts=None, extra_opts=None,
27 | crash_mode=False, use_qemu=True,
28 | run_timeout=None, login_json_fn=""
29 | ):
30 | """
31 | :param target: path to the script to fuzz (from AFL)
32 | :param seeds: list of inputs to seed fuzzing with (from AFL)
33 | :param dictionary: a list of bytes objects to seed the dictionary with (from AFL)
34 | :param create_dictionary: create a dictionary from the string references in the binary (from AFL)
35 | :param work_dir: the work directory which contains fuzzing jobs, our job directory will go here (from AFL)
36 |
37 | :param resume: resume the prior run, if possible (from AFL)
38 | :param afl_count:
39 |
40 | :param memory: AFL child process memory limit (default: "8G")
41 | :param afl_count: number of AFL jobs total to spin up for the binary
42 | :param timeout: timeout for individual runs within AFL
43 |
44 | :param library_path: library path to use, if none is specified a default is chosen
45 | :param target_opts: extra options to pass to the target
46 | :param extra_opts: extra options to pass to AFL when starting up
47 |
48 | :param crash_mode: if set to True AFL is set to crash explorer mode, and seed will be expected to be a crashing input
49 | :param use_qemu: Utilize QEMU for instrumentation of binary.
50 |
51 | :param run_timeout: amount of time for AFL to wait for a single execution to finish
52 | :param login_json_fn: login configuration file path for automatically craeting a login session and performing other initial tasks
53 |
54 | """
55 | super().__init__(
56 | target=target, work_dir=work_dir, seeds=seeds, afl_count=afl_count,
57 | create_dictionary=create_dictionary, timeout=timeout,
58 | memory=memory, dictionary=dictionary, use_qemu=use_qemu,
59 | target_opts=target_opts, resume=resume, crash_mode=crash_mode, extra_opts=extra_opts,
60 | run_timeout=run_timeout
61 | )
62 |
63 | self.login_json_fn = login_json_fn
64 |
65 | self.used_sessions = set()
66 | self.session_name = ""
67 | self.bearer = ""
68 |
69 | if "AFL_PATH" in os.environ:
70 | afl_fuzz_bin = os.path.join(os.environ['AFL_PATH'], "afl-fuzz")
71 | if os.path.exists(afl_fuzz_bin):
72 | self.afl_path = afl_fuzz_bin
73 | else:
74 | raise ValueError(
75 | f"error, have AFL_PATH but cannot find afl-fuzz at {os.environ['AFL_PATH']} with {afl_fuzz_bin}")
76 |
77 | def _start_afl_instance(self, instance_cnt=0):
78 |
79 | args, fuzzer_id = self.build_args()
80 |
81 | my_env = os.environ.copy()
82 |
83 | target_opts = []
84 | for op in self.target_opts:
85 | target_opts.append(op.replace("~~", "--").replace("@PORT@", my_env["PORT"]))
86 | args += target_opts
87 |
88 | self._get_login(my_env)
89 |
90 | my_env["AFL_BASE"] = os.path.join(self.work_dir, fuzzer_id)
91 | my_env["STRICT"] = "true"
92 |
93 | if "METHOD" not in my_env:
94 | my_env["METHOD"] = "POST"
95 |
96 | # print(f"[WC] my word dir {self.work_dir} AFL_BASE={my_env['AFL_BASE']}")
97 |
98 | self.log_command(args, fuzzer_id, my_env)
99 |
100 | logpath = os.path.join(self.work_dir, fuzzer_id + ".log")
101 | l.debug("execing: %s > %s", ' '.join(args), logpath)
102 |
103 | # set core affinity if environment variable is set
104 | if "AFL_SET_AFFINITY" in my_env:
105 | tempint = int(my_env["AFL_SET_AFFINITY"])
106 | tempint += instance_cnt
107 | my_env["AFL_SET_AFFINITY"] = str(tempint)
108 |
109 | with open(logpath, "w") as fp:
110 | return subprocess.Popen(args, stdout=fp, stderr=fp, close_fds=True, env=my_env)
111 |
112 | def _check_for_authorized_response(self, body, headers, loginconfig):
113 | return WitcherAFL._check_body(body, loginconfig) and WitcherAFL._check_headers(headers, loginconfig)
114 |
115 | @staticmethod
116 | def _check_body(self, body, loginconfig):
117 | if "positiveBody" in loginconfig and len(loginconfig["positiveBody"]) > 1:
118 | pattern = re.compile(loginconfig["positiveBody"])
119 | return pattern.search(body) is None
120 | return True
121 |
122 | @staticmethod
123 | def _check_headers(self, headers, loginconfig):
124 | if "postiveHeaders" in loginconfig:
125 | posHeaders = loginconfig["positiveHeaders"]
126 | for ph in posHeaders:
127 | for posname, posvalue in ph:
128 | found = False
129 | for headername, headervalue in headers:
130 | if posname == headername and posvalue == headervalue:
131 | found = True
132 | if not found:
133 | return False
134 | return True
135 |
136 | def _save_session(self, session_cookie, loginconfig):
137 | session_cookie_locations = ["/tmp","/var/lib/php/sessions"]
138 | if "loginSessionCookie" in loginconfig:
139 | session_name = loginconfig["loginSessionCookie"]
140 | else:
141 | session_name = r".*"
142 | if "cookieLocations" in loginconfig:
143 | for cl in loginconfig["cookeLocations"]:
144 | session_cookie_locations.append(cl)
145 |
146 | sessidrex = re.compile(rf"{session_name}=(?P[a-z0-9]{{24,40}})")
147 | sessid = sessidrex.match(session_cookie).group("sessid")
148 | if not sessid:
149 | return False
150 |
151 | # print("[WC] sessidrex " + sessid)
152 | actual_sess_fn = ""
153 | for f in session_cookie_locations:
154 |
155 | sfile = f"*{sessid}"
156 | sesmask = os.path.join(f,sfile)
157 | for sfn in glob.glob(sesmask):
158 | if os.path.isfile(sfn):
159 | actual_sess_fn = sfn
160 | break
161 | if len(actual_sess_fn) > 0:
162 | break
163 |
164 | if len(actual_sess_fn) == 0:
165 | return False
166 |
167 | saved_sess_fn = f"/tmp/save_{sessid}"
168 | if os.path.isfile(actual_sess_fn):
169 | shutil.copyfile(actual_sess_fn, saved_sess_fn)
170 | os.chmod(saved_sess_fn, stat.S_IRWXO | stat.S_IRWXG | stat.S_IRWXU)
171 | self.used_sessions.add(saved_sess_fn)
172 | return True
173 | return False
174 |
175 | def _extract_authdata(self, headers, loginconfig):
176 | authdata = []
177 | for headername, headervalue in headers:
178 | if headername.upper() == "SET-COOKIE":
179 | # Uses special authdata header so that the value prepends all other cookie values and
180 | # random data from AFL does not interfere
181 |
182 | if self._save_session(headervalue, loginconfig):
183 | authdata.append(("LOGIN_COOKIE", headervalue))
184 |
185 |
186 | if headername.upper() == "AUTHORIZATION":
187 | self.bearer = [(headername, headervalue)]
188 | authdata.append((headername, headervalue))
189 |
190 | return authdata
191 |
192 | def _do_local_cgi_req_login(self, loginconfig):
193 |
194 | login_cmd = [loginconfig["cgiBinary"]]
195 |
196 | # print("[WC] \033[34m starting with command " + str(login_cmd) + "\033[0m")
197 | myenv = os.environ.copy()
198 | if "AFL_BASE" in myenv:
199 | del myenv["AFL_BASE"]
200 |
201 | myenv["METHOD"] = loginconfig["method"]
202 | myenv["STRICT"] = "1"
203 | myenv["SCRIPT_FILENAME"] = loginconfig["url"]
204 |
205 | if "afl_preload" in loginconfig:
206 | myenv["LD_PRELOAD"] = loginconfig["afl_preload"]
207 | if "ld_library_path" in loginconfig:
208 | myenv["LD_LIBRARY_PATH"] = loginconfig["ld_library_path"]
209 |
210 | cookieData = loginconfig["cookieData"] if "cookieData" in loginconfig else ""
211 | getData = loginconfig["getData"] if "getData" in loginconfig else ""
212 | postData = loginconfig["postData"] if "postData" in loginconfig else ""
213 |
214 | httpdata = f'{cookieData}\x00{getData}\x00{postData}\x00'
215 |
216 | open("/tmp/login_req.dat", "wb").write(httpdata.encode())
217 |
218 | login_req_file = open("/tmp/login_req.dat", "r")
219 |
220 | p = subprocess.Popen(login_cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, stdin=login_req_file,
221 | env=myenv)
222 |
223 | nbsr = NonBlockingStreamReader(p.stdout)
224 | strout = ""
225 |
226 | while not nbsr.is_finished:
227 |
228 | line = nbsr.readline(0.1)
229 | if line is not None:
230 | inp = line.decode('latin-1')
231 | strout += inp
232 | # print("\033[32m", end="")
233 | # print(inp, end="")
234 | # print("\033[0m", end="")
235 |
236 | p.wait()
237 |
238 | headers = []
239 | body = ""
240 | inbody = False
241 | for respline in strout.splitlines():
242 | if len(respline) == 0:
243 | inbody = True
244 | continue
245 | if inbody:
246 | body += respline + "\n"
247 | else:
248 | header = respline.split(":")
249 | if len(header) > 1 and inbody:
250 | headername = header[0].strip()
251 | headerval = ":".join(header[1:])
252 | headerval = headerval.lstrip()
253 | headers.append((headername, headerval))
254 |
255 | if not self._check_for_authorized_response(body, headers, loginconfig):
256 | return []
257 |
258 | return self._extract_authdata(headers, loginconfig)
259 |
260 | def _do_http_req_login(self, loginconfig):
261 |
262 | url = loginconfig["url"]
263 |
264 | if "getData" in loginconfig:
265 | url += f"?{loginconfig['getData']}"
266 |
267 | post_data = loginconfig["postData"] if "postData" in loginconfig else ""
268 | post_data = post_data.encode('ascii')
269 |
270 | req_headers = loginconfig["headers"] if "headers" in loginconfig else {}
271 | opener = urllib.request.build_opener(NoRedirection)
272 | urllib.request.install_opener(opener)
273 |
274 | req = urllib.request.Request(url, post_data, req_headers)
275 | response = urllib.request.urlopen(req)
276 |
277 | headers = response.getheaders()
278 | body = response.read()
279 |
280 | if not self._check_for_authorized_response(body, headers, loginconfig):
281 | return []
282 |
283 | return self._extract_authdata(headers, loginconfig)
284 |
285 | @staticmethod
286 | def _do_authorized_requests(self, loginconfig, authdata):
287 | extra_requests = loginconfig["extra_authorized_requests"] if "postData" in loginconfig else []
288 |
289 | for auth_request in extra_requests:
290 | url = auth_request["url"]
291 |
292 | if "getData" in auth_request:
293 | url += f"?{auth_request['getData']}"
294 |
295 | post_data = auth_request["postData"] if "postData" in auth_request else ""
296 | post_data = post_data.encode('ascii')
297 |
298 | req_headers = auth_request["headers"] if "headers" in auth_request else {}
299 | for adname, advalue in authdata:
300 | adname = adname.replace("LOGIN_COOKIE","Cookie")
301 | req_headers[adname] = advalue
302 | req = urllib.request.Request(url, post_data, req_headers)
303 | urllib.request.urlopen(req)
304 |
305 | def _get_login(self, my_env):
306 | if self.login_json_fn == "":
307 | return
308 | if len(self.bearer) > 0:
309 | for bname, bvalue in self.bearer:
310 | my_env[bname] = bvalue
311 | return
312 |
313 | with open(self.login_json_fn, "r") as jfile:
314 | jdata = json.load(jfile)
315 | if jdata["direct"]["url"] == "NO_LOGIN":
316 | return
317 | loginconfig = jdata["direct"]
318 |
319 | saved_session_id = self._get_saved_session()
320 | if len(saved_session_id) > 0:
321 | saved_session_name = loginconfig["loginSessionCookie"]
322 | my_env["LOGIN_COOKIE"] = f"{saved_session_name}:{saved_session_id}"
323 | return
324 |
325 | authdata = None
326 | for _ in range(0, 10):
327 | if loginconfig["url"].startswith("http"):
328 | authdata = self._do_http_req_login(loginconfig)
329 | WitcherAFL._do_authorized_requests(loginconfig, authdata)
330 | else:
331 | authdata = self._do_local_cgi_req_login(loginconfig)
332 | if authdata is not None:
333 | break
334 | time.sleep(5)
335 |
336 | if authdata is None:
337 | raise ValueError("Login failed to return authenticated cookie/bearer value")
338 |
339 | for authname, authvalue in authdata:
340 |
341 | my_env[authname] = authvalue
342 |
343 | def _get_saved_session(self):
344 | # if we have an unused session file, we are done for this worker.
345 | for saved_sess_fn in glob.iglob("/tmp/save_????????????????????*"):
346 | if saved_sess_fn not in self.used_sessions:
347 | sess_fn = saved_sess_fn.replace("save", "sess")
348 | # print("sess_fn=" + sess_fn)
349 | self.used_sessions.add(saved_sess_fn)
350 | shutil.copyfile(saved_sess_fn, sess_fn)
351 |
352 | saved_session_id = saved_sess_fn.split("_")[1]
353 | return saved_session_id
354 | return ""
355 |
356 |
357 | class NoRedirection(urllib.request.HTTPErrorProcessor):
358 |
359 | def http_response(self, request, response):
360 | return response
361 |
362 | https_response = http_response
363 |
364 |
365 | class NonBlockingStreamReader:
366 |
367 | def __init__(self, stream):
368 | '''
369 | stream: the stream to read from.
370 | Usually a process' stdout or stderr.
371 | '''
372 |
373 | self._s = stream
374 | self._q = Queue()
375 | self._finished = False
376 |
377 | def _populateQueue(stream, queue):
378 | '''
379 | Collect lines from 'stream' and put them in 'quque'.
380 | '''
381 |
382 | while True:
383 | line = stream.readline()
384 | if line:
385 | queue.put(line)
386 | else:
387 | self._finished = True
388 | #raise UnexpectedEndOfStream
389 |
390 | self._t = Thread(target = _populateQueue,
391 | args = (self._s, self._q))
392 | self._t.daemon = True
393 | self._t.start() #start collecting lines from the stream
394 |
395 | @property
396 | def is_finished(self):
397 | return self._finished
398 |
399 | def readline(self, timeout = None):
400 | try:
401 | if self._finished:
402 | return None
403 | return self._q.get(block = timeout is not None,
404 | timeout = timeout)
405 | except Empty:
406 | return None
407 |
408 |
409 | class UnexpectedEndOfStream(Exception):
410 | pass
411 |
--------------------------------------------------------------------------------
/phuzzer/phuzzers/afl.py:
--------------------------------------------------------------------------------
1 | import contextlib
2 | import glob
3 | import logging
4 | import os
5 | import shutil
6 | import signal
7 | import subprocess
8 |
9 | from collections import defaultdict
10 | from ..errors import InstallError
11 | from ..util import hexescape
12 | from . import Phuzzer
13 |
14 | l = logging.getLogger("phuzzer.phuzzers.afl")
15 |
16 |
17 | class AFL(Phuzzer):
18 | """ Phuzzer object, spins up a fuzzing job on a binary """
19 |
20 | def __init__(
21 | self, target, seeds=None, dictionary=None, create_dictionary=None,
22 | work_dir=None, seeds_dir=None, resume=False,
23 | afl_count=1, memory="8G", timeout=None,
24 | library_path=None, target_opts=None, extra_opts=None,
25 | crash_mode=False, use_qemu=True,
26 | run_timeout=None
27 | ):
28 | """
29 | :param target: path to the binary to fuzz. List or tuple for multi-CB.
30 | :param seeds: list of inputs to seed fuzzing with
31 | :param dictionary: a list of bytes objects to seed the dictionary with
32 | :param create_dictionary: create a dictionary from the string references in the binary
33 |
34 | :param work_dir: the work directory which contains fuzzing jobs, our job directory will go here
35 | :param resume: resume the prior run, if possible
36 |
37 | :param memory: AFL child process memory limit (default: "8G")
38 | :param afl_count: number of AFL jobs total to spin up for the binary
39 | :param timeout: timeout for individual runs within AFL
40 |
41 | :param library_path: library path to use, if none is specified a default is chosen
42 | :param target_opts: extra options to pass to the target
43 | :param extra_opts: extra options to pass to AFL when starting up
44 |
45 | :param crash_mode: if set to True AFL is set to crash explorer mode, and seed will be expected to be a
46 | crashing input :param use_qemu: Utilize QEMU for instrumentation of binary.
47 |
48 | :param run_timeout: amount of time for AFL to wait for a single execution to finish
49 |
50 | """
51 | super().__init__(target=target, seeds=seeds, dictionary=dictionary, create_dictionary=create_dictionary,
52 | timeout=timeout)
53 |
54 | self.work_dir = work_dir or os.path.join("/tmp", "phuzzer", os.path.basename(str(target)))
55 | if resume and os.path.isdir(self.work_dir):
56 | self.in_dir = "-"
57 | else:
58 | l.info("could resume, but starting over upon request")
59 | with contextlib.suppress(FileNotFoundError):
60 | shutil.rmtree(self.work_dir)
61 | self.in_dir = seeds_dir or os.path.join(self.work_dir, "initial_seeds")
62 | with contextlib.suppress(FileExistsError):
63 | os.makedirs(self.in_dir)
64 |
65 | self.afl_count = afl_count
66 | self.memory = memory
67 |
68 | self.library_path = library_path
69 | self.target_opts = target_opts or []
70 | self.extra_opts = extra_opts if type(extra_opts) is list else extra_opts.split() if type(
71 | extra_opts) is str else []
72 |
73 | self.crash_mode = crash_mode
74 | self.use_qemu = use_qemu
75 |
76 | self.run_timeout = run_timeout
77 |
78 | # sanity check crash mode
79 | if self.crash_mode:
80 | if seeds is None:
81 | raise ValueError("Seeds must be specified if using the fuzzer in crash mode")
82 | l.info("AFL will be started in crash mode")
83 |
84 | # set up the paths
85 | self.afl_phuzzer_bin_path = self.choose_afl()
86 |
87 | #
88 | # Overrides
89 | #
90 |
91 | def create_dictionary(self):
92 | d = super().create_dictionary()
93 |
94 | # AFL has a limit of 128 bytes per dictionary entries
95 | valid_strings = []
96 | for s in d:
97 | if len(s) <= 128:
98 | valid_strings.append(s)
99 | for s_atom in s.split():
100 | if len(s_atom) <= 128:
101 | valid_strings.append(s_atom)
102 | else:
103 | valid_strings.append(s[:128])
104 |
105 | return valid_strings
106 |
107 | #
108 | # AFL functionality
109 | #
110 |
111 | @property
112 | def dictionary_file(self):
113 | return os.path.join(self.work_dir, "dict.txt")
114 |
115 | def start(self):
116 | """
117 | start fuzzing
118 | """
119 |
120 | super().start()
121 |
122 | # check for the existence of the AFL Directory
123 | if self.afl_bin_dir is None or not os.path.isdir(self.afl_bin_dir):
124 | l.error("AFL Bin Directory does not exist at: %s.", self.afl_bin_dir)
125 |
126 | # create the directory
127 | with contextlib.suppress(FileExistsError):
128 | os.makedirs(self.work_dir)
129 |
130 | # write the dictionary
131 | if self.dictionary:
132 | with open(self.dictionary_file, "w") as df:
133 | for i, s in enumerate(set(self.dictionary)):
134 | if len(s) == 0:
135 | continue
136 | s_val = hexescape(s)
137 | df.write("string_%d=\"%s\"" % (i, s_val) + "\n")
138 |
139 | # write the seeds
140 | if self.in_dir != "-":
141 | if not self.seeds:
142 | l.warning("No seeds provided - using 'fuzz'")
143 | template = os.path.join(self.in_dir, "seed-%d")
144 | for i, seed in enumerate(self.seeds or [b"fuzz"]):
145 | with open(template % i, "wb") as f:
146 | f.write(seed)
147 |
148 | # spin up the master AFL instance
149 | master = self._start_afl_instance() # the master fuzzer
150 | self.processes.append(master)
151 |
152 | # only spins up an AFL instances if afl_count > 1
153 | for i in range(1, self.afl_count):
154 | self.processes.append(self._start_afl_instance(i))
155 |
156 | return self
157 |
158 | @property
159 | def alive(self):
160 | if not len(self.stats):
161 | return False
162 |
163 | alive_cnt = 0
164 | for fuzzer in self.stats:
165 | try:
166 | os.kill(int(self.stats[fuzzer]['fuzzer_pid']), 0)
167 | alive_cnt += 1
168 | except (OSError, KeyError):
169 | pass
170 |
171 | return bool(alive_cnt)
172 |
173 | @property
174 | def summary_stats(self):
175 | stats = self.stats
176 | summary_stats = defaultdict(lambda: 0)
177 | for _, fuzzstats in stats.items():
178 | for fstat, value in fuzzstats.items():
179 | try:
180 | fvalue = float(value)
181 | if fstat == "paths_total":
182 | summary_stats[fstat] = max(summary_stats[fstat], int(fvalue))
183 | else:
184 | summary_stats[fstat] += fvalue
185 | except ValueError:
186 | pass
187 | return summary_stats
188 |
189 | @property
190 | def stats(self):
191 |
192 | # collect stats into dictionary
193 | stats = {}
194 | if os.path.isdir(self.work_dir):
195 | for fuzzer_dir in os.listdir(self.work_dir):
196 | stat_path = os.path.join(self.work_dir, fuzzer_dir, "fuzzer_stats")
197 | if os.path.isfile(stat_path):
198 | stats[fuzzer_dir] = {}
199 |
200 | with open(stat_path, "r") as f:
201 | stat_blob = f.read()
202 | stat_lines = stat_blob.split("\n")[:-1]
203 | for stat in stat_lines:
204 | if ":" in stat:
205 | try:
206 |
207 | key, val = stat.split(":")
208 | except:
209 | index = stat.find(":")
210 | key = stat[:index]
211 | val = stat[index + 1:]
212 |
213 | else:
214 | print(f"Skipping stat '${stat}' in \n${stat_lines} because no split value")
215 | continue
216 | stats[fuzzer_dir][key.strip()] = val.strip()
217 |
218 | return stats
219 |
220 | #
221 | # Helpers
222 | #
223 |
224 | def _get_crashing_inputs(self, signals):
225 | """
226 | Retrieve the crashes discovered by AFL. Only return those crashes which
227 | recieved a signal within 'signals' as the kill signal.
228 |
229 | :param signals: list of valid kill signal numbers
230 | :return: a list of strings which are crashing inputs
231 | """
232 |
233 | crashes = set()
234 | for fuzzer in os.listdir(self.work_dir):
235 | crashes_dir = glob.iglob(f"{self.work_dir}/fuzzer-*/crashes*")
236 | for crash_dir in crashes_dir:
237 | if not os.path.isdir(crash_dir):
238 | # if this entry doesn't have a crashes directory, just skip it
239 | continue
240 |
241 | for crash in os.listdir(crash_dir):
242 | if crash == "README.txt":
243 | # skip the readme entry
244 | continue
245 |
246 | attrs = dict(map(lambda x: (x[0], x[-1]), map(lambda y: y.split(":"), crash.split(","))))
247 |
248 | if int(attrs['sig']) not in signals:
249 | continue
250 |
251 | crash_path = os.path.join(crash_dir, crash)
252 | with open(crash_path, 'rb') as f:
253 | crashes.add(f.read())
254 |
255 | return list(crashes)
256 |
257 | #
258 | # AFL-specific
259 | #
260 |
261 | def bitmap(self, fuzzer='fuzzer-master'):
262 | """
263 | retrieve the bitmap for the fuzzer `fuzzer`.
264 | :return: a string containing the contents of the bitmap.
265 | """
266 |
267 | if not fuzzer in os.listdir(self.work_dir):
268 | raise ValueError("fuzzer '%s' does not exist" % fuzzer)
269 |
270 | bitmap_path = os.path.join(self.work_dir, fuzzer, "fuzz_bitmap")
271 |
272 | bdata = None
273 | try:
274 | with open(bitmap_path, "rb") as f:
275 | bdata = f.read()
276 | except IOError:
277 | pass
278 |
279 | return bdata
280 |
281 | #
282 | # Interface
283 | #
284 |
285 | @staticmethod
286 | def _check_environment():
287 | err = ""
288 | # check for afl sensitive settings
289 | with open("/proc/sys/kernel/core_pattern") as f:
290 | if not "core" in f.read():
291 | err += "echo core | sudo tee /proc/sys/kernel/core_pattern\n"
292 |
293 | # This file is based on a driver not all systems use
294 | # http://unix.stackexchange.com/questions/153693/cant-use-userspace-cpufreq-governor-and-set-cpu-frequency
295 | # TODO: Perform similar performance check for other default drivers.
296 | if os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"):
297 | with open("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor") as f:
298 | if not "performance" in f.read():
299 | err += "echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor\n"
300 |
301 | # TODO: test, to be sure it doesn't mess things up
302 | if os.path.exists("/proc/sys/kernel/sched_child_runs_first"):
303 | with open("/proc/sys/kernel/sched_child_runs_first") as f:
304 | if not "1" in f.read():
305 | err += "echo 1 | sudo tee /proc/sys/kernel/sched_child_runs_first\n"
306 |
307 | if err:
308 | raise InstallError(err)
309 |
310 | def add_core(self):
311 | """
312 | add one fuzzer
313 | """
314 |
315 | self.processes.append(self._start_afl_instance())
316 |
317 | def remove_core(self):
318 | """
319 | remove one fuzzer
320 | """
321 |
322 | try:
323 | f = self.processes.pop()
324 | except IndexError:
325 | l.error("no fuzzer to remove")
326 | raise ValueError("no fuzzer to remove")
327 |
328 | f.kill()
329 |
330 | def crashes(self, signals=(signal.SIGSEGV, signal.SIGILL)):
331 | """
332 | Retrieve the crashes discovered by AFL. Since we are now detecting flag
333 | page leaks (via SIGUSR1) we will not return these leaks as crashes.
334 | Instead, these 'crashes' can be found with the leaks function.
335 |
336 | :param signals: list of valid kill signal numbers to override the default (SIGSEGV and SIGILL)
337 | :return: a list of strings which are crashing inputs
338 | """
339 |
340 | return self._get_crashing_inputs(signals)
341 |
342 | def queue(self, fuzzer='fuzzer-master'):
343 | """
344 | retrieve the current queue of inputs from a fuzzer
345 | :return: a list of strings which represent a fuzzer's queue
346 | """
347 |
348 | if not fuzzer in os.listdir(self.work_dir):
349 | raise ValueError(f"fuzzer '{fuzzer}' does not exist")
350 |
351 | queue_path = os.path.join(self.work_dir, fuzzer, 'queue')
352 | queue_files = list(filter(lambda x: x != ".state", os.listdir(queue_path)))
353 |
354 | queue_l = []
355 | for q in queue_files:
356 | with open(os.path.join(queue_path, q), 'rb') as f:
357 | queue_l.append(f.read())
358 |
359 | return queue_l
360 |
361 | def pollenate(self, *testcases):
362 | """
363 | pollenate a fuzzing job with new testcases
364 |
365 | :param testcases: list of bytes objects representing new inputs to introduce
366 | """
367 |
368 | nectary_queue_directory = os.path.join(self.work_dir, 'pollen', 'queue')
369 | if not 'pollen' in os.listdir(self.work_dir):
370 | os.makedirs(nectary_queue_directory)
371 |
372 | pollen_cnt = len(os.listdir(nectary_queue_directory))
373 |
374 | for tcase in testcases:
375 | with open(os.path.join(nectary_queue_directory, "id:%06d,src:pollenation" % pollen_cnt), "wb") as f:
376 | f.write(tcase)
377 |
378 | pollen_cnt += 1
379 |
380 | #
381 | # AFL launchers
382 | #
383 | def build_args(self):
384 | args = [self.afl_phuzzer_bin_path]
385 |
386 | args += ["-i", self.in_dir]
387 | args += ["-o", self.work_dir]
388 | args += ["-m", self.memory]
389 |
390 | if self.use_qemu:
391 | args += ["-Q"]
392 |
393 | if self.crash_mode:
394 | args += ["-C"]
395 |
396 | if len(self.processes) == 0:
397 | fuzzer_id = "fuzzer-master"
398 | args += ["-M", fuzzer_id]
399 | else:
400 | fuzzer_id = f"fuzzer-{len(self.processes)}"
401 | args += ["-S", fuzzer_id]
402 |
403 | if os.path.exists(self.dictionary_file):
404 | args += ["-x", self.dictionary_file]
405 |
406 | args += self.extra_opts
407 |
408 | if self.run_timeout is not None:
409 | args += ["-t", "%d+" % self.run_timeout]
410 | args += ["--"]
411 | args += [self.target]
412 | target_opts = []
413 |
414 | for op in self.target_opts:
415 | target_opts.append(op.replace("~~", "--").replace("~", "-"))
416 |
417 | args += target_opts
418 |
419 | return args, fuzzer_id
420 |
421 | def _start_afl_instance(self, instance_cnt=0):
422 |
423 | args, fuzzer_id = self.build_args()
424 |
425 | my_env = os.environ.copy()
426 |
427 | self.log_command(args, fuzzer_id, my_env)
428 |
429 | logpath = os.path.join(self.work_dir, fuzzer_id + ".log")
430 | l.debug("execing: %s > %s", ' '.join(args), logpath)
431 |
432 | with open(logpath, "w") as fp:
433 | return subprocess.Popen(args, stdout=fp, stderr=fp, close_fds=True, env=my_env)
434 |
435 | def log_command(self, args, fuzzer_id, my_env):
436 | with open(os.path.join(self.work_dir, fuzzer_id + ".cmd"), "w") as cf:
437 | cf.write(" ".join(args) + "\n")
438 | listvars = [f"{k}={v}" for k, v in my_env.items()]
439 | listvars.sort()
440 | cf.write("\n" + "\n".join(listvars))
441 |
442 | def choose_afl(self):
443 | """
444 | Chooses the right AFL and sets up some environment.
445 | """
446 |
447 | afl_dir, qemu_arch_name = Phuzzer.init_afl_config(self.target)
448 |
449 | directory = None
450 | if qemu_arch_name == "aarch64":
451 | directory = "arm64"
452 | if qemu_arch_name == "i386":
453 | directory = "i386"
454 | if qemu_arch_name == "x86_64":
455 | directory = "x86_64"
456 | if qemu_arch_name == "mips":
457 | directory = "mips"
458 | if qemu_arch_name == "mipsel":
459 | directory = "mipsel"
460 | if qemu_arch_name == "ppc":
461 | directory = "powerpc"
462 | if qemu_arch_name == "arm":
463 | # some stuff qira uses to determine the which libs to use for arm
464 | with open(self.target, "rb") as f:
465 | progdata = f.read(0x800)
466 | if b"/lib/ld-linux.so.3" in progdata:
467 | directory = "armel"
468 | elif b"/lib/ld-linux-armhf.so.3" in progdata:
469 | directory = "armhf"
470 |
471 | if directory is None and qemu_arch_name != "":
472 | l.warning("architecture \"%s\" has no installed libraries", qemu_arch_name)
473 | elif directory is not None:
474 | libpath = os.path.join(afl_dir, "..", "fuzzer-libs", directory)
475 |
476 | l.debug(f"exporting QEMU_LD_PREFIX of '{libpath}'")
477 | os.environ['QEMU_LD_PREFIX'] = libpath
478 |
479 | # return the AFL path
480 | # import ipdb
481 | # ipdb.set_trace()
482 | afl_bin = os.path.join(afl_dir, "afl-fuzz")
483 | l.debug(f"afl_bin={afl_bin}")
484 | return afl_bin
485 |
--------------------------------------------------------------------------------