├── .gitignore ├── LICENSE ├── README.md ├── bin └── yara-multi-rules.py ├── requirements.txt ├── setup.py └── yararules.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info/ 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | NOTICE 2 | 3 | This software (or technical data) was produced for the U. S. Government under 4 | contract, and is subject to the Rights in Data-General Clause 52.227-14, 5 | Alt. IV (DEC 2007) 6 | 7 | 8 | Copyright (c) 2018-2020, The MITRE Corporation. All rights reserved. 9 | Approved for Public Release; Distribution Unlimited. Case Number 18-0989 10 | 11 | 12 | Licensed under the Apache License, Version 2.0 (the "License"); 13 | you may not use this file except in compliance with the License. 14 | You may obtain a copy of the License at 15 | 16 | http://www.apache.org/licenses/LICENSE-2.0 17 | 18 | Unless required by applicable law or agreed to in writing, software 19 | distributed under the License is distributed on an "AS IS" BASIS, 20 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | See the License for the specific language governing permissions and 22 | limitations under the License. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Scan files and directories with multiple rules files, without cross-file rule name collision! 2 | 3 | Files containing rules can be provided on the command-line, as a list in one or more text 4 | files, as a directory containing (just) rules files, or in a config dir. Each option 5 | (-d -f -l) can be provided multiple times. 6 | 7 | Default output is space-separated RULE NAME, RULE FILE, and MATCH FILE. Use CSV option 8 | for comma-separated values. 9 | 10 | ## Installation 11 | ```bash 12 | pip install . 13 | ``` 14 | 15 | ## Usage 16 | ~~~~ 17 | usage: yara-multi-rules.py [-h] [-d SIGDIRS] [-f SIGFILES] [-l LISTFILES] [-v] 18 | [--csv] [-m] [--quiet] [--init] 19 | [--config-dir CONFIGDIR] [--fail-on-warnings] 20 | FILE [FILE ...] 21 | 22 | positional arguments: 23 | FILE file(s) to scan 24 | 25 | optional arguments: 26 | -h, --help show this help message and exit 27 | -d SIGDIRS Directory containing rules 28 | -f SIGFILES rule file (allowed multiple times for list) 29 | -l LISTFILES file containing path to rule files, one per line 30 | -v verbose output 31 | --csv output in CSV format 32 | -m only show matches 33 | --quiet, -q only display match/none, no informational messages 34 | --init Create a blank config (default: ~/.yara/) 35 | --config-dir CONFIGDIR 36 | Use/create configuration in given directory. 37 | --fail-on-warnings Error on warnings during rule compilation 38 | ~~~~ 39 | 40 | 41 | ## Copyright 42 | 43 | Copyright (c) 2018-2020, The MITRE Corporation. All rights reserved. 44 | 45 | Approved for Public Release; Distribution Unlimited. Case Number 18-0989 46 | -------------------------------------------------------------------------------- /bin/yara-multi-rules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Scan files with Yara rules from multiple sources with ease! 4 | 5 | https://github.com/MITRE/yararules-python 6 | 7 | 8 | Copyright (c) 2018, The MITRE Corporation. All rights reserved. 9 | """ 10 | 11 | from __future__ import print_function 12 | 13 | import os 14 | import sys 15 | import csv 16 | #import hashlib 17 | import binascii 18 | 19 | import yararules 20 | 21 | 22 | def main(args): 23 | if not args.files: 24 | return 25 | # get sig files 26 | sigfiles = [] 27 | if args.sigfiles: 28 | # copy the list 29 | sigfiles = list(args.sigfiles) 30 | if args.sigdirs: 31 | for d in args.sigdirs: 32 | # TODO: find files and apply args.filter 33 | # TODO: recurse if args.recurse 34 | for root, dirs, files in os.walk(d): 35 | for name in files: 36 | filepath = os.path.join(root, name) 37 | #print('INFO: adding sig {}'.format(filepath)) 38 | #namespace = hashlib.md5(filepath).hexdigest()[:16] 39 | sigfiles.append(filepath) 40 | # ignore git dir 41 | if '.git' in dirs: 42 | dirs.remove('.git') 43 | if args.listfiles: 44 | for lf in args.listfiles: 45 | with open(lf, 'r') as fh: 46 | for line in fh: 47 | line = line.strip() 48 | if line and not line.startswith('#'): 49 | sigfiles.append(line) 50 | if args.csv: 51 | csv_writer = csv.writer(sys.stdout) 52 | for match, filepath in yararules.match_files( 53 | args.files, 54 | sigfiles, 55 | raise_on_warn=args.error_on_warn): 56 | if args.only_matches and match.rule is None: 57 | continue 58 | if args.csv: 59 | csv_writer.writerow([match.rule, match.namespace, filepath]) 60 | else: 61 | print(match.rule, match.namespace, filepath) 62 | if args.print_strings and match.strings: 63 | for sm in match.strings: 64 | for smi in sm.instances: 65 | s = binascii.b2a_qp(smi.plaintext()) 66 | s = s.decode('ascii') 67 | s = s.replace('=','\\x') 68 | if sm.is_xor(): 69 | print(f"0x{smi.offset:x}:{sm.identifier}: {s} (xor 0x{smi.xorkey:x})") 70 | else: 71 | print(f"0x{smi.offset:x}:{sm.identifier}: {s}") 72 | 73 | if __name__ == '__main__': 74 | import argparse 75 | parser = argparse.ArgumentParser() 76 | parser.add_argument('-d', dest='sigdirs', help='Directory containing rules', action='append') 77 | #parser.add_argument('-r',dest='recurse',help='Recurse SIGDIR directory',action='store_true') 78 | #parser.add_argument('--filter',dest='filter',help='Filename filter') 79 | parser.add_argument( 80 | '-f', 81 | dest='sigfiles', 82 | help='rule file (allowed multiple times for list)', 83 | action='append') 84 | parser.add_argument( 85 | '-l', 86 | dest='listfiles', 87 | help='file containing path to rule files, one per line', 88 | action='append') 89 | parser.add_argument('-v', dest='verbose', help='verbose output', action='count') 90 | parser.add_argument('--csv', dest='csv', help='output in CSV format', action='store_true') 91 | parser.add_argument('-m', dest='only_matches', help='only show matches', action='store_true') 92 | parser.add_argument('files', metavar='FILE', nargs='+', help='file(s) to scan') 93 | parser.add_argument( 94 | '--quiet', 95 | '-q', 96 | help='only display match/none, no informational messages', 97 | action='store_true') 98 | parser.add_argument( 99 | '--init', 100 | action='store_true', 101 | help='Create a blank config (default: ~/.yara/)') 102 | parser.add_argument( 103 | '--config-dir', 104 | dest='configdir', 105 | help='Use/create configuration in given directory.') 106 | parser.add_argument( 107 | '--fail-on-warnings', 108 | dest='error_on_warn', 109 | action='store_true', 110 | default=False, 111 | help="Error on warnings during rule compilation") 112 | parser.add_argument( 113 | '--print-strings', 114 | dest='print_strings', 115 | action='store_true', 116 | default=False, 117 | help="Print strings in offset:var:string format") 118 | args = parser.parse_args() 119 | config_base = os.path.join(os.path.expanduser('~'), '.yara') 120 | if args.configdir: 121 | config_base = args.configdir 122 | if args.init: 123 | os.makedirs(os.path.join(config_base, 'rulesets')) 124 | os.makedirs(os.path.join(config_base, 'blacklists')) 125 | sys.exit(0) 126 | # if no rules given on command line 127 | if not args.sigdirs and not args.sigfiles and not args.listfiles: 128 | # check for sets in user dir 129 | if not args.quiet: 130 | print('No rulesets given; checking user-specific config...') 131 | sets_dir = os.path.join(config_base, 'rulesets') 132 | if os.path.exists(sets_dir): 133 | args.listfiles = [] 134 | for root, dirs, files in os.walk(sets_dir): 135 | for name in files: 136 | filepath = os.path.join(root, name) 137 | args.listfiles.append(filepath) 138 | if args.listfiles and not args.quiet: 139 | print('Rulesets found: {}'.format(len(args.listfiles))) 140 | if not args.listfiles and not args.quiet: 141 | print('No Rulesets found in {}'.format(config_base)) 142 | else: 143 | if not args.quiet: 144 | print('Configuration directory not found!') 145 | sys.exit(1) 146 | main(args) 147 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | yara-python 2 | python-magic 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='yararules', 5 | version='0.3.0', 6 | py_modules=['yararules'], 7 | scripts=['bin/yara-multi-rules.py'], 8 | install_requires=[ 9 | 'yara-python>=4.3.0', 10 | 'python-magic', 11 | ], 12 | ) 13 | -------------------------------------------------------------------------------- /yararules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | yararules makes using multiple sources of Yara rules easier 4 | by using sane defaults for externals and placing each rule 5 | file into its own namespace to avoid rule name conflicts. 6 | 7 | https://github.com/MITRE/yararules-python 8 | 9 | 10 | Copyright (c) 2018-2020, The MITRE Corporation. All rights reserved. 11 | """ 12 | 13 | 14 | from __future__ import print_function 15 | 16 | import os 17 | import sys 18 | import yara 19 | import magic 20 | 21 | 22 | class FakeMatch(object): 23 | """A fake Match class that mimics the yara Match object. 24 | Used to indicate no match. 25 | """ 26 | rule = None 27 | namespace = None 28 | strings = None 29 | 30 | def make_externals(filepath='', filename='', fileext='', dirname='', base_externals=None, include_type=False): 31 | """Given a file name, extension, and dir OR a full file path string, return 32 | a dictionary suitable for the yara match() function externals argument. 33 | If base_externals dictionary provided, then initialize the externals with it. 34 | 35 | The externals created by this function are: 36 | filepath 37 | filename 38 | extension 39 | """ 40 | # initialize return dict with optionally given values 41 | d = dict() 42 | if base_externals: 43 | d.update(base_externals) 44 | # if not filepath, but we do have filename and dirname 45 | if not filepath and filename and dirname: 46 | filepath = os.path.join(dirname, filename) 47 | # if no extension, but do have filename or filepath 48 | if not fileext: 49 | if filename: 50 | _, fileext = os.path.splitext(filename) 51 | elif filepath: 52 | _, fileext = os.path.splitext(filepath) 53 | # if no filename, but we have filepath 54 | if not filename and filepath: 55 | _, filename = os.path.split(filepath) 56 | ftype = '' 57 | if include_type: 58 | m = magic.from_file(filepath) 59 | if m: 60 | ftype = m.split()[0] 61 | # update return dict with common externals when processing a file 62 | d.update({'filepath': filepath, 'filename': filename, 'extension': fileext, 'filetype': ftype}) 63 | # return the computed externals 64 | return d 65 | 66 | 67 | def yara_matches(compiled_sigs, filepath, externals=None): 68 | try: 69 | if externals: 70 | matches = compiled_sigs.match(filepath, externals=externals) 71 | else: 72 | matches = compiled_sigs.match(filepath) 73 | except yara.Error: 74 | print('Exception matching on file "{}"'.format(filepath), file=sys.stderr) 75 | raise 76 | if not matches: 77 | yield FakeMatch(), filepath 78 | for m in matches: 79 | yield m, filepath 80 | 81 | 82 | def compile_files(rule_files, externals=None): 83 | """Given a list of files containing rules, return a list of warnings 84 | and a compiled object as one would receive from yara.compile(). 85 | The rules from each file are put into their own namespaces. For 86 | example, all of the rules in the '/tmp/alice.yara' file will be 87 | compiled into the '/tmp/alice.yara' namespace. This prevents 88 | rule name collisions. 89 | """ 90 | if not rule_files: 91 | return (None, None) 92 | # compile rules 93 | rules = {} 94 | warnings = list() 95 | for filepath in rule_files: 96 | rules[filepath] = filepath 97 | try: 98 | compiled_rules = yara.compile( 99 | filepaths=rules, 100 | externals=make_externals(base_externals=externals), 101 | error_on_warning=True 102 | ) 103 | except yara.WarningError as e: 104 | compiled_rules = yara.compile( 105 | filepaths=rules, 106 | externals=make_externals(base_externals=externals) 107 | ) 108 | warnings.append('{}'.format(e)) 109 | except yara.Error as e: 110 | print('Error compiling {} rules: {}'.format( 111 | len(rules), 112 | ' '.join([rules[i] for i in rules]) 113 | ), file=sys.stderr) 114 | raise 115 | return warnings, compiled_rules 116 | 117 | 118 | def match_files(files, rule_files=None, compiled_rules=None, externals=None, raise_on_warn=False): 119 | """Given iterator of files to match against and either a list of files 120 | containing rules or a compiled rules object, 121 | YIELD a tuple of matches and filename. 122 | 123 | Optionally, if given an externals dict, use that as the initial 124 | externals values. This function will add the following definitions: 125 | filename : name of file without directories 126 | filepath : full path including directories and filename 127 | extension : the filename's extension, if present 128 | """ 129 | if not compiled_rules: 130 | # compile rules 131 | try: 132 | warnings, compiled_rules = compile_files( 133 | rule_files, 134 | make_externals(base_externals=externals) 135 | ) 136 | except yara.Error as e: 137 | print( 138 | 'Error compiling {} rule files: {}'.format(len(rule_files), e), 139 | file=sys.stderr 140 | ) 141 | raise 142 | if warnings and raise_on_warn: 143 | raise Exception('\n'.join(warnings)) 144 | if not compiled_rules: 145 | raise Exception('Rules not compiled') 146 | # iterate files to scan 147 | for fname in files: 148 | if os.path.isdir(fname): 149 | for root, _, walk_files in os.walk(fname): 150 | for name in walk_files: 151 | filepath = os.path.join(root, name) 152 | extern_d = make_externals( 153 | filename=name, 154 | filepath=filepath, 155 | base_externals=externals 156 | ) 157 | for m, f in yara_matches(compiled_rules, filepath, extern_d): 158 | yield m, f 159 | else: 160 | extern_d = make_externals(filepath=fname, base_externals=externals) 161 | for m, f in yara_matches(compiled_rules, fname, extern_d): 162 | yield m, f 163 | --------------------------------------------------------------------------------