├── .gitignore ├── AUTHORS ├── CHANGELOG.md ├── CONTRIBUTORS ├── LICENSE ├── LICENSE.cti-pattern-matcher ├── LICENSE.cti-pattern-validator ├── MANIFEST.in ├── README.md ├── compile_stix_grammar.py ├── dendrol ├── __init__.py ├── debug.py ├── lang │ ├── .gitignore │ ├── STIXPattern.g4 │ ├── STIXPattern.interp │ ├── STIXPattern.tokens │ ├── STIXPatternLexer.interp │ ├── STIXPatternLexer.py │ ├── STIXPatternLexer.tokens │ ├── STIXPatternListener.py │ ├── STIXPatternParser.py │ ├── STIXPatternVisitor.py │ └── __init__.py ├── parse.py ├── transform.py ├── tz.py └── version.py ├── pytest.ini ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── test_debug.py └── test_transform.py /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | *.egg-info/ 3 | *.pyc 4 | .tox 5 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the list of dendrol authors for copyright purposes. 2 | # 3 | # This does not necessarily list everyone who has contributed code, since in 4 | # some cases, their employer may be the copyright holder. To see the full list 5 | # of contributors, see the revision history in source control. 6 | 7 | Perch Security, Inc. 8 | OASIS Open 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | 10 | ## [0.0.5] - 2019-03-06 11 | ### Fixed 12 | - Relax PyYAML requirement (see [yaml/pyyaml#152](https://github.com/yaml/pyyaml/issues/152). thank you, [@traut](https://github.com/traut)) 13 | 14 | 15 | ## [0.0.4] - 2018-11-30 16 | ### Fixed 17 | - Update logo markup for compatibility with PyPI (see [pypa/readme_renderer#126](https://github.com/pypa/readme_renderer/issues/126)) 18 | 19 | 20 | ## [0.0.3] - 2018-11-30 21 | ### Changed 22 | - Update logo 23 | 24 | 25 | ## [0.0.2] - 2018-11-26 26 | ### Added 27 | - Registered rove classifiers to _setup.py_ for display and indexing on PyPI 28 | - Added development instructions to README 29 | 30 | 31 | ## [0.0.1] - 2018-11-25 32 | ### Added 33 | - Initial commit: STIX2 pattern expression -> AST parser 34 | - Initial commit: YAML loader/dumper for AST 35 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This file contains information about people who have contributed to various 2 | # parts of the library. Names are listed in alphabetical order, by first name. 3 | 4 | 5 | Chris Lenk @clenk 6 | - STIX2 Pattern grammar 7 | - Conversions of STIX2 literals to Python primitives 8 | 9 | 10 | Danny Elliott @delliott90 11 | - STIX2 Pattern grammar 12 | 13 | 14 | Greg Back @gtback 15 | - cti-pattern-validator / cti-pattern-matcher organization 16 | 17 | 18 | Michael Chisholm @chisholm 19 | - STIX2 Pattern grammar 20 | - STIX2 Pattern ANTLR parsing routines 21 | - Conversions of STIX2 literals to Python primitives 22 | 23 | 24 | Zach Kanzler @they4kman 25 | - PatternTree Visitor 26 | - PatternTree YAML DSL 27 | - STIX2 Pattern grammar compiling tool 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Perch Security 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE.cti-pattern-matcher: -------------------------------------------------------------------------------- 1 | Copyright (c) [2016], OASIS Open 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /LICENSE.cti-pattern-validator: -------------------------------------------------------------------------------- 1 | Copyright (c) [2016], OASIS Open 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.py 2 | include README.md 3 | recursive-include dendrol *.py 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | dendrol 5 |

6 |

7 |

8 | 9 | dendrol PyPI version 10 | 11 | 12 | PyPI pyversions 13 | 14 | 15 | PyPI license 16 | 17 |

18 | 19 |

20 | dendrol parses STIX2 pattern expressions into basic Python structures 21 |

22 | 23 |

24 | 25 | Iconic STIX2 Pattern visualization 26 | 27 |

28 | 29 | This iconic [STIX2 Pattern](http://docs.oasis-open.org/cti/stix/v2.0/cs01/part5-stix-patterning/stix-v2.0-cs01-part5-stix-patterning.html#_t7hu3hrkvmff) visualization is based upon this example expression: 30 | ``` 31 | ( 32 | [ipv4-addr:value = '198.51.100.1/32' OR 33 | ipv4-addr:value = '203.0.113.33/32' OR 34 | ipv6-addr:value = '2001:0db8:dead:beef:dead:beef:dead:0001/128'] 35 | 36 | FOLLOWEDBY [ 37 | domain-name:value = 'example.com'] 38 | 39 | ) WITHIN 600 SECONDS 40 | ``` 41 | 42 | Using the formal STIX2 Pattern grammar, that expression is converted into this parse tree: 43 | 44 | ![example expression parse tree](https://user-images.githubusercontent.com/33840/47131306-eaf41b00-d26b-11e8-83ab-82a6932b95cf.png) 45 | 46 | dendrol will convert that expression (by way of that parse tree) into this more human-readable and machine-actionable form: 47 | ```yaml 48 | pattern: 49 | expression: 50 | join: FOLLOWEDBY 51 | qualifiers: 52 | - within: 53 | value: 600 54 | unit: SECONDS 55 | expressions: 56 | - observation: 57 | objects: 58 | ? ipv4-addr 59 | ? ipv6-addr 60 | join: OR 61 | qualifiers: 62 | expressions: 63 | - comparison: 64 | object: ipv4-addr 65 | path: [value] 66 | negated: 67 | operator: '=' 68 | value: 198.51.100.1/32 69 | - comparison: 70 | object: ipv4-addr 71 | path: [value] 72 | negated: 73 | operator: '=' 74 | value: 203.0.113.33/32 75 | - comparison: 76 | object: ipv6-addr 77 | path: [value] 78 | negated: 79 | operator: '=' 80 | value: 2001:0db8:dead:beef:dead:beef:dead:0001/128 81 | - observation: 82 | objects: {domain-name} 83 | join: 84 | qualifiers: 85 | expressions: 86 | - comparison: 87 | object: domain-name 88 | path: [value] 89 | negated: 90 | operator: '=' 91 | value: example.com 92 | ``` 93 | 94 | 95 | # How do I use it? 96 | 97 | dendrol provides an interface for parsing STIX2 Pattern Expressions much like [cti-pattern-validator](https://github.com/oasis-open/cti-pattern-validator), with the `dendrol.Pattern` class. This class has a method, `to_dict_tree()`, which converts the ANTLR parse tree to a dict-based tree structure, PatternTree. 98 | 99 | ```py 100 | from dendrol import Pattern 101 | 102 | pattern = Pattern("[domain-name:value = 'http://xyz.com/download']") 103 | 104 | assert pattern.to_dict_tree() == { 105 | 'pattern': { 106 | 'observation': { 107 | 'objects': {'domain-name'}, 108 | 'join': None, 109 | 'qualifiers': None, 110 | 'expressions': [ 111 | {'comparison': { 112 | 'object': 'domain-name', 113 | 'path': ['value'], 114 | 'negated': None, 115 | 'operator': '=', 116 | 'value': 'http://xyz.com/download', 117 | }} 118 | ] 119 | } 120 | } 121 | } 122 | ``` 123 | 124 | A specialized YAML representation is also proposed, to make visualization of this data a little less cumbersome: 125 | 126 | ```py 127 | from dendrol import Pattern 128 | 129 | pattern = Pattern("[domain-name:value = 'http://xyz.com/download']") 130 | 131 | assert str(pattern.to_dict_tree()) == '''\ 132 | pattern: 133 | observation: 134 | objects: {domain-name} 135 | join: 136 | qualifiers: 137 | expressions: 138 | - comparison: 139 | object: domain-name 140 | path: [value] 141 | negated: 142 | operator: '=' 143 | value: http://xyz.com/download 144 | ''' 145 | ``` 146 | 147 | For more info, read [The Spec](#the-spec) below, or check out the tests. 148 | 149 | 150 | # Development 151 | To develop dendrol and run its tests, first clone the repo. Then install the dev and testing dependencies: 152 | ```bash 153 | pip install .[dev] .[test] 154 | ``` 155 | 156 | pytest is used for testing: 157 | ```bash 158 | py.test 159 | ``` 160 | 161 | Reported issues and pull requests welcomed! From new features and suggestions to typo fixes and poor naming choices, fresh eyes bolster software eternally in development. 162 | 163 | If submitting a pull request, please add yourself to the CONTRIBUTORS file for a piece of that sweet, sweet street cred! 164 | 165 | 166 | # The Spec 167 | ## Brief 168 | A PatternTree begins with a `'pattern'` key. Below it is an observation expression, with an `'observation'` or `'expression'` key (which may contain more observation expressions joined by AND/OR/FOLLOWEDBY). Below `'observation'` keys are comparison expressions, marked by a `'comparison'` or `'expression'` key (which may contain more comparison expressions joined by AND/OR). `'comparison'` keys denote a single comparison between an object property and a literal value. 169 | 170 | ## Pattern 171 | ```py 172 | {'pattern': {...}} 173 | ``` 174 | ```yaml 175 | pattern: 176 | ... 177 | ``` 178 | A PatternTree is a dict with one top-level key, `'pattern'`. This paradigm of a dict with a single key identifying its contents is seen throughout this spec. 179 | 180 | The value of this `'pattern'` key is an [observation expression](#spec-observation-expressions). 181 | 182 | 183 | ## Observation Expressions 184 | 185 | An Observation Expression is a dict with a single key of either `'expression'` or `'observation'`. An [`'expression'`](#spec-observation-expressions-expression) SHALL contain two or more observation expressions joined by AND/OR/FOLLOWEDBY, whereas an [`'observation'`](#spec-observation-expressions-observation) SHALL contain only comparison expressions. 186 | 187 | 188 | ### Expression 189 | ```py 190 | {'expression': { 191 | 'join': oneOf('AND', 'OR', 'FOLLOWEDBY'), 192 | 'qualifiers': [...], 193 | 'expressions': [...], 194 | }} 195 | ``` 196 | ```yaml 197 | expression: 198 | join: AND | OR | FOLLOWEDBY 199 | qualifiers: 200 | expressions: 201 | - a 202 | - b 203 | - ... 204 | ``` 205 | An `'expression'` is a container for other observation expressions, joined by an observation operator in `'join'`. It MAY have a list of qualifiers in the `'qualifiers'` key, or `None` if there are none. 206 | 207 | Its children are in `'expressions'`, whose values SHALL be dicts with single keys (of either `'observation'` or `'expression'`). 208 | 209 | 210 | ### Observation 211 | ```py 212 | {'observation': { 213 | 'objects': {'ipv4-addr', 'ipv6-addr', ...}, 214 | 'join': oneOf('AND', 'OR'), 215 | 'qualifiers': [...], 216 | 'expressions': [...], 217 | }} 218 | ``` 219 | ```yaml 220 | observation: 221 | objects: 222 | ? ipv4-addr 223 | ? ipv6-addr 224 | ? ... 225 | join: AND | OR 226 | qualifiers: 227 | expressions: 228 | - a 229 | - ... 230 | ``` 231 | An `'observation'` is analogous to square brackets in STIX2 Pattern Expressions, e.g.: `[ipv4-addr:value = '1.2.3.4']`. Children of an observation (in the `'expressions'` key) SHALL only be comparisons or comparison expressions. 232 | 233 | An `'observation'` MAY have qualifiers, but its children MUST NOT. 234 | 235 | An `'observation'` MAY have a join method, which denotes how its child [comparison expressions](#spec-comparison-expressions) are to be joined. This method MAY be AND or OR, but MUST NOT be FOLLOWEDBY, because the join method applies to comparison expressions, not observation expressions. If there is only a single child comparison expression, `'join'` MAY be `None`. 236 | 237 | An `'observation'` SHALL contain a set of all the object types of its child comparison expressions. This is mainly for human consumption. A STIX2 observation is allowed to contain comparisons on disparate object types, provided they're joined by OR— this is why `'objects'` is a set, not a single string. 238 | 239 | If `'objects'` contains only a single object type, it MAY be compacted into set literal form: 240 | ```yaml 241 | observation: 242 | objects: {ipv4-addr} 243 | join: AND | OR 244 | qualifiers: 245 | expressions: 246 | - a 247 | - ... 248 | ``` 249 | 250 | 251 | ### Qualifiers 252 | A Qualifier is a dict having a single key identifying its Qualifier type. Currently, this SHALL be one of: 253 | - [`'start_stop'`](#spec-observation-expressions-qualifiers-start-stop) 254 | - [`'within'`](#spec-observation-expressions-qualifiers-within) 255 | - [`'repeats'`](#spec-observation-expressions-qualifiers-repeats) 256 | 257 | 258 | #### Start/Stop Qualifier 259 | ```py 260 | {'start_stop': { 261 | 'start': datetime(2018, 10, 7, 0, 0, tzinfo=tzutc()), 262 | 'stop': datetime(2018, 10, 7, 23, 59, tzinfo=tzutc()), 263 | }} 264 | ``` 265 | ```yaml 266 | start_stop: 267 | start: 2018-10-07T00:00:00Z 268 | stop: 2018-10-08T23:59:00Z 269 | ``` 270 | The `'start_stop'` qualifier constrains the timeframe in which its associated observation expressions MUST occur within to evaluate true. Unlike `WITHIN`, `START ... STOP ...` denotes absolute points in time, using datetime literals. 271 | 272 | Example STIX2 expression: 273 | ``` 274 | [a:b = 12] START t'2018-10-07T00:00:00Z' STOP t'2018-10-08T23:59:00Z' 275 | ``` 276 | In STIX2 Pattern Expressions, all datetimes MUST be in [RFC3339](https://tools.ietf.org/html/rfc3339) format, and MUST be in UTC timezone. datetime literals resemble Python strings with `t` as their modifying char (like an f-string, or a bytestring). Because they must be in UTC timezone, datetime literals MUST end with the `Z` char. 277 | 278 | When parsed into Python, they SHALL have a `tzinfo` object with a `dstoffset` of 0. 279 | 280 | 281 | #### Within Qualifier 282 | ```py 283 | {'within': { 284 | 'value': 600, 285 | 'unit': 'SECONDS', 286 | }} 287 | ``` 288 | ```yaml 289 | within: 290 | value: 600 291 | unit: SECONDS 292 | ``` 293 | The `'within'` qualifier constrains the timeframe in which its associated observation expressions MUST occur within to evaluate true. Unlike `START ... STOP ...`, `WITHIN` denotes relative timeframes, where the latest observation expression MUST occur within the specified number of seconds from the earliest observation expression. 294 | 295 | Example STIX2 expression: 296 | ``` 297 | [a:b = 12] WITHIN 600 SECONDS 298 | ``` 299 | 300 | `SECONDS` is hard-coded into the STIX2 Pattern Expression grammar, and MUST be included in pattern expressions. However, to avoid ambiguity for the reader, and to allow for future STIX2 spec changes, the unit is also included in the Pattern Tree. 301 | 302 | 303 | #### Repeated Qualifier 304 | ```py 305 | {'repeats': { 306 | 'value': 9000, 307 | }} 308 | ``` 309 | ```yaml 310 | repeats: 311 | value: 9000 312 | ``` 313 | The `'repeats'` qualifier REQUIRES that its associated observation expressions evaluate true at different occasions, for a specified number of times. 314 | 315 | Example STIX2 expression: 316 | ``` 317 | [a:b = 12] REPEATS 9000 TIMES 318 | ``` 319 | 320 | `TIMES` is hard-coded into the STIX2 Pattern Expression grammar, and MUST be included in pattern expressions. However, since there aren't any other obvious units of multiplicity, other than "X times", it has been omitted from the Pattern Tree output — unlike `SECONDS` of `WITHIN`. 321 | 322 | 323 | ## Comparison Expressions 324 | A Comparison Expression is a dict with a single key of either `'expression'` or `'comparison'`. An [`'expression'`](#spec-comparison-expressions-expression) SHALL contain two or more comparison expressions joined by AND/OR, whereas a [`'comparison'`](#spec-comparison-expressions-comparison) contains no children, and only marks a comparison of one variable to one literal value. 325 | 326 | 327 | ### Expression 328 | ```py 329 | {'expression': { 330 | 'join': oneOf('AND', 'OR'), 331 | 'expressions': [a, b, ...], 332 | }} 333 | ``` 334 | ```yaml 335 | expression: 336 | join: AND | OR 337 | expressions: 338 | - a 339 | - b 340 | - ... 341 | ``` 342 | An `'expression'` is a container for other comparison expressions, joined by either AND or OR in `'join'` — comparison expressions do not have FOLLOWEDBY, as they are intended to reference a single object at a single point in time. 343 | 344 | An `'expression'` MUST NOT have qualifiers. 345 | 346 | Its children are in `'expressions'`, whose values SHALL be dicts with single keys (of either `'comparison'` or `'expression'`). 347 | 348 | 349 | ### Comparison 350 | ```py 351 | {'comparison': { 352 | 'object': 'email-message', 353 | 'path': ['from_ref', 'value'], 354 | 'negated': None, 355 | 'operator': 'MATCHES', 356 | 'value': '.+@malicio\\.us', 357 | }} 358 | ``` 359 | ```yaml 360 | comparison: 361 | object: email-message 362 | path: 363 | - from_ref 364 | - value 365 | negated: 366 | operator: MATCHES 367 | value: .+@malicio\.us 368 | ``` 369 | A `'comparison'` represents a single comparison between a STIX2 object property and a literal value. A single string object type SHALL be placed in the `'object'` key. 370 | 371 | `'path'` SHALL be a list beginning with a top-level property of the object type denoted in `'object'`, as a string. Following this MAY be any number of child properties, as strings, or list index components/dereferences, denoted as Python `slice()` objects, where `[1]` is equivalent to `slice(start=None, stop=1, step=None)`. The special _match any_ list index from STIX2 (e.g. `file:sections[*]`) is equivalent to `slice(start=None, stop='*', step=None)`. 372 | 373 | `'negated'` SHALL be a bool denoting whether the operator SHALL be negated during evaluation. STIX2 allows a `NOT` keyword before the operator: `file:name NOT MATCHES 'james.*'`. If the operator is not negated, `'negated'` MAY be `None`. (This allows for a more compact YAML representation — where the value may simply be omitted.) 374 | 375 | `'operator'` SHALL be a string representing the operator, e.g. `'>'`, `'LIKE'`, or `'='`. 376 | 377 | `'value'` MAY be any static Python value. Currently, only strings, bools, ints, floats, datetimes, and bytes are outputted, but this could change in the future (e.g. if compiled regular expressions are deemed useful). 378 | 379 | If `'path'` contains only a single property, it MAY be compacted into list literal form: 380 | ```yaml 381 | comparison: 382 | object: domain-name 383 | path: [value] 384 | negated: 385 | operator: = 386 | value: cnn.com 387 | ``` 388 | -------------------------------------------------------------------------------- /compile_stix_grammar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import subprocess 3 | import sys 4 | from pathlib import Path 5 | from typing import Iterable, Union 6 | 7 | import click 8 | import requests 9 | from tqdm import tqdm 10 | 11 | 12 | @click.command() 13 | @click.option('-o', '--output', 14 | type=click.Path(exists=True, file_okay=False, resolve_path=True), 15 | default='./dendrol/lang/', 16 | help='Directory to save the generated Python modules into') 17 | @click.option('--force-antlr-download', 18 | default=False, 19 | help='Force the ANTLR .jar to be downloaded, even if it already exists') 20 | @click.option('--progress', 21 | default=True) 22 | def compile_stix_grammar(output, force_antlr_download: bool, progress: bool): 23 | output = Path(output) 24 | 25 | if not is_java_installed(): 26 | click.echo('Java could not be found. It is required to run the ANTLR ' 27 | 'compiler. Please install it, and ensure the "java" ' 28 | 'executable is on your PATH.', color='red') 29 | sys.exit(2) 30 | 31 | jar_path = get_antlr_jar_path(output) 32 | will_download_antlr = False 33 | 34 | if force_antlr_download: 35 | will_download_antlr = True 36 | 37 | elif not is_antlr_jar_saved(output): 38 | will_download_antlr = True 39 | click.echo(f'ANTLR .jar not found at: {jar_path}', color='orange') 40 | 41 | if will_download_antlr: 42 | click.echo(f'Downloading ANTLR .jar to: {jar_path}') 43 | save_antlr_jar(directory=output, display_progress=progress) 44 | 45 | compile_grammar(output_dir=output) 46 | click.echo(f'Successfully compiled grammar to: {output}', color='green') 47 | 48 | 49 | def is_java_installed() -> bool: 50 | """Simple check to detect existence of Java.""" 51 | try: 52 | retcode = subprocess.check_call( 53 | ['java', '-version'], 54 | stderr=subprocess.DEVNULL, 55 | ) 56 | except (subprocess.CalledProcessError, OSError): 57 | return False 58 | else: 59 | return retcode == 0 60 | 61 | 62 | def compile_grammar(output_dir: Path = None) -> bool: 63 | """Generate lexer, parser, listener, and visitor classes for the STIX grammar 64 | 65 | If output_dir is not passed, the directory of this module is used. 66 | """ 67 | jar_path = get_antlr_jar_path(output_dir) 68 | 69 | retcode = subprocess.check_call( 70 | cwd=output_dir, 71 | env={'CLASSPATH': str(jar_path)}, 72 | args=[ 73 | 'java', 74 | 'org.antlr.v4.Tool', 75 | '-Dlanguage=Python3', 76 | '-visitor', 77 | '-listener', 78 | '-package', get_antlr_jar_path.__module__, 79 | '-o', output_dir, 80 | 'STIXPattern.g4', 81 | ], 82 | ) 83 | return retcode == 0 84 | 85 | 86 | def save_antlr_jar(directory: Path, display_progress=False) -> Path: 87 | """Download and save the ANTLR .jar 88 | """ 89 | path = get_antlr_jar_path(directory) 90 | content = stream_antlr_jar() 91 | 92 | if display_progress: 93 | url = get_antlr_jar_download_url() 94 | head_res = requests.head(url) 95 | head_res.raise_for_status() 96 | file_size = int(head_res.headers['Content-Length']) 97 | 98 | pbar = tqdm( 99 | iterable=content, 100 | total=file_size, 101 | unit='B', 102 | unit_scale=True, 103 | desc=path.name, 104 | ) 105 | content = _iter_with_progress(pbar) 106 | 107 | with path.open('wb') as fp: 108 | for chunk in content: 109 | fp.write(chunk) 110 | 111 | return path 112 | 113 | 114 | def _iter_with_progress(pbar): 115 | for chunk in pbar: 116 | if chunk: 117 | pbar.update(len(chunk)) 118 | yield chunk 119 | pbar.close() 120 | 121 | 122 | def stream_antlr_jar(chunk_size=1024) -> Iterable[bytes]: 123 | """Retrieve the ANTLR .jar in chunks from the internet 124 | """ 125 | url = get_antlr_jar_download_url() 126 | response = requests.get(url, stream=True) 127 | response.raise_for_status() 128 | 129 | for chunk in response.iter_content(chunk_size=chunk_size): 130 | yield chunk 131 | 132 | 133 | def get_antlr_jar_path(directory: Path) -> Path: 134 | return directory / 'antlr-4.7.1-complete.jar' 135 | 136 | 137 | def is_antlr_jar_saved(directory: Path) -> bool: 138 | return get_antlr_jar_path(directory).exists() 139 | 140 | 141 | def get_antlr_jar_download_url() -> str: 142 | return 'https://www.antlr.org/download/antlr-4.7.1-complete.jar' 143 | 144 | 145 | if __name__ == '__main__': 146 | compile_stix_grammar() 147 | -------------------------------------------------------------------------------- /dendrol/__init__.py: -------------------------------------------------------------------------------- 1 | from .debug import print_tree, dump_tree, load_tree 2 | from .parse import parse, Pattern 3 | from .transform import PatternTree 4 | -------------------------------------------------------------------------------- /dendrol/debug.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from contextlib import contextmanager 3 | from datetime import datetime 4 | from typing import Union, Any 5 | 6 | import yaml 7 | 8 | from .transform import CompactibleObject, PatternTree 9 | 10 | 11 | DEFAULT_SET_TAG = 'tag:yaml.org,2002:set' 12 | DEFAULT_SLICE_TAG = '!slice' 13 | 14 | 15 | def load_tree(s: str) -> PatternTree: 16 | d = yaml.load(s, Loader=PatternTreeLoader) 17 | return PatternTree.from_dict(d) 18 | 19 | 20 | def dump_tree(tree: Union[Any, PatternTree]) -> str: 21 | return yaml.dump( 22 | tree, 23 | Dumper=PatternTreeDumper, 24 | allow_unicode=True, 25 | indent=2, 26 | default_flow_style=False, 27 | ) 28 | 29 | 30 | def print_tree(tree: Union[dict, PatternTree]): 31 | print(dump_tree(tree)) 32 | 33 | 34 | class PatternTreeDumper(yaml.Dumper): 35 | """Pretty-printer for STIX2 Pattern Trees 36 | 37 | This dumper: 38 | - Indents list items past their parent keys 39 | - Prints empty space instead of `null` 40 | - Prints OrderedDicts as regular dicts 41 | - Prints datetimes in RFC 3339 / ISO 8601 format, with a hard requirement 42 | on including the UTC "Z" for timezone, as required by STIX2 spec [1] 43 | - Prints short lists/sets as single-line literals, instead of dashed items 44 | (e.g. 'path: [name]' instead of 'path:\n - name') 45 | - Prints Python slice objects as plain representations of their syntax 46 | (e.g. slice(1) == [1] and slice(2, None, 7) == [2::7]) 47 | - Omits mapping values with set literals 48 | 49 | [1]: http://docs.oasis-open.org/cti/stix/v2.0/cs01/part1-stix-core/stix-v2.0-cs01-part1-stix-core.html#_Toc496709271 50 | """ 51 | 52 | def __init__(self, *args, **kwargs): 53 | super().__init__(*args, **kwargs) 54 | 55 | # Whether to omit mapping values (for set literals). See expect_flow_mapping 56 | self.in_flow_set = False 57 | 58 | # Whether to omit mapping values and use '?' instead of '-' (for 59 | # multiline sets). See expect_block_mapping 60 | self.in_block_set = False 61 | 62 | def process_tag(self): 63 | """Disable the emission of tags (e.g. !dict)""" 64 | pass # noop 65 | 66 | def increase_indent(self, flow=False, indentless=False): 67 | """Indent dashes in lists, not just items 68 | 69 | This will output: 70 | 71 | my_list: 72 | - item 1 73 | - item 2 74 | 75 | Instead of the default: 76 | 77 | my_list: 78 | - item 1 79 | - item 2 80 | 81 | """ 82 | return super(PatternTreeDumper, self).increase_indent(flow, False) 83 | 84 | def represent_none(self, data): 85 | """Don't print "null" for None; only """ 86 | return self.represent_scalar('tag:yaml.org,2002:null', '') 87 | 88 | def represent_pattern_tree(self, data: PatternTree): 89 | return self.represent_mapping('!dict', data) 90 | 91 | def represent_ordered_dict(self, data: OrderedDict): 92 | return self.represent_mapping('!dict', data.items()) 93 | 94 | def represent_datetime(self, data: datetime): 95 | value = data.strftime('%Y-%m-%dT%H:%M:%SZ') 96 | return self.represent_scalar('tag:yaml.org,2002:timestamp', value) 97 | 98 | def represent_compactible_object(self, data: CompactibleObject): 99 | data_type = data.get_literal_type() 100 | representer = self.yaml_representers.get(data_type) or self.__class__.represent_data 101 | node = representer(self, data) 102 | node.flow_style = data.is_eligible_for_compaction() 103 | return node 104 | 105 | def represent_slice(self, data: slice): 106 | if data.step is not None: 107 | components = [data.start, data.stop, data.step] 108 | elif data.start is not None: 109 | components = [data.start, data.stop] 110 | else: 111 | components = [data.stop] 112 | 113 | s = ':'.join( 114 | str(comp) if comp is not None else '' 115 | for comp in components 116 | ) 117 | 118 | return self.represent_scalar(DEFAULT_SLICE_TAG, f'[{s}]', style='') 119 | 120 | def expect_flow_mapping(self): 121 | if self.event.tag == DEFAULT_SET_TAG: 122 | self.in_flow_set = True 123 | super().expect_flow_mapping() 124 | 125 | def expect_first_flow_mapping_key(self): 126 | if isinstance(self.event, yaml.MappingEndEvent): 127 | self.in_flow_set = False 128 | super().expect_first_flow_mapping_key() 129 | 130 | def expect_flow_mapping_key(self): 131 | if isinstance(self.event, yaml.MappingEndEvent): 132 | self.in_flow_set = False 133 | super().expect_first_flow_mapping_key() 134 | 135 | def expect_flow_mapping_simple_value(self): 136 | if self.in_flow_set: 137 | # This branch skips writing ":" with `write_indicator(':')`, the 138 | # value with `expect_node(mapping=True)`, and jumps straight to the 139 | # next key by setting state to expect_flow_mapping_key 140 | self.state = self.expect_flow_mapping_key 141 | else: 142 | super().expect_flow_mapping_simple_value() 143 | 144 | def expect_block_mapping(self): 145 | if self.event.tag == DEFAULT_SET_TAG: 146 | self.in_block_set = True 147 | super().expect_block_mapping() 148 | 149 | def expect_block_mapping_key(self, first=False): 150 | if not first and isinstance(self.event, yaml.MappingEndEvent): 151 | self.in_block_set = False 152 | super().expect_block_mapping_key(first=first) 153 | 154 | def expect_block_mapping_value(self): 155 | if not self.in_block_set: 156 | self.write_indent() 157 | self.write_indicator(':', True, indention=True) 158 | self.states.append(self.expect_block_mapping_key) 159 | self.expect_node(mapping=True) 160 | 161 | def analyze_scalar(self, scalar): 162 | analysis = super().analyze_scalar(scalar) 163 | 164 | if self.in_block_set: 165 | # multiline is used, in part, to determine whether the block mapping 166 | # key is "simple". In case of sets, "simple" means "?" is not used 167 | # to prefix items — and we *do* want the "?" 168 | analysis.multiline = True 169 | 170 | if self.event.tag == DEFAULT_SLICE_TAG: 171 | # YAML disallows flow style (AKA literals) if the scalar begins 172 | # with "[". We overload the loader to allow this, so we're okay 173 | # allowing scalars beginning with [. 174 | analysis.allow_flow_plain = True 175 | self.style = '' 176 | 177 | return analysis 178 | 179 | 180 | PatternTreeDumper.add_representer(type(None), PatternTreeDumper.represent_none) 181 | PatternTreeDumper.add_representer(PatternTree, PatternTreeDumper.represent_pattern_tree) 182 | PatternTreeDumper.add_representer(OrderedDict, PatternTreeDumper.represent_ordered_dict) 183 | PatternTreeDumper.add_representer(datetime, PatternTreeDumper.represent_datetime) 184 | PatternTreeDumper.add_multi_representer(CompactibleObject, PatternTreeDumper.represent_compactible_object) 185 | PatternTreeDumper.add_representer(slice, PatternTreeDumper.represent_slice) 186 | 187 | 188 | class PatternTreeLoader(yaml.Loader): 189 | def __init__(self, *args, **kwargs): 190 | super().__init__(*args, **kwargs) 191 | 192 | # Whether we've recognized an ObjectPath, and are about to descend 193 | # into it 194 | self.is_object_path = False 195 | 196 | # Whether we've descended into an ObjectPath, which we use to 197 | # construct slices instead of lists for, e.g., [1] and [*]. 198 | self.in_object_path = False 199 | 200 | def compose_node(self, parent, index): 201 | event = self.peek_event() 202 | key_name = getattr(index, 'value', None) 203 | state = {} 204 | 205 | # Heuristic: any sequence whose key is "path" is presumed to be an 206 | # ObjectPath. 207 | if isinstance(event, yaml.SequenceStartEvent) and key_name == 'path': 208 | state['is_object_path'] = True 209 | 210 | with override(self, state): 211 | node = super().compose_node(parent, index) 212 | 213 | # Heuristic: any mapping whose key is "objects" is presumed to be 214 | # underneath an observation/expression. 215 | if isinstance(event, yaml.MappingStartEvent) and key_name == 'objects': 216 | # Heuristic: if the value of the "objects" key has no explicit YAML 217 | # tag, and it's using a map literal (flow_style=True), we presume 218 | # it's a set literal 219 | if event.implicit and event.flow_style: 220 | node.tag = DEFAULT_SET_TAG 221 | 222 | return node 223 | 224 | def compose_sequence_node(self, anchor): 225 | state = {} 226 | if self.is_object_path: 227 | state['in_object_path'] = True 228 | 229 | with override(self, state): 230 | node = super().compose_sequence_node(anchor) 231 | 232 | # Any sequences inside object paths are actually slice notation. 233 | # e.g. [1] is not list([1]), but slice(1) 234 | if self.in_object_path: 235 | assert isinstance(node.value, list) 236 | assert len(node.value) == 1 237 | 238 | return yaml.ScalarNode( 239 | DEFAULT_SLICE_TAG, 240 | f'[{node.value[0].value}]', 241 | node.start_mark, 242 | node.end_mark, 243 | ) 244 | 245 | return node 246 | 247 | def scan_anchor(self, TokenClass): 248 | # Partial heuristic: check for start of list literal (AKA flow sequence) 249 | if len(self.tokens) == 1 and isinstance(self.tokens[0], yaml.FlowSequenceStartToken): 250 | indicator = self.peek() 251 | next_char = self.peek(1) 252 | 253 | # Heuristic: if list literal consists entirely of [*], skip treating 254 | # it as an alias (which asterisk normally represents) 255 | if indicator == '*' and next_char == ']': 256 | # Eat our asterisk 257 | self.forward(1) 258 | 259 | # Tell the scanner we're done with our "sequence". This will 260 | # append the flow sequence end token, which we'll ditch. 261 | self.fetch_flow_sequence_end() 262 | 263 | start_mark = self.tokens[0].start_mark 264 | end_mark = self.tokens[1].end_mark 265 | 266 | # Fabricated tag to ease construction, by explicitly marking 267 | # the type of our slice scalar — instead of relying on 268 | # heuristics of the ['*'] value later. 269 | tag_token = yaml.TagToken(('!', 'slice'), start_mark, end_mark) 270 | 271 | slice_token = yaml.ScalarToken( 272 | value='[*]', 273 | plain=False, 274 | start_mark=start_mark, 275 | end_mark=end_mark, 276 | style='', 277 | ) 278 | 279 | # Wipe out the flow sequence start and end tokens and replace 280 | # with our tag token, effectively removing all traces of the 281 | # flow sequence, and fabricating the existence of our slice 282 | self.tokens[:] = [tag_token] 283 | 284 | # This return value is appended to self.tokens 285 | return slice_token 286 | 287 | return super().scan_anchor(TokenClass) 288 | 289 | def construct_slice(self, node): 290 | contents = node.value[1:-1] # remove [ and ] 291 | split_contents = contents.split(':') 292 | 293 | components = [] 294 | for value in split_contents: 295 | try: 296 | value = int(value) 297 | except (ValueError, TypeError): 298 | pass 299 | components.append(value) 300 | 301 | if len(components) == 1: 302 | stop = components[0] 303 | start = step = None 304 | elif len(components) == 2: 305 | start, stop = components 306 | step = None 307 | elif len(components) == 3: 308 | start, stop, step = components 309 | else: 310 | raise AssertionError( 311 | 'slices may only have three components: [start:stop:step]') 312 | 313 | return slice(start, stop, step) 314 | 315 | 316 | PatternTreeLoader.add_constructor(DEFAULT_SLICE_TAG, PatternTreeLoader.construct_slice) 317 | 318 | 319 | @contextmanager 320 | def override(obj, _state=None, **extra): 321 | """Change attrs of obj within manager, then revert afterward 322 | 323 | Usage: 324 | 325 | self.stuff = 1337 326 | 327 | with override(self, stuff=12): 328 | print(self.stuff) 329 | # 12 330 | 331 | print(self.stuff) 332 | # 1337 333 | """ 334 | if _state is None: 335 | _state = {} 336 | state = { 337 | **_state, 338 | **extra, 339 | } 340 | 341 | old_state = {k: getattr(obj, k, None) for k in state} 342 | 343 | for attr, value in state.items(): 344 | setattr(obj, attr, value) 345 | 346 | yield 347 | 348 | for attr, value in old_state.items(): 349 | setattr(obj, attr, value) 350 | -------------------------------------------------------------------------------- /dendrol/lang/.gitignore: -------------------------------------------------------------------------------- 1 | # ignore the ANTLR JAR 2 | *.jar 3 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPattern.g4: -------------------------------------------------------------------------------- 1 | // This is an ANTLR4 grammar for the STIX Patterning Language. 2 | // 3 | // http://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part5-stix-patterning.html 4 | 5 | grammar STIXPattern; 6 | 7 | pattern 8 | : observationExpressions EOF 9 | ; 10 | 11 | observationExpressions 12 | : observationExpressions FOLLOWEDBY observationExpressions 13 | | observationExpressionOr 14 | ; 15 | 16 | observationExpressionOr 17 | : observationExpressionOr OR observationExpressionOr 18 | | observationExpressionAnd 19 | ; 20 | 21 | observationExpressionAnd 22 | : observationExpressionAnd AND observationExpressionAnd 23 | | observationExpression 24 | ; 25 | 26 | observationExpression 27 | : LBRACK comparisonExpression RBRACK # observationExpressionSimple 28 | | LPAREN observationExpressions RPAREN # observationExpressionCompound 29 | | observationExpression startStopQualifier # observationExpressionStartStop 30 | | observationExpression withinQualifier # observationExpressionWithin 31 | | observationExpression repeatedQualifier # observationExpressionRepeated 32 | ; 33 | 34 | comparisonExpression 35 | : comparisonExpression OR comparisonExpression 36 | | comparisonExpressionAnd 37 | ; 38 | 39 | comparisonExpressionAnd 40 | : comparisonExpressionAnd AND comparisonExpressionAnd 41 | | propTest 42 | ; 43 | 44 | propTest 45 | : objectPath NOT? (EQ|NEQ) primitiveLiteral # propTestEqual 46 | | objectPath NOT? (GT|LT|GE|LE) orderableLiteral # propTestOrder 47 | | objectPath NOT? IN setLiteral # propTestSet 48 | | objectPath NOT? LIKE StringLiteral # propTestLike 49 | | objectPath NOT? MATCHES StringLiteral # propTestRegex 50 | | objectPath NOT? ISSUBSET StringLiteral # propTestIsSubset 51 | | objectPath NOT? ISSUPERSET StringLiteral # propTestIsSuperset 52 | | LPAREN comparisonExpression RPAREN # propTestParen 53 | ; 54 | 55 | startStopQualifier 56 | : START TimestampLiteral STOP TimestampLiteral 57 | ; 58 | 59 | withinQualifier 60 | : WITHIN (IntPosLiteral|FloatPosLiteral) SECONDS 61 | ; 62 | 63 | repeatedQualifier 64 | : REPEATS IntPosLiteral TIMES 65 | ; 66 | 67 | objectPath 68 | : objectType COLON firstPathComponent objectPathComponent? 69 | ; 70 | 71 | objectType 72 | : IdentifierWithoutHyphen 73 | | IdentifierWithHyphen 74 | ; 75 | 76 | firstPathComponent 77 | : IdentifierWithoutHyphen 78 | | StringLiteral 79 | ; 80 | 81 | objectPathComponent 82 | : objectPathComponent objectPathComponent # pathStep 83 | | '.' (IdentifierWithoutHyphen | StringLiteral) # keyPathStep 84 | | LBRACK (IntPosLiteral|IntNegLiteral|ASTERISK) RBRACK # indexPathStep 85 | ; 86 | 87 | setLiteral 88 | : LPAREN RPAREN 89 | | LPAREN primitiveLiteral (COMMA primitiveLiteral)* RPAREN 90 | ; 91 | 92 | primitiveLiteral 93 | : orderableLiteral 94 | | BoolLiteral 95 | ; 96 | 97 | orderableLiteral 98 | : IntPosLiteral 99 | | IntNegLiteral 100 | | FloatPosLiteral 101 | | FloatNegLiteral 102 | | StringLiteral 103 | | BinaryLiteral 104 | | HexLiteral 105 | | TimestampLiteral 106 | ; 107 | 108 | IntNegLiteral : 109 | '-' ('0' | [1-9] [0-9]*) 110 | ; 111 | 112 | IntPosLiteral : 113 | '+'? ('0' | [1-9] [0-9]*) 114 | ; 115 | 116 | FloatNegLiteral : 117 | '-' [0-9]* '.' [0-9]+ 118 | ; 119 | 120 | FloatPosLiteral : 121 | '+'? [0-9]* '.' [0-9]+ 122 | ; 123 | 124 | HexLiteral : 125 | 'h' QUOTE TwoHexDigits* QUOTE 126 | ; 127 | 128 | BinaryLiteral : 129 | 'b' QUOTE 130 | ( Base64Char Base64Char Base64Char Base64Char )* 131 | ( (Base64Char Base64Char Base64Char Base64Char ) 132 | | (Base64Char Base64Char Base64Char ) '=' 133 | | (Base64Char Base64Char ) '==' 134 | ) 135 | QUOTE 136 | ; 137 | 138 | StringLiteral : 139 | QUOTE ( ~['\\] | '\\\'' | '\\\\' )* QUOTE 140 | ; 141 | 142 | BoolLiteral : 143 | TRUE | FALSE 144 | ; 145 | 146 | TimestampLiteral : 147 | 't' QUOTE 148 | [0-9] [0-9] [0-9] [0-9] HYPHEN 149 | ( ('0' [1-9]) | ('1' [012]) ) HYPHEN 150 | ( ('0' [1-9]) | ([12] [0-9]) | ('3' [01]) ) 151 | 'T' 152 | ( ([01] [0-9]) | ('2' [0-3]) ) COLON 153 | [0-5] [0-9] COLON 154 | ([0-5] [0-9] | '60') 155 | (DOT [0-9]+)? 156 | 'Z' 157 | QUOTE 158 | ; 159 | 160 | ////////////////////////////////////////////// 161 | // Keywords 162 | 163 | AND: 'AND' ; 164 | OR: 'OR' ; 165 | NOT: 'NOT' ; 166 | FOLLOWEDBY: 'FOLLOWEDBY'; 167 | LIKE: 'LIKE' ; 168 | MATCHES: 'MATCHES' ; 169 | ISSUPERSET: 'ISSUPERSET' ; 170 | ISSUBSET: 'ISSUBSET' ; 171 | LAST: 'LAST' ; 172 | IN: 'IN' ; 173 | START: 'START' ; 174 | STOP: 'STOP' ; 175 | SECONDS: 'SECONDS' ; 176 | TRUE: 'true' ; 177 | FALSE: 'false' ; 178 | WITHIN: 'WITHIN' ; 179 | REPEATS: 'REPEATS' ; 180 | TIMES: 'TIMES' ; 181 | 182 | // After keywords, so the lexer doesn't tokenize them as identifiers. 183 | // Object types may have unquoted hyphens, but property names 184 | // (in object paths) cannot. 185 | IdentifierWithoutHyphen : 186 | [a-zA-Z_] [a-zA-Z0-9_]* 187 | ; 188 | 189 | IdentifierWithHyphen : 190 | [a-zA-Z_] [a-zA-Z0-9_-]* 191 | ; 192 | 193 | EQ : '=' | '=='; 194 | NEQ : '!=' | '<>'; 195 | LT : '<'; 196 | LE : '<='; 197 | GT : '>'; 198 | GE : '>='; 199 | 200 | QUOTE : '\''; 201 | COLON : ':' ; 202 | DOT : '.' ; 203 | COMMA : ',' ; 204 | RPAREN : ')' ; 205 | LPAREN : '(' ; 206 | RBRACK : ']' ; 207 | LBRACK : '[' ; 208 | PLUS : '+' ; 209 | HYPHEN : MINUS ; 210 | MINUS : '-' ; 211 | POWER_OP : '^' ; 212 | DIVIDE : '/' ; 213 | ASTERISK : '*'; 214 | 215 | fragment HexDigit: [A-Fa-f0-9]; 216 | fragment TwoHexDigits: HexDigit HexDigit; 217 | fragment Base64Char: [A-Za-z0-9+/]; 218 | 219 | // Whitespace and comments 220 | // 221 | WS : [ \t\r\n\u000B\u000C\u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000]+ -> skip 222 | ; 223 | 224 | COMMENT 225 | : '/*' .*? '*/' -> skip 226 | ; 227 | 228 | LINE_COMMENT 229 | : '//' ~[\r\n]* -> skip 230 | ; 231 | 232 | // Catch-all to prevent lexer from silently eating unusable characters. 233 | InvalidCharacter 234 | : . 235 | ; 236 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPattern.interp: -------------------------------------------------------------------------------- 1 | token literal names: 2 | null 3 | null 4 | null 5 | null 6 | null 7 | null 8 | null 9 | null 10 | null 11 | null 12 | 'AND' 13 | 'OR' 14 | 'NOT' 15 | 'FOLLOWEDBY' 16 | 'LIKE' 17 | 'MATCHES' 18 | 'ISSUPERSET' 19 | 'ISSUBSET' 20 | 'LAST' 21 | 'IN' 22 | 'START' 23 | 'STOP' 24 | 'SECONDS' 25 | 'true' 26 | 'false' 27 | 'WITHIN' 28 | 'REPEATS' 29 | 'TIMES' 30 | null 31 | null 32 | null 33 | null 34 | '<' 35 | '<=' 36 | '>' 37 | '>=' 38 | '\'' 39 | ':' 40 | '.' 41 | ',' 42 | ')' 43 | '(' 44 | ']' 45 | '[' 46 | '+' 47 | null 48 | '-' 49 | '^' 50 | '/' 51 | '*' 52 | null 53 | null 54 | null 55 | null 56 | 57 | token symbolic names: 58 | null 59 | IntNegLiteral 60 | IntPosLiteral 61 | FloatNegLiteral 62 | FloatPosLiteral 63 | HexLiteral 64 | BinaryLiteral 65 | StringLiteral 66 | BoolLiteral 67 | TimestampLiteral 68 | AND 69 | OR 70 | NOT 71 | FOLLOWEDBY 72 | LIKE 73 | MATCHES 74 | ISSUPERSET 75 | ISSUBSET 76 | LAST 77 | IN 78 | START 79 | STOP 80 | SECONDS 81 | TRUE 82 | FALSE 83 | WITHIN 84 | REPEATS 85 | TIMES 86 | IdentifierWithoutHyphen 87 | IdentifierWithHyphen 88 | EQ 89 | NEQ 90 | LT 91 | LE 92 | GT 93 | GE 94 | QUOTE 95 | COLON 96 | DOT 97 | COMMA 98 | RPAREN 99 | LPAREN 100 | RBRACK 101 | LBRACK 102 | PLUS 103 | HYPHEN 104 | MINUS 105 | POWER_OP 106 | DIVIDE 107 | ASTERISK 108 | WS 109 | COMMENT 110 | LINE_COMMENT 111 | InvalidCharacter 112 | 113 | rule names: 114 | pattern 115 | observationExpressions 116 | observationExpressionOr 117 | observationExpressionAnd 118 | observationExpression 119 | comparisonExpression 120 | comparisonExpressionAnd 121 | propTest 122 | startStopQualifier 123 | withinQualifier 124 | repeatedQualifier 125 | objectPath 126 | objectType 127 | firstPathComponent 128 | objectPathComponent 129 | setLiteral 130 | primitiveLiteral 131 | orderableLiteral 132 | 133 | 134 | atn: 135 | [3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 3, 55, 233, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 7, 3, 48, 10, 3, 12, 3, 14, 3, 51, 11, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 3, 4, 7, 4, 59, 10, 4, 12, 4, 14, 4, 62, 11, 4, 3, 5, 3, 5, 3, 5, 3, 5, 3, 5, 3, 5, 7, 5, 70, 10, 5, 12, 5, 14, 5, 73, 11, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 5, 6, 84, 10, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 7, 6, 92, 10, 6, 12, 6, 14, 6, 95, 11, 6, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 103, 10, 7, 12, 7, 14, 7, 106, 11, 7, 3, 8, 3, 8, 3, 8, 3, 8, 3, 8, 3, 8, 7, 8, 114, 10, 8, 12, 8, 14, 8, 117, 11, 8, 3, 9, 3, 9, 5, 9, 121, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 128, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 135, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 142, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 149, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 156, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 163, 10, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 3, 9, 5, 9, 172, 10, 9, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, 3, 13, 5, 13, 191, 10, 13, 3, 14, 3, 14, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 5, 16, 203, 10, 16, 3, 16, 3, 16, 7, 16, 207, 10, 16, 12, 16, 14, 16, 210, 11, 16, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 7, 17, 218, 10, 17, 12, 17, 14, 17, 221, 11, 17, 3, 17, 3, 17, 5, 17, 225, 10, 17, 3, 18, 3, 18, 5, 18, 229, 10, 18, 3, 19, 3, 19, 3, 19, 2, 9, 4, 6, 8, 10, 12, 14, 30, 20, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 2, 9, 3, 2, 32, 33, 3, 2, 34, 37, 4, 2, 4, 4, 6, 6, 3, 2, 30, 31, 4, 2, 9, 9, 30, 30, 4, 2, 3, 4, 51, 51, 4, 2, 3, 9, 11, 11, 2, 243, 2, 38, 3, 2, 2, 2, 4, 41, 3, 2, 2, 2, 6, 52, 3, 2, 2, 2, 8, 63, 3, 2, 2, 2, 10, 83, 3, 2, 2, 2, 12, 96, 3, 2, 2, 2, 14, 107, 3, 2, 2, 2, 16, 171, 3, 2, 2, 2, 18, 173, 3, 2, 2, 2, 20, 178, 3, 2, 2, 2, 22, 182, 3, 2, 2, 2, 24, 186, 3, 2, 2, 2, 26, 192, 3, 2, 2, 2, 28, 194, 3, 2, 2, 2, 30, 202, 3, 2, 2, 2, 32, 224, 3, 2, 2, 2, 34, 228, 3, 2, 2, 2, 36, 230, 3, 2, 2, 2, 38, 39, 5, 4, 3, 2, 39, 40, 7, 2, 2, 3, 40, 3, 3, 2, 2, 2, 41, 42, 8, 3, 1, 2, 42, 43, 5, 6, 4, 2, 43, 49, 3, 2, 2, 2, 44, 45, 12, 4, 2, 2, 45, 46, 7, 15, 2, 2, 46, 48, 5, 4, 3, 5, 47, 44, 3, 2, 2, 2, 48, 51, 3, 2, 2, 2, 49, 47, 3, 2, 2, 2, 49, 50, 3, 2, 2, 2, 50, 5, 3, 2, 2, 2, 51, 49, 3, 2, 2, 2, 52, 53, 8, 4, 1, 2, 53, 54, 5, 8, 5, 2, 54, 60, 3, 2, 2, 2, 55, 56, 12, 4, 2, 2, 56, 57, 7, 13, 2, 2, 57, 59, 5, 6, 4, 5, 58, 55, 3, 2, 2, 2, 59, 62, 3, 2, 2, 2, 60, 58, 3, 2, 2, 2, 60, 61, 3, 2, 2, 2, 61, 7, 3, 2, 2, 2, 62, 60, 3, 2, 2, 2, 63, 64, 8, 5, 1, 2, 64, 65, 5, 10, 6, 2, 65, 71, 3, 2, 2, 2, 66, 67, 12, 4, 2, 2, 67, 68, 7, 12, 2, 2, 68, 70, 5, 8, 5, 5, 69, 66, 3, 2, 2, 2, 70, 73, 3, 2, 2, 2, 71, 69, 3, 2, 2, 2, 71, 72, 3, 2, 2, 2, 72, 9, 3, 2, 2, 2, 73, 71, 3, 2, 2, 2, 74, 75, 8, 6, 1, 2, 75, 76, 7, 45, 2, 2, 76, 77, 5, 12, 7, 2, 77, 78, 7, 44, 2, 2, 78, 84, 3, 2, 2, 2, 79, 80, 7, 43, 2, 2, 80, 81, 5, 4, 3, 2, 81, 82, 7, 42, 2, 2, 82, 84, 3, 2, 2, 2, 83, 74, 3, 2, 2, 2, 83, 79, 3, 2, 2, 2, 84, 93, 3, 2, 2, 2, 85, 86, 12, 5, 2, 2, 86, 92, 5, 18, 10, 2, 87, 88, 12, 4, 2, 2, 88, 92, 5, 20, 11, 2, 89, 90, 12, 3, 2, 2, 90, 92, 5, 22, 12, 2, 91, 85, 3, 2, 2, 2, 91, 87, 3, 2, 2, 2, 91, 89, 3, 2, 2, 2, 92, 95, 3, 2, 2, 2, 93, 91, 3, 2, 2, 2, 93, 94, 3, 2, 2, 2, 94, 11, 3, 2, 2, 2, 95, 93, 3, 2, 2, 2, 96, 97, 8, 7, 1, 2, 97, 98, 5, 14, 8, 2, 98, 104, 3, 2, 2, 2, 99, 100, 12, 4, 2, 2, 100, 101, 7, 13, 2, 2, 101, 103, 5, 12, 7, 5, 102, 99, 3, 2, 2, 2, 103, 106, 3, 2, 2, 2, 104, 102, 3, 2, 2, 2, 104, 105, 3, 2, 2, 2, 105, 13, 3, 2, 2, 2, 106, 104, 3, 2, 2, 2, 107, 108, 8, 8, 1, 2, 108, 109, 5, 16, 9, 2, 109, 115, 3, 2, 2, 2, 110, 111, 12, 4, 2, 2, 111, 112, 7, 12, 2, 2, 112, 114, 5, 14, 8, 5, 113, 110, 3, 2, 2, 2, 114, 117, 3, 2, 2, 2, 115, 113, 3, 2, 2, 2, 115, 116, 3, 2, 2, 2, 116, 15, 3, 2, 2, 2, 117, 115, 3, 2, 2, 2, 118, 120, 5, 24, 13, 2, 119, 121, 7, 14, 2, 2, 120, 119, 3, 2, 2, 2, 120, 121, 3, 2, 2, 2, 121, 122, 3, 2, 2, 2, 122, 123, 9, 2, 2, 2, 123, 124, 5, 34, 18, 2, 124, 172, 3, 2, 2, 2, 125, 127, 5, 24, 13, 2, 126, 128, 7, 14, 2, 2, 127, 126, 3, 2, 2, 2, 127, 128, 3, 2, 2, 2, 128, 129, 3, 2, 2, 2, 129, 130, 9, 3, 2, 2, 130, 131, 5, 36, 19, 2, 131, 172, 3, 2, 2, 2, 132, 134, 5, 24, 13, 2, 133, 135, 7, 14, 2, 2, 134, 133, 3, 2, 2, 2, 134, 135, 3, 2, 2, 2, 135, 136, 3, 2, 2, 2, 136, 137, 7, 21, 2, 2, 137, 138, 5, 32, 17, 2, 138, 172, 3, 2, 2, 2, 139, 141, 5, 24, 13, 2, 140, 142, 7, 14, 2, 2, 141, 140, 3, 2, 2, 2, 141, 142, 3, 2, 2, 2, 142, 143, 3, 2, 2, 2, 143, 144, 7, 16, 2, 2, 144, 145, 7, 9, 2, 2, 145, 172, 3, 2, 2, 2, 146, 148, 5, 24, 13, 2, 147, 149, 7, 14, 2, 2, 148, 147, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 151, 7, 17, 2, 2, 151, 152, 7, 9, 2, 2, 152, 172, 3, 2, 2, 2, 153, 155, 5, 24, 13, 2, 154, 156, 7, 14, 2, 2, 155, 154, 3, 2, 2, 2, 155, 156, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, 157, 158, 7, 19, 2, 2, 158, 159, 7, 9, 2, 2, 159, 172, 3, 2, 2, 2, 160, 162, 5, 24, 13, 2, 161, 163, 7, 14, 2, 2, 162, 161, 3, 2, 2, 2, 162, 163, 3, 2, 2, 2, 163, 164, 3, 2, 2, 2, 164, 165, 7, 18, 2, 2, 165, 166, 7, 9, 2, 2, 166, 172, 3, 2, 2, 2, 167, 168, 7, 43, 2, 2, 168, 169, 5, 12, 7, 2, 169, 170, 7, 42, 2, 2, 170, 172, 3, 2, 2, 2, 171, 118, 3, 2, 2, 2, 171, 125, 3, 2, 2, 2, 171, 132, 3, 2, 2, 2, 171, 139, 3, 2, 2, 2, 171, 146, 3, 2, 2, 2, 171, 153, 3, 2, 2, 2, 171, 160, 3, 2, 2, 2, 171, 167, 3, 2, 2, 2, 172, 17, 3, 2, 2, 2, 173, 174, 7, 22, 2, 2, 174, 175, 7, 11, 2, 2, 175, 176, 7, 23, 2, 2, 176, 177, 7, 11, 2, 2, 177, 19, 3, 2, 2, 2, 178, 179, 7, 27, 2, 2, 179, 180, 9, 4, 2, 2, 180, 181, 7, 24, 2, 2, 181, 21, 3, 2, 2, 2, 182, 183, 7, 28, 2, 2, 183, 184, 7, 4, 2, 2, 184, 185, 7, 29, 2, 2, 185, 23, 3, 2, 2, 2, 186, 187, 5, 26, 14, 2, 187, 188, 7, 39, 2, 2, 188, 190, 5, 28, 15, 2, 189, 191, 5, 30, 16, 2, 190, 189, 3, 2, 2, 2, 190, 191, 3, 2, 2, 2, 191, 25, 3, 2, 2, 2, 192, 193, 9, 5, 2, 2, 193, 27, 3, 2, 2, 2, 194, 195, 9, 6, 2, 2, 195, 29, 3, 2, 2, 2, 196, 197, 8, 16, 1, 2, 197, 198, 7, 40, 2, 2, 198, 203, 9, 6, 2, 2, 199, 200, 7, 45, 2, 2, 200, 201, 9, 7, 2, 2, 201, 203, 7, 44, 2, 2, 202, 196, 3, 2, 2, 2, 202, 199, 3, 2, 2, 2, 203, 208, 3, 2, 2, 2, 204, 205, 12, 5, 2, 2, 205, 207, 5, 30, 16, 6, 206, 204, 3, 2, 2, 2, 207, 210, 3, 2, 2, 2, 208, 206, 3, 2, 2, 2, 208, 209, 3, 2, 2, 2, 209, 31, 3, 2, 2, 2, 210, 208, 3, 2, 2, 2, 211, 212, 7, 43, 2, 2, 212, 225, 7, 42, 2, 2, 213, 214, 7, 43, 2, 2, 214, 219, 5, 34, 18, 2, 215, 216, 7, 41, 2, 2, 216, 218, 5, 34, 18, 2, 217, 215, 3, 2, 2, 2, 218, 221, 3, 2, 2, 2, 219, 217, 3, 2, 2, 2, 219, 220, 3, 2, 2, 2, 220, 222, 3, 2, 2, 2, 221, 219, 3, 2, 2, 2, 222, 223, 7, 42, 2, 2, 223, 225, 3, 2, 2, 2, 224, 211, 3, 2, 2, 2, 224, 213, 3, 2, 2, 2, 225, 33, 3, 2, 2, 2, 226, 229, 5, 36, 19, 2, 227, 229, 7, 10, 2, 2, 228, 226, 3, 2, 2, 2, 228, 227, 3, 2, 2, 2, 229, 35, 3, 2, 2, 2, 230, 231, 9, 8, 2, 2, 231, 37, 3, 2, 2, 2, 24, 49, 60, 71, 83, 91, 93, 104, 115, 120, 127, 134, 141, 148, 155, 162, 171, 190, 202, 208, 219, 224, 228] -------------------------------------------------------------------------------- /dendrol/lang/STIXPattern.tokens: -------------------------------------------------------------------------------- 1 | IntNegLiteral=1 2 | IntPosLiteral=2 3 | FloatNegLiteral=3 4 | FloatPosLiteral=4 5 | HexLiteral=5 6 | BinaryLiteral=6 7 | StringLiteral=7 8 | BoolLiteral=8 9 | TimestampLiteral=9 10 | AND=10 11 | OR=11 12 | NOT=12 13 | FOLLOWEDBY=13 14 | LIKE=14 15 | MATCHES=15 16 | ISSUPERSET=16 17 | ISSUBSET=17 18 | LAST=18 19 | IN=19 20 | START=20 21 | STOP=21 22 | SECONDS=22 23 | TRUE=23 24 | FALSE=24 25 | WITHIN=25 26 | REPEATS=26 27 | TIMES=27 28 | IdentifierWithoutHyphen=28 29 | IdentifierWithHyphen=29 30 | EQ=30 31 | NEQ=31 32 | LT=32 33 | LE=33 34 | GT=34 35 | GE=35 36 | QUOTE=36 37 | COLON=37 38 | DOT=38 39 | COMMA=39 40 | RPAREN=40 41 | LPAREN=41 42 | RBRACK=42 43 | LBRACK=43 44 | PLUS=44 45 | HYPHEN=45 46 | MINUS=46 47 | POWER_OP=47 48 | DIVIDE=48 49 | ASTERISK=49 50 | WS=50 51 | COMMENT=51 52 | LINE_COMMENT=52 53 | InvalidCharacter=53 54 | 'AND'=10 55 | 'OR'=11 56 | 'NOT'=12 57 | 'FOLLOWEDBY'=13 58 | 'LIKE'=14 59 | 'MATCHES'=15 60 | 'ISSUPERSET'=16 61 | 'ISSUBSET'=17 62 | 'LAST'=18 63 | 'IN'=19 64 | 'START'=20 65 | 'STOP'=21 66 | 'SECONDS'=22 67 | 'true'=23 68 | 'false'=24 69 | 'WITHIN'=25 70 | 'REPEATS'=26 71 | 'TIMES'=27 72 | '<'=32 73 | '<='=33 74 | '>'=34 75 | '>='=35 76 | '\''=36 77 | ':'=37 78 | '.'=38 79 | ','=39 80 | ')'=40 81 | '('=41 82 | ']'=42 83 | '['=43 84 | '+'=44 85 | '-'=46 86 | '^'=47 87 | '/'=48 88 | '*'=49 89 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPatternLexer.interp: -------------------------------------------------------------------------------- 1 | token literal names: 2 | null 3 | null 4 | null 5 | null 6 | null 7 | null 8 | null 9 | null 10 | null 11 | null 12 | 'AND' 13 | 'OR' 14 | 'NOT' 15 | 'FOLLOWEDBY' 16 | 'LIKE' 17 | 'MATCHES' 18 | 'ISSUPERSET' 19 | 'ISSUBSET' 20 | 'LAST' 21 | 'IN' 22 | 'START' 23 | 'STOP' 24 | 'SECONDS' 25 | 'true' 26 | 'false' 27 | 'WITHIN' 28 | 'REPEATS' 29 | 'TIMES' 30 | null 31 | null 32 | null 33 | null 34 | '<' 35 | '<=' 36 | '>' 37 | '>=' 38 | '\'' 39 | ':' 40 | '.' 41 | ',' 42 | ')' 43 | '(' 44 | ']' 45 | '[' 46 | '+' 47 | null 48 | '-' 49 | '^' 50 | '/' 51 | '*' 52 | null 53 | null 54 | null 55 | null 56 | 57 | token symbolic names: 58 | null 59 | IntNegLiteral 60 | IntPosLiteral 61 | FloatNegLiteral 62 | FloatPosLiteral 63 | HexLiteral 64 | BinaryLiteral 65 | StringLiteral 66 | BoolLiteral 67 | TimestampLiteral 68 | AND 69 | OR 70 | NOT 71 | FOLLOWEDBY 72 | LIKE 73 | MATCHES 74 | ISSUPERSET 75 | ISSUBSET 76 | LAST 77 | IN 78 | START 79 | STOP 80 | SECONDS 81 | TRUE 82 | FALSE 83 | WITHIN 84 | REPEATS 85 | TIMES 86 | IdentifierWithoutHyphen 87 | IdentifierWithHyphen 88 | EQ 89 | NEQ 90 | LT 91 | LE 92 | GT 93 | GE 94 | QUOTE 95 | COLON 96 | DOT 97 | COMMA 98 | RPAREN 99 | LPAREN 100 | RBRACK 101 | LBRACK 102 | PLUS 103 | HYPHEN 104 | MINUS 105 | POWER_OP 106 | DIVIDE 107 | ASTERISK 108 | WS 109 | COMMENT 110 | LINE_COMMENT 111 | InvalidCharacter 112 | 113 | rule names: 114 | IntNegLiteral 115 | IntPosLiteral 116 | FloatNegLiteral 117 | FloatPosLiteral 118 | HexLiteral 119 | BinaryLiteral 120 | StringLiteral 121 | BoolLiteral 122 | TimestampLiteral 123 | AND 124 | OR 125 | NOT 126 | FOLLOWEDBY 127 | LIKE 128 | MATCHES 129 | ISSUPERSET 130 | ISSUBSET 131 | LAST 132 | IN 133 | START 134 | STOP 135 | SECONDS 136 | TRUE 137 | FALSE 138 | WITHIN 139 | REPEATS 140 | TIMES 141 | IdentifierWithoutHyphen 142 | IdentifierWithHyphen 143 | EQ 144 | NEQ 145 | LT 146 | LE 147 | GT 148 | GE 149 | QUOTE 150 | COLON 151 | DOT 152 | COMMA 153 | RPAREN 154 | LPAREN 155 | RBRACK 156 | LBRACK 157 | PLUS 158 | HYPHEN 159 | MINUS 160 | POWER_OP 161 | DIVIDE 162 | ASTERISK 163 | HexDigit 164 | TwoHexDigits 165 | Base64Char 166 | WS 167 | COMMENT 168 | LINE_COMMENT 169 | InvalidCharacter 170 | 171 | channel names: 172 | DEFAULT_TOKEN_CHANNEL 173 | HIDDEN 174 | 175 | mode names: 176 | DEFAULT_MODE 177 | 178 | atn: 179 | [3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 55, 495, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 4, 33, 9, 33, 4, 34, 9, 34, 4, 35, 9, 35, 4, 36, 9, 36, 4, 37, 9, 37, 4, 38, 9, 38, 4, 39, 9, 39, 4, 40, 9, 40, 4, 41, 9, 41, 4, 42, 9, 42, 4, 43, 9, 43, 4, 44, 9, 44, 4, 45, 9, 45, 4, 46, 9, 46, 4, 47, 9, 47, 4, 48, 9, 48, 4, 49, 9, 49, 4, 50, 9, 50, 4, 51, 9, 51, 4, 52, 9, 52, 4, 53, 9, 53, 4, 54, 9, 54, 4, 55, 9, 55, 4, 56, 9, 56, 4, 57, 9, 57, 3, 2, 3, 2, 3, 2, 3, 2, 7, 2, 120, 10, 2, 12, 2, 14, 2, 123, 11, 2, 5, 2, 125, 10, 2, 3, 3, 5, 3, 128, 10, 3, 3, 3, 3, 3, 3, 3, 7, 3, 133, 10, 3, 12, 3, 14, 3, 136, 11, 3, 5, 3, 138, 10, 3, 3, 4, 3, 4, 7, 4, 142, 10, 4, 12, 4, 14, 4, 145, 11, 4, 3, 4, 3, 4, 6, 4, 149, 10, 4, 13, 4, 14, 4, 150, 3, 5, 5, 5, 154, 10, 5, 3, 5, 7, 5, 157, 10, 5, 12, 5, 14, 5, 160, 11, 5, 3, 5, 3, 5, 6, 5, 164, 10, 5, 13, 5, 14, 5, 165, 3, 6, 3, 6, 3, 6, 7, 6, 171, 10, 6, 12, 6, 14, 6, 174, 11, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 185, 10, 7, 12, 7, 14, 7, 188, 11, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 3, 7, 5, 7, 207, 10, 7, 3, 7, 3, 7, 3, 8, 3, 8, 3, 8, 3, 8, 3, 8, 3, 8, 7, 8, 217, 10, 8, 12, 8, 14, 8, 220, 11, 8, 3, 8, 3, 8, 3, 9, 3, 9, 5, 9, 226, 10, 9, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 5, 10, 239, 10, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 5, 10, 248, 10, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 5, 10, 255, 10, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 5, 10, 265, 10, 10, 3, 10, 3, 10, 6, 10, 269, 10, 10, 13, 10, 14, 10, 270, 5, 10, 273, 10, 10, 3, 10, 3, 10, 3, 10, 3, 11, 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, 3, 13, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 20, 3, 20, 3, 20, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 22, 3, 22, 3, 22, 3, 22, 3, 22, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 24, 3, 24, 3, 24, 3, 24, 3, 24, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 26, 3, 26, 3, 26, 3, 26, 3, 26, 3, 26, 3, 26, 3, 27, 3, 27, 3, 27, 3, 27, 3, 27, 3, 27, 3, 27, 3, 27, 3, 28, 3, 28, 3, 28, 3, 28, 3, 28, 3, 28, 3, 29, 3, 29, 7, 29, 394, 10, 29, 12, 29, 14, 29, 397, 11, 29, 3, 30, 3, 30, 7, 30, 401, 10, 30, 12, 30, 14, 30, 404, 11, 30, 3, 31, 3, 31, 3, 31, 5, 31, 409, 10, 31, 3, 32, 3, 32, 3, 32, 3, 32, 5, 32, 415, 10, 32, 3, 33, 3, 33, 3, 34, 3, 34, 3, 34, 3, 35, 3, 35, 3, 36, 3, 36, 3, 36, 3, 37, 3, 37, 3, 38, 3, 38, 3, 39, 3, 39, 3, 40, 3, 40, 3, 41, 3, 41, 3, 42, 3, 42, 3, 43, 3, 43, 3, 44, 3, 44, 3, 45, 3, 45, 3, 46, 3, 46, 3, 47, 3, 47, 3, 48, 3, 48, 3, 49, 3, 49, 3, 50, 3, 50, 3, 51, 3, 51, 3, 52, 3, 52, 3, 52, 3, 53, 3, 53, 3, 54, 6, 54, 463, 10, 54, 13, 54, 14, 54, 464, 3, 54, 3, 54, 3, 55, 3, 55, 3, 55, 3, 55, 7, 55, 473, 10, 55, 12, 55, 14, 55, 476, 11, 55, 3, 55, 3, 55, 3, 55, 3, 55, 3, 55, 3, 56, 3, 56, 3, 56, 3, 56, 7, 56, 487, 10, 56, 12, 56, 14, 56, 490, 11, 56, 3, 56, 3, 56, 3, 57, 3, 57, 3, 474, 2, 58, 3, 3, 5, 4, 7, 5, 9, 6, 11, 7, 13, 8, 15, 9, 17, 10, 19, 11, 21, 12, 23, 13, 25, 14, 27, 15, 29, 16, 31, 17, 33, 18, 35, 19, 37, 20, 39, 21, 41, 22, 43, 23, 45, 24, 47, 25, 49, 26, 51, 27, 53, 28, 55, 29, 57, 30, 59, 31, 61, 32, 63, 33, 65, 34, 67, 35, 69, 36, 71, 37, 73, 38, 75, 39, 77, 40, 79, 41, 81, 42, 83, 43, 85, 44, 87, 45, 89, 46, 91, 47, 93, 48, 95, 49, 97, 50, 99, 51, 101, 2, 103, 2, 105, 2, 107, 52, 109, 53, 111, 54, 113, 55, 3, 2, 17, 3, 2, 51, 59, 3, 2, 50, 59, 4, 2, 41, 41, 94, 94, 3, 2, 50, 52, 3, 2, 51, 52, 3, 2, 50, 51, 3, 2, 50, 53, 3, 2, 50, 55, 5, 2, 67, 92, 97, 97, 99, 124, 6, 2, 50, 59, 67, 92, 97, 97, 99, 124, 7, 2, 47, 47, 50, 59, 67, 92, 97, 97, 99, 124, 5, 2, 50, 59, 67, 72, 99, 104, 6, 2, 45, 45, 49, 59, 67, 92, 99, 124, 12, 2, 11, 15, 34, 34, 135, 135, 162, 162, 5762, 5762, 8194, 8204, 8234, 8235, 8241, 8241, 8289, 8289, 12290, 12290, 4, 2, 12, 12, 15, 15, 2, 523, 2, 3, 3, 2, 2, 2, 2, 5, 3, 2, 2, 2, 2, 7, 3, 2, 2, 2, 2, 9, 3, 2, 2, 2, 2, 11, 3, 2, 2, 2, 2, 13, 3, 2, 2, 2, 2, 15, 3, 2, 2, 2, 2, 17, 3, 2, 2, 2, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 2, 65, 3, 2, 2, 2, 2, 67, 3, 2, 2, 2, 2, 69, 3, 2, 2, 2, 2, 71, 3, 2, 2, 2, 2, 73, 3, 2, 2, 2, 2, 75, 3, 2, 2, 2, 2, 77, 3, 2, 2, 2, 2, 79, 3, 2, 2, 2, 2, 81, 3, 2, 2, 2, 2, 83, 3, 2, 2, 2, 2, 85, 3, 2, 2, 2, 2, 87, 3, 2, 2, 2, 2, 89, 3, 2, 2, 2, 2, 91, 3, 2, 2, 2, 2, 93, 3, 2, 2, 2, 2, 95, 3, 2, 2, 2, 2, 97, 3, 2, 2, 2, 2, 99, 3, 2, 2, 2, 2, 107, 3, 2, 2, 2, 2, 109, 3, 2, 2, 2, 2, 111, 3, 2, 2, 2, 2, 113, 3, 2, 2, 2, 3, 115, 3, 2, 2, 2, 5, 127, 3, 2, 2, 2, 7, 139, 3, 2, 2, 2, 9, 153, 3, 2, 2, 2, 11, 167, 3, 2, 2, 2, 13, 177, 3, 2, 2, 2, 15, 210, 3, 2, 2, 2, 17, 225, 3, 2, 2, 2, 19, 227, 3, 2, 2, 2, 21, 277, 3, 2, 2, 2, 23, 281, 3, 2, 2, 2, 25, 284, 3, 2, 2, 2, 27, 288, 3, 2, 2, 2, 29, 299, 3, 2, 2, 2, 31, 304, 3, 2, 2, 2, 33, 312, 3, 2, 2, 2, 35, 323, 3, 2, 2, 2, 37, 332, 3, 2, 2, 2, 39, 337, 3, 2, 2, 2, 41, 340, 3, 2, 2, 2, 43, 346, 3, 2, 2, 2, 45, 351, 3, 2, 2, 2, 47, 359, 3, 2, 2, 2, 49, 364, 3, 2, 2, 2, 51, 370, 3, 2, 2, 2, 53, 377, 3, 2, 2, 2, 55, 385, 3, 2, 2, 2, 57, 391, 3, 2, 2, 2, 59, 398, 3, 2, 2, 2, 61, 408, 3, 2, 2, 2, 63, 414, 3, 2, 2, 2, 65, 416, 3, 2, 2, 2, 67, 418, 3, 2, 2, 2, 69, 421, 3, 2, 2, 2, 71, 423, 3, 2, 2, 2, 73, 426, 3, 2, 2, 2, 75, 428, 3, 2, 2, 2, 77, 430, 3, 2, 2, 2, 79, 432, 3, 2, 2, 2, 81, 434, 3, 2, 2, 2, 83, 436, 3, 2, 2, 2, 85, 438, 3, 2, 2, 2, 87, 440, 3, 2, 2, 2, 89, 442, 3, 2, 2, 2, 91, 444, 3, 2, 2, 2, 93, 446, 3, 2, 2, 2, 95, 448, 3, 2, 2, 2, 97, 450, 3, 2, 2, 2, 99, 452, 3, 2, 2, 2, 101, 454, 3, 2, 2, 2, 103, 456, 3, 2, 2, 2, 105, 459, 3, 2, 2, 2, 107, 462, 3, 2, 2, 2, 109, 468, 3, 2, 2, 2, 111, 482, 3, 2, 2, 2, 113, 493, 3, 2, 2, 2, 115, 124, 7, 47, 2, 2, 116, 125, 7, 50, 2, 2, 117, 121, 9, 2, 2, 2, 118, 120, 9, 3, 2, 2, 119, 118, 3, 2, 2, 2, 120, 123, 3, 2, 2, 2, 121, 119, 3, 2, 2, 2, 121, 122, 3, 2, 2, 2, 122, 125, 3, 2, 2, 2, 123, 121, 3, 2, 2, 2, 124, 116, 3, 2, 2, 2, 124, 117, 3, 2, 2, 2, 125, 4, 3, 2, 2, 2, 126, 128, 7, 45, 2, 2, 127, 126, 3, 2, 2, 2, 127, 128, 3, 2, 2, 2, 128, 137, 3, 2, 2, 2, 129, 138, 7, 50, 2, 2, 130, 134, 9, 2, 2, 2, 131, 133, 9, 3, 2, 2, 132, 131, 3, 2, 2, 2, 133, 136, 3, 2, 2, 2, 134, 132, 3, 2, 2, 2, 134, 135, 3, 2, 2, 2, 135, 138, 3, 2, 2, 2, 136, 134, 3, 2, 2, 2, 137, 129, 3, 2, 2, 2, 137, 130, 3, 2, 2, 2, 138, 6, 3, 2, 2, 2, 139, 143, 7, 47, 2, 2, 140, 142, 9, 3, 2, 2, 141, 140, 3, 2, 2, 2, 142, 145, 3, 2, 2, 2, 143, 141, 3, 2, 2, 2, 143, 144, 3, 2, 2, 2, 144, 146, 3, 2, 2, 2, 145, 143, 3, 2, 2, 2, 146, 148, 7, 48, 2, 2, 147, 149, 9, 3, 2, 2, 148, 147, 3, 2, 2, 2, 149, 150, 3, 2, 2, 2, 150, 148, 3, 2, 2, 2, 150, 151, 3, 2, 2, 2, 151, 8, 3, 2, 2, 2, 152, 154, 7, 45, 2, 2, 153, 152, 3, 2, 2, 2, 153, 154, 3, 2, 2, 2, 154, 158, 3, 2, 2, 2, 155, 157, 9, 3, 2, 2, 156, 155, 3, 2, 2, 2, 157, 160, 3, 2, 2, 2, 158, 156, 3, 2, 2, 2, 158, 159, 3, 2, 2, 2, 159, 161, 3, 2, 2, 2, 160, 158, 3, 2, 2, 2, 161, 163, 7, 48, 2, 2, 162, 164, 9, 3, 2, 2, 163, 162, 3, 2, 2, 2, 164, 165, 3, 2, 2, 2, 165, 163, 3, 2, 2, 2, 165, 166, 3, 2, 2, 2, 166, 10, 3, 2, 2, 2, 167, 168, 7, 106, 2, 2, 168, 172, 5, 73, 37, 2, 169, 171, 5, 103, 52, 2, 170, 169, 3, 2, 2, 2, 171, 174, 3, 2, 2, 2, 172, 170, 3, 2, 2, 2, 172, 173, 3, 2, 2, 2, 173, 175, 3, 2, 2, 2, 174, 172, 3, 2, 2, 2, 175, 176, 5, 73, 37, 2, 176, 12, 3, 2, 2, 2, 177, 178, 7, 100, 2, 2, 178, 186, 5, 73, 37, 2, 179, 180, 5, 105, 53, 2, 180, 181, 5, 105, 53, 2, 181, 182, 5, 105, 53, 2, 182, 183, 5, 105, 53, 2, 183, 185, 3, 2, 2, 2, 184, 179, 3, 2, 2, 2, 185, 188, 3, 2, 2, 2, 186, 184, 3, 2, 2, 2, 186, 187, 3, 2, 2, 2, 187, 206, 3, 2, 2, 2, 188, 186, 3, 2, 2, 2, 189, 190, 5, 105, 53, 2, 190, 191, 5, 105, 53, 2, 191, 192, 5, 105, 53, 2, 192, 193, 5, 105, 53, 2, 193, 207, 3, 2, 2, 2, 194, 195, 5, 105, 53, 2, 195, 196, 5, 105, 53, 2, 196, 197, 5, 105, 53, 2, 197, 198, 3, 2, 2, 2, 198, 199, 7, 63, 2, 2, 199, 207, 3, 2, 2, 2, 200, 201, 5, 105, 53, 2, 201, 202, 5, 105, 53, 2, 202, 203, 3, 2, 2, 2, 203, 204, 7, 63, 2, 2, 204, 205, 7, 63, 2, 2, 205, 207, 3, 2, 2, 2, 206, 189, 3, 2, 2, 2, 206, 194, 3, 2, 2, 2, 206, 200, 3, 2, 2, 2, 207, 208, 3, 2, 2, 2, 208, 209, 5, 73, 37, 2, 209, 14, 3, 2, 2, 2, 210, 218, 5, 73, 37, 2, 211, 217, 10, 4, 2, 2, 212, 213, 7, 94, 2, 2, 213, 217, 7, 41, 2, 2, 214, 215, 7, 94, 2, 2, 215, 217, 7, 94, 2, 2, 216, 211, 3, 2, 2, 2, 216, 212, 3, 2, 2, 2, 216, 214, 3, 2, 2, 2, 217, 220, 3, 2, 2, 2, 218, 216, 3, 2, 2, 2, 218, 219, 3, 2, 2, 2, 219, 221, 3, 2, 2, 2, 220, 218, 3, 2, 2, 2, 221, 222, 5, 73, 37, 2, 222, 16, 3, 2, 2, 2, 223, 226, 5, 47, 24, 2, 224, 226, 5, 49, 25, 2, 225, 223, 3, 2, 2, 2, 225, 224, 3, 2, 2, 2, 226, 18, 3, 2, 2, 2, 227, 228, 7, 118, 2, 2, 228, 229, 5, 73, 37, 2, 229, 230, 9, 3, 2, 2, 230, 231, 9, 3, 2, 2, 231, 232, 9, 3, 2, 2, 232, 233, 9, 3, 2, 2, 233, 238, 5, 91, 46, 2, 234, 235, 7, 50, 2, 2, 235, 239, 9, 2, 2, 2, 236, 237, 7, 51, 2, 2, 237, 239, 9, 5, 2, 2, 238, 234, 3, 2, 2, 2, 238, 236, 3, 2, 2, 2, 239, 240, 3, 2, 2, 2, 240, 247, 5, 91, 46, 2, 241, 242, 7, 50, 2, 2, 242, 248, 9, 2, 2, 2, 243, 244, 9, 6, 2, 2, 244, 248, 9, 3, 2, 2, 245, 246, 7, 53, 2, 2, 246, 248, 9, 7, 2, 2, 247, 241, 3, 2, 2, 2, 247, 243, 3, 2, 2, 2, 247, 245, 3, 2, 2, 2, 248, 249, 3, 2, 2, 2, 249, 254, 7, 86, 2, 2, 250, 251, 9, 7, 2, 2, 251, 255, 9, 3, 2, 2, 252, 253, 7, 52, 2, 2, 253, 255, 9, 8, 2, 2, 254, 250, 3, 2, 2, 2, 254, 252, 3, 2, 2, 2, 255, 256, 3, 2, 2, 2, 256, 257, 5, 75, 38, 2, 257, 258, 9, 9, 2, 2, 258, 259, 9, 3, 2, 2, 259, 264, 5, 75, 38, 2, 260, 261, 9, 9, 2, 2, 261, 265, 9, 3, 2, 2, 262, 263, 7, 56, 2, 2, 263, 265, 7, 50, 2, 2, 264, 260, 3, 2, 2, 2, 264, 262, 3, 2, 2, 2, 265, 272, 3, 2, 2, 2, 266, 268, 5, 77, 39, 2, 267, 269, 9, 3, 2, 2, 268, 267, 3, 2, 2, 2, 269, 270, 3, 2, 2, 2, 270, 268, 3, 2, 2, 2, 270, 271, 3, 2, 2, 2, 271, 273, 3, 2, 2, 2, 272, 266, 3, 2, 2, 2, 272, 273, 3, 2, 2, 2, 273, 274, 3, 2, 2, 2, 274, 275, 7, 92, 2, 2, 275, 276, 5, 73, 37, 2, 276, 20, 3, 2, 2, 2, 277, 278, 7, 67, 2, 2, 278, 279, 7, 80, 2, 2, 279, 280, 7, 70, 2, 2, 280, 22, 3, 2, 2, 2, 281, 282, 7, 81, 2, 2, 282, 283, 7, 84, 2, 2, 283, 24, 3, 2, 2, 2, 284, 285, 7, 80, 2, 2, 285, 286, 7, 81, 2, 2, 286, 287, 7, 86, 2, 2, 287, 26, 3, 2, 2, 2, 288, 289, 7, 72, 2, 2, 289, 290, 7, 81, 2, 2, 290, 291, 7, 78, 2, 2, 291, 292, 7, 78, 2, 2, 292, 293, 7, 81, 2, 2, 293, 294, 7, 89, 2, 2, 294, 295, 7, 71, 2, 2, 295, 296, 7, 70, 2, 2, 296, 297, 7, 68, 2, 2, 297, 298, 7, 91, 2, 2, 298, 28, 3, 2, 2, 2, 299, 300, 7, 78, 2, 2, 300, 301, 7, 75, 2, 2, 301, 302, 7, 77, 2, 2, 302, 303, 7, 71, 2, 2, 303, 30, 3, 2, 2, 2, 304, 305, 7, 79, 2, 2, 305, 306, 7, 67, 2, 2, 306, 307, 7, 86, 2, 2, 307, 308, 7, 69, 2, 2, 308, 309, 7, 74, 2, 2, 309, 310, 7, 71, 2, 2, 310, 311, 7, 85, 2, 2, 311, 32, 3, 2, 2, 2, 312, 313, 7, 75, 2, 2, 313, 314, 7, 85, 2, 2, 314, 315, 7, 85, 2, 2, 315, 316, 7, 87, 2, 2, 316, 317, 7, 82, 2, 2, 317, 318, 7, 71, 2, 2, 318, 319, 7, 84, 2, 2, 319, 320, 7, 85, 2, 2, 320, 321, 7, 71, 2, 2, 321, 322, 7, 86, 2, 2, 322, 34, 3, 2, 2, 2, 323, 324, 7, 75, 2, 2, 324, 325, 7, 85, 2, 2, 325, 326, 7, 85, 2, 2, 326, 327, 7, 87, 2, 2, 327, 328, 7, 68, 2, 2, 328, 329, 7, 85, 2, 2, 329, 330, 7, 71, 2, 2, 330, 331, 7, 86, 2, 2, 331, 36, 3, 2, 2, 2, 332, 333, 7, 78, 2, 2, 333, 334, 7, 67, 2, 2, 334, 335, 7, 85, 2, 2, 335, 336, 7, 86, 2, 2, 336, 38, 3, 2, 2, 2, 337, 338, 7, 75, 2, 2, 338, 339, 7, 80, 2, 2, 339, 40, 3, 2, 2, 2, 340, 341, 7, 85, 2, 2, 341, 342, 7, 86, 2, 2, 342, 343, 7, 67, 2, 2, 343, 344, 7, 84, 2, 2, 344, 345, 7, 86, 2, 2, 345, 42, 3, 2, 2, 2, 346, 347, 7, 85, 2, 2, 347, 348, 7, 86, 2, 2, 348, 349, 7, 81, 2, 2, 349, 350, 7, 82, 2, 2, 350, 44, 3, 2, 2, 2, 351, 352, 7, 85, 2, 2, 352, 353, 7, 71, 2, 2, 353, 354, 7, 69, 2, 2, 354, 355, 7, 81, 2, 2, 355, 356, 7, 80, 2, 2, 356, 357, 7, 70, 2, 2, 357, 358, 7, 85, 2, 2, 358, 46, 3, 2, 2, 2, 359, 360, 7, 118, 2, 2, 360, 361, 7, 116, 2, 2, 361, 362, 7, 119, 2, 2, 362, 363, 7, 103, 2, 2, 363, 48, 3, 2, 2, 2, 364, 365, 7, 104, 2, 2, 365, 366, 7, 99, 2, 2, 366, 367, 7, 110, 2, 2, 367, 368, 7, 117, 2, 2, 368, 369, 7, 103, 2, 2, 369, 50, 3, 2, 2, 2, 370, 371, 7, 89, 2, 2, 371, 372, 7, 75, 2, 2, 372, 373, 7, 86, 2, 2, 373, 374, 7, 74, 2, 2, 374, 375, 7, 75, 2, 2, 375, 376, 7, 80, 2, 2, 376, 52, 3, 2, 2, 2, 377, 378, 7, 84, 2, 2, 378, 379, 7, 71, 2, 2, 379, 380, 7, 82, 2, 2, 380, 381, 7, 71, 2, 2, 381, 382, 7, 67, 2, 2, 382, 383, 7, 86, 2, 2, 383, 384, 7, 85, 2, 2, 384, 54, 3, 2, 2, 2, 385, 386, 7, 86, 2, 2, 386, 387, 7, 75, 2, 2, 387, 388, 7, 79, 2, 2, 388, 389, 7, 71, 2, 2, 389, 390, 7, 85, 2, 2, 390, 56, 3, 2, 2, 2, 391, 395, 9, 10, 2, 2, 392, 394, 9, 11, 2, 2, 393, 392, 3, 2, 2, 2, 394, 397, 3, 2, 2, 2, 395, 393, 3, 2, 2, 2, 395, 396, 3, 2, 2, 2, 396, 58, 3, 2, 2, 2, 397, 395, 3, 2, 2, 2, 398, 402, 9, 10, 2, 2, 399, 401, 9, 12, 2, 2, 400, 399, 3, 2, 2, 2, 401, 404, 3, 2, 2, 2, 402, 400, 3, 2, 2, 2, 402, 403, 3, 2, 2, 2, 403, 60, 3, 2, 2, 2, 404, 402, 3, 2, 2, 2, 405, 409, 7, 63, 2, 2, 406, 407, 7, 63, 2, 2, 407, 409, 7, 63, 2, 2, 408, 405, 3, 2, 2, 2, 408, 406, 3, 2, 2, 2, 409, 62, 3, 2, 2, 2, 410, 411, 7, 35, 2, 2, 411, 415, 7, 63, 2, 2, 412, 413, 7, 62, 2, 2, 413, 415, 7, 64, 2, 2, 414, 410, 3, 2, 2, 2, 414, 412, 3, 2, 2, 2, 415, 64, 3, 2, 2, 2, 416, 417, 7, 62, 2, 2, 417, 66, 3, 2, 2, 2, 418, 419, 7, 62, 2, 2, 419, 420, 7, 63, 2, 2, 420, 68, 3, 2, 2, 2, 421, 422, 7, 64, 2, 2, 422, 70, 3, 2, 2, 2, 423, 424, 7, 64, 2, 2, 424, 425, 7, 63, 2, 2, 425, 72, 3, 2, 2, 2, 426, 427, 7, 41, 2, 2, 427, 74, 3, 2, 2, 2, 428, 429, 7, 60, 2, 2, 429, 76, 3, 2, 2, 2, 430, 431, 7, 48, 2, 2, 431, 78, 3, 2, 2, 2, 432, 433, 7, 46, 2, 2, 433, 80, 3, 2, 2, 2, 434, 435, 7, 43, 2, 2, 435, 82, 3, 2, 2, 2, 436, 437, 7, 42, 2, 2, 437, 84, 3, 2, 2, 2, 438, 439, 7, 95, 2, 2, 439, 86, 3, 2, 2, 2, 440, 441, 7, 93, 2, 2, 441, 88, 3, 2, 2, 2, 442, 443, 7, 45, 2, 2, 443, 90, 3, 2, 2, 2, 444, 445, 5, 93, 47, 2, 445, 92, 3, 2, 2, 2, 446, 447, 7, 47, 2, 2, 447, 94, 3, 2, 2, 2, 448, 449, 7, 96, 2, 2, 449, 96, 3, 2, 2, 2, 450, 451, 7, 49, 2, 2, 451, 98, 3, 2, 2, 2, 452, 453, 7, 44, 2, 2, 453, 100, 3, 2, 2, 2, 454, 455, 9, 13, 2, 2, 455, 102, 3, 2, 2, 2, 456, 457, 5, 101, 51, 2, 457, 458, 5, 101, 51, 2, 458, 104, 3, 2, 2, 2, 459, 460, 9, 14, 2, 2, 460, 106, 3, 2, 2, 2, 461, 463, 9, 15, 2, 2, 462, 461, 3, 2, 2, 2, 463, 464, 3, 2, 2, 2, 464, 462, 3, 2, 2, 2, 464, 465, 3, 2, 2, 2, 465, 466, 3, 2, 2, 2, 466, 467, 8, 54, 2, 2, 467, 108, 3, 2, 2, 2, 468, 469, 7, 49, 2, 2, 469, 470, 7, 44, 2, 2, 470, 474, 3, 2, 2, 2, 471, 473, 11, 2, 2, 2, 472, 471, 3, 2, 2, 2, 473, 476, 3, 2, 2, 2, 474, 475, 3, 2, 2, 2, 474, 472, 3, 2, 2, 2, 475, 477, 3, 2, 2, 2, 476, 474, 3, 2, 2, 2, 477, 478, 7, 44, 2, 2, 478, 479, 7, 49, 2, 2, 479, 480, 3, 2, 2, 2, 480, 481, 8, 55, 2, 2, 481, 110, 3, 2, 2, 2, 482, 483, 7, 49, 2, 2, 483, 484, 7, 49, 2, 2, 484, 488, 3, 2, 2, 2, 485, 487, 10, 16, 2, 2, 486, 485, 3, 2, 2, 2, 487, 490, 3, 2, 2, 2, 488, 486, 3, 2, 2, 2, 488, 489, 3, 2, 2, 2, 489, 491, 3, 2, 2, 2, 490, 488, 3, 2, 2, 2, 491, 492, 8, 56, 2, 2, 492, 112, 3, 2, 2, 2, 493, 494, 11, 2, 2, 2, 494, 114, 3, 2, 2, 2, 32, 2, 121, 124, 127, 134, 137, 143, 150, 153, 158, 165, 172, 186, 206, 216, 218, 225, 238, 247, 254, 264, 270, 272, 395, 402, 408, 414, 464, 474, 488, 3, 8, 2, 2] -------------------------------------------------------------------------------- /dendrol/lang/STIXPatternLexer.py: -------------------------------------------------------------------------------- 1 | # Generated from STIXPattern.g4 by ANTLR 4.7.1 2 | from antlr4 import * 3 | from io import StringIO 4 | from typing.io import TextIO 5 | import sys 6 | 7 | 8 | def serializedATN(): 9 | with StringIO() as buf: 10 | buf.write("\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2\67") 11 | buf.write("\u01ef\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7") 12 | buf.write("\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t\r") 13 | buf.write("\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22\4\23") 14 | buf.write("\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30") 15 | buf.write("\4\31\t\31\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36") 16 | buf.write("\t\36\4\37\t\37\4 \t \4!\t!\4\"\t\"\4#\t#\4$\t$\4%\t%") 17 | buf.write("\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t+\4,\t,\4-\t-\4.") 18 | buf.write("\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64") 19 | buf.write("\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\3\2\3") 20 | buf.write("\2\3\2\3\2\7\2x\n\2\f\2\16\2{\13\2\5\2}\n\2\3\3\5\3\u0080") 21 | buf.write("\n\3\3\3\3\3\3\3\7\3\u0085\n\3\f\3\16\3\u0088\13\3\5\3") 22 | buf.write("\u008a\n\3\3\4\3\4\7\4\u008e\n\4\f\4\16\4\u0091\13\4\3") 23 | buf.write("\4\3\4\6\4\u0095\n\4\r\4\16\4\u0096\3\5\5\5\u009a\n\5") 24 | buf.write("\3\5\7\5\u009d\n\5\f\5\16\5\u00a0\13\5\3\5\3\5\6\5\u00a4") 25 | buf.write("\n\5\r\5\16\5\u00a5\3\6\3\6\3\6\7\6\u00ab\n\6\f\6\16\6") 26 | buf.write("\u00ae\13\6\3\6\3\6\3\7\3\7\3\7\3\7\3\7\3\7\3\7\7\7\u00b9") 27 | buf.write("\n\7\f\7\16\7\u00bc\13\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3") 28 | buf.write("\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\5\7\u00cf\n\7\3") 29 | buf.write("\7\3\7\3\b\3\b\3\b\3\b\3\b\3\b\7\b\u00d9\n\b\f\b\16\b") 30 | buf.write("\u00dc\13\b\3\b\3\b\3\t\3\t\5\t\u00e2\n\t\3\n\3\n\3\n") 31 | buf.write("\3\n\3\n\3\n\3\n\3\n\3\n\3\n\3\n\5\n\u00ef\n\n\3\n\3\n") 32 | buf.write("\3\n\3\n\3\n\3\n\3\n\5\n\u00f8\n\n\3\n\3\n\3\n\3\n\3\n") 33 | buf.write("\5\n\u00ff\n\n\3\n\3\n\3\n\3\n\3\n\3\n\3\n\3\n\5\n\u0109") 34 | buf.write("\n\n\3\n\3\n\6\n\u010d\n\n\r\n\16\n\u010e\5\n\u0111\n") 35 | buf.write("\n\3\n\3\n\3\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\3\r\3\r") 36 | buf.write("\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16") 37 | buf.write("\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\20") 38 | buf.write("\3\20\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21") 39 | buf.write("\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22") 40 | buf.write("\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\25") 41 | buf.write("\3\25\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\26\3\27") 42 | buf.write("\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30") 43 | buf.write("\3\30\3\31\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32") 44 | buf.write("\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33") 45 | buf.write("\3\34\3\34\3\34\3\34\3\34\3\34\3\35\3\35\7\35\u018a\n") 46 | buf.write("\35\f\35\16\35\u018d\13\35\3\36\3\36\7\36\u0191\n\36\f") 47 | buf.write("\36\16\36\u0194\13\36\3\37\3\37\3\37\5\37\u0199\n\37\3") 48 | buf.write(" \3 \3 \3 \5 \u019f\n \3!\3!\3\"\3\"\3\"\3#\3#\3$\3$\3") 49 | buf.write("$\3%\3%\3&\3&\3\'\3\'\3(\3(\3)\3)\3*\3*\3+\3+\3,\3,\3") 50 | buf.write("-\3-\3.\3.\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3") 51 | buf.write("\63\3\64\3\64\3\64\3\65\3\65\3\66\6\66\u01cf\n\66\r\66") 52 | buf.write("\16\66\u01d0\3\66\3\66\3\67\3\67\3\67\3\67\7\67\u01d9") 53 | buf.write("\n\67\f\67\16\67\u01dc\13\67\3\67\3\67\3\67\3\67\3\67") 54 | buf.write("\38\38\38\38\78\u01e7\n8\f8\168\u01ea\138\38\38\39\39") 55 | buf.write("\3\u01da\2:\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25") 56 | buf.write("\f\27\r\31\16\33\17\35\20\37\21!\22#\23%\24\'\25)\26+") 57 | buf.write("\27-\30/\31\61\32\63\33\65\34\67\359\36;\37= ?!A\"C#E") 58 | buf.write("$G%I&K\'M(O)Q*S+U,W-Y.[/]\60_\61a\62c\63e\2g\2i\2k\64") 59 | buf.write("m\65o\66q\67\3\2\21\3\2\63;\3\2\62;\4\2))^^\3\2\62\64") 60 | buf.write("\3\2\63\64\3\2\62\63\3\2\62\65\3\2\62\67\5\2C\\aac|\6") 61 | buf.write("\2\62;C\\aac|\7\2//\62;C\\aac|\5\2\62;CHch\6\2--\61;C") 62 | buf.write("\\c|\f\2\13\17\"\"\u0087\u0087\u00a2\u00a2\u1682\u1682") 63 | buf.write("\u2002\u200c\u202a\u202b\u2031\u2031\u2061\u2061\u3002") 64 | buf.write("\u3002\4\2\f\f\17\17\2\u020b\2\3\3\2\2\2\2\5\3\2\2\2\2") 65 | buf.write("\7\3\2\2\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3") 66 | buf.write("\2\2\2\2\21\3\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2") 67 | buf.write("\2\2\2\31\3\2\2\2\2\33\3\2\2\2\2\35\3\2\2\2\2\37\3\2\2") 68 | buf.write("\2\2!\3\2\2\2\2#\3\2\2\2\2%\3\2\2\2\2\'\3\2\2\2\2)\3\2") 69 | buf.write("\2\2\2+\3\2\2\2\2-\3\2\2\2\2/\3\2\2\2\2\61\3\2\2\2\2\63") 70 | buf.write("\3\2\2\2\2\65\3\2\2\2\2\67\3\2\2\2\29\3\2\2\2\2;\3\2\2") 71 | buf.write("\2\2=\3\2\2\2\2?\3\2\2\2\2A\3\2\2\2\2C\3\2\2\2\2E\3\2") 72 | buf.write("\2\2\2G\3\2\2\2\2I\3\2\2\2\2K\3\2\2\2\2M\3\2\2\2\2O\3") 73 | buf.write("\2\2\2\2Q\3\2\2\2\2S\3\2\2\2\2U\3\2\2\2\2W\3\2\2\2\2Y") 74 | buf.write("\3\2\2\2\2[\3\2\2\2\2]\3\2\2\2\2_\3\2\2\2\2a\3\2\2\2\2") 75 | buf.write("c\3\2\2\2\2k\3\2\2\2\2m\3\2\2\2\2o\3\2\2\2\2q\3\2\2\2") 76 | buf.write("\3s\3\2\2\2\5\177\3\2\2\2\7\u008b\3\2\2\2\t\u0099\3\2") 77 | buf.write("\2\2\13\u00a7\3\2\2\2\r\u00b1\3\2\2\2\17\u00d2\3\2\2\2") 78 | buf.write("\21\u00e1\3\2\2\2\23\u00e3\3\2\2\2\25\u0115\3\2\2\2\27") 79 | buf.write("\u0119\3\2\2\2\31\u011c\3\2\2\2\33\u0120\3\2\2\2\35\u012b") 80 | buf.write("\3\2\2\2\37\u0130\3\2\2\2!\u0138\3\2\2\2#\u0143\3\2\2") 81 | buf.write("\2%\u014c\3\2\2\2\'\u0151\3\2\2\2)\u0154\3\2\2\2+\u015a") 82 | buf.write("\3\2\2\2-\u015f\3\2\2\2/\u0167\3\2\2\2\61\u016c\3\2\2") 83 | buf.write("\2\63\u0172\3\2\2\2\65\u0179\3\2\2\2\67\u0181\3\2\2\2") 84 | buf.write("9\u0187\3\2\2\2;\u018e\3\2\2\2=\u0198\3\2\2\2?\u019e\3") 85 | buf.write("\2\2\2A\u01a0\3\2\2\2C\u01a2\3\2\2\2E\u01a5\3\2\2\2G\u01a7") 86 | buf.write("\3\2\2\2I\u01aa\3\2\2\2K\u01ac\3\2\2\2M\u01ae\3\2\2\2") 87 | buf.write("O\u01b0\3\2\2\2Q\u01b2\3\2\2\2S\u01b4\3\2\2\2U\u01b6\3") 88 | buf.write("\2\2\2W\u01b8\3\2\2\2Y\u01ba\3\2\2\2[\u01bc\3\2\2\2]\u01be") 89 | buf.write("\3\2\2\2_\u01c0\3\2\2\2a\u01c2\3\2\2\2c\u01c4\3\2\2\2") 90 | buf.write("e\u01c6\3\2\2\2g\u01c8\3\2\2\2i\u01cb\3\2\2\2k\u01ce\3") 91 | buf.write("\2\2\2m\u01d4\3\2\2\2o\u01e2\3\2\2\2q\u01ed\3\2\2\2s|") 92 | buf.write("\7/\2\2t}\7\62\2\2uy\t\2\2\2vx\t\3\2\2wv\3\2\2\2x{\3\2") 93 | buf.write("\2\2yw\3\2\2\2yz\3\2\2\2z}\3\2\2\2{y\3\2\2\2|t\3\2\2\2") 94 | buf.write("|u\3\2\2\2}\4\3\2\2\2~\u0080\7-\2\2\177~\3\2\2\2\177\u0080") 95 | buf.write("\3\2\2\2\u0080\u0089\3\2\2\2\u0081\u008a\7\62\2\2\u0082") 96 | buf.write("\u0086\t\2\2\2\u0083\u0085\t\3\2\2\u0084\u0083\3\2\2\2") 97 | buf.write("\u0085\u0088\3\2\2\2\u0086\u0084\3\2\2\2\u0086\u0087\3") 98 | buf.write("\2\2\2\u0087\u008a\3\2\2\2\u0088\u0086\3\2\2\2\u0089\u0081") 99 | buf.write("\3\2\2\2\u0089\u0082\3\2\2\2\u008a\6\3\2\2\2\u008b\u008f") 100 | buf.write("\7/\2\2\u008c\u008e\t\3\2\2\u008d\u008c\3\2\2\2\u008e") 101 | buf.write("\u0091\3\2\2\2\u008f\u008d\3\2\2\2\u008f\u0090\3\2\2\2") 102 | buf.write("\u0090\u0092\3\2\2\2\u0091\u008f\3\2\2\2\u0092\u0094\7") 103 | buf.write("\60\2\2\u0093\u0095\t\3\2\2\u0094\u0093\3\2\2\2\u0095") 104 | buf.write("\u0096\3\2\2\2\u0096\u0094\3\2\2\2\u0096\u0097\3\2\2\2") 105 | buf.write("\u0097\b\3\2\2\2\u0098\u009a\7-\2\2\u0099\u0098\3\2\2") 106 | buf.write("\2\u0099\u009a\3\2\2\2\u009a\u009e\3\2\2\2\u009b\u009d") 107 | buf.write("\t\3\2\2\u009c\u009b\3\2\2\2\u009d\u00a0\3\2\2\2\u009e") 108 | buf.write("\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a1\3\2\2\2") 109 | buf.write("\u00a0\u009e\3\2\2\2\u00a1\u00a3\7\60\2\2\u00a2\u00a4") 110 | buf.write("\t\3\2\2\u00a3\u00a2\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5") 111 | buf.write("\u00a3\3\2\2\2\u00a5\u00a6\3\2\2\2\u00a6\n\3\2\2\2\u00a7") 112 | buf.write("\u00a8\7j\2\2\u00a8\u00ac\5I%\2\u00a9\u00ab\5g\64\2\u00aa") 113 | buf.write("\u00a9\3\2\2\2\u00ab\u00ae\3\2\2\2\u00ac\u00aa\3\2\2\2") 114 | buf.write("\u00ac\u00ad\3\2\2\2\u00ad\u00af\3\2\2\2\u00ae\u00ac\3") 115 | buf.write("\2\2\2\u00af\u00b0\5I%\2\u00b0\f\3\2\2\2\u00b1\u00b2\7") 116 | buf.write("d\2\2\u00b2\u00ba\5I%\2\u00b3\u00b4\5i\65\2\u00b4\u00b5") 117 | buf.write("\5i\65\2\u00b5\u00b6\5i\65\2\u00b6\u00b7\5i\65\2\u00b7") 118 | buf.write("\u00b9\3\2\2\2\u00b8\u00b3\3\2\2\2\u00b9\u00bc\3\2\2\2") 119 | buf.write("\u00ba\u00b8\3\2\2\2\u00ba\u00bb\3\2\2\2\u00bb\u00ce\3") 120 | buf.write("\2\2\2\u00bc\u00ba\3\2\2\2\u00bd\u00be\5i\65\2\u00be\u00bf") 121 | buf.write("\5i\65\2\u00bf\u00c0\5i\65\2\u00c0\u00c1\5i\65\2\u00c1") 122 | buf.write("\u00cf\3\2\2\2\u00c2\u00c3\5i\65\2\u00c3\u00c4\5i\65\2") 123 | buf.write("\u00c4\u00c5\5i\65\2\u00c5\u00c6\3\2\2\2\u00c6\u00c7\7") 124 | buf.write("?\2\2\u00c7\u00cf\3\2\2\2\u00c8\u00c9\5i\65\2\u00c9\u00ca") 125 | buf.write("\5i\65\2\u00ca\u00cb\3\2\2\2\u00cb\u00cc\7?\2\2\u00cc") 126 | buf.write("\u00cd\7?\2\2\u00cd\u00cf\3\2\2\2\u00ce\u00bd\3\2\2\2") 127 | buf.write("\u00ce\u00c2\3\2\2\2\u00ce\u00c8\3\2\2\2\u00cf\u00d0\3") 128 | buf.write("\2\2\2\u00d0\u00d1\5I%\2\u00d1\16\3\2\2\2\u00d2\u00da") 129 | buf.write("\5I%\2\u00d3\u00d9\n\4\2\2\u00d4\u00d5\7^\2\2\u00d5\u00d9") 130 | buf.write("\7)\2\2\u00d6\u00d7\7^\2\2\u00d7\u00d9\7^\2\2\u00d8\u00d3") 131 | buf.write("\3\2\2\2\u00d8\u00d4\3\2\2\2\u00d8\u00d6\3\2\2\2\u00d9") 132 | buf.write("\u00dc\3\2\2\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2") 133 | buf.write("\u00db\u00dd\3\2\2\2\u00dc\u00da\3\2\2\2\u00dd\u00de\5") 134 | buf.write("I%\2\u00de\20\3\2\2\2\u00df\u00e2\5/\30\2\u00e0\u00e2") 135 | buf.write("\5\61\31\2\u00e1\u00df\3\2\2\2\u00e1\u00e0\3\2\2\2\u00e2") 136 | buf.write("\22\3\2\2\2\u00e3\u00e4\7v\2\2\u00e4\u00e5\5I%\2\u00e5") 137 | buf.write("\u00e6\t\3\2\2\u00e6\u00e7\t\3\2\2\u00e7\u00e8\t\3\2\2") 138 | buf.write("\u00e8\u00e9\t\3\2\2\u00e9\u00ee\5[.\2\u00ea\u00eb\7\62") 139 | buf.write("\2\2\u00eb\u00ef\t\2\2\2\u00ec\u00ed\7\63\2\2\u00ed\u00ef") 140 | buf.write("\t\5\2\2\u00ee\u00ea\3\2\2\2\u00ee\u00ec\3\2\2\2\u00ef") 141 | buf.write("\u00f0\3\2\2\2\u00f0\u00f7\5[.\2\u00f1\u00f2\7\62\2\2") 142 | buf.write("\u00f2\u00f8\t\2\2\2\u00f3\u00f4\t\6\2\2\u00f4\u00f8\t") 143 | buf.write("\3\2\2\u00f5\u00f6\7\65\2\2\u00f6\u00f8\t\7\2\2\u00f7") 144 | buf.write("\u00f1\3\2\2\2\u00f7\u00f3\3\2\2\2\u00f7\u00f5\3\2\2\2") 145 | buf.write("\u00f8\u00f9\3\2\2\2\u00f9\u00fe\7V\2\2\u00fa\u00fb\t") 146 | buf.write("\7\2\2\u00fb\u00ff\t\3\2\2\u00fc\u00fd\7\64\2\2\u00fd") 147 | buf.write("\u00ff\t\b\2\2\u00fe\u00fa\3\2\2\2\u00fe\u00fc\3\2\2\2") 148 | buf.write("\u00ff\u0100\3\2\2\2\u0100\u0101\5K&\2\u0101\u0102\t\t") 149 | buf.write("\2\2\u0102\u0103\t\3\2\2\u0103\u0108\5K&\2\u0104\u0105") 150 | buf.write("\t\t\2\2\u0105\u0109\t\3\2\2\u0106\u0107\78\2\2\u0107") 151 | buf.write("\u0109\7\62\2\2\u0108\u0104\3\2\2\2\u0108\u0106\3\2\2") 152 | buf.write("\2\u0109\u0110\3\2\2\2\u010a\u010c\5M\'\2\u010b\u010d") 153 | buf.write("\t\3\2\2\u010c\u010b\3\2\2\2\u010d\u010e\3\2\2\2\u010e") 154 | buf.write("\u010c\3\2\2\2\u010e\u010f\3\2\2\2\u010f\u0111\3\2\2\2") 155 | buf.write("\u0110\u010a\3\2\2\2\u0110\u0111\3\2\2\2\u0111\u0112\3") 156 | buf.write("\2\2\2\u0112\u0113\7\\\2\2\u0113\u0114\5I%\2\u0114\24") 157 | buf.write("\3\2\2\2\u0115\u0116\7C\2\2\u0116\u0117\7P\2\2\u0117\u0118") 158 | buf.write("\7F\2\2\u0118\26\3\2\2\2\u0119\u011a\7Q\2\2\u011a\u011b") 159 | buf.write("\7T\2\2\u011b\30\3\2\2\2\u011c\u011d\7P\2\2\u011d\u011e") 160 | buf.write("\7Q\2\2\u011e\u011f\7V\2\2\u011f\32\3\2\2\2\u0120\u0121") 161 | buf.write("\7H\2\2\u0121\u0122\7Q\2\2\u0122\u0123\7N\2\2\u0123\u0124") 162 | buf.write("\7N\2\2\u0124\u0125\7Q\2\2\u0125\u0126\7Y\2\2\u0126\u0127") 163 | buf.write("\7G\2\2\u0127\u0128\7F\2\2\u0128\u0129\7D\2\2\u0129\u012a") 164 | buf.write("\7[\2\2\u012a\34\3\2\2\2\u012b\u012c\7N\2\2\u012c\u012d") 165 | buf.write("\7K\2\2\u012d\u012e\7M\2\2\u012e\u012f\7G\2\2\u012f\36") 166 | buf.write("\3\2\2\2\u0130\u0131\7O\2\2\u0131\u0132\7C\2\2\u0132\u0133") 167 | buf.write("\7V\2\2\u0133\u0134\7E\2\2\u0134\u0135\7J\2\2\u0135\u0136") 168 | buf.write("\7G\2\2\u0136\u0137\7U\2\2\u0137 \3\2\2\2\u0138\u0139") 169 | buf.write("\7K\2\2\u0139\u013a\7U\2\2\u013a\u013b\7U\2\2\u013b\u013c") 170 | buf.write("\7W\2\2\u013c\u013d\7R\2\2\u013d\u013e\7G\2\2\u013e\u013f") 171 | buf.write("\7T\2\2\u013f\u0140\7U\2\2\u0140\u0141\7G\2\2\u0141\u0142") 172 | buf.write("\7V\2\2\u0142\"\3\2\2\2\u0143\u0144\7K\2\2\u0144\u0145") 173 | buf.write("\7U\2\2\u0145\u0146\7U\2\2\u0146\u0147\7W\2\2\u0147\u0148") 174 | buf.write("\7D\2\2\u0148\u0149\7U\2\2\u0149\u014a\7G\2\2\u014a\u014b") 175 | buf.write("\7V\2\2\u014b$\3\2\2\2\u014c\u014d\7N\2\2\u014d\u014e") 176 | buf.write("\7C\2\2\u014e\u014f\7U\2\2\u014f\u0150\7V\2\2\u0150&\3") 177 | buf.write("\2\2\2\u0151\u0152\7K\2\2\u0152\u0153\7P\2\2\u0153(\3") 178 | buf.write("\2\2\2\u0154\u0155\7U\2\2\u0155\u0156\7V\2\2\u0156\u0157") 179 | buf.write("\7C\2\2\u0157\u0158\7T\2\2\u0158\u0159\7V\2\2\u0159*\3") 180 | buf.write("\2\2\2\u015a\u015b\7U\2\2\u015b\u015c\7V\2\2\u015c\u015d") 181 | buf.write("\7Q\2\2\u015d\u015e\7R\2\2\u015e,\3\2\2\2\u015f\u0160") 182 | buf.write("\7U\2\2\u0160\u0161\7G\2\2\u0161\u0162\7E\2\2\u0162\u0163") 183 | buf.write("\7Q\2\2\u0163\u0164\7P\2\2\u0164\u0165\7F\2\2\u0165\u0166") 184 | buf.write("\7U\2\2\u0166.\3\2\2\2\u0167\u0168\7v\2\2\u0168\u0169") 185 | buf.write("\7t\2\2\u0169\u016a\7w\2\2\u016a\u016b\7g\2\2\u016b\60") 186 | buf.write("\3\2\2\2\u016c\u016d\7h\2\2\u016d\u016e\7c\2\2\u016e\u016f") 187 | buf.write("\7n\2\2\u016f\u0170\7u\2\2\u0170\u0171\7g\2\2\u0171\62") 188 | buf.write("\3\2\2\2\u0172\u0173\7Y\2\2\u0173\u0174\7K\2\2\u0174\u0175") 189 | buf.write("\7V\2\2\u0175\u0176\7J\2\2\u0176\u0177\7K\2\2\u0177\u0178") 190 | buf.write("\7P\2\2\u0178\64\3\2\2\2\u0179\u017a\7T\2\2\u017a\u017b") 191 | buf.write("\7G\2\2\u017b\u017c\7R\2\2\u017c\u017d\7G\2\2\u017d\u017e") 192 | buf.write("\7C\2\2\u017e\u017f\7V\2\2\u017f\u0180\7U\2\2\u0180\66") 193 | buf.write("\3\2\2\2\u0181\u0182\7V\2\2\u0182\u0183\7K\2\2\u0183\u0184") 194 | buf.write("\7O\2\2\u0184\u0185\7G\2\2\u0185\u0186\7U\2\2\u01868\3") 195 | buf.write("\2\2\2\u0187\u018b\t\n\2\2\u0188\u018a\t\13\2\2\u0189") 196 | buf.write("\u0188\3\2\2\2\u018a\u018d\3\2\2\2\u018b\u0189\3\2\2\2") 197 | buf.write("\u018b\u018c\3\2\2\2\u018c:\3\2\2\2\u018d\u018b\3\2\2") 198 | buf.write("\2\u018e\u0192\t\n\2\2\u018f\u0191\t\f\2\2\u0190\u018f") 199 | buf.write("\3\2\2\2\u0191\u0194\3\2\2\2\u0192\u0190\3\2\2\2\u0192") 200 | buf.write("\u0193\3\2\2\2\u0193<\3\2\2\2\u0194\u0192\3\2\2\2\u0195") 201 | buf.write("\u0199\7?\2\2\u0196\u0197\7?\2\2\u0197\u0199\7?\2\2\u0198") 202 | buf.write("\u0195\3\2\2\2\u0198\u0196\3\2\2\2\u0199>\3\2\2\2\u019a") 203 | buf.write("\u019b\7#\2\2\u019b\u019f\7?\2\2\u019c\u019d\7>\2\2\u019d") 204 | buf.write("\u019f\7@\2\2\u019e\u019a\3\2\2\2\u019e\u019c\3\2\2\2") 205 | buf.write("\u019f@\3\2\2\2\u01a0\u01a1\7>\2\2\u01a1B\3\2\2\2\u01a2") 206 | buf.write("\u01a3\7>\2\2\u01a3\u01a4\7?\2\2\u01a4D\3\2\2\2\u01a5") 207 | buf.write("\u01a6\7@\2\2\u01a6F\3\2\2\2\u01a7\u01a8\7@\2\2\u01a8") 208 | buf.write("\u01a9\7?\2\2\u01a9H\3\2\2\2\u01aa\u01ab\7)\2\2\u01ab") 209 | buf.write("J\3\2\2\2\u01ac\u01ad\7<\2\2\u01adL\3\2\2\2\u01ae\u01af") 210 | buf.write("\7\60\2\2\u01afN\3\2\2\2\u01b0\u01b1\7.\2\2\u01b1P\3\2") 211 | buf.write("\2\2\u01b2\u01b3\7+\2\2\u01b3R\3\2\2\2\u01b4\u01b5\7*") 212 | buf.write("\2\2\u01b5T\3\2\2\2\u01b6\u01b7\7_\2\2\u01b7V\3\2\2\2") 213 | buf.write("\u01b8\u01b9\7]\2\2\u01b9X\3\2\2\2\u01ba\u01bb\7-\2\2") 214 | buf.write("\u01bbZ\3\2\2\2\u01bc\u01bd\5]/\2\u01bd\\\3\2\2\2\u01be") 215 | buf.write("\u01bf\7/\2\2\u01bf^\3\2\2\2\u01c0\u01c1\7`\2\2\u01c1") 216 | buf.write("`\3\2\2\2\u01c2\u01c3\7\61\2\2\u01c3b\3\2\2\2\u01c4\u01c5") 217 | buf.write("\7,\2\2\u01c5d\3\2\2\2\u01c6\u01c7\t\r\2\2\u01c7f\3\2") 218 | buf.write("\2\2\u01c8\u01c9\5e\63\2\u01c9\u01ca\5e\63\2\u01cah\3") 219 | buf.write("\2\2\2\u01cb\u01cc\t\16\2\2\u01ccj\3\2\2\2\u01cd\u01cf") 220 | buf.write("\t\17\2\2\u01ce\u01cd\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0") 221 | buf.write("\u01ce\3\2\2\2\u01d0\u01d1\3\2\2\2\u01d1\u01d2\3\2\2\2") 222 | buf.write("\u01d2\u01d3\b\66\2\2\u01d3l\3\2\2\2\u01d4\u01d5\7\61") 223 | buf.write("\2\2\u01d5\u01d6\7,\2\2\u01d6\u01da\3\2\2\2\u01d7\u01d9") 224 | buf.write("\13\2\2\2\u01d8\u01d7\3\2\2\2\u01d9\u01dc\3\2\2\2\u01da") 225 | buf.write("\u01db\3\2\2\2\u01da\u01d8\3\2\2\2\u01db\u01dd\3\2\2\2") 226 | buf.write("\u01dc\u01da\3\2\2\2\u01dd\u01de\7,\2\2\u01de\u01df\7") 227 | buf.write("\61\2\2\u01df\u01e0\3\2\2\2\u01e0\u01e1\b\67\2\2\u01e1") 228 | buf.write("n\3\2\2\2\u01e2\u01e3\7\61\2\2\u01e3\u01e4\7\61\2\2\u01e4") 229 | buf.write("\u01e8\3\2\2\2\u01e5\u01e7\n\20\2\2\u01e6\u01e5\3\2\2") 230 | buf.write("\2\u01e7\u01ea\3\2\2\2\u01e8\u01e6\3\2\2\2\u01e8\u01e9") 231 | buf.write("\3\2\2\2\u01e9\u01eb\3\2\2\2\u01ea\u01e8\3\2\2\2\u01eb") 232 | buf.write("\u01ec\b8\2\2\u01ecp\3\2\2\2\u01ed\u01ee\13\2\2\2\u01ee") 233 | buf.write("r\3\2\2\2 \2y|\177\u0086\u0089\u008f\u0096\u0099\u009e") 234 | buf.write("\u00a5\u00ac\u00ba\u00ce\u00d8\u00da\u00e1\u00ee\u00f7") 235 | buf.write("\u00fe\u0108\u010e\u0110\u018b\u0192\u0198\u019e\u01d0") 236 | buf.write("\u01da\u01e8\3\b\2\2") 237 | return buf.getvalue() 238 | 239 | 240 | class STIXPatternLexer(Lexer): 241 | 242 | atn = ATNDeserializer().deserialize(serializedATN()) 243 | 244 | decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] 245 | 246 | IntNegLiteral = 1 247 | IntPosLiteral = 2 248 | FloatNegLiteral = 3 249 | FloatPosLiteral = 4 250 | HexLiteral = 5 251 | BinaryLiteral = 6 252 | StringLiteral = 7 253 | BoolLiteral = 8 254 | TimestampLiteral = 9 255 | AND = 10 256 | OR = 11 257 | NOT = 12 258 | FOLLOWEDBY = 13 259 | LIKE = 14 260 | MATCHES = 15 261 | ISSUPERSET = 16 262 | ISSUBSET = 17 263 | LAST = 18 264 | IN = 19 265 | START = 20 266 | STOP = 21 267 | SECONDS = 22 268 | TRUE = 23 269 | FALSE = 24 270 | WITHIN = 25 271 | REPEATS = 26 272 | TIMES = 27 273 | IdentifierWithoutHyphen = 28 274 | IdentifierWithHyphen = 29 275 | EQ = 30 276 | NEQ = 31 277 | LT = 32 278 | LE = 33 279 | GT = 34 280 | GE = 35 281 | QUOTE = 36 282 | COLON = 37 283 | DOT = 38 284 | COMMA = 39 285 | RPAREN = 40 286 | LPAREN = 41 287 | RBRACK = 42 288 | LBRACK = 43 289 | PLUS = 44 290 | HYPHEN = 45 291 | MINUS = 46 292 | POWER_OP = 47 293 | DIVIDE = 48 294 | ASTERISK = 49 295 | WS = 50 296 | COMMENT = 51 297 | LINE_COMMENT = 52 298 | InvalidCharacter = 53 299 | 300 | channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN" ] 301 | 302 | modeNames = [ "DEFAULT_MODE" ] 303 | 304 | literalNames = [ "", 305 | "'AND'", "'OR'", "'NOT'", "'FOLLOWEDBY'", "'LIKE'", "'MATCHES'", 306 | "'ISSUPERSET'", "'ISSUBSET'", "'LAST'", "'IN'", "'START'", "'STOP'", 307 | "'SECONDS'", "'true'", "'false'", "'WITHIN'", "'REPEATS'", "'TIMES'", 308 | "'<'", "'<='", "'>'", "'>='", "'''", "':'", "'.'", "','", "')'", 309 | "'('", "']'", "'['", "'+'", "'-'", "'^'", "'/'", "'*'" ] 310 | 311 | symbolicNames = [ "", 312 | "IntNegLiteral", "IntPosLiteral", "FloatNegLiteral", "FloatPosLiteral", 313 | "HexLiteral", "BinaryLiteral", "StringLiteral", "BoolLiteral", 314 | "TimestampLiteral", "AND", "OR", "NOT", "FOLLOWEDBY", "LIKE", 315 | "MATCHES", "ISSUPERSET", "ISSUBSET", "LAST", "IN", "START", 316 | "STOP", "SECONDS", "TRUE", "FALSE", "WITHIN", "REPEATS", "TIMES", 317 | "IdentifierWithoutHyphen", "IdentifierWithHyphen", "EQ", "NEQ", 318 | "LT", "LE", "GT", "GE", "QUOTE", "COLON", "DOT", "COMMA", "RPAREN", 319 | "LPAREN", "RBRACK", "LBRACK", "PLUS", "HYPHEN", "MINUS", "POWER_OP", 320 | "DIVIDE", "ASTERISK", "WS", "COMMENT", "LINE_COMMENT", "InvalidCharacter" ] 321 | 322 | ruleNames = [ "IntNegLiteral", "IntPosLiteral", "FloatNegLiteral", "FloatPosLiteral", 323 | "HexLiteral", "BinaryLiteral", "StringLiteral", "BoolLiteral", 324 | "TimestampLiteral", "AND", "OR", "NOT", "FOLLOWEDBY", 325 | "LIKE", "MATCHES", "ISSUPERSET", "ISSUBSET", "LAST", "IN", 326 | "START", "STOP", "SECONDS", "TRUE", "FALSE", "WITHIN", 327 | "REPEATS", "TIMES", "IdentifierWithoutHyphen", "IdentifierWithHyphen", 328 | "EQ", "NEQ", "LT", "LE", "GT", "GE", "QUOTE", "COLON", 329 | "DOT", "COMMA", "RPAREN", "LPAREN", "RBRACK", "LBRACK", 330 | "PLUS", "HYPHEN", "MINUS", "POWER_OP", "DIVIDE", "ASTERISK", 331 | "HexDigit", "TwoHexDigits", "Base64Char", "WS", "COMMENT", 332 | "LINE_COMMENT", "InvalidCharacter" ] 333 | 334 | grammarFileName = "STIXPattern.g4" 335 | 336 | def __init__(self, input=None, output:TextIO = sys.stdout): 337 | super().__init__(input, output) 338 | self.checkVersion("4.7.1") 339 | self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) 340 | self._actions = None 341 | self._predicates = None 342 | 343 | 344 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPatternLexer.tokens: -------------------------------------------------------------------------------- 1 | IntNegLiteral=1 2 | IntPosLiteral=2 3 | FloatNegLiteral=3 4 | FloatPosLiteral=4 5 | HexLiteral=5 6 | BinaryLiteral=6 7 | StringLiteral=7 8 | BoolLiteral=8 9 | TimestampLiteral=9 10 | AND=10 11 | OR=11 12 | NOT=12 13 | FOLLOWEDBY=13 14 | LIKE=14 15 | MATCHES=15 16 | ISSUPERSET=16 17 | ISSUBSET=17 18 | LAST=18 19 | IN=19 20 | START=20 21 | STOP=21 22 | SECONDS=22 23 | TRUE=23 24 | FALSE=24 25 | WITHIN=25 26 | REPEATS=26 27 | TIMES=27 28 | IdentifierWithoutHyphen=28 29 | IdentifierWithHyphen=29 30 | EQ=30 31 | NEQ=31 32 | LT=32 33 | LE=33 34 | GT=34 35 | GE=35 36 | QUOTE=36 37 | COLON=37 38 | DOT=38 39 | COMMA=39 40 | RPAREN=40 41 | LPAREN=41 42 | RBRACK=42 43 | LBRACK=43 44 | PLUS=44 45 | HYPHEN=45 46 | MINUS=46 47 | POWER_OP=47 48 | DIVIDE=48 49 | ASTERISK=49 50 | WS=50 51 | COMMENT=51 52 | LINE_COMMENT=52 53 | InvalidCharacter=53 54 | 'AND'=10 55 | 'OR'=11 56 | 'NOT'=12 57 | 'FOLLOWEDBY'=13 58 | 'LIKE'=14 59 | 'MATCHES'=15 60 | 'ISSUPERSET'=16 61 | 'ISSUBSET'=17 62 | 'LAST'=18 63 | 'IN'=19 64 | 'START'=20 65 | 'STOP'=21 66 | 'SECONDS'=22 67 | 'true'=23 68 | 'false'=24 69 | 'WITHIN'=25 70 | 'REPEATS'=26 71 | 'TIMES'=27 72 | '<'=32 73 | '<='=33 74 | '>'=34 75 | '>='=35 76 | '\''=36 77 | ':'=37 78 | '.'=38 79 | ','=39 80 | ')'=40 81 | '('=41 82 | ']'=42 83 | '['=43 84 | '+'=44 85 | '-'=46 86 | '^'=47 87 | '/'=48 88 | '*'=49 89 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPatternListener.py: -------------------------------------------------------------------------------- 1 | # Generated from STIXPattern.g4 by ANTLR 4.7.1 2 | from antlr4 import * 3 | if __name__ is not None and "." in __name__: 4 | from .STIXPatternParser import STIXPatternParser 5 | else: 6 | from STIXPatternParser import STIXPatternParser 7 | 8 | # This class defines a complete listener for a parse tree produced by STIXPatternParser. 9 | class STIXPatternListener(ParseTreeListener): 10 | 11 | # Enter a parse tree produced by STIXPatternParser#pattern. 12 | def enterPattern(self, ctx:STIXPatternParser.PatternContext): 13 | pass 14 | 15 | # Exit a parse tree produced by STIXPatternParser#pattern. 16 | def exitPattern(self, ctx:STIXPatternParser.PatternContext): 17 | pass 18 | 19 | 20 | # Enter a parse tree produced by STIXPatternParser#observationExpressions. 21 | def enterObservationExpressions(self, ctx:STIXPatternParser.ObservationExpressionsContext): 22 | pass 23 | 24 | # Exit a parse tree produced by STIXPatternParser#observationExpressions. 25 | def exitObservationExpressions(self, ctx:STIXPatternParser.ObservationExpressionsContext): 26 | pass 27 | 28 | 29 | # Enter a parse tree produced by STIXPatternParser#observationExpressionOr. 30 | def enterObservationExpressionOr(self, ctx:STIXPatternParser.ObservationExpressionOrContext): 31 | pass 32 | 33 | # Exit a parse tree produced by STIXPatternParser#observationExpressionOr. 34 | def exitObservationExpressionOr(self, ctx:STIXPatternParser.ObservationExpressionOrContext): 35 | pass 36 | 37 | 38 | # Enter a parse tree produced by STIXPatternParser#observationExpressionAnd. 39 | def enterObservationExpressionAnd(self, ctx:STIXPatternParser.ObservationExpressionAndContext): 40 | pass 41 | 42 | # Exit a parse tree produced by STIXPatternParser#observationExpressionAnd. 43 | def exitObservationExpressionAnd(self, ctx:STIXPatternParser.ObservationExpressionAndContext): 44 | pass 45 | 46 | 47 | # Enter a parse tree produced by STIXPatternParser#observationExpressionRepeated. 48 | def enterObservationExpressionRepeated(self, ctx:STIXPatternParser.ObservationExpressionRepeatedContext): 49 | pass 50 | 51 | # Exit a parse tree produced by STIXPatternParser#observationExpressionRepeated. 52 | def exitObservationExpressionRepeated(self, ctx:STIXPatternParser.ObservationExpressionRepeatedContext): 53 | pass 54 | 55 | 56 | # Enter a parse tree produced by STIXPatternParser#observationExpressionSimple. 57 | def enterObservationExpressionSimple(self, ctx:STIXPatternParser.ObservationExpressionSimpleContext): 58 | pass 59 | 60 | # Exit a parse tree produced by STIXPatternParser#observationExpressionSimple. 61 | def exitObservationExpressionSimple(self, ctx:STIXPatternParser.ObservationExpressionSimpleContext): 62 | pass 63 | 64 | 65 | # Enter a parse tree produced by STIXPatternParser#observationExpressionCompound. 66 | def enterObservationExpressionCompound(self, ctx:STIXPatternParser.ObservationExpressionCompoundContext): 67 | pass 68 | 69 | # Exit a parse tree produced by STIXPatternParser#observationExpressionCompound. 70 | def exitObservationExpressionCompound(self, ctx:STIXPatternParser.ObservationExpressionCompoundContext): 71 | pass 72 | 73 | 74 | # Enter a parse tree produced by STIXPatternParser#observationExpressionWithin. 75 | def enterObservationExpressionWithin(self, ctx:STIXPatternParser.ObservationExpressionWithinContext): 76 | pass 77 | 78 | # Exit a parse tree produced by STIXPatternParser#observationExpressionWithin. 79 | def exitObservationExpressionWithin(self, ctx:STIXPatternParser.ObservationExpressionWithinContext): 80 | pass 81 | 82 | 83 | # Enter a parse tree produced by STIXPatternParser#observationExpressionStartStop. 84 | def enterObservationExpressionStartStop(self, ctx:STIXPatternParser.ObservationExpressionStartStopContext): 85 | pass 86 | 87 | # Exit a parse tree produced by STIXPatternParser#observationExpressionStartStop. 88 | def exitObservationExpressionStartStop(self, ctx:STIXPatternParser.ObservationExpressionStartStopContext): 89 | pass 90 | 91 | 92 | # Enter a parse tree produced by STIXPatternParser#comparisonExpression. 93 | def enterComparisonExpression(self, ctx:STIXPatternParser.ComparisonExpressionContext): 94 | pass 95 | 96 | # Exit a parse tree produced by STIXPatternParser#comparisonExpression. 97 | def exitComparisonExpression(self, ctx:STIXPatternParser.ComparisonExpressionContext): 98 | pass 99 | 100 | 101 | # Enter a parse tree produced by STIXPatternParser#comparisonExpressionAnd. 102 | def enterComparisonExpressionAnd(self, ctx:STIXPatternParser.ComparisonExpressionAndContext): 103 | pass 104 | 105 | # Exit a parse tree produced by STIXPatternParser#comparisonExpressionAnd. 106 | def exitComparisonExpressionAnd(self, ctx:STIXPatternParser.ComparisonExpressionAndContext): 107 | pass 108 | 109 | 110 | # Enter a parse tree produced by STIXPatternParser#propTestEqual. 111 | def enterPropTestEqual(self, ctx:STIXPatternParser.PropTestEqualContext): 112 | pass 113 | 114 | # Exit a parse tree produced by STIXPatternParser#propTestEqual. 115 | def exitPropTestEqual(self, ctx:STIXPatternParser.PropTestEqualContext): 116 | pass 117 | 118 | 119 | # Enter a parse tree produced by STIXPatternParser#propTestOrder. 120 | def enterPropTestOrder(self, ctx:STIXPatternParser.PropTestOrderContext): 121 | pass 122 | 123 | # Exit a parse tree produced by STIXPatternParser#propTestOrder. 124 | def exitPropTestOrder(self, ctx:STIXPatternParser.PropTestOrderContext): 125 | pass 126 | 127 | 128 | # Enter a parse tree produced by STIXPatternParser#propTestSet. 129 | def enterPropTestSet(self, ctx:STIXPatternParser.PropTestSetContext): 130 | pass 131 | 132 | # Exit a parse tree produced by STIXPatternParser#propTestSet. 133 | def exitPropTestSet(self, ctx:STIXPatternParser.PropTestSetContext): 134 | pass 135 | 136 | 137 | # Enter a parse tree produced by STIXPatternParser#propTestLike. 138 | def enterPropTestLike(self, ctx:STIXPatternParser.PropTestLikeContext): 139 | pass 140 | 141 | # Exit a parse tree produced by STIXPatternParser#propTestLike. 142 | def exitPropTestLike(self, ctx:STIXPatternParser.PropTestLikeContext): 143 | pass 144 | 145 | 146 | # Enter a parse tree produced by STIXPatternParser#propTestRegex. 147 | def enterPropTestRegex(self, ctx:STIXPatternParser.PropTestRegexContext): 148 | pass 149 | 150 | # Exit a parse tree produced by STIXPatternParser#propTestRegex. 151 | def exitPropTestRegex(self, ctx:STIXPatternParser.PropTestRegexContext): 152 | pass 153 | 154 | 155 | # Enter a parse tree produced by STIXPatternParser#propTestIsSubset. 156 | def enterPropTestIsSubset(self, ctx:STIXPatternParser.PropTestIsSubsetContext): 157 | pass 158 | 159 | # Exit a parse tree produced by STIXPatternParser#propTestIsSubset. 160 | def exitPropTestIsSubset(self, ctx:STIXPatternParser.PropTestIsSubsetContext): 161 | pass 162 | 163 | 164 | # Enter a parse tree produced by STIXPatternParser#propTestIsSuperset. 165 | def enterPropTestIsSuperset(self, ctx:STIXPatternParser.PropTestIsSupersetContext): 166 | pass 167 | 168 | # Exit a parse tree produced by STIXPatternParser#propTestIsSuperset. 169 | def exitPropTestIsSuperset(self, ctx:STIXPatternParser.PropTestIsSupersetContext): 170 | pass 171 | 172 | 173 | # Enter a parse tree produced by STIXPatternParser#propTestParen. 174 | def enterPropTestParen(self, ctx:STIXPatternParser.PropTestParenContext): 175 | pass 176 | 177 | # Exit a parse tree produced by STIXPatternParser#propTestParen. 178 | def exitPropTestParen(self, ctx:STIXPatternParser.PropTestParenContext): 179 | pass 180 | 181 | 182 | # Enter a parse tree produced by STIXPatternParser#startStopQualifier. 183 | def enterStartStopQualifier(self, ctx:STIXPatternParser.StartStopQualifierContext): 184 | pass 185 | 186 | # Exit a parse tree produced by STIXPatternParser#startStopQualifier. 187 | def exitStartStopQualifier(self, ctx:STIXPatternParser.StartStopQualifierContext): 188 | pass 189 | 190 | 191 | # Enter a parse tree produced by STIXPatternParser#withinQualifier. 192 | def enterWithinQualifier(self, ctx:STIXPatternParser.WithinQualifierContext): 193 | pass 194 | 195 | # Exit a parse tree produced by STIXPatternParser#withinQualifier. 196 | def exitWithinQualifier(self, ctx:STIXPatternParser.WithinQualifierContext): 197 | pass 198 | 199 | 200 | # Enter a parse tree produced by STIXPatternParser#repeatedQualifier. 201 | def enterRepeatedQualifier(self, ctx:STIXPatternParser.RepeatedQualifierContext): 202 | pass 203 | 204 | # Exit a parse tree produced by STIXPatternParser#repeatedQualifier. 205 | def exitRepeatedQualifier(self, ctx:STIXPatternParser.RepeatedQualifierContext): 206 | pass 207 | 208 | 209 | # Enter a parse tree produced by STIXPatternParser#objectPath. 210 | def enterObjectPath(self, ctx:STIXPatternParser.ObjectPathContext): 211 | pass 212 | 213 | # Exit a parse tree produced by STIXPatternParser#objectPath. 214 | def exitObjectPath(self, ctx:STIXPatternParser.ObjectPathContext): 215 | pass 216 | 217 | 218 | # Enter a parse tree produced by STIXPatternParser#objectType. 219 | def enterObjectType(self, ctx:STIXPatternParser.ObjectTypeContext): 220 | pass 221 | 222 | # Exit a parse tree produced by STIXPatternParser#objectType. 223 | def exitObjectType(self, ctx:STIXPatternParser.ObjectTypeContext): 224 | pass 225 | 226 | 227 | # Enter a parse tree produced by STIXPatternParser#firstPathComponent. 228 | def enterFirstPathComponent(self, ctx:STIXPatternParser.FirstPathComponentContext): 229 | pass 230 | 231 | # Exit a parse tree produced by STIXPatternParser#firstPathComponent. 232 | def exitFirstPathComponent(self, ctx:STIXPatternParser.FirstPathComponentContext): 233 | pass 234 | 235 | 236 | # Enter a parse tree produced by STIXPatternParser#indexPathStep. 237 | def enterIndexPathStep(self, ctx:STIXPatternParser.IndexPathStepContext): 238 | pass 239 | 240 | # Exit a parse tree produced by STIXPatternParser#indexPathStep. 241 | def exitIndexPathStep(self, ctx:STIXPatternParser.IndexPathStepContext): 242 | pass 243 | 244 | 245 | # Enter a parse tree produced by STIXPatternParser#pathStep. 246 | def enterPathStep(self, ctx:STIXPatternParser.PathStepContext): 247 | pass 248 | 249 | # Exit a parse tree produced by STIXPatternParser#pathStep. 250 | def exitPathStep(self, ctx:STIXPatternParser.PathStepContext): 251 | pass 252 | 253 | 254 | # Enter a parse tree produced by STIXPatternParser#keyPathStep. 255 | def enterKeyPathStep(self, ctx:STIXPatternParser.KeyPathStepContext): 256 | pass 257 | 258 | # Exit a parse tree produced by STIXPatternParser#keyPathStep. 259 | def exitKeyPathStep(self, ctx:STIXPatternParser.KeyPathStepContext): 260 | pass 261 | 262 | 263 | # Enter a parse tree produced by STIXPatternParser#setLiteral. 264 | def enterSetLiteral(self, ctx:STIXPatternParser.SetLiteralContext): 265 | pass 266 | 267 | # Exit a parse tree produced by STIXPatternParser#setLiteral. 268 | def exitSetLiteral(self, ctx:STIXPatternParser.SetLiteralContext): 269 | pass 270 | 271 | 272 | # Enter a parse tree produced by STIXPatternParser#primitiveLiteral. 273 | def enterPrimitiveLiteral(self, ctx:STIXPatternParser.PrimitiveLiteralContext): 274 | pass 275 | 276 | # Exit a parse tree produced by STIXPatternParser#primitiveLiteral. 277 | def exitPrimitiveLiteral(self, ctx:STIXPatternParser.PrimitiveLiteralContext): 278 | pass 279 | 280 | 281 | # Enter a parse tree produced by STIXPatternParser#orderableLiteral. 282 | def enterOrderableLiteral(self, ctx:STIXPatternParser.OrderableLiteralContext): 283 | pass 284 | 285 | # Exit a parse tree produced by STIXPatternParser#orderableLiteral. 286 | def exitOrderableLiteral(self, ctx:STIXPatternParser.OrderableLiteralContext): 287 | pass 288 | 289 | 290 | -------------------------------------------------------------------------------- /dendrol/lang/STIXPatternVisitor.py: -------------------------------------------------------------------------------- 1 | # Generated from STIXPattern.g4 by ANTLR 4.7.1 2 | from antlr4 import * 3 | if __name__ is not None and "." in __name__: 4 | from .STIXPatternParser import STIXPatternParser 5 | else: 6 | from STIXPatternParser import STIXPatternParser 7 | 8 | # This class defines a complete generic visitor for a parse tree produced by STIXPatternParser. 9 | 10 | class STIXPatternVisitor(ParseTreeVisitor): 11 | 12 | # Visit a parse tree produced by STIXPatternParser#pattern. 13 | def visitPattern(self, ctx:STIXPatternParser.PatternContext): 14 | return self.visitChildren(ctx) 15 | 16 | 17 | # Visit a parse tree produced by STIXPatternParser#observationExpressions. 18 | def visitObservationExpressions(self, ctx:STIXPatternParser.ObservationExpressionsContext): 19 | return self.visitChildren(ctx) 20 | 21 | 22 | # Visit a parse tree produced by STIXPatternParser#observationExpressionOr. 23 | def visitObservationExpressionOr(self, ctx:STIXPatternParser.ObservationExpressionOrContext): 24 | return self.visitChildren(ctx) 25 | 26 | 27 | # Visit a parse tree produced by STIXPatternParser#observationExpressionAnd. 28 | def visitObservationExpressionAnd(self, ctx:STIXPatternParser.ObservationExpressionAndContext): 29 | return self.visitChildren(ctx) 30 | 31 | 32 | # Visit a parse tree produced by STIXPatternParser#observationExpressionRepeated. 33 | def visitObservationExpressionRepeated(self, ctx:STIXPatternParser.ObservationExpressionRepeatedContext): 34 | return self.visitChildren(ctx) 35 | 36 | 37 | # Visit a parse tree produced by STIXPatternParser#observationExpressionSimple. 38 | def visitObservationExpressionSimple(self, ctx:STIXPatternParser.ObservationExpressionSimpleContext): 39 | return self.visitChildren(ctx) 40 | 41 | 42 | # Visit a parse tree produced by STIXPatternParser#observationExpressionCompound. 43 | def visitObservationExpressionCompound(self, ctx:STIXPatternParser.ObservationExpressionCompoundContext): 44 | return self.visitChildren(ctx) 45 | 46 | 47 | # Visit a parse tree produced by STIXPatternParser#observationExpressionWithin. 48 | def visitObservationExpressionWithin(self, ctx:STIXPatternParser.ObservationExpressionWithinContext): 49 | return self.visitChildren(ctx) 50 | 51 | 52 | # Visit a parse tree produced by STIXPatternParser#observationExpressionStartStop. 53 | def visitObservationExpressionStartStop(self, ctx:STIXPatternParser.ObservationExpressionStartStopContext): 54 | return self.visitChildren(ctx) 55 | 56 | 57 | # Visit a parse tree produced by STIXPatternParser#comparisonExpression. 58 | def visitComparisonExpression(self, ctx:STIXPatternParser.ComparisonExpressionContext): 59 | return self.visitChildren(ctx) 60 | 61 | 62 | # Visit a parse tree produced by STIXPatternParser#comparisonExpressionAnd. 63 | def visitComparisonExpressionAnd(self, ctx:STIXPatternParser.ComparisonExpressionAndContext): 64 | return self.visitChildren(ctx) 65 | 66 | 67 | # Visit a parse tree produced by STIXPatternParser#propTestEqual. 68 | def visitPropTestEqual(self, ctx:STIXPatternParser.PropTestEqualContext): 69 | return self.visitChildren(ctx) 70 | 71 | 72 | # Visit a parse tree produced by STIXPatternParser#propTestOrder. 73 | def visitPropTestOrder(self, ctx:STIXPatternParser.PropTestOrderContext): 74 | return self.visitChildren(ctx) 75 | 76 | 77 | # Visit a parse tree produced by STIXPatternParser#propTestSet. 78 | def visitPropTestSet(self, ctx:STIXPatternParser.PropTestSetContext): 79 | return self.visitChildren(ctx) 80 | 81 | 82 | # Visit a parse tree produced by STIXPatternParser#propTestLike. 83 | def visitPropTestLike(self, ctx:STIXPatternParser.PropTestLikeContext): 84 | return self.visitChildren(ctx) 85 | 86 | 87 | # Visit a parse tree produced by STIXPatternParser#propTestRegex. 88 | def visitPropTestRegex(self, ctx:STIXPatternParser.PropTestRegexContext): 89 | return self.visitChildren(ctx) 90 | 91 | 92 | # Visit a parse tree produced by STIXPatternParser#propTestIsSubset. 93 | def visitPropTestIsSubset(self, ctx:STIXPatternParser.PropTestIsSubsetContext): 94 | return self.visitChildren(ctx) 95 | 96 | 97 | # Visit a parse tree produced by STIXPatternParser#propTestIsSuperset. 98 | def visitPropTestIsSuperset(self, ctx:STIXPatternParser.PropTestIsSupersetContext): 99 | return self.visitChildren(ctx) 100 | 101 | 102 | # Visit a parse tree produced by STIXPatternParser#propTestParen. 103 | def visitPropTestParen(self, ctx:STIXPatternParser.PropTestParenContext): 104 | return self.visitChildren(ctx) 105 | 106 | 107 | # Visit a parse tree produced by STIXPatternParser#startStopQualifier. 108 | def visitStartStopQualifier(self, ctx:STIXPatternParser.StartStopQualifierContext): 109 | return self.visitChildren(ctx) 110 | 111 | 112 | # Visit a parse tree produced by STIXPatternParser#withinQualifier. 113 | def visitWithinQualifier(self, ctx:STIXPatternParser.WithinQualifierContext): 114 | return self.visitChildren(ctx) 115 | 116 | 117 | # Visit a parse tree produced by STIXPatternParser#repeatedQualifier. 118 | def visitRepeatedQualifier(self, ctx:STIXPatternParser.RepeatedQualifierContext): 119 | return self.visitChildren(ctx) 120 | 121 | 122 | # Visit a parse tree produced by STIXPatternParser#objectPath. 123 | def visitObjectPath(self, ctx:STIXPatternParser.ObjectPathContext): 124 | return self.visitChildren(ctx) 125 | 126 | 127 | # Visit a parse tree produced by STIXPatternParser#objectType. 128 | def visitObjectType(self, ctx:STIXPatternParser.ObjectTypeContext): 129 | return self.visitChildren(ctx) 130 | 131 | 132 | # Visit a parse tree produced by STIXPatternParser#firstPathComponent. 133 | def visitFirstPathComponent(self, ctx:STIXPatternParser.FirstPathComponentContext): 134 | return self.visitChildren(ctx) 135 | 136 | 137 | # Visit a parse tree produced by STIXPatternParser#indexPathStep. 138 | def visitIndexPathStep(self, ctx:STIXPatternParser.IndexPathStepContext): 139 | return self.visitChildren(ctx) 140 | 141 | 142 | # Visit a parse tree produced by STIXPatternParser#pathStep. 143 | def visitPathStep(self, ctx:STIXPatternParser.PathStepContext): 144 | return self.visitChildren(ctx) 145 | 146 | 147 | # Visit a parse tree produced by STIXPatternParser#keyPathStep. 148 | def visitKeyPathStep(self, ctx:STIXPatternParser.KeyPathStepContext): 149 | return self.visitChildren(ctx) 150 | 151 | 152 | # Visit a parse tree produced by STIXPatternParser#setLiteral. 153 | def visitSetLiteral(self, ctx:STIXPatternParser.SetLiteralContext): 154 | return self.visitChildren(ctx) 155 | 156 | 157 | # Visit a parse tree produced by STIXPatternParser#primitiveLiteral. 158 | def visitPrimitiveLiteral(self, ctx:STIXPatternParser.PrimitiveLiteralContext): 159 | return self.visitChildren(ctx) 160 | 161 | 162 | # Visit a parse tree produced by STIXPatternParser#orderableLiteral. 163 | def visitOrderableLiteral(self, ctx:STIXPatternParser.OrderableLiteralContext): 164 | return self.visitChildren(ctx) 165 | 166 | 167 | 168 | del STIXPatternParser -------------------------------------------------------------------------------- /dendrol/lang/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PerchSecurity/dendrol/b3770beb90547fc7f8e3009794fcf3774404d795/dendrol/lang/__init__.py -------------------------------------------------------------------------------- /dendrol/parse.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import antlr4 4 | import antlr4.error.Errors 5 | import antlr4.error.ErrorListener 6 | import six 7 | 8 | from .debug import print_tree 9 | from .lang.STIXPatternLexer import STIXPatternLexer 10 | from .lang.STIXPatternListener import STIXPatternListener 11 | from .lang.STIXPatternParser import STIXPatternParser 12 | from .lang.STIXPatternVisitor import STIXPatternVisitor 13 | from .transform import PatternTreeVisitor, PatternTree 14 | 15 | 16 | class ParserErrorListener(antlr4.error.ErrorListener.ErrorListener): 17 | """ 18 | Simple error listener which just remembers the last error message received. 19 | """ 20 | def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): 21 | self.error_message = f'{line}:{column}: {msg}' 22 | 23 | 24 | class ParseException(Exception): 25 | """Represents a parse error.""" 26 | pass 27 | 28 | 29 | def parse(pattern_str: str, trace: bool = False) -> STIXPatternParser.PatternContext: 30 | """ 31 | Parses the given pattern and returns the antlr parse tree. 32 | 33 | NOTE: most of this code and comments were lifted from oasis-open/cti-pattern-validator 34 | 35 | :param pattern_str: The STIX pattern 36 | :param trace: Whether to enable debug tracing while parsing. 37 | :return: The parse tree 38 | :raises ParseException: If there is a parse error 39 | """ 40 | in_ = antlr4.InputStream(pattern_str) 41 | lexer = STIXPatternLexer(in_) 42 | lexer.removeErrorListeners() # remove the default "console" listener 43 | token_stream = antlr4.CommonTokenStream(lexer) 44 | 45 | parser = STIXPatternParser(token_stream) 46 | parser.removeErrorListeners() # remove the default "console" listener 47 | error_listener = ParserErrorListener() 48 | parser.addErrorListener(error_listener) 49 | 50 | # I found no public API for this... 51 | # The default error handler tries to keep parsing, and I don't 52 | # think that's appropriate here. (These error handlers are only for 53 | # handling the built-in RecognitionException errors.) 54 | parser._errHandler = antlr4.BailErrorStrategy() 55 | 56 | # To improve error messages, replace "" in the literal 57 | # names with symbolic names. This is a hack, but seemed like 58 | # the simplest workaround. 59 | for i, lit_name in enumerate(parser.literalNames): 60 | if lit_name == u"": 61 | parser.literalNames[i] = parser.symbolicNames[i] 62 | 63 | parser.setTrace(trace) 64 | 65 | try: 66 | tree = parser.pattern() 67 | except antlr4.error.Errors.ParseCancellationException as e: 68 | # The cancellation exception wraps the real RecognitionException 69 | # which caused the parser to bail. 70 | real_exc = e.args[0] 71 | 72 | # I want to bail when the first error is hit. But I also want 73 | # a decent error message. When an error is encountered in 74 | # Parser.match(), the BailErrorStrategy produces the 75 | # ParseCancellationException. It is not a subclass of 76 | # RecognitionException, so none of the 'except' clauses which would 77 | # normally report an error are invoked. 78 | # 79 | # Error message creation is buried in the ErrorStrategy, and I can 80 | # (ab)use the API to get a message: register an error listener with 81 | # the parser, force an error report, then get the message out of the 82 | # listener. Error listener registration is above; now we force its 83 | # invocation. Wish this could be cleaner... 84 | parser._errHandler.reportError(parser, real_exc) 85 | 86 | # should probably chain exceptions if we can... 87 | # Should I report the cancellation or recognition exception as the 88 | # cause...? 89 | six.raise_from(ParseException(error_listener.error_message), 90 | real_exc) 91 | else: 92 | return tree 93 | 94 | 95 | class Pattern: 96 | """A parsed pattern expression, with traversal and representation methods 97 | """ 98 | tree: STIXPatternParser.PatternContext 99 | 100 | def __init__(self, pattern_str: str): 101 | """ 102 | Compile a pattern. 103 | 104 | :param pattern_str: The pattern to compile 105 | :raises ParseException: If there is a parse error 106 | """ 107 | self.tree = parse(pattern_str) 108 | 109 | def walk(self, listener: STIXPatternListener): 110 | """Walk all nodes of the parse tree 111 | """ 112 | antlr4.ParseTreeWalker.DEFAULT.walk(listener, self.tree) 113 | 114 | def visit(self, visitor: STIXPatternVisitor) -> Any: 115 | """Visit nodes in the parse tree and return a value 116 | 117 | Unlike the listener pattern, which enumerates all nodes and gossips to 118 | the listener about them, a visitor determines which nodes to descend 119 | into, and is able to return a value for each. 120 | """ 121 | return self.tree.accept(visitor) 122 | 123 | def to_dict_tree(self) -> PatternTree: 124 | """Convert the parse tree to a simplified dict tree 125 | 126 | See dendrol.transform.DictVisitor for more info 127 | """ 128 | visitor = PatternTreeVisitor() 129 | return visitor.visit(self.tree) 130 | 131 | def print_dict_tree(self): 132 | """Print a human-consumable representation of the dict tree to console 133 | """ 134 | print_tree(self.to_dict_tree()) 135 | -------------------------------------------------------------------------------- /dendrol/transform.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import binascii 3 | from collections import OrderedDict 4 | from datetime import date, datetime, tzinfo 5 | from typing import Iterable, List, Type, Union, Set, TypeVar, NewType, Dict 6 | 7 | from . import tz 8 | from .lang.STIXPatternParser import STIXPatternParser, ParserRuleContext 9 | from .lang.STIXPatternVisitor import STIXPatternVisitor 10 | 11 | 12 | ConvertedStixLiteral = NewType('ConvertedStixLiteral', Union[ 13 | int, 14 | str, 15 | bool, 16 | float, 17 | bytes, 18 | datetime, 19 | ]) 20 | 21 | 22 | class PatternTree(dict): 23 | """dict-based tree structure representing a STIX2 Pattern Expression 24 | 25 | OrderedDicts and some specialized subclasses of Python primitives are used 26 | to to offer a human-consumable YAML representation. 27 | See dendrol.debug.PatternTreeDumper for more info. 28 | """ 29 | 30 | def __str__(self): 31 | return self.serialize() 32 | 33 | def serialize(self): 34 | from .debug import dump_tree 35 | return dump_tree(self) 36 | 37 | @classmethod 38 | def from_dict(cls, d: dict) -> 'PatternTree': 39 | """Transform a regular Python dict to PatternTree's types and orderings 40 | 41 | XXX: a more holistic/less context-dependent way of doing this (and all 42 | the format methods below) might be using OrderedDict subclasses 43 | for each node type (CompositeObservation, SimpleComparison, etc) 44 | """ 45 | if d.keys() != {'pattern'}: 46 | raise ValueError('Expected a dict with one top-level key, "pattern"') 47 | 48 | tree = PatternTree(d) 49 | 50 | # We will walk the tree, updating nodes as we go, and collecting next 51 | # nodes to visit in these lists 52 | observations = [d['pattern']] 53 | qualifiers = [] 54 | comparisons = [] 55 | 56 | OBSERVATION_TYPES = { 57 | 'expression': cls.format_composite_observation, 58 | 'observation': cls.format_simple_observation, 59 | } 60 | 61 | # Walk all observations, formatting each as we go, saving child 62 | # observations for successive iterations, and any qualifiers and 63 | # comparisons for the next stages 64 | while observations: 65 | node = observations.pop() 66 | assert len(node) == 1, \ 67 | 'Each observation must be a dict with a single key-value pair' 68 | 69 | node_type, body = next(iter(node.items())) 70 | if node_type not in OBSERVATION_TYPES: 71 | raise ValueError( 72 | f'Unexpected observation type {repr(node_type)}. ' 73 | f'Expected one of: {OBSERVATION_TYPES.keys()}') 74 | 75 | formatter = OBSERVATION_TYPES[node_type] 76 | node.update(formatter(**body)) 77 | new_body = next(iter(node.values())) 78 | 79 | if node_type == 'expression': 80 | observations.extend(new_body['expressions']) 81 | elif node_type == 'observation': 82 | comparisons.extend(new_body['expressions']) 83 | 84 | qualifiers.extend(new_body['qualifiers'] or ()) 85 | 86 | QUALIFIER_TYPES = { 87 | 'start_stop': cls.format_start_stop_qualifier, 88 | 'within': cls.format_within_qualifier, 89 | 'repeats': cls.format_repeats_qualifier, 90 | } 91 | 92 | # Because qualifiers are not nested, we can iterate through them with 93 | # a simple for-loop 94 | for qualifier in qualifiers: 95 | assert len(qualifier) == 1, \ 96 | 'Each qualifier must be a dict with a single key-value pair' 97 | 98 | qualifier_type, body = next(iter(qualifier.items())) 99 | if qualifier_type not in QUALIFIER_TYPES: 100 | raise ValueError( 101 | f'Unexpected qualifier type {repr(qualifier_type)}. ' 102 | f'Expected one of: {QUALIFIER_TYPES.keys()}') 103 | 104 | formatter = QUALIFIER_TYPES[qualifier_type] 105 | qualifier.update(formatter(**body)) 106 | 107 | COMPARISON_TYPES = { 108 | 'expression': cls.format_composite_comparison, 109 | 'comparison': cls.format_simple_comparison, 110 | } 111 | 112 | # Comparisons may have children, so we convert each node as we iterate, 113 | # and collect any children for successive iterations 114 | while comparisons: 115 | node = comparisons.pop() 116 | assert len(node) == 1, \ 117 | 'Each comparison must be a dict with a single key-value pair' 118 | 119 | node_type, body = next(iter(node.items())) 120 | if node_type not in COMPARISON_TYPES: 121 | raise ValueError( 122 | f'Unexpected comparison type {repr(node_type)}. ' 123 | f'Expected one of: {COMPARISON_TYPES.keys()}') 124 | 125 | formatter = COMPARISON_TYPES[node_type] 126 | node.update(formatter(**body)) 127 | new_body = next(iter(node.values())) 128 | 129 | if 'expression' in node: 130 | comparisons.extend(new_body['expressions']) 131 | 132 | return tree 133 | 134 | @classmethod 135 | def format_pattern(cls, *, root: dict): 136 | return cls(pattern=root) 137 | 138 | @classmethod 139 | def format_composite_observation(cls, *, 140 | expressions: List[Dict[str, dict]], 141 | join: str = None, 142 | qualifiers: List[Dict[str, dict]] = None, 143 | ) -> Dict[str, OrderedDict]: 144 | return { 145 | 'expression': OrderedDict([ 146 | ('join', join), 147 | ('qualifiers', qualifiers), 148 | ('expressions', expressions), 149 | ]), 150 | } 151 | 152 | @classmethod 153 | def format_simple_observation(cls, *, 154 | objects: Iterable[str], 155 | expressions: List[Dict[str, dict]], 156 | join: str = None, 157 | qualifiers: List[Dict[str, dict]] = None, 158 | ) -> Dict[str, OrderedDict]: 159 | if not isinstance(objects, ObjectTypeSet): 160 | objects = ObjectTypeSet(objects) 161 | 162 | return { 163 | 'observation': OrderedDict([ 164 | ('objects', objects), 165 | ('join', join), # XXX: should this be allowed? 166 | # I kinda figured it would be convenient to rollup the first 167 | # composite comparison into its parent observation... but this 168 | # may make traversal implementations cumbersome. 169 | ('qualifiers', qualifiers), 170 | ('expressions', expressions), 171 | ]), 172 | } 173 | 174 | @classmethod 175 | def format_composite_comparison(cls, *, 176 | expressions: List[Dict[str, dict]], 177 | join: str = None, 178 | ) -> Dict[str, OrderedDict]: 179 | return { 180 | 'expression': OrderedDict([ 181 | ('join', join), 182 | ('expressions', expressions), 183 | ]), 184 | } 185 | 186 | @classmethod 187 | def format_simple_comparison(cls, *, 188 | object: str, 189 | path: List[Union[str, slice]], 190 | operator: str, 191 | value: ConvertedStixLiteral, 192 | negated: bool = None, 193 | ) -> Dict[str, OrderedDict]: 194 | if not isinstance(path, ObjectPath): 195 | # Converting path to our special type ensures uniform string 196 | # representations, enabling useful text diffing. 197 | path = ObjectPath(path) 198 | 199 | return { 200 | 'comparison': OrderedDict([ 201 | ('object', object), 202 | ('path', path), 203 | ('negated', negated), 204 | ('operator', operator), 205 | ('value', cls.format_literal(value)), 206 | ]), 207 | } 208 | 209 | @classmethod 210 | def format_start_stop_qualifier(cls, *, 211 | start: datetime, 212 | stop: datetime, 213 | ) -> Dict[str, OrderedDict]: 214 | return { 215 | 'start_stop': OrderedDict([ 216 | ('start', cls.format_literal(start)), 217 | ('stop', cls.format_literal(stop)), 218 | ]) 219 | } 220 | 221 | @classmethod 222 | def format_within_qualifier(cls, *, 223 | value: int, 224 | unit: str = 'SECONDS', 225 | ) -> Dict[str, OrderedDict]: 226 | return { 227 | 'within': OrderedDict([ 228 | ('value', cls.format_literal(value)), 229 | ('unit', unit), 230 | ]) 231 | } 232 | 233 | @classmethod 234 | def format_repeats_qualifier(cls, *, value: int) -> Dict[str, OrderedDict]: 235 | return { 236 | 'repeats': OrderedDict([ 237 | ('value', cls.format_literal(value)), 238 | ]) 239 | } 240 | 241 | @classmethod 242 | def format_object_path(cls, *, object: str, path: List[Union[str, slice]]): 243 | return OrderedDict([ 244 | ('object', object), 245 | ('path', path), 246 | ]) 247 | 248 | @classmethod 249 | def format_literal(cls, literal): 250 | if isinstance(literal, datetime): 251 | if literal.tzinfo is None: 252 | literal = literal.replace(tzinfo=tz.utc()) 253 | elif not is_utc(literal): 254 | raise ValueError('All datetimes must be in UTC') 255 | return literal 256 | 257 | 258 | class CompactibleObject: 259 | """Base class which instructs the YAML dumper when to use literal syntax 260 | 261 | For example, lists with single component can be dumped as list literals by 262 | overriding and returning True from is_eligible_for_compaction(). 263 | 264 | >>> class CompactMe(CompactibleObject, list): 265 | ... def is_eligible_for_compaction(self): 266 | ... return True 267 | ... 268 | >>> from dendrol import print_tree 269 | >>> print_tree({'path': CompactMe(['name'])}) 270 | path: [name] 271 | >>> class DontCompactOnMe(CompactibleObject, list): 272 | ... def is_eligible_for_compaction(self): 273 | ... return False 274 | ... 275 | >>> print_tree({'path': DontCompactOnMe(['name'])}) 276 | path: 277 | - name 278 | """ 279 | 280 | def is_eligible_for_compaction(self) -> bool: 281 | """Whether this object could be represented on one line""" 282 | return False 283 | 284 | def get_literal_type(self) -> Type: 285 | """Return the type of literal this object is represented as 286 | 287 | By default, this returns the first non-CompactibleObject base class in 288 | the object's method resolution order (MRO). 289 | """ 290 | for klass in self.__class__.__mro__: 291 | if issubclass(klass, CompactibleObject): 292 | continue 293 | 294 | # Heuristic: ignore type-hinting classes 295 | if klass.__module__ == 'typing': 296 | continue 297 | 298 | return klass 299 | 300 | raise NotImplementedError( 301 | 'This CompactibleObject has no obvious literal type. ' 302 | 'Please override the get_literal_type() method.') 303 | 304 | 305 | class CompactibleList(CompactibleObject, list): 306 | pass 307 | 308 | 309 | class CompactibleSet(CompactibleObject, set): 310 | pass 311 | 312 | 313 | class ObjectTypeSet(CompactibleSet, Set[str]): 314 | def is_eligible_for_compaction(self) -> bool: 315 | return len(self) == 1 316 | 317 | 318 | class ObjectPath(CompactibleList): 319 | def is_eligible_for_compaction(self) -> bool: 320 | return len(self) == 1 321 | 322 | 323 | class PatternTreeVisitor(STIXPatternVisitor): 324 | """Converts a STIX2 Pattern into a PatternTree, by selectively descending 325 | 326 | BACKGROUND :: 327 | 328 | The Visitor pattern differs from the Listener pattern (which has the 329 | characteristic enterXYZ/exitXYZ methods) in that instead of visiting 330 | each symbol (and calling those enter/exit methods for each), a Visitor 331 | is responsible for selecting which symbols to descend into. By way of 332 | analogy, this is akin to finding the quickest route through a forest 333 | by following the trail markers, rather than trying all routes, drawing 334 | a map of them, and ignoring all the long ones. 335 | 336 | Unlike the Listener pattern, where any output must be a side effect — 337 | some state stored in the class, built up incrementally — a Visitor is 338 | in control of its return value. This makes development of parse tree 339 | transformers a breeze: just figure out what a symbol ought to look like 340 | in the end result, then figure out how those things ought to be 341 | combined — rinse, repeat until all symbols are covered. 342 | 343 | 344 | IMPLEMENTATION :: 345 | 346 | The self.visit method is responsible for accepting an arbitrary symbol 347 | and calling the appropriate self.visitXYZ method specific to the type — 348 | AKA dispatch. 349 | 350 | The root node in all pattern expressions is the Pattern symbol. 351 | Execution of PatternTreeVisitor thusly always begins in visitPattern. 352 | 353 | Step through with a debugger, and it can seem like execution jumps all 354 | around the file — behind the scenes, though, the Visitor acts like a 355 | decision tree: for each symbol, a specific visitXYZ or generic emitYYZ 356 | method must decide which components to ignore, which to include as data 357 | in the PatternTree, and which to descend into (so the same can be done 358 | with them). 359 | 360 | Bit by bit, each symbol is transformed into PatternTree form, till 361 | visitPattern finally returns with the end result. 362 | 363 | """ 364 | 365 | def visitPattern(self, 366 | ctx: STIXPatternParser.PatternContext, 367 | ) -> PatternTree: 368 | """Convert the root node, Pattern, into a PatternTree""" 369 | return self.emitPattern(ctx) 370 | 371 | 372 | ########################## 373 | # COMPOSITE OBSERVATIONS # 374 | ########################## 375 | # 376 | # The STIX2 grammar uses parent symbols to join 1 or more 377 | # observations/expressions together by one of the FOLLOWEDBY, OR, or AND 378 | # operators. All observations, regardless of whether they're joined to 379 | # another expression, are children to these parent link symbols. 380 | # 381 | # FOLLOWEDBY corresponds to the observationExpressions symbol; 382 | # OR to observationExpressionOR; and AND to observationExpressionAnd. 383 | # 384 | # Though the grammar does not formally name this class of symbols, 385 | # I (zkanzler) figured "composite observation" described it well, and made 386 | # it easy to discern from other symbol classes used in PatternTree parsing. 387 | # 388 | # The emitCompositeObservation method is used to convert each type of 389 | # parent symbols into PatternTree form, simply storing the join operator's 390 | # text in the PatternTree to differentiate each symbol type. 391 | # 392 | def visitObservationExpressions(self, ctx: STIXPatternParser.ObservationExpressionsContext): 393 | """Convert FOLLOWEDBY into PatternTree form""" 394 | return self.emitCompositeObservation(ctx) 395 | 396 | def visitObservationExpressionOr(self, ctx: STIXPatternParser.ObservationExpressionOrContext): 397 | """Convert OR into PatternTree form""" 398 | return self.emitCompositeObservation(ctx) 399 | 400 | def visitObservationExpressionAnd(self, ctx: STIXPatternParser.ObservationExpressionAndContext): 401 | """Convert AND into PatternTree form""" 402 | return self.emitCompositeObservation(ctx) 403 | 404 | def visitObservationExpressionCompound(self, ctx: STIXPatternParser.ObservationExpressionCompoundContext): 405 | """Ditch parens around an observation expression""" 406 | lparen, expr, rparen = ctx.getChildren() 407 | return self.visit(expr) 408 | 409 | ####################### 410 | # SIMPLE OBSERVATIONS # 411 | ####################### 412 | # 413 | # Whereas a "composite observation" joins two or more observations and/or 414 | # expressions by an operator, a "simple observation" is a bare observation, 415 | # below which (in the tree) no further composite observations may occur — 416 | # only comparisons/expressions may be found. 417 | # 418 | # The grammar deems this observationExpressionSimple, the naming of which 419 | # inspired the terms used in PatternTree parsing for other classes of 420 | # symbols (e.g. simple comparison, composite comparison). 421 | # 422 | def visitObservationExpressionSimple(self, ctx: STIXPatternParser.ObservationExpressionSimpleContext): 423 | return self.emitSimpleObservation(ctx) 424 | 425 | ########################## 426 | # OBSERVATION QUALIFIERS # 427 | ########################## 428 | # 429 | # The STIX2 grammar uses a parent symbol to link an observation/expression 430 | # to an actual qualifier symbol. These parent symbols are named in the form 431 | # observationExpression. The qualifier symbols are named 432 | # Qualifier. 433 | # 434 | # We use the generic emitObservationQualifier method to add all the 435 | # qualifiers of an observation expression (by flattening the subtree) to 436 | # its PatternTree form. The qualifiers are converted to PatternTree form 437 | # in the overridden visitQualifier methods. 438 | # 439 | def visitObservationExpressionStartStop(self, ctx: STIXPatternParser.ObservationExpressionStartStopContext): 440 | return self.emitObservationQualifier(ctx) 441 | 442 | def visitObservationExpressionWithin(self, ctx: STIXPatternParser.ObservationExpressionWithinContext): 443 | return self.emitObservationQualifier(ctx) 444 | 445 | def visitObservationExpressionRepeated(self, ctx: STIXPatternParser.ObservationExpressionRepeatedContext): 446 | return self.emitObservationQualifier(ctx) 447 | 448 | def visitStartStopQualifier(self, ctx: STIXPatternParser.StartStopQualifierContext): 449 | """Convert START STOP qualifier into PatternTree form 450 | 451 | See PatternTree.format_start_stop_qualifier for returned structure 452 | """ 453 | start, start_dt, stop, stop_dt = ctx.getChildren() 454 | return PatternTree.format_start_stop_qualifier( 455 | start=self.visit(start_dt), 456 | stop=self.visit(stop_dt), 457 | ) 458 | 459 | def visitWithinQualifier(self, ctx: STIXPatternParser.WithinQualifierContext): 460 | """Convert WITHIN SECONDS qualifier into PatternTree form 461 | """ 462 | within, number, unit = ctx.getChildren() 463 | return PatternTree.format_within_qualifier( 464 | value=self.visit(number), 465 | unit=self.visit(unit), 466 | ) 467 | 468 | def visitRepeatedQualifier(self, ctx:STIXPatternParser.RepeatedQualifierContext): 469 | repeats, number, times = ctx.getChildren() 470 | return PatternTree.format_repeats_qualifier( 471 | value=self.visit(number), 472 | ) 473 | 474 | ######################### 475 | # COMPOSITE COMPARISONS # 476 | ######################### 477 | # 478 | # The STIX2 grammar uses the comparisonExpression symbol to denote 1 or 479 | # more simple or composite comparisons, joined by OR. Below this in the 480 | # hierarchy, comparisonExpressionAnd does the same for AND. 481 | # 482 | # NOTE: this means all comparison expressions, regardless of whether they 483 | # are joined by OR/AND, sit below a comparisonExpression and 484 | # comparisonExpressionAnd symbol. 485 | # 486 | # Though the grammar does not formally name this type of symbol 487 | # representing a joined group of comparisons & comparison expressions, 488 | # I (zkanzler) figured "composite comparison" 489 | # 490 | # Composite comparisons are handled by the generic emitCompositeComparison, 491 | # which is able to determine which join operator is used by reading the 492 | # symbol type. 493 | # 494 | def visitComparisonExpression(self, ctx: STIXPatternParser.ComparisonExpressionContext): 495 | return self.emitCompositeComparison(ctx) 496 | 497 | def visitComparisonExpressionAnd(self, ctx: STIXPatternParser.ComparisonExpressionAndContext): 498 | return self.emitCompositeComparison(ctx) 499 | 500 | ###################### 501 | # SIMPLE COMPARISONS # 502 | ###################### 503 | # 504 | # PropTests are the actual comparisons of an object's property to a literal 505 | # value, e.g. `file:name = "test"`. There is one symbol for each type of 506 | # comparison operator. 507 | # 508 | # STIX2 deems them "property tests", which are aptly named for their 509 | # function — but I (zkanzler) think "simple comparison" uniquely identifies 510 | # their role in reference to all other types of expressions. 511 | # 512 | # All types of comparisons are handled by the generic emitSimpleComparison, 513 | # which, at time of writing (2018/11/20), simply stores the text of the 514 | # operator to differentiate each symbol type — each propTest symbol has the 515 | # same ? syntax. 516 | # 517 | def visitPropTestEqual(self, ctx: STIXPatternParser.PropTestEqualContext): 518 | return self.emitSimpleComparison(ctx) 519 | 520 | def visitPropTestOrder(self, ctx: STIXPatternParser.PropTestOrderContext): 521 | return self.emitSimpleComparison(ctx) 522 | 523 | def visitPropTestSet(self, ctx: STIXPatternParser.PropTestSetContext): 524 | return self.emitSimpleComparison(ctx) 525 | 526 | def visitPropTestLike(self, ctx: STIXPatternParser.PropTestLikeContext): 527 | return self.emitSimpleComparison(ctx) 528 | 529 | def visitPropTestRegex(self, ctx: STIXPatternParser.PropTestRegexContext): 530 | return self.emitSimpleComparison(ctx) 531 | 532 | def visitPropTestIsSubset(self, ctx: STIXPatternParser.PropTestIsSubsetContext): 533 | return self.emitSimpleComparison(ctx) 534 | 535 | def visitPropTestIsSuperset(self, ctx: STIXPatternParser.PropTestIsSupersetContext): 536 | return self.emitSimpleComparison(ctx) 537 | 538 | def visitPropTestParen(self, ctx: STIXPatternParser.PropTestParenContext): 539 | """Strips a parenthesized cmp expr and processes only its body 540 | 541 | Conversion is deferred to other visitPropTestXYZ methods (dispatch is 542 | handled by self.visit). 543 | 544 | Example (pseudo-code): 545 | 546 | visitPropTestParen('(file:name = "test" AND file:size > 12)') == 547 | visit('file:name = "test" AND file:size > 12') 548 | 549 | """ 550 | lparen, expr, rparen = ctx.getChildren() 551 | return self.visit(expr) 552 | 553 | def visitObjectPath(self, ctx: STIXPatternParser.ObjectPathContext): 554 | """Split an object path into component "object" and "path" parts 555 | 556 | Example (pseudo-code): 557 | 558 | visitObjectPath('file:name.decoded') == { 559 | 'object': 'file', 560 | 'path': ['name', 'decoded'], 561 | } 562 | 563 | """ 564 | # NOTE: path will contain 0 or 1 symbols 565 | object_type, colon, property, *path = ctx.getChildren() 566 | assert not path or len(path) == 1 567 | 568 | # This will store the converted object path symbols, flattened from 569 | # tree form into a convenient list. 570 | full_path = [self.visit(property)] 571 | 572 | # Past the first property, which is a special case, if the object path 573 | # has two or more components (three or more in total), path[0] will be 574 | # a PathStep symbol, which is flattened into a list by visitPathStep. 575 | # If there is only one component (two in total), path[0] will be a 576 | # single symbol, requiring no flattening. 577 | if path: 578 | path_component: STIXPatternParser.ObjectPathComponentContext = path[0] 579 | if isinstance(path_component, STIXPatternParser.PathStepContext): 580 | full_path += self.visit(path_component) 581 | else: 582 | full_path.append(self.visit(path_component)) 583 | 584 | return PatternTree.format_object_path( 585 | object=object_type.getText(), 586 | path=ObjectPath(full_path), 587 | ) 588 | 589 | def visitPathStep(self, 590 | ctx: STIXPatternParser.PathStepContext, 591 | ) -> List[Union[ConvertedStixLiteral, slice]]: 592 | """Flatten object path steps into list of literals or slices (for [1] notation) 593 | 594 | Example (pseudo-code): 595 | 596 | visitPathStep('test.muffin[2]') == ['test', 'muffin', slice(None, 2, None)] 597 | 598 | """ 599 | children = flatten_left(ctx) 600 | return [ 601 | self.visit(child) 602 | for child in children 603 | ] 604 | 605 | def visitIndexPathStep(self, ctx: STIXPatternParser.IndexPathStepContext) -> slice: 606 | """Convert array/object-prop notation into a Python slice 607 | 608 | NOTE: the special [*] case (for "match ANY items in the array/object") 609 | is converted into slice(None, '*', None) 610 | """ 611 | lbracket, index, rbracket = ctx.getChildren() 612 | return slice(self.emitLiteral(index)) 613 | 614 | def visitFirstPathComponent(self, ctx: STIXPatternParser.FirstPathComponentContext) -> ConvertedStixLiteral: 615 | """Convert the first step of an object path into string form 616 | 617 | BACKGROUND: (I speculate) the reason for a separate, special path 618 | symbol for the first step is so the grammar can enforce the 619 | constraint that all object paths must have at least *one* 620 | step. 621 | """ 622 | return self.emitLiteral(ctx.getChild(0)) 623 | 624 | def visitKeyPathStep(self, ctx: STIXPatternParser.KeyPathStepContext) -> ConvertedStixLiteral: 625 | """Convert a regular property path step (past the first) into a string 626 | """ 627 | dot, key = ctx.getChildren() 628 | return self.emitLiteral(key) 629 | 630 | def visitTerminal(self, node) -> ConvertedStixLiteral: 631 | """Convert non-symbol nodes to Python literals 632 | 633 | Non-symbol nodes include string literals, names, etc 634 | """ 635 | return self.emitLiteral(node) 636 | 637 | def emitPattern(self, ctx: STIXPatternParser.PatternContext): 638 | """Convert a Pattern symbol into PatternTree form 639 | 640 | See PatternTree.format_pattern for returned structure 641 | """ 642 | observations, eof = ctx.getChildren() 643 | return PatternTree.format_pattern( 644 | root=self.visit(observations), 645 | ) 646 | 647 | def emitCompositeObservation(self, 648 | ctx: Union[STIXPatternParser.ObservationExpressionsContext, 649 | STIXPatternParser.ObservationExpressionOrContext, 650 | STIXPatternParser.ObservationExpressionAndContext], 651 | ) -> dict: 652 | """Convert a group of joined observations into PatternTree form 653 | 654 | A "composite observation" is any observations joined by FOLLOWEDBY, OR, 655 | or AND. 656 | 657 | See PatternTree.format_composite_observation for returned structure 658 | """ 659 | if ctx.getChildCount() == 1: 660 | return self.visit(ctx.getChild(0)) 661 | 662 | op = ctx.getChild(1) 663 | children = flatten_left(ctx) 664 | 665 | return PatternTree.format_composite_observation( 666 | join=op.getText().upper(), 667 | qualifiers=None, # if there are any qualifiers, they will be added 668 | # by the parent node (which contains this 669 | # expression and the qualifier) 670 | expressions=[ 671 | self.visit(child) 672 | for child in children 673 | ], 674 | ) 675 | 676 | def emitSimpleObservation(self, ctx: STIXPatternParser.ObservationExpressionSimpleContext): 677 | """Convert a non-grouped observation to its PatternTree form 678 | 679 | A "simple observation" is a whole observation and its child 680 | comparisons, including the square brackets. 681 | 682 | See PatternTree.format_simple_observation for returned structure 683 | """ 684 | lbracket, child, rbracket = ctx.getChildren() 685 | root = self.visit(child) 686 | 687 | if 'expression' in root: 688 | expression = root['expression'] 689 | join = expression['join'] 690 | expressions = expression['expressions'] 691 | else: 692 | join = None 693 | expressions = [root] 694 | 695 | object_types = self.findObjectTypes(expressions) 696 | 697 | return PatternTree.format_simple_observation( 698 | objects=object_types, 699 | join=join, 700 | expressions=expressions, 701 | ) 702 | 703 | def emitObservationQualifier(self, ctx: Union[STIXPatternParser.ObservationExpressionStartStopContext, 704 | STIXPatternParser.ObservationExpressionWithinContext, 705 | STIXPatternParser.ObservationExpressionRepeatedContext]): 706 | """Add an observation expression's qualifiers in its PatternTree form 707 | 708 | The STIX2 Pattern grammar places the symbol linking an observation 709 | expression to its qualifier(s) above the observation expression in the 710 | tree. Because an observation expression can exist without qualifiers 711 | (and without a qualifier link symbol), we use self.visit() to grab the 712 | PatternTree form of the observation expression and update it with the 713 | PatternTree-converted qualifiers — instead of generating a stub of the 714 | observation expression's structure here, then filling it when 715 | processing the observation expression. 716 | 717 | NOTE: see PatternTree.format_composite_observation 718 | and PatternTree.format_simple_observation for returned structure 719 | """ 720 | # NOTE: qualifiers will have 1 or more qualifier symbols 721 | expr, *qualifiers = flatten_left(ctx, [ 722 | STIXPatternParser.ObservationExpressionStartStopContext, 723 | STIXPatternParser.ObservationExpressionWithinContext, 724 | STIXPatternParser.ObservationExpressionRepeatedContext, 725 | ]) 726 | 727 | # node will either be {"expression": body} or {"observation": body} 728 | node: dict = self.visit(expr) 729 | assert isinstance(node, dict) and len(node) == 1 # sanity check 730 | 731 | # body will contain the meat of the observation/expression, which we 732 | # will fill with our qualifier info. 733 | body = next(iter(node.values())) 734 | 735 | body['qualifiers'] = [ 736 | self.visit(qualifier) 737 | for qualifier in qualifiers 738 | ] 739 | return node 740 | 741 | 742 | def emitCompositeComparison(self, ctx: Union[STIXPatternParser.ComparisonExpressionContext, 743 | STIXPatternParser.ComparisonExpressionAndContext]): 744 | """Convert a group of joined comparisons into PatternTree form 745 | 746 | See PatternTree.format_composite_comparison for returned structure. 747 | """ 748 | if ctx.getChildCount() == 1: 749 | return self.visit(ctx.getChild(0)) 750 | 751 | op = ctx.getChild(1) 752 | children = flatten_left(ctx) 753 | 754 | return PatternTree.format_composite_comparison( 755 | join=op.getText().upper(), 756 | expressions=[ 757 | self.visit(child) 758 | for child in children 759 | ], 760 | ) 761 | 762 | def emitSimpleComparison(self, ctx: STIXPatternParser.PropTestContext) -> dict: 763 | """Convert a comparison (AKA propTest) into PatternTree form 764 | 765 | See PatternTree.format_simple_comparison for returned structure. 766 | """ 767 | lhs, *nots, op, rhs = ctx.getChildren() 768 | 769 | return PatternTree.format_simple_comparison( 770 | **self.visit(lhs), 771 | negated=True if nots else None, # using None makes for an uncluttered yaml tree, 772 | # where the field will just be empty -- we only 773 | # need to see the value if it's True. 774 | operator=op.getText(), 775 | value=self.visit(rhs), 776 | ) 777 | 778 | def emitLiteral(self, literal) -> ConvertedStixLiteral: 779 | """Convert terminal node or identifier-like sym into Python primitives 780 | """ 781 | text = literal.getText() 782 | symbol_type = literal.getSymbol().type 783 | return coerce_literal(text, symbol_type) 784 | 785 | def findObjectTypes(self, comparison_expressions: List[dict]) -> ObjectTypeSet: 786 | """Find the object types used by the given comparison expressions 787 | """ 788 | encountered_types = ObjectTypeSet() 789 | to_visit = list(comparison_expressions) 790 | 791 | while to_visit: 792 | node = to_visit.pop() 793 | assert isinstance(node, dict) and len(node) == 1 794 | 795 | node_type, body = next(iter(node.items())) 796 | 797 | if node_type == 'expression': 798 | to_visit.extend(body['expressions']) 799 | elif node_type == 'comparison': 800 | encountered_types.add(body['object']) 801 | 802 | return encountered_types 803 | 804 | 805 | T = TypeVar('T', bound=ParserRuleContext) 806 | 807 | 808 | def flatten_left(ctx: ParserRuleContext, 809 | rules: Iterable[Type[T]]=None, 810 | ) -> List[T]: 811 | r"""Flatten left-associative symbols 812 | 813 | Composite expressions joined with AND/OR/etc are left-associative 814 | (their trees expand recursively on the left-hand side). This method 815 | traverses down the left, recording all symbols of the same rule type 816 | and returning a flat representation of a homogeneous tree. 817 | 818 | NOTE: this method preserves only the rightmost node of each matching rule 819 | 820 | Example: a FOLLOWEDBY b FOLLOWEDBY c AND d 821 | 822 | Parens: (a FOLLOWEDBY (b FOLLOWEDBY (c AND d)) 823 | 824 | Binary Tree (max two children): 825 | FOLLOWEDBY <-+ 826 | / \ |--- Each FOLLOWEDBY has two children 827 | FOLLOWEDBY AND <-+ 828 | / \ / \ 829 | a b c d 830 | 831 | Regular Tree: 832 | FOLLOWEDBY 833 | / | \ 834 | a b AND <--- Single FOLLOWEDBY with all children 835 | / \ 836 | c d 837 | """ 838 | rules = tuple(rules or (type(ctx),)) 839 | 840 | flattened = [] 841 | last_lhs = ctx 842 | while True: 843 | lhs, *others = last_lhs.getChildren() 844 | if others: 845 | flattened.append(others[-1]) 846 | 847 | if isinstance(lhs, rules): 848 | last_lhs = lhs 849 | continue 850 | else: 851 | flattened.append(lhs) 852 | break 853 | 854 | # Because insertion is O(n) in CPython, we use the O(1) append and return 855 | # the list in reverse. 856 | # ref: https://wiki.python.org/moin/TimeComplexity#list 857 | return list(reversed(flattened)) 858 | 859 | 860 | # NOTE: this was lifted from oasis-open/cti-pattern-matcher 861 | def convert_stix_datetime(timestamp_str: str, ignore_case: bool=False) -> datetime: 862 | """ 863 | Convert a timestamp string from a pattern to a datetime.datetime object. 864 | If conversion fails, raises a ValueError. 865 | """ 866 | 867 | # strptime() appears to work case-insensitively. I think we require 868 | # case-sensitivity for timestamp literals inside patterns and JSON 869 | # (for the "T" and "Z" chars). So check case first. 870 | if not ignore_case and any(c.islower() for c in timestamp_str): 871 | raise ValueError(f'Invalid timestamp format (require upper case): {timestamp_str}') 872 | 873 | # Can't create a pattern with an optional part... so use two patterns 874 | if '.' in timestamp_str: 875 | fmt = '%Y-%m-%dT%H:%M:%S.%fZ' 876 | else: 877 | fmt = '%Y-%m-%dT%H:%M:%SZ' 878 | 879 | dt = datetime.strptime(timestamp_str, fmt) 880 | dt = dt.replace(tzinfo=tz.utc()) 881 | 882 | return dt 883 | 884 | 885 | def is_utc(o: Union[tzinfo, datetime, date]) -> bool: 886 | """Return whether the timezone or date/time object is (in) UTC 887 | """ 888 | if isinstance(o, (datetime, date)): 889 | if o.tzinfo is None: 890 | return False 891 | o = o.tzinfo 892 | 893 | arbitrary_dt = datetime.now().replace(tzinfo=o) 894 | return arbitrary_dt.utcoffset().total_seconds() == 0 895 | 896 | 897 | # NOTE: this was lifted from oasis-open/cti-pattern-matcher 898 | PRIMITIVE_COERCERS = { 899 | STIXPatternParser.IntPosLiteral: int, 900 | STIXPatternParser.IntNegLiteral: int, 901 | STIXPatternParser.StringLiteral: lambda s: s[1:-1].replace("\\'", "'").replace('\\\\', '\\'), 902 | STIXPatternParser.BoolLiteral: lambda s: s.lower() == 'true', 903 | STIXPatternParser.FloatPosLiteral: float, 904 | STIXPatternParser.FloatNegLiteral: float, 905 | STIXPatternParser.BinaryLiteral: lambda s: base64.standard_b64decode(s[2:-1]), 906 | STIXPatternParser.HexLiteral: lambda s: binascii.a2b_hex(s[2:-1]), 907 | STIXPatternParser.TimestampLiteral: lambda t: convert_stix_datetime(t[2:-1]), 908 | } 909 | 910 | 911 | def coerce_literal(text: str, 912 | symbol_type: int, 913 | ) -> ConvertedStixLiteral: 914 | """Convert a parsed literal symbol into a Python primitive 915 | 916 | NOTE: If the literal type is not recognized, the original text is returned. 917 | """ 918 | coercer = PRIMITIVE_COERCERS.get(symbol_type) 919 | return coercer(text) if coercer else text 920 | -------------------------------------------------------------------------------- /dendrol/tz.py: -------------------------------------------------------------------------------- 1 | from datetime import tzinfo, timedelta 2 | 3 | 4 | class utc(tzinfo): 5 | def tzname(self,**kwargs): 6 | return 'UTC' 7 | 8 | def utcoffset(self, dt): 9 | return timedelta(0) 10 | -------------------------------------------------------------------------------- /dendrol/version.py: -------------------------------------------------------------------------------- 1 | version_info = (0, 0, 5) 2 | 3 | __version__ = '.'.join(str(n) for n in version_info) 4 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts: 3 | # Verbosity 4 | # 0 - hide test names while running (only showing ".", "F", etc) 5 | # 1 - show full test names while running 6 | # 2 - don't truncate assertion failure printouts (includes full diffs) 7 | --verbosity=2 8 | 9 | # Show an abbreviated traceback format 10 | --tb=short 11 | 12 | 13 | # Only find tests underneath the tests/ directory 14 | testpaths = tests 15 | 16 | # Only search for tests within files matching these patterns 17 | python_files = tests.py test_*.py 18 | 19 | # Show times of log messages 20 | log_format = %(asctime)s.%(msecs)f %(name)-25s %(levelname)-8s %(message)s 21 | log_date_format = %H:%M:%S 22 | 23 | 24 | filterwarnings = 25 | # hide warnings for yaml library (we have no control over) 26 | ignore::DeprecationWarning:yaml 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from setuptools import setup 4 | 5 | 6 | PROJECT_DIR = Path(__file__).parent.resolve() 7 | PACKAGE_DIR = PROJECT_DIR / 'dendrol' 8 | 9 | README_PATH = PROJECT_DIR / 'README.md' 10 | VERSION_PATH = PACKAGE_DIR / 'version.py' 11 | 12 | 13 | def get_readme() -> str: 14 | with open(README_PATH) as fp: 15 | return fp.read() 16 | 17 | 18 | def get_version() -> str: 19 | with open(VERSION_PATH) as fp: 20 | source = fp.read() 21 | 22 | context = {} 23 | exec(source, context) 24 | return context['__version__'] 25 | 26 | 27 | setup( 28 | name='dendrol', 29 | version=get_version(), 30 | packages=['dendrol', 'dendrol.lang'], 31 | url='https://github.com/usePF/dendrol', 32 | license='MIT', 33 | author='Perch Security', 34 | author_email='hello@perchsecurity.com', 35 | maintainer='Zach "theY4Kman" Kanzler', 36 | maintainer_email='z@perchsecurity.com', 37 | description='The STIX2 Pattern expression library for humans', 38 | long_description=get_readme(), 39 | long_description_content_type="text/markdown", 40 | install_requires=[ 41 | # TODO: laxer req versions 42 | 'antlr4-python3-runtime==4.7', 43 | 'PyYAML>=3.12,<4', 44 | ], 45 | extras_require={ 46 | 'test': [ 47 | 'icdiff', 48 | 'pytest>=3.8,<4.0', 49 | ], 50 | 'dev': [ 51 | 'dendrol[test]', 52 | 'click>=7.0', 53 | 'requests>=2.20.1', 54 | 'tqdm>=4.28.1', 55 | ] 56 | }, 57 | tests_require=[ 58 | 'dendrol[test]', 59 | ], 60 | classifiers=[ 61 | 'Development Status :: 3 - Alpha', 62 | 'License :: OSI Approved :: MIT License', 63 | 'Programming Language :: Python :: 3 :: Only', 64 | 'Programming Language :: Python :: 3.6', 65 | 'Programming Language :: Python :: 3.7', 66 | 'Topic :: Security', 67 | ], 68 | ) 69 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PerchSecurity/dendrol/b3770beb90547fc7f8e3009794fcf3774404d795/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from _pytest.config import Config 2 | from icdiff import ConsoleDiff 3 | from py._io.terminalwriter import get_terminal_width 4 | 5 | 6 | # This is done during initialization, before any tests are run, instead of 7 | # within our assertrepr hook — because the assertrepr hook is called while 8 | # terminal capturing is enabled and all calls to get_terminal_width() return 80 9 | # ref: https://github.com/pytest-dev/pytest/issues/4030#issuecomment-425672782 10 | INITIAL_TERMWIDTH = get_terminal_width() 11 | 12 | 13 | def pytest_assertrepr_compare(config: Config, op: str, left, right): 14 | # Local import to avoid preclusion of pytest's assertion rewriting 15 | from dendrol import PatternTree 16 | 17 | # Resets the red color from the "E" at the start of each pytest 18 | # exception/assertion traceback line -- markup() appends a reset 19 | # character after its output, so we give it an empty string, 20 | # because we only care about that reset. 21 | terminal_writer = config.get_terminal_writer() 22 | reset_colors = lambda s: terminal_writer.markup('', white=True) + s 23 | 24 | if op == '==' and isinstance(left, PatternTree) and isinstance(right, PatternTree): 25 | left_desc = 'PatternTree()' 26 | right_desc = 'PatternTree()' 27 | rewritten_assert = f'{left_desc} {op} {right_desc}' 28 | 29 | summary = 'The pattern trees are not equivalent. Full diff:' 30 | 31 | left_repr = left.serialize() 32 | right_repr = right.serialize() 33 | 34 | differ = ConsoleDiff(tabsize=4, cols=120) 35 | diff = differ.make_table( 36 | fromdesc=left_desc, 37 | fromlines=left_repr.splitlines(), 38 | todesc=right_desc, 39 | tolines=right_repr.splitlines(), 40 | ) 41 | 42 | lines = [ 43 | rewritten_assert, 44 | '', 45 | summary, 46 | '', 47 | ] 48 | lines.extend( 49 | reset_colors(diff_line) 50 | for diff_line in diff 51 | ) 52 | return lines 53 | -------------------------------------------------------------------------------- /tests/test_debug.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Tuple, Iterable 3 | 4 | import pytest 5 | 6 | from dendrol import PatternTree, load_tree 7 | 8 | 9 | # NOTE: whitespace is important here! 10 | # these structures will be compared char-by-char, so mind trailing spaces. 11 | TESTS = ''' 12 | 13 | name: simple 14 | pattern: 15 | observation: 16 | objects: {ipv4-addr} 17 | join: 18 | qualifiers: 19 | expressions: 20 | - comparison: 21 | object: ipv4-addr 22 | path: [value] 23 | negated: 24 | operator: '=' 25 | value: 1.2.3.4 26 | 27 | --- 28 | 29 | name: multiple-object-types 30 | pattern: 31 | expression: 32 | join: FOLLOWEDBY 33 | qualifiers: 34 | - within: 35 | value: 600 36 | unit: SECONDS 37 | expressions: 38 | - observation: 39 | objects: 40 | ? ipv4-addr 41 | ? ipv6-addr 42 | join: OR 43 | qualifiers: 44 | expressions: 45 | - comparison: 46 | object: ipv4-addr 47 | path: [value] 48 | negated: 49 | operator: '=' 50 | value: 198.51.100.1/32 51 | - comparison: 52 | object: ipv4-addr 53 | path: [value] 54 | negated: 55 | operator: '=' 56 | value: 203.0.113.33/32 57 | - comparison: 58 | object: ipv6-addr 59 | path: [value] 60 | negated: 61 | operator: '=' 62 | value: 2001:0db8:dead:beef:dead:beef:dead:0001/128 63 | - observation: 64 | objects: {domain-name} 65 | join: 66 | qualifiers: 67 | expressions: 68 | - comparison: 69 | object: domain-name 70 | path: [value] 71 | negated: 72 | operator: '=' 73 | value: example.com 74 | 75 | --- 76 | 77 | name: list-path-explicit 78 | pattern: 79 | observation: 80 | objects: {file} 81 | join: 82 | qualifiers: 83 | expressions: 84 | - comparison: 85 | object: file 86 | path: 87 | - extensions 88 | - windows-pebinary-ext 89 | - sections 90 | - [12] 91 | - entropy 92 | negated: 93 | operator: '>' 94 | value: 7.0 95 | 96 | --- 97 | 98 | name: list-path-any 99 | pattern: 100 | observation: 101 | objects: {file} 102 | join: 103 | qualifiers: 104 | expressions: 105 | - comparison: 106 | object: file 107 | path: 108 | - extensions 109 | - windows-pebinary-ext 110 | - sections 111 | - [*] 112 | - entropy 113 | negated: 114 | operator: '>' 115 | value: 7.0 116 | 117 | ''' 118 | 119 | 120 | RGX_TEST_NAME = re.compile(r'^name: (\S+)\s+', re.MULTILINE) 121 | 122 | 123 | def get_tests() -> Iterable[Tuple[str, str, PatternTree]]: 124 | docs = TESTS.split('\n---\n') 125 | for i, doc in enumerate(docs): 126 | doc = doc.strip() 127 | 128 | name_match = RGX_TEST_NAME.match(doc) 129 | if name_match: 130 | name = name_match.group(1) 131 | doc = RGX_TEST_NAME.sub('', doc) 132 | else: 133 | name = f'doc {i}' 134 | 135 | tree = load_tree(doc) 136 | yield name, doc, tree 137 | 138 | 139 | @pytest.mark.parametrize('source,tree', [ 140 | pytest.param(source, tree, id=name) 141 | for name, source, tree in get_tests() 142 | ]) 143 | def test_yaml_repr(source: str, tree: PatternTree): 144 | expected = source 145 | actual = tree.serialize().strip() 146 | assert expected == actual 147 | -------------------------------------------------------------------------------- /tests/test_transform.py: -------------------------------------------------------------------------------- 1 | from typing import Iterable, Tuple 2 | 3 | import pytest 4 | import yaml 5 | 6 | from dendrol import Pattern, PatternTree 7 | from dendrol.debug import PatternTreeLoader 8 | 9 | 10 | TESTS = yaml.load(''' 11 | 12 | simple-comparison: 13 | expression: > 14 | [ipv4-addr:value = '1.2.3.4'] 15 | 16 | pattern: 17 | observation: 18 | objects: {ipv4-addr} 19 | join: 20 | qualifiers: 21 | expressions: 22 | - comparison: 23 | object: ipv4-addr 24 | path: [value] 25 | negated: 26 | operator: '=' 27 | value: 1.2.3.4 28 | 29 | 30 | joined-comparisons: 31 | expression: > 32 | [email-message:subject = 'Yo!' AND email-message:body LIKE '%malicious%'] 33 | 34 | pattern: 35 | observation: 36 | objects: {email-message} 37 | join: AND 38 | qualifiers: 39 | expressions: 40 | - comparison: 41 | object: email-message 42 | path: [subject] 43 | negated: 44 | operator: '=' 45 | value: Yo! 46 | - comparison: 47 | object: email-message 48 | path: [body] 49 | negated: 50 | operator: LIKE 51 | value: '%malicious%' 52 | 53 | 54 | complex-comparisons: 55 | expression: > 56 | [file:name = 'test.exe' AND (file:size < 4 OR file:size > 4096)] 57 | 58 | pattern: 59 | observation: 60 | objects: {file} 61 | join: AND 62 | qualifiers: 63 | expressions: 64 | - comparison: 65 | object: file 66 | path: [name] 67 | negated: 68 | operator: '=' 69 | value: test.exe 70 | - expression: 71 | join: OR 72 | expressions: 73 | - comparison: 74 | object: file 75 | path: [size] 76 | negated: 77 | operator: '<' 78 | value: 4 79 | - comparison: 80 | object: file 81 | path: [size] 82 | negated: 83 | operator: '>' 84 | value: 4096 85 | 86 | 87 | dictionary-object-properties: 88 | expression: > 89 | [email-message:from_ref.value MATCHES '.*'] 90 | 91 | pattern: 92 | observation: 93 | objects: {email-message} 94 | join: 95 | qualifiers: 96 | expressions: 97 | - comparison: 98 | object: email-message 99 | path: 100 | - from_ref 101 | - value 102 | negated: 103 | operator: MATCHES 104 | value: .* 105 | 106 | 107 | list-object-properties: 108 | expression: > 109 | [file:extensions.'windows-pebinary-ext'.sections[*].entropy > 7.0] 110 | 111 | pattern: 112 | observation: 113 | objects: {file} 114 | join: 115 | qualifiers: 116 | expressions: 117 | - comparison: 118 | object: file 119 | path: 120 | - extensions 121 | - windows-pebinary-ext 122 | - sections 123 | - [*] 124 | - entropy 125 | negated: 126 | operator: '>' 127 | value: 7.0 128 | 129 | 130 | start-stop-qualifier: 131 | expression: > 132 | [ipv4-addr:value = '1.2.3.4'] START t'2017-06-29T00:00:00Z' STOP t'2017-12-05T00:00:00Z' 133 | 134 | pattern: 135 | observation: 136 | objects: {ipv4-addr} 137 | join: 138 | qualifiers: 139 | - start_stop: 140 | start: 2017-06-29 00:00:00 141 | stop: 2017-12-05 00:00:00 142 | expressions: 143 | - comparison: 144 | object: ipv4-addr 145 | path: [value] 146 | negated: 147 | operator: '=' 148 | value: '1.2.3.4' 149 | 150 | 151 | within-qualifier: 152 | expression: > 153 | [ipv4-addr:value = '1.2.3.4'] WITHIN 10 SECONDS 154 | 155 | pattern: 156 | observation: 157 | objects: {ipv4-addr} 158 | join: 159 | qualifiers: 160 | - within: 161 | value: 10 162 | unit: SECONDS 163 | expressions: 164 | - comparison: 165 | object: ipv4-addr 166 | path: [value] 167 | negated: 168 | operator: '=' 169 | value: '1.2.3.4' 170 | 171 | 172 | repeated-qualifier: 173 | expression: > 174 | [ipv4-addr:value = '1.2.3.4'] REPEATS 5 TIMES 175 | 176 | pattern: 177 | observation: 178 | objects: {ipv4-addr} 179 | join: 180 | qualifiers: 181 | - repeats: 182 | value: 5 183 | expressions: 184 | - comparison: 185 | object: ipv4-addr 186 | path: [value] 187 | negated: 188 | operator: '=' 189 | value: '1.2.3.4' 190 | 191 | 192 | multiple-qualifiers: 193 | expression: > 194 | [ipv4-addr:value = '1.2.3.4'] REPEATS 5 TIMES WITHIN 10 SECONDS 195 | 196 | pattern: 197 | observation: 198 | objects: {ipv4-addr} 199 | join: 200 | qualifiers: 201 | - repeats: 202 | value: 5 203 | - within: 204 | value: 10 205 | unit: SECONDS 206 | expressions: 207 | - comparison: 208 | object: ipv4-addr 209 | path: [value] 210 | negated: 211 | operator: '=' 212 | value: '1.2.3.4' 213 | 214 | 215 | joined-observation: 216 | expression: > 217 | [domain-name:value = 'xyz.com'] AND 218 | [file:name = 'test.exe'] 219 | 220 | pattern: 221 | expression: 222 | join: AND 223 | qualifiers: 224 | expressions: 225 | - observation: 226 | objects: {domain-name} 227 | join: 228 | qualifiers: 229 | expressions: 230 | - comparison: 231 | object: domain-name 232 | path: [value] 233 | negated: 234 | operator: '=' 235 | value: xyz.com 236 | - observation: 237 | objects: {file} 238 | join: 239 | qualifiers: 240 | expressions: 241 | - comparison: 242 | object: file 243 | path: [name] 244 | negated: 245 | operator: '=' 246 | value: test.exe 247 | 248 | 249 | stix2-patterning-example: 250 | source: 251 | expression: > 252 | ( 253 | [ipv4-addr:value = '198.51.100.1/32' OR 254 | ipv4-addr:value = '203.0.113.33/32' OR 255 | ipv6-addr:value = '2001:0db8:dead:beef:dead:beef:dead:0001/128'] 256 | 257 | FOLLOWEDBY [ 258 | domain-name:value = 'example.com'] 259 | 260 | ) WITHIN 600 SECONDS 261 | 262 | pattern: 263 | expression: 264 | join: FOLLOWEDBY 265 | qualifiers: 266 | - within: 267 | value: 600 268 | unit: SECONDS 269 | expressions: 270 | - observation: 271 | objects: 272 | ? ipv4-addr 273 | ? ipv6-addr 274 | join: OR 275 | qualifiers: 276 | expressions: 277 | - comparison: 278 | object: ipv4-addr 279 | path: [value] 280 | negated: 281 | operator: '=' 282 | value: 198.51.100.1/32 283 | - comparison: 284 | object: ipv4-addr 285 | path: [value] 286 | negated: 287 | operator: '=' 288 | value: 203.0.113.33/32 289 | - comparison: 290 | object: ipv6-addr 291 | path: [value] 292 | negated: 293 | operator: '=' 294 | value: 2001:0db8:dead:beef:dead:beef:dead:0001/128 295 | - observation: 296 | objects: {domain-name} 297 | join: 298 | qualifiers: 299 | expressions: 300 | - comparison: 301 | object: domain-name 302 | path: [value] 303 | negated: 304 | operator: '=' 305 | value: example.com 306 | 307 | ''', Loader=PatternTreeLoader) 308 | 309 | 310 | def get_tests() -> Iterable[Tuple[str, str, dict]]: 311 | for name, test in TESTS.items(): 312 | expression = test['expression'] 313 | 314 | pattern = test['pattern'] 315 | tree = {'pattern': pattern} 316 | expected = PatternTree.from_dict(tree) 317 | 318 | yield name, expression, expected 319 | 320 | 321 | @pytest.mark.parametrize('input,expected', [ 322 | pytest.param(expression, expected, id=name) 323 | for name, expression, expected in get_tests() 324 | ]) 325 | def test_dict_tree_visitor(input, expected): 326 | pattern = Pattern(input) 327 | dict_tree = pattern.to_dict_tree() 328 | 329 | expected = expected 330 | actual = dict_tree 331 | assert expected == actual 332 | --------------------------------------------------------------------------------