├── __init__.py ├── test ├── __init__.py ├── unit │ ├── test_logchecker.py │ └── test_ruleset.py ├── integration │ ├── test_multipart.py │ ├── test_expecterror.py │ ├── EXPECTERRORFIXTURE.yaml │ ├── LOGCONTAINSFIXTURE.yaml │ ├── NOLOGCONTAINSFIXTURE.yaml │ ├── test_cookie.py │ ├── test_nologcontains.py │ ├── test_logcontains.py │ ├── HTMLCONTAINSFIXTURE.yaml │ ├── MULTIPART.yaml │ ├── test_htmlcontains.py │ ├── COOKIEFIXTURE.yaml │ └── test_http.py ├── test_default.py ├── test_modsecurityv2.py └── yaml │ └── EXAMPLE.yaml ├── ftw ├── __init__.py ├── errors.py ├── util │ ├── output │ │ ├── b00-01-normal.yaml │ │ ├── path-00-baseline.yaml │ │ ├── path-13-NUL-bare.yaml │ │ ├── path-01-url-encoding.yaml │ │ ├── path-02-u-encoding.yaml │ │ ├── path-05-u-bestfit.yaml │ │ ├── path-12-NUL-encoded.yaml │ │ ├── path-03-utf8-encoded.yaml │ │ ├── path-08-invalid-url-encoding.yaml │ │ ├── path-09-invalid-u-encoding.yaml │ │ ├── path-14-backslash-separator.yaml │ │ ├── path-36-double-url-decoding.yaml │ │ ├── b01-01-query-string.yaml │ │ ├── b05-01-request-method.yaml │ │ ├── b06-01-request-protocol.yaml │ │ ├── path-07-utf8-encoded-bestfit.yaml │ │ ├── path-10-valid-invalid-urle-preference.yaml │ │ ├── path-11-valid-invalid-u-preference.yaml │ │ ├── path-19-control-chars-encoded.yaml │ │ ├── path-20-control-chars-bare.yaml │ │ ├── path-21-utf8-overlong-encoded-2.yaml │ │ ├── path-33-u-fullwidth-mapping.yaml │ │ ├── path-34-utf8-invalid-encoding.yaml │ │ ├── b04-01-request-filename.yaml │ │ ├── path-04-utf8-bare.yaml │ │ ├── path-17-backslash-separator-url-encoded.yaml │ │ ├── path-22-utf8-overlong-encoded-3.yaml │ │ ├── b03-01-header.yaml │ │ ├── path-16-forward-slash-separator-u-encoded.yaml │ │ ├── path-18-backslash-separator-u-encoded.yaml │ │ ├── path-23-utf8-overlong-encoded-4.yaml │ │ ├── path-35-utf8-encoded-fullwidth-mapping.yaml │ │ ├── path-37-unicode-normalization.yaml │ │ ├── b02-01-request-hostname-uri.yaml │ │ ├── path-06-utf8-bare-bestfit.yaml │ │ ├── path-15-forward-slash-separator-url-encoded.yaml │ │ ├── path-27-utf8-separators-overlong-encoded-2.yaml │ │ ├── path-28-utf8-separators-overlong-encoded-3.yaml │ │ ├── b03-03-header-referer.yaml │ │ ├── b03-04-header-cookie.yaml │ │ ├── path-29-utf8-separators-overlong-encoded-4.yaml │ │ ├── path-38-utf8-bare-fullwidth-mapping.yaml │ │ ├── b03-02-header-user-agent.yaml │ │ ├── b02-02-request-hostname-header.yaml │ │ ├── path-24-utf8-overlong-bare-2.yaml │ │ ├── path-25-utf8-overlong-bare-3.yaml │ │ ├── path-26-utf8-overlong-bare-4.yaml │ │ ├── path-30-utf8-separators-overlong-bare-2.yaml │ │ ├── path-31-utf8-separators-overlong-bare-3.yaml │ │ ├── path-32-utf8-separators-overlong-bare-4.yaml │ │ ├── b03-05-header-authorization-username.yaml │ │ ├── b03-06-header-authorization-password.yaml │ │ ├── b08-01-request-body-urlencoded-param-value.yaml │ │ ├── b08-02-request-body-urlencoded-param-name.yaml │ │ ├── b09-01-request-body-json.yaml │ │ ├── b07-01-trailing-header-cookie.yaml │ │ ├── m16-lf-line-endings.yaml │ │ ├── b10-03-multipart-param-filename.yaml │ │ ├── b10-04-multipart-file-contents.yaml │ │ ├── m18-cr-line.yaml │ │ ├── m19-multiple-ct-headers.yaml │ │ ├── b10-02-multipart-param-name.yaml │ │ ├── b10-01-multipart-preamble.yaml │ │ ├── b10-05-multipart-epilogue.yaml │ │ ├── m00-01-normal.yaml │ │ ├── m03-02-multiple-boundaries.yaml │ │ ├── m03-04-multiple-boundaries.yaml │ │ ├── m01-01-invalid-separator.yaml │ │ ├── m01-02-invalid-separator.yaml │ │ ├── m08-01-quoted-boundary.yaml │ │ ├── m08-06-partial-quote.yaml │ │ ├── m08-07-partial-quote.yaml │ │ ├── m02-01-invalid-parameter-name.yaml │ │ ├── m10-boundary-case-sensitivity.yaml │ │ ├── m11-01-invalid-multipart-type.yaml │ │ ├── m11-02-invalid-multipart-type.yaml │ │ ├── m11-04-invalid-multipart-type.yaml │ │ ├── m11-05-invalid-multipart-type.yaml │ │ ├── m13-06-disposition-folding.yaml │ │ ├── m17-01-first-boundary-lf.yaml │ │ ├── m02-02-invalid-parameter-name.yaml │ │ ├── m08-08-whitespace-after-boundary.yaml │ │ ├── m11-03-invalid-multipart-type.yaml │ │ ├── m11-06-invalid-multipart-type.yaml │ │ ├── m11-07-invalid-multipart-type.yaml │ │ ├── m12-02-disposition-name-no-quotes.yaml │ │ ├── m13-01-disposition-folding.yaml │ │ ├── m13-03-disposition-folding.yaml │ │ ├── m13-04-disposition-folding.yaml │ │ ├── m13-05-disposition-folding.yaml │ │ ├── m17-02-first-boundary-crlf.yaml │ │ ├── m04-whitespace-after-parameter-name.yaml │ │ ├── m05-whitespace-before-parameter-value.yaml │ │ ├── m06-whitespace-after-parameter-value.yaml │ │ ├── m07-01-special-chars-in-boundary.yaml │ │ ├── m07-02-special-chars-in-boundary.yaml │ │ ├── m08-02-whitespace-in-quoted-boundary.yaml │ │ ├── m08-03-whitespace-in-quoted-boundary.yaml │ │ ├── m12-03-disposition-name-single-quotes.yaml │ │ ├── m12-04-disposition-name-partial-quote.yaml │ │ ├── m12-05-disposition-name-partial-quote.yaml │ │ ├── m03-01-multiple-boundaries.yaml │ │ ├── m03-03-multiple-boundaries.yaml │ │ ├── m08-04-quote-in-quoted-boundary.yaml │ │ ├── m08-05-quote-in-quoted-boundary.yaml │ │ ├── m13-02-disposition-folding-isspace.yaml │ │ ├── m14-01-disposition-php-quoting.yaml │ │ ├── m12-01-disposition-multiple-param-names.yaml │ │ ├── m15-01-invalid-part.yaml │ │ └── m09-data-after-last-boundary.yaml │ ├── out.yaml │ ├── ironbee.py │ └── request_to_yaml.py ├── logchecker.py ├── util.py ├── testrunner.py ├── pytest_plugin.py ├── ruleset.py └── http.py ├── requirements.txt ├── .gitignore ├── setup.py ├── .travis.yml ├── CONTRIBUTORS.md ├── README.md ├── LICENSE └── docs ├── ExtendingFTW.md └── YAMLFormat.md /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ftw/__init__.py: -------------------------------------------------------------------------------- 1 | import http 2 | import logchecker 3 | import ruleset 4 | import testrunner 5 | import util 6 | import errors 7 | -------------------------------------------------------------------------------- /ftw/errors.py: -------------------------------------------------------------------------------- 1 | class TestError(Exception): 2 | def __init___(self, msg, context_args): 3 | Exception.__init__(self, "{0} {1}".format(msg, context_args)) 4 | -------------------------------------------------------------------------------- /test/unit/test_logchecker.py: -------------------------------------------------------------------------------- 1 | from ftw import logchecker 2 | import pytest 3 | 4 | def test_logchecker(): 5 | with pytest.raises(TypeError) as excinfo: 6 | checker = logchecker.LogChecker() 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | astroid==1.4.5 2 | colorama==0.3.7 3 | IPy==0.83 4 | lazy-object-proxy==1.2.2 5 | py==1.4.31 6 | pylint==1.5.5 7 | pytest==2.9.1 8 | PyYAML==3.11 9 | requests==2.9.1 10 | six==1.10.0 11 | wrapt==1.10.8 12 | -------------------------------------------------------------------------------- /test/integration/test_multipart.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, logchecker, testrunner 2 | import pytest 3 | import random 4 | import threading 5 | 6 | def test_multipart(ruleset, test): 7 | runner = testrunner.TestRunner() 8 | for stage in test.stages: 9 | runner.run_stage(stage) 10 | -------------------------------------------------------------------------------- /test/integration/test_expecterror.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, testrunner, http, errors 2 | import pytest 3 | import re 4 | import random 5 | import threading 6 | 7 | def test_expecterror(ruleset, test): 8 | runner = testrunner.TestRunner() 9 | for stage in test.stages: 10 | runner.run_stage(stage) 11 | -------------------------------------------------------------------------------- /ftw/util/output/b00-01-normal.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /?b00-01 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-00-baseline.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-13-NUL-bare.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/test.txt 15 | version: .txt HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/out.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "xyz\r\n\r\n" 12 | headers: 13 | User-Agent: test 14 | method: GET 15 | uri: / 16 | version: HTTP/1.1 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-01-url-encoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%74est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-02-u-encoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%u0074est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-05-u-bestfit.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/Risti%u0107.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-12-NUL-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/test.txt%00.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-03-utf8-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/Risti%c4%87.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-08-invalid-url-encoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/te%st.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-09-invalid-u-encoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-14-backslash-separator.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et\test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-36-double-url-decoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%2574est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b01-01-query-string.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /?b01-01%20UNION%20SELECT 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b05-01-request-method.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "\r\n" 12 | headers: {} 13 | method: UNION%20SELECT 14 | uri: /?b05-01 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b06-01-request-protocol.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "\r\n" 12 | headers: {} 13 | method: GET 14 | uri: /?b06-01 15 | version: UNION%20SELECT/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-07-utf8-encoded-bestfit.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/Risti%c4%87.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-10-valid-invalid-urle-preference.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%64.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-11-valid-invalid-u-preference.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%u0064.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-19-control-chars-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%01test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-20-control-chars-bare.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: "/et/\x01test.txt" 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-21-utf8-overlong-encoded-2.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%c1%b4est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-33-u-fullwidth-mapping.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%uff54est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-34-utf8-invalid-encoding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%c0%01test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b04-01-request-filename.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "\r\n" 12 | headers: {} 13 | method: GET 14 | uri: /UNION%20SELECT?b04-01 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-04-utf8-bare.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!python/str "/et/Risti\u0107.txt" 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-17-backslash-separator-url-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%5ctest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-22-utf8-overlong-encoded-3.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%e0%81%b4est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b03-01-header.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Host: UNION%20SELECT 14 | method: GET 15 | uri: /?b03-01 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-16-forward-slash-separator-u-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%2ftest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-18-backslash-separator-u-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%u005ctest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-23-utf8-overlong-encoded-4.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%f0%80%81%b4est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-35-utf8-encoded-fullwidth-mapping.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/%ef%bd%94est.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-37-unicode-normalization.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et/bogus/%u2025/test.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b02-01-request-hostname-uri.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: http://UNION%20SELECT.com/?b02-01 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-06-utf8-bare-bestfit.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!python/str "/et/Risti\u0107.txt" 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-15-forward-slash-separator-url-encoded.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%u002ftest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-27-utf8-separators-overlong-encoded-2.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%c0%aftest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-28-utf8-separators-overlong-encoded-3.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%e0%80%aftest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b03-03-header-referer.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Referer: UNION%20SELECT 14 | method: GET 15 | uri: /?b03-03 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/b03-04-header-cookie.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Cookie: name=UNION%20SELECT 14 | method: GET 15 | uri: /?b03-04 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-29-utf8-separators-overlong-encoded-4.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: /et%f0%80%80%aftest.txt 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/path-38-utf8-bare-fullwidth-mapping.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!python/str "/et/\uFF54est.txt" 15 | version: HTTP/1.0 16 | output: 17 | status: 200 18 | -------------------------------------------------------------------------------- /ftw/util/output/b03-02-header-user-agent.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | User-Agent: UNION%20SELECT 14 | method: GET 15 | uri: /?b03-02 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/b02-02-request-hostname-header.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Host: UNION%20SELECT.com 14 | method: GET 15 | uri: /?b02-02 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-24-utf8-overlong-bare-2.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V0L8G0ZXN0LnR4dA== 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-25-utf8-overlong-bare-3.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V0L+CBtGVzdC50eHQ= 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-26-utf8-overlong-bare-4.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V0L/CAgbRlc3QudHh0 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-30-utf8-separators-overlong-bare-2.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V0wK90ZXN0LnR4dA== 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-31-utf8-separators-overlong-bare-3.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V04ICvdGVzdC50eHQ= 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/path-32-utf8-separators-overlong-bare-4.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: {} 13 | method: GET 14 | uri: !!binary | 15 | L2V08ICAr3Rlc3QudHh0 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/b03-05-header-authorization-username.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Authorization: Basic VU5JT04gU0VMRUNUOnRlc3Q= 14 | method: GET 15 | uri: /?b03-05 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/b03-06-header-authorization-password.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: '' 12 | headers: 13 | Authorization: Basic dGVzdDpVTklPTiBTRUxFQ1Q= 14 | method: GET 15 | uri: /?b03-06 16 | version: HTTP/1.0 17 | output: 18 | status: 200 19 | -------------------------------------------------------------------------------- /ftw/util/output/b08-01-request-body-urlencoded-param-value.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "name=UNION%20SELECT\r\n\r\n\r\n" 12 | headers: 13 | Content-Length: '19' 14 | Content-Type: application/x-www-form-urlencoded 15 | User-Agent: Mozilla 16 | method: POST 17 | uri: /?b08-1 18 | version: HTTP/1.1 19 | output: 20 | status: 200 21 | -------------------------------------------------------------------------------- /ftw/util/output/b08-02-request-body-urlencoded-param-name.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "UNION%20SELECT=value\r\n\r\n\r\n" 12 | headers: 13 | Content-Length: '20' 14 | Content-Type: application/x-www-form-urlencoded 15 | User-Agent: Mozilla 16 | method: POST 17 | uri: /?b08-2 18 | version: HTTP/1.1 19 | output: 20 | status: 200 21 | -------------------------------------------------------------------------------- /ftw/util/output/b09-01-request-body-json.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "{\r\n \"name\": \"\\u0055NION SE\\u004cECT\"\r\n}\r\n\r\n\r\n" 12 | headers: 13 | Content-Length: '41' 14 | Content-Type: application/json 15 | User-Agent: Mozilla 16 | method: POST 17 | uri: /?b09-1 18 | version: HTTP/1.1 19 | output: 20 | status: 200 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Common ignores 2 | *.pyc 3 | *.swp 4 | *.swo 5 | 6 | # Ignore custom config files and logs 7 | crits/config/database.py 8 | crits/config/overrides.py 9 | logs/*.log* 10 | 11 | # Ignore build files 12 | documentation/src/_build/ 13 | extras/www/static/admin/ 14 | 15 | # Ignore Apple and brew 16 | .DS_Store 17 | INSTALL_RECEIPT.json 18 | 19 | # Ignore aptitude cache 20 | .apt_cache 21 | 22 | # Ignore pip cache 23 | .pip_cache 24 | 25 | # Ignore vagrant dir 26 | .vagrant 27 | 28 | # Ignore MongoDB download 29 | *mongodb-* 30 | 31 | # Ignore VirtualEnv 32 | venv/* 33 | 34 | # Ignore test cache 35 | test/.cache/* 36 | -------------------------------------------------------------------------------- /ftw/util/output/b07-01-trailing-header-cookie.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "9\r\n012345678\r\n1\r\n9\r\n0\r\nCookie: name=UNION%20SELECT\r\n\r\n\ 12 | \r\n" 13 | headers: 14 | Content-Type: application/x-www-form-urlencoded 15 | Cookie: name=value 16 | Transfer-Encoding: chunked 17 | User-Agent: Mozilla 18 | method: POST 19 | uri: /?b07-1 20 | version: HTTP/1.1 21 | output: 22 | status: 200 23 | -------------------------------------------------------------------------------- /test/integration/EXPECTERRORFIXTURE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "csanders-git" 4 | enabled: true 5 | name: "EXPECTERRORFIXTURE.yaml" 6 | description: "Tests expect_error functionality within a test" 7 | tests: 8 | - 9 | test_title: "expect_error(1)" 10 | stages: 11 | - 12 | stage: 13 | input: 14 | dest_addr: "example.com" 15 | method: "GET" 16 | port: 80 17 | headers: 18 | User-Agent: "Foo" 19 | Host: "example.com" 20 | Content-Length: "100" 21 | uri: "/" 22 | output: 23 | expect_error: True -------------------------------------------------------------------------------- /test/test_default.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ftw import testrunner, errors 3 | 4 | def test_default(ruleset, test, destaddr): 5 | """ 6 | Default tester with no logger obj. Useful for HTML contains and Status code 7 | Not useful for testing loggers 8 | """ 9 | runner = testrunner.TestRunner() 10 | try: 11 | for stage in test.stages: 12 | if destaddr is not None: 13 | stage.input.dest_addr = destaddr 14 | runner.run_stage(stage, None) 15 | except errors.TestError as e: 16 | e.args[1]['meta'] = ruleset.meta 17 | pytest.fail('Failure! Message -> {0}, Context -> {1}' 18 | .format(e.args[0],e.args[1])) 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup(name='ftw', 6 | version='1.0.2', 7 | description='Framework for Testing WAFs', 8 | author='Chaim Sanders, Zack Allen', 9 | author_email='zallen@fastly.com, chaim.sanders@gmail.com', 10 | url='https://www.github.com/fastly/ftw', 11 | download_url='https://github.com/fastly/ftw/tarball/1.0.2', 12 | entry_points = { 13 | 'pytest11': [ 14 | 'ftw = ftw.pytest_plugin' 15 | ] 16 | }, 17 | packages=['ftw'], 18 | keywords=['waf'], 19 | install_requires=[ 20 | 'IPy', 21 | 'pytest==2.9.1', 22 | 'PyYAML' 23 | ], 24 | ) 25 | -------------------------------------------------------------------------------- /test/integration/LOGCONTAINSFIXTURE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "Zack" 4 | enabled: true 5 | name: "LOGCONTAINSFIXTURE.yaml" 6 | description: "Tests log contains functionality with an integration object" 7 | tests: 8 | - 9 | test_title: "log_contains" 10 | stages: 11 | - 12 | stage: 13 | input: 14 | dest_addr: "example.com" 15 | method: "GET" 16 | port: 80 17 | headers: 18 | User-Agent: "Foo" 19 | Host: "example.com" 20 | protocol: "http" 21 | uri: "/" 22 | output: 23 | status: 200 24 | log_contains: "rule-id-[0-9]{2}" 25 | -------------------------------------------------------------------------------- /test/integration/NOLOGCONTAINSFIXTURE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "chaim" 4 | enabled: true 5 | name: "NOLOGCONTAINSFIXTURE.yaml" 6 | description: "Tests the no log contains functionality with an integration object" 7 | tests: 8 | - 9 | test_title: 1234 10 | stages: 11 | - 12 | stage: 13 | input: 14 | dest_addr: "example.com" 15 | method: "GET" 16 | port: 80 17 | headers: 18 | User-Agent: "Foo" 19 | Host: "example.com" 20 | protocol: "http" 21 | uri: "/" 22 | output: 23 | status: 200 24 | no_log_contains: "xyz-id-[0-9]{2}" 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | install: "pip install -e ." 5 | script: 6 | - py.test test/unit/ -s -v 7 | - py.test test/integration/test_logcontains.py -s -v --rule=test/integration/LOGCONTAINSFIXTURE.yaml 8 | - py.test test/integration/test_nologcontains.py -s -v --rule=test/integration/NOLOGCONTAINSFIXTURE.yaml 9 | - py.test test/integration/test_htmlcontains.py -s -v --rule=test/integration/HTMLCONTAINSFIXTURE.yaml 10 | - py.test test/integration/test_http.py -s -v 11 | - py.test test/integration/test_cookie.py -s -v --rule=test/integration/COOKIEFIXTURE.yaml 12 | - py.test test/integration/test_multipart.py -s -v --rule=test/integration/MULTIPART.yaml 13 | - py.test test/integration/test_expecterror.py -s -v --rule=test/integration/EXPECTERRORFIXTURE.yaml 14 | -------------------------------------------------------------------------------- /test/test_modsecurityv2.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from ftw import testrunner, errors 3 | 4 | @pytest.fixture 5 | def modsec_logger_obj(): 6 | """ 7 | Modsec logger object to access in the verifier 8 | @TODO 9 | """ 10 | return None 11 | 12 | def test_modsecurityv2(modsec_logger_obj, ruleset, test, destaddr): 13 | """ 14 | Modsec specific test 15 | """ 16 | runner = testrunner.TestRunner() 17 | try: 18 | for stage in test.stages: 19 | if destaddr is not None: 20 | stage.input.dest_addr = destaddr 21 | runner.run_stage(stage, modsec_logger_obj) 22 | except errors.TestError as e: 23 | e.args[1]['meta'] = ruleset.meta 24 | pytest.fail('Failure! Message -> {0}, Context -> {1}' 25 | .format(e.args[0],e.args[1])) 26 | -------------------------------------------------------------------------------- /ftw/util/ironbee.py: -------------------------------------------------------------------------------- 1 | import os 2 | import request_to_yaml 3 | 4 | filelist = [] 5 | for root, dirs, files in os.walk("waf-research", topdown=False): 6 | for name in files: 7 | extension = name[-5:] 8 | if extension == ".test": 9 | filelist.append(os.path.join(root, name)) 10 | for fname in filelist: 11 | f = open(fname,'r') 12 | request = "" 13 | for line in f.readlines(): 14 | if line[0] != '#': 15 | request += line 16 | req = request_to_yaml.Request() 17 | request = request.replace('\n','\r\n') 18 | req.get_request_line(request) 19 | req.get_headers(request) 20 | req.get_data(request) 21 | yaml_out = req.generate_yaml() 22 | newfname = (fname.split('/')[-1]).split('.')[0] 23 | print newfname 24 | req.write_yaml('output/'+newfname+'.yaml',yaml_out) 25 | #print yaml_out 26 | 27 | -------------------------------------------------------------------------------- /ftw/util/output/m16-lf-line-endings.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\nContent-Disposition: form-data; name=\"name\"\r\n\r\nJohn\ 12 | \ Smith\r\n--0000\r\nContent-Disposition: form-data; name=\"email\"\r\n\r\ 13 | \njohn.smith@example.com\r\n--0000\r\nContent-Disposition: form-data; name=\"\ 14 | image\"; filename=\"image.jpg\"\r\nContent-Type: image/jpeg\r\n\r\nBINARYDATA\r\ 15 | \n--0000--\r\n" 16 | headers: 17 | Content-Length: '259' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m16 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/b10-03-multipart-param-filename.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smit\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smit@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"UNION SELECT\"\r\r\nContent-Type:\ 15 | \ image/jpeg\r\r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '274' 18 | Content-Type: multipart/form-data; boundary=0000 19 | method: POST 20 | uri: /?b10-03 21 | version: HTTP/1.0 22 | output: 23 | status: 200 24 | -------------------------------------------------------------------------------- /ftw/util/output/b10-04-multipart-file-contents.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smit@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nUNION SELECT\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '274' 18 | Content-Type: multipart/form-data; boundary=0000 19 | method: POST 20 | uri: /?b10-04 21 | version: HTTP/1.0 22 | output: 23 | status: 200 24 | -------------------------------------------------------------------------------- /ftw/util/output/m18-cr-line.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '272' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m18 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m19-multiple-ct-headers.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m01-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/b10-02-multipart-param-name.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"UNION SELECT\"\r\ 12 | \r\n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smit@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n\r\n" 16 | headers: 17 | Content-Length: '281' 18 | Content-Type: multipart/form-data; boundary=0000 19 | method: POST 20 | uri: /?b10-02 21 | version: HTTP/1.0 22 | output: 23 | status: 200 24 | -------------------------------------------------------------------------------- /ftw/util/output/b10-01-multipart-preamble.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "UNION SELECT\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 12 | name\"\r\r\n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data;\ 13 | \ name=\"email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '286' 18 | Content-Type: multipart/form-data; boundary=0000 19 | method: POST 20 | uri: /?b10-01 21 | version: HTTP/1.0 22 | output: 23 | status: 200 24 | -------------------------------------------------------------------------------- /ftw/util/output/b10-05-multipart-epilogue.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\nUNION SELECT\r\n\r\n" 16 | headers: 17 | Content-Length: '287' 18 | Content-Type: multipart/form-data; boundary=0000 19 | method: POST 20 | uri: /?b10-05 21 | version: HTTP/1.0 22 | output: 23 | status: 200 24 | -------------------------------------------------------------------------------- /ftw/util/output/m00-01-normal.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m00-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/logchecker.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class LogChecker(): 5 | """ 6 | LogChecker is an abstract class that integrations with WAFs MUST implement. 7 | This class is used by the testrunner to test log lines against an expected 8 | regex 9 | """ 10 | __metaclass__ = abc.ABCMeta 11 | 12 | def __init__(self): 13 | self.start = None 14 | self.end = None 15 | 16 | def set_times(self, start, end): 17 | self.start = start 18 | self.end = end 19 | 20 | @abc.abstractmethod 21 | def get_logs(self): 22 | """ 23 | MUST be implemented, MUST return an array of strings 24 | These strings represent distinct log lines that were pulled from an 25 | outside logfile. The times are used by the testrunner to assist the 26 | implementers in pulling out the correct lines from the log file 27 | """ 28 | pass 29 | -------------------------------------------------------------------------------- /ftw/util/output/m03-02-multiple-boundaries.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\nContent-Disposition: form-data; name=\"name\"\r\n\r\nJohn\ 12 | \ Smith\r\n--0000\r\nContent-Disposition: form-data; name=\"email\"\r\n\r\ 13 | \njohn.smith@example.com\r\n--0000\r\nContent-Disposition: form-data; name=\"\ 14 | image\"; filename=\"image.jpg\"\r\nContent-Type: image/jpeg\r\n\r\nBINARYDATA\r\ 15 | \n--0000--\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary="0000"; boundary=1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m03-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m03-04-multiple-boundaries.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000;\r\nContent-Disposition: form-data; name=\"name\"\r\n\r\nJohn\ 12 | \ Smith\r\n--0000;\r\nContent-Disposition: form-data; name=\"email\"\r\n\ 13 | \r\njohn.smith@example.com\r\n--0000;\r\nContent-Disposition: form-data;\ 14 | \ name=\"image\"; filename=\"image.jpg\"\r\nContent-Type: image/jpeg\r\n\ 15 | \r\nBINARYDATA\r\n--0000;--\r\n" 16 | headers: 17 | Content-Length: '263' 18 | Content-Type: multipart/form-data; boundary=0000; boundary=1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m03-04 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m01-01-invalid-separator.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data, boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m01-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m01-02-invalid-separator.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m01-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-01-quoted-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary="0000" 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-06-partial-quote.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary="0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-06 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-07-partial-quote.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000" 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-07 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m02-01-invalid-parameter-name.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; bOundAry=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m02-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m10-boundary-case-sensitivity.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--abcd\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--abcd\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--abcd\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--abcd--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=aBcD 19 | Host: localhost 20 | method: POST 21 | uri: /?m10-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-01-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/evasion; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-02-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/mixed; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-04-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/digest; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-04 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-05-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: mulTIpart/FORM-dATA; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-05 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-06-disposition-folding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\n Content-Type:\ 15 | \ image/jpeg\r\r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '274' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-06 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m17-01-first-boundary-lf.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "\r\n--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\ 12 | \r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '274' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m17-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m02-02-invalid-parameter-name.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary123=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m02-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-08-whitespace-after-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m00-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-03-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/alternative; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-03 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-06-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-datax; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-06 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m11-07-invalid-multipart-type.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: application/octet-stream; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m11-07 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m12-02-disposition-name-no-quotes.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=name\r\r\n\r\r\nJohn\ 12 | \ Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\r\ 13 | \r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition: form-data;\ 14 | \ name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '271' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m12-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-01-disposition-folding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data;\r\r\n name=\"name\"\r\r\ 12 | \n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '275' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-03-disposition-folding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n \r\r\ 12 | \n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '276' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-03 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-04-disposition-folding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\";\r\r\n filename=\"image.jpg\"\r\r\nContent-Type:\ 15 | \ image/jpeg\r\r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '275' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-05-disposition-folding.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data;\r\r\nname=\"name\"\r\r\n\ 12 | \r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '274' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-05 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m17-02-first-boundary-crlf.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\ 12 | \n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '275' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m17-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /test/integration/test_cookie.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, testrunner, http, errors 2 | import pytest 3 | 4 | def test_default(ruleset, test, destaddr): 5 | """ 6 | Default tester with no logger obj. Useful for HTML contains and Status code 7 | Not useful for testing loggers 8 | """ 9 | runner = testrunner.TestRunner() 10 | try: 11 | last_ua = http.HttpUA() 12 | for stage in test.stages: 13 | if destaddr is not None: 14 | stage.input.dest_addr = destaddr 15 | if stage.input.save_cookie: 16 | runner.run_stage(stage, http_ua=last_ua) 17 | else: 18 | runner.run_stage(stage, logger_obj=None, http_ua=None) 19 | except errors.TestError as e: 20 | e.args[1]['meta'] = ruleset.meta 21 | pytest.fail('Failure! Message -> {0}, Context -> {1}' 22 | .format(e.args[0],e.args[1])) 23 | -------------------------------------------------------------------------------- /ftw/util/output/m04-whitespace-after-parameter-name.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary =0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m04-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m05-whitespace-before-parameter-value.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary= 0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m05-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m06-whitespace-after-parameter-value.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m06-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m07-01-special-chars-in-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000 1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m07-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m07-02-special-chars-in-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000,1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m07-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-02-whitespace-in-quoted-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=" 0000" 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-03-whitespace-in-quoted-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary="0000 " 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-03 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m12-03-disposition-name-single-quotes.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name='name'\r\r\n\r\r\n\ 12 | John Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m12-03 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m12-04-disposition-name-partial-quote.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\r\r\n\r\r\n\ 12 | John Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '272' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m12-04 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m12-05-disposition-name-partial-quote.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=name\"\r\r\n\r\r\n\ 12 | John Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '272' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m12-05 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m03-01-multiple-boundaries.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000; boundary=1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m03-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m03-03-multiple-boundaries.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '273' 18 | Content-Type: multipart/form-data; boundary=0000, boundary=1111 19 | Host: localhost 20 | method: POST 21 | uri: /?m03-03 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-04-quote-in-quoted-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--00\"00\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\ 12 | \r\nJohn Smith\r\r\n--00\"00\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--00\"00\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--00\"00--\r\r\n" 16 | headers: 17 | Content-Length: '277' 18 | Content-Type: multipart/form-data; boundary="00\"00" 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-04 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m08-05-quote-in-quoted-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--00\"00\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\ 12 | \r\nJohn Smith\r\r\n--00\"00\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--00\"00\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--00\"00--\r\r\n" 16 | headers: 17 | Content-Length: '277' 18 | Content-Type: multipart/form-data; boundary="00""00" 19 | Host: localhost 20 | method: POST 21 | uri: /?m08-05 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m13-02-disposition-folding-isspace.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data;\r\r\n\vname=\"name\"\r\r\ 12 | \n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 13 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '275' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m13-02 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m14-01-disposition-php-quoting.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=x';filename=\"';name=name;\"\ 12 | \r\r\n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data;\ 13 | \ name=\"email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '293' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m14-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /ftw/util/output/m12-01-disposition-multiple-param-names.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"XXXX\"; name=\"\ 12 | name\"\r\r\n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data;\ 13 | \ name=\"email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 16 | headers: 17 | Content-Length: '286' 18 | Content-Type: multipart/form-data; boundary=0000 19 | Host: localhost 20 | method: POST 21 | uri: /?m12-01 22 | version: HTTP/1.0 23 | output: 24 | status: 200 25 | -------------------------------------------------------------------------------- /test/yaml/EXAMPLE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "Zack" 4 | enabled: true 5 | name: "EXAMPLE.yaml" 6 | description: "Description" 7 | tests: 8 | - 9 | test_title: 1234 10 | stages: 11 | - 12 | stage: 13 | input: 14 | method: "GET" 15 | port: 80 16 | headers: 17 | User-Agent: "Foo" 18 | Host: "localhost" 19 | protocol: "http" 20 | uri: "/" 21 | output: 22 | status: 200 23 | - 24 | test_title: 1235 25 | stages: 26 | - 27 | stage: 28 | input: 29 | method: "GET" 30 | port: 80 31 | headers: 32 | User-Agent: "Foo" 33 | Host: "localhost" 34 | protocol: "http" 35 | uri: "/fail.html" 36 | output: 37 | status: 404 38 | -------------------------------------------------------------------------------- /ftw/util/output/m15-01-invalid-part.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n--0000\r\ 12 | \r\nContent-Disposition: form-data; name=\"name\"; filename=\"dummy\"\r\r\ 13 | \n\r\r\nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"\ 14 | email\"\r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 15 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 16 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\n" 17 | headers: 18 | Content-Length: '344' 19 | Content-Type: multipart/form-data; boundary=0000 20 | Host: localhost 21 | method: POST 22 | uri: /?m00-01 23 | version: HTTP/1.0 24 | output: 25 | status: 200 26 | -------------------------------------------------------------------------------- /ftw/util.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import glob 4 | import yaml 5 | 6 | 7 | def get_files(directory, extension): 8 | """ 9 | Take a directory and an extension and return the files 10 | that match the extension 11 | """ 12 | return glob.glob('%s/*.%s' % (directory, extension)) 13 | 14 | 15 | def extract_yaml(yaml_files): 16 | """ 17 | Take a list of yaml_files and load them to return back 18 | to the testing program 19 | """ 20 | loaded_yaml = [] 21 | for yaml_file in yaml_files: 22 | try: 23 | with open(yaml_file, 'r') as fd: 24 | loaded_yaml.append(yaml.safe_load(fd)) 25 | except IOError as e: 26 | print('Error reading file', yaml_file) 27 | raise e 28 | except yaml.YAMLError as e: 29 | print('Error parsing file', yaml_file) 30 | raise e 31 | except Exception as e: 32 | print('General error') 33 | raise e 34 | return loaded_yaml 35 | -------------------------------------------------------------------------------- /ftw/util/output/m09-data-after-last-boundary.yaml: -------------------------------------------------------------------------------- 1 | meta: 2 | author: Zack 3 | description: Description 4 | enabled: true 5 | name: EXAMPLE.yaml 6 | tests: 7 | - rule_id: 1234 8 | stages: 9 | - stage: 10 | input: 11 | data: "--0000\r\r\nContent-Disposition: form-data; name=\"name\"\r\r\n\r\r\ 12 | \nJohn Smith\r\r\n--0000\r\r\nContent-Disposition: form-data; name=\"email\"\ 13 | \r\r\n\r\r\njohn.smith@example.com\r\r\n--0000\r\r\nContent-Disposition:\ 14 | \ form-data; name=\"image\"; filename=\"image.jpg\"\r\r\nContent-Type: image/jpeg\r\ 15 | \r\n\r\r\nBINARYDATA\r\r\n--0000--\r\r\nContent-Disposition: form-data;\ 16 | \ name=\"name\"\r\r\n\r\r\nSmith John\r\r\n--0000--\r\r\nSome text here\r\ 17 | \n" 18 | headers: 19 | Content-Length: '357' 20 | Content-Type: multipart/form-data; boundary=0000 21 | Host: localhost 22 | method: POST 23 | uri: /?m09-01 24 | version: HTTP/1.0 25 | output: 26 | status: 200 27 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | How to contribute 2 | ============================================ 3 | 4 | 1. Open issues and tag them appropriately based on bug/feature request etc 5 | 2. Follow PEP-8 compliance for code review 6 | 3. `py.test` is used for testing, write integration/unit tests where necessary 7 | 4. +1 required by @csanders-git or @zmallen for merge 8 | 5. CI (eventually) required for merge 9 | 10 | FTW contributors (sorted alphabetically) 11 | ============================================ 12 | 13 | * **[Chaim Sanders](https://github.com/csanders-git)** 14 | 15 | * HTTP request/response parser 16 | * Rule Design 17 | 18 | * **[Ezekiel Templin](https://github.com/ezkl/)** 19 | 20 | * Rule design 21 | * Documentation 22 | 23 | * **[Jared Stroud](https://github.com/jaredestroud/)** 24 | * HTTP Library 25 | 26 | * **[Zack Allen](https://github.com/zmallen/)** 27 | 28 | * Rule design 29 | * Testing format 30 | * Framework refactoring 31 | * Error handling 32 | 33 | * **[Christian Peron](https://github.com/csjperon)** 34 | 35 | * HTTP Library 36 | * Test environment 37 | -------------------------------------------------------------------------------- /test/integration/test_nologcontains.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, logchecker, testrunner 2 | import pytest 3 | import random 4 | import threading 5 | 6 | class LoggerTestObj(logchecker.LogChecker): 7 | def __init__(self): 8 | self.do_nothing = False 9 | 10 | def generate_random_logs(self): 11 | if self.do_nothing: 12 | return [] 13 | else: 14 | return [str(self.start) + ' rule-id-' + str(random.randint(10,99))] 15 | 16 | def get_logs(self): 17 | logs = self.generate_random_logs() 18 | return logs 19 | 20 | @pytest.fixture 21 | def logchecker_obj(): 22 | """ 23 | Returns a LoggerTest Integration object 24 | """ 25 | return LoggerTestObj() 26 | 27 | def test_logcontains_withlog(logchecker_obj, ruleset, test): 28 | runner = testrunner.TestRunner() 29 | for stage in test.stages: 30 | runner.run_stage(stage, logchecker_obj) 31 | 32 | def test_logcontains_nolog(logchecker_obj, ruleset, test): 33 | logchecker_obj.do_nothing = True 34 | runner = testrunner.TestRunner() 35 | for stage in test.stages: 36 | runner.run_stage(stage, logchecker_obj) 37 | -------------------------------------------------------------------------------- /test/integration/test_logcontains.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, logchecker, testrunner 2 | import pytest 3 | import random 4 | import threading 5 | 6 | class LoggerTestObj(logchecker.LogChecker): 7 | def __init__(self): 8 | self.do_nothing = False 9 | 10 | def generate_random_logs(self): 11 | if self.do_nothing: 12 | return [] 13 | else: 14 | return [str(self.start) + ' rule-id-' + str(random.randint(10,99))] 15 | 16 | def get_logs(self): 17 | logs = self.generate_random_logs() 18 | return logs 19 | 20 | @pytest.fixture 21 | def logchecker_obj(): 22 | """ 23 | Returns a LoggerTest Integration object 24 | """ 25 | return LoggerTestObj() 26 | 27 | def test_logcontains_withlog(logchecker_obj, ruleset, test): 28 | runner = testrunner.TestRunner() 29 | for stage in test.stages: 30 | runner.run_stage(stage, logchecker_obj) 31 | 32 | def test_logcontains_nolog(logchecker_obj, ruleset, test): 33 | logchecker_obj.do_nothing = True 34 | runner = testrunner.TestRunner() 35 | with(pytest.raises(AssertionError)): 36 | for stage in test.stages: 37 | runner.run_stage(stage, logchecker_obj) 38 | -------------------------------------------------------------------------------- /test/integration/HTMLCONTAINSFIXTURE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "Chaim" 4 | enabled: true 5 | name: "HTMLCONTAINSFIXTURE.yaml" 6 | description: "Tests html contains functionality with an integration object" 7 | tests: 8 | - 9 | test_title: "response_contains(1)" 10 | stages: 11 | - 12 | stage: 13 | input: 14 | dest_addr: "example.com" 15 | method: "GET" 16 | port: 80 17 | headers: 18 | User-Agent: "Foo" 19 | Host: "example.com" 20 | protocol: "http" 21 | uri: "/" 22 | output: 23 | status: 200 24 | response_contains: "established to be used for" 25 | - 26 | test_title: "response_contains(2)" 27 | stages: 28 | - 29 | stage: 30 | input: 31 | dest_addr: "example.com" 32 | method: "GET" 33 | port: 80 34 | headers: 35 | User-Agent: "Foo" 36 | Host: "example.com" 37 | protocol: "http" 38 | uri: "/" 39 | output: 40 | status: 200 41 | response_contains: "Content-Type: text/html" 42 | -------------------------------------------------------------------------------- /test/unit/test_ruleset.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, errors 2 | import pytest 3 | 4 | def test_output(): 5 | with pytest.raises(errors.TestError) as excinfo: 6 | output = ruleset.Output({}) 7 | assert(excinfo.value.args[0].startswith('Need at least')) 8 | with pytest.raises(ValueError) as excinfo: 9 | output = ruleset.Output({'status': 'derp'}) 10 | assert(excinfo.value.message.startswith('invalid literal')) 11 | 12 | with pytest.raises(TypeError) as excinfo: 13 | output = ruleset.Output({'log_contains': 10}) 14 | 15 | def test_input(): 16 | input_1 = ruleset.Input() 17 | assert(input_1.uri == '/') 18 | headers = {'Host': 'domain.com', 'User-Agent': 'Zack'} 19 | dictionary = {} 20 | dictionary['headers'] = headers 21 | input_2 = ruleset.Input(**dictionary) 22 | assert(len(input_2.headers.keys()) == 2) 23 | dictionary_2 = {'random_key': 'bar'} 24 | with pytest.raises(TypeError) as excinfo: 25 | input_3 = ruleset.Input(**dictionary_2) 26 | 27 | def test_testobj(): 28 | with pytest.raises(KeyError) as excinfo: 29 | test = ruleset.Test({},{}) 30 | assert(excinfo.value.message == 'test_title') 31 | stages_dict = {'test_title': 1, 'stages':[{'stage': {'output':{'log_contains':'foo'}, 'input': {}}}]} 32 | test = ruleset.Test(stages_dict, {}) 33 | 34 | def test_ruleset(): 35 | with pytest.raises(KeyError) as excinfo: 36 | ruleset_1 = ruleset.Ruleset({}) 37 | -------------------------------------------------------------------------------- /test/integration/MULTIPART.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "csanders-git" 4 | enabled: true 5 | name: "MULTIPART.yaml" 6 | description: "This is meant to test the data parameter with its list format on a multipart request" 7 | tests: 8 | - 9 | test_title: "Data(list)" 10 | stages: 11 | - 12 | stage: 13 | input: 14 | dest_addr: "www.example.com" 15 | method: "POST" 16 | port: 80 17 | headers: 18 | User-Agent: "ModSecurity CRS 3 Tests" 19 | Host: "example.com" 20 | Accept: "*/*" 21 | Accept-Language: "en" 22 | Connection: "close" 23 | Referer: "http://localhost/" 24 | Content-Type: "multipart/form-data; boundary=--------397236876" 25 | data: 26 | - "----------397236876" 27 | - "Content-Disposition: form-data; name=\"text\";" 28 | - "" 29 | - "test default" 30 | - "----------397236876" 31 | - "Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\"" 32 | - "Content-Type: text/plain" 33 | - "" 34 | - "Content of a.txt." 35 | - "" 36 | - "----------397236876--" 37 | protocol: "http" 38 | output: 39 | status: 200 40 | -------------------------------------------------------------------------------- /test/integration/test_htmlcontains.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, testrunner, http, errors 2 | import pytest 3 | import re 4 | import random 5 | import threading 6 | 7 | def test_logcontains(ruleset, test): 8 | runner = testrunner.TestRunner() 9 | for stage in test.stages: 10 | runner.run_stage(stage) 11 | 12 | # Should return a test error because its searching before response 13 | def test_search1(): 14 | runner = testrunner.TestRunner() 15 | x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) 16 | http_ua = http.HttpUA() 17 | with pytest.raises(errors.TestError): 18 | runner.test_response(http_ua.response_object,re.compile('dog')) 19 | 20 | # Should return a failure because it is searching for a word not there 21 | def test_search2(): 22 | runner = testrunner.TestRunner() 23 | x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) 24 | http_ua = http.HttpUA() 25 | http_ua.send_request(x) 26 | with pytest.raises(AssertionError): 27 | runner.test_response(http_ua.response_object,re.compile('dog')) 28 | 29 | # Should return a success because it is searching for a word not there 30 | def test_search3(): 31 | runner = testrunner.TestRunner() 32 | x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) 33 | http_ua = http.HttpUA() 34 | http_ua.send_request(x) 35 | runner.test_response(http_ua.response_object,re.compile('established to be used for')) 36 | 37 | # Should return a success because we found our regex 38 | def test_search4(): 39 | runner = testrunner.TestRunner() 40 | x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) 41 | http_ua = http.HttpUA() 42 | http_ua.send_request(x) 43 | runner.test_response(http_ua.response_object,re.compile('.*')) 44 | 45 | -------------------------------------------------------------------------------- /test/integration/COOKIEFIXTURE.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | meta: 3 | author: "Chaim" 4 | enabled: true 5 | name: "COOKIEFIXTURE.yaml" 6 | description: "Tests cookie saving functionality" 7 | tests: 8 | - 9 | test_title: "Multi-Stage w\\ Cookie" 10 | stages: 11 | - 12 | stage: 13 | input: 14 | save_cookie: true 15 | dest_addr: "ieee.org" 16 | method: "GET" 17 | port: 80 18 | headers: 19 | User-Agent: "Foo" 20 | Host: "ieee.org" 21 | protocol: "http" 22 | uri: "/" 23 | output: 24 | status: 302 25 | html_contains: "Set-Cookie: TS01293935=" 26 | stage: 27 | input: 28 | save_cookie: true 29 | dest_addr: "ieee.org" 30 | method: "GET" 31 | port: 80 32 | headers: 33 | User-Agent: "Foo" 34 | Host: "ieee.org" 35 | protocol: "http" 36 | uri: "/" 37 | output: 38 | status: 302 39 | html_contains: "Set-Cookie: TS01293935=" 40 | - 41 | test_title: "Multi-Stage w\\ Cookie rule disabled" 42 | enabled: false 43 | stages: 44 | - 45 | stage: 46 | input: 47 | save_cookie: true 48 | dest_addr: "ieee.org" 49 | method: "GET" 50 | port: 80 51 | headers: 52 | User-Agent: "Foo" 53 | Host: "ieee.org" 54 | protocol: "http" 55 | uri: "/" 56 | output: 57 | status: 302 58 | html_contains: "Set-Cookie: TS01293935=" 59 | stage: 60 | input: 61 | save_cookie: true 62 | dest_addr: "ieee.org" 63 | method: "GET" 64 | port: 80 65 | headers: 66 | User-Agent: "Foo" 67 | Host: "ieee.org" 68 | protocol: "http" 69 | uri: "/" 70 | output: 71 | status: 302 72 | html_contains: "Set-Cookie: TS01293935=" 73 | -------------------------------------------------------------------------------- /ftw/util/request_to_yaml.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | class Request(): 4 | def __init__(self): 5 | self.input = {} 6 | self.output = {} 7 | self.data_start = 0 8 | def double_quote(self,mystr): 9 | return mystr 10 | return "\"" + str(mystr) + "\"" 11 | def generate_yaml(self): 12 | data = dict( 13 | meta = dict( 14 | author = "Zack", 15 | enabled = True, 16 | name = "EXAMPLE.yaml", 17 | description = "Description" 18 | ), 19 | tests = [dict( 20 | rule_id = 1234, 21 | stages = [dict( 22 | stage = dict( 23 | input = self.input, 24 | output = dict( 25 | status = 200 26 | ) 27 | ) 28 | )] 29 | )] 30 | ) 31 | return yaml.dump(data, default_flow_style=False) 32 | def get_request_line(self, req): 33 | req = req.split('\r\n')[0] 34 | req = req.split(' ',2) 35 | self.input["method"] = self.double_quote(req[0]) 36 | self.input["uri"] = self.double_quote(req[1]) 37 | self.input["version"] = self.double_quote(req[2]) 38 | 39 | def get_headers(self, req): 40 | req = req.split('\r\n')[1:] 41 | header = {} 42 | for num in range(len(req)): 43 | if req[num] == '': 44 | self.data_start = num 45 | break 46 | head = req[num].split(':') 47 | header[head[0]] = head[1].strip() 48 | self.input["headers"] = self.double_quote(header) 49 | def get_data(self, req): 50 | req = req.split('\r\n')[1:] 51 | self.input["data"] = self.double_quote("\r\n".join(req[self.data_start+1:])) 52 | 53 | def write_yaml(self,fname, yaml_out): 54 | f = open(fname,'w') 55 | f.write(yaml_out) 56 | 57 | # Example Usage 58 | #req = Request() 59 | # 60 | #request = """GET / HTTP/1.1 61 | #User-Agent: test:/data 62 | # 63 | #xyz 64 | # 65 | #""" 66 | #request = request.replace('\n','\r\n') 67 | #req.get_request_line(request) 68 | #req.get_headers(request) 69 | #req.get_data(request) 70 | #yaml_out = req.generate_yaml() 71 | #write_yaml('out.yaml', yaml_out) 72 | -------------------------------------------------------------------------------- /ftw/testrunner.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import errors 3 | import http 4 | import pytest 5 | import ruleset 6 | import util 7 | import re 8 | 9 | class TestRunner(object): 10 | """ 11 | Runner that accepts stages of a test and verifies expected and actual 12 | responses 13 | @TODO 14 | Accept logger objects for assertions 15 | """ 16 | def test_status(self, expected_status, actual_status): 17 | """ 18 | Compares the expected output against actual output of test and stage 19 | In a separate function to make debugging easy with py.test 20 | """ 21 | assert expected_status == actual_status 22 | 23 | def test_log(self, lines, log_contains, negate): 24 | """ 25 | Checks if a series of log lines contains a regex specified in the 26 | output stage. It will flag true on the first log_contains regex match 27 | and then assert on the flag at the end of the function 28 | """ 29 | found = False 30 | for line in lines: 31 | if log_contains.search(line): 32 | found = True 33 | break 34 | if negate: 35 | assert not found 36 | else: 37 | assert found 38 | 39 | def test_response(self, response_object, regex): 40 | """ 41 | Checks if the response response contains a regex specified in the 42 | output stage. It will assert that the regex is present. 43 | """ 44 | if response_object is None: 45 | raise errors.TestError( 46 | 'Searching before response received', 47 | { 48 | 'regex': regex, 49 | 'response_object': response_object, 50 | 'function': 'testrunner.TestRunner.test_response' 51 | }) 52 | if regex.search(response_object.response): 53 | assert True 54 | else: 55 | assert False 56 | 57 | def run_stage(self, stage, logger_obj=None, http_ua=None): 58 | """ 59 | Runs a stage in a test by building an httpua object with the stage 60 | input, waits for output then compares expected vs actual output 61 | http_ua can be passed in to persist cookies 62 | """ 63 | 64 | # Send our request (exceptions caught as needed) 65 | if stage.output.expect_error: 66 | with pytest.raises(errors.TestError) as excinfo: 67 | if not http_ua: 68 | http_ua = http.HttpUA() 69 | start = datetime.datetime.now() 70 | http_ua.send_request(stage.input) 71 | end = datetime.datetime.now() 72 | print '\nExpected Error: %s' % str(excinfo) 73 | else: 74 | if not http_ua: 75 | http_ua = http.HttpUA() 76 | start = datetime.datetime.now() 77 | http_ua.send_request(stage.input) 78 | end = datetime.datetime.now() 79 | if (stage.output.log_contains_str or stage.output.no_log_contains_str) \ 80 | and logger_obj is not None: 81 | logger_obj.set_times(start, end) 82 | lines = logger_obj.get_logs() 83 | if stage.output.log_contains_str: 84 | self.test_log(lines, stage.output.log_contains_str, False) 85 | if stage.output.no_log_contains_str: 86 | # The last argument means that we should negate the resp 87 | self.test_log(lines, stage.output.no_log_contains_str, True) 88 | if stage.output.response_contains_str: 89 | self.test_response(http_ua.response_object, 90 | stage.output.response_contains_str) 91 | if stage.output.status: 92 | self.test_status(stage.output.status, 93 | http_ua.response_object.status) 94 | -------------------------------------------------------------------------------- /ftw/pytest_plugin.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import ruleset 3 | import util 4 | import os 5 | 6 | def get_rulesets(ruledir, recurse): 7 | """ 8 | List of ruleset objects extracted from the yaml directory 9 | """ 10 | if os.path.isdir(ruledir) and recurse: 11 | yaml_files = [] 12 | for root, dirs, files in os.walk(ruledir): 13 | for name in files: 14 | filename, file_extension = os.path.splitext(name) 15 | if file_extension == '.yaml': 16 | yaml_files.append(os.path.join(root, name)) 17 | if os.path.isdir(ruledir) and not recurse: 18 | yaml_files = util.get_files(ruledir, 'yaml') 19 | elif os.path.isfile(ruledir): 20 | yaml_files = [ruledir] 21 | extracted_files = util.extract_yaml(yaml_files) 22 | rulesets = [] 23 | for extracted_yaml in extracted_files: 24 | rulesets.append(ruleset.Ruleset(extracted_yaml)) 25 | return rulesets 26 | 27 | def get_testdata(rulesets): 28 | """ 29 | In order to do test-level parametrization (is this a word?), we have to 30 | bundle the test data from rulesets into tuples so py.test can understand 31 | how to run tests across the whole suite of rulesets 32 | """ 33 | testdata = [] 34 | for ruleset in rulesets: 35 | for test in ruleset.tests: 36 | if test.enabled: 37 | testdata.append((ruleset, test)) 38 | 39 | return testdata 40 | 41 | def test_id(val): 42 | """ 43 | Dynamically names tests, useful for when we are running dozens to hundreds 44 | of tests 45 | """ 46 | if isinstance(val, (dict,ruleset.Test,)): 47 | # We must be carful here because errors are swallowed and defaults returned 48 | if 'name' in val.ruleset_meta.keys(): 49 | return '%s -- %s' % (val.ruleset_meta['name'], val.test_title) 50 | else: 51 | return '%s -- %s' % ("Unnamed_Test", val.test_title) 52 | 53 | 54 | @pytest.fixture 55 | def destaddr(request): 56 | """ 57 | Destination address override for tests 58 | """ 59 | return request.config.getoption('--destaddr') 60 | 61 | @pytest.fixture 62 | def http_serv_obj(): 63 | """ 64 | Return an HTTP object listening on localhost port 80 for testing 65 | """ 66 | return HTTPServer(('localhost', 80), SimpleHTTPRequestHandler) 67 | 68 | def pytest_addoption(parser): 69 | """ 70 | Adds command line options to py.test 71 | """ 72 | parser.addoption('--ruledir', action='store', default=None, 73 | help='rule directory that holds YAML files for testing') 74 | parser.addoption('--destaddr', action='store', default=None, 75 | help='destination address to direct tests towards') 76 | parser.addoption('--rule', action='store', default=None, 77 | help='fully qualified path to one rule') 78 | parser.addoption('--ruledir_recurse', action='store', default=None, 79 | help='walk the directory structure finding YAML files') 80 | 81 | def pytest_generate_tests(metafunc): 82 | """ 83 | Pre-test configurations, mostly used for parametrization 84 | """ 85 | options = ['ruledir','ruledir_recurse','rule'] 86 | args = metafunc.config.option.__dict__ 87 | # Check if we have any arguments by creating a list of supplied args we want 88 | if [i for i in options if i in args and args[i] != None] : 89 | if metafunc.config.option.ruledir: 90 | rulesets = get_rulesets(metafunc.config.option.ruledir, False) 91 | if metafunc.config.option.ruledir_recurse: 92 | rulesets = get_rulesets(metafunc.config.option.ruledir_recurse, True) 93 | if metafunc.config.option.rule: 94 | rulesets = get_rulesets(metafunc.config.option.rule, False) 95 | if 'ruleset' in metafunc.fixturenames and 'test' in metafunc.fixturenames: 96 | metafunc.parametrize('ruleset,test', get_testdata(rulesets), 97 | ids=test_id) 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Framework for Testing WAFs (FTW) 2 | [![Build Status](https://travis-ci.org/fastly/ftw.svg?branch=master)](https://travis-ci.org/fastly/ftw) 3 | 4 | ##### Purpose 5 | This project was created by researchers from ModSecurity and Fastly to help provide rigorous tests for WAF rules. It uses the OWASP Core Ruleset V3 as a baseline to test rules on a WAF. Each rule from the ruleset is loaded into a YAML file that issues HTTP requests that will trigger these rules. 6 | 7 | Goals / Use cases include: 8 | 9 | * Find regressions in WAF deployments by using continuous integration and issuing repeatable attacks to a WAF 10 | * Provide a testing framework for new rules into ModSecurity, if a rule is submitted it MUST have corresponding positive & negative tests 11 | * Evaluate WAFs against a common, agreeable baseline ruleset (OWASP) 12 | * Test and verify custom rules for WAFs that are not part of the core rule set 13 | 14 | ## Installation 15 | * `git clone git@github.com:fastly/ftw.git` 16 | * `cd ftw` 17 | * Make sure that pip is installed `apt-get install python-pip` 18 | * `pip install -r requirements.txt` 19 | 20 | ## Running Tests with HTML contains and Status code checks only 21 | * Create YAML files that point to your webserver with a WAF in front of it 22 | * `py.test test/test_default.py --ruledir test/yaml` 23 | 24 | ## Provisioning Apache+Modsecurity+OWASP CRS 25 | If you require an environment for testing WAF rules, there has been one created with Apache, Modsecurity and version 3.0.0 of the OWASP core ruleset. This can be deployed by: 26 | 27 | * Checking out the repository: ``git clone https://github.com/fastly/waf_testbed.git``` 28 | * Typeing ```vagrant up``` 29 | 30 | ## Running Tests while overriding destination address in the yaml files to custom domain 31 | * *start your test web server* 32 | * `py.test test/test_default.py --ruledir=test/yaml --destaddr=domain.com` 33 | 34 | ## Run integration test, local webserver, may have to use sudo 35 | * `py.test test/integration/test_logcontains.py -s --ruledir=test/integration/` 36 | 37 | ## HOW TO INTEGRATE LOGS 38 | 1. Create a `*.py` file with the necessary imports, an example is shown in `test/integration/test_logcontains.py` 39 | 2. All functions with `test*` in the beginning will be ran by `py.test`, so make a function `def test_somewaf` 40 | 3. Implement a class that inherits `LogChecker` 41 | 1. Implement the `get_logs()` function. FTW will call this function after it runs the test, and it will set datetimes of `self.start` and `self.end` 42 | 2. Use the information from the datetime variables to retrieve the files from your WAF, whether its a file or an API call 43 | 3. Get the logs, store them in an array of strings and return it from `get_logs()` 44 | 4. Make use of `py.test fixtures`. Use a function decorator `@pytest.fixture`, return your new `LogChecker` object. Whenever you use a function argument in your tests that matches the name of that `@pytest.fixture`, it will instantiate your object and make it easier to run tests. An example of this is in the python file from step 1. 45 | 5. Write a testing configuration in the `*.yaml` format as seen in `test/integration/LOGCONTAINSFIXTURE.yaml`, the `log_contains` line requires a string that is a regex. FTW will compile the `log_contains` string from each stage in the YAML file into a regex. This regex will then be used alongside the lines of logs passed in from `get_logs()` to look for a match. The `log_contains` string, then, should be a unique rule-id as FTW is greedy and will pass on the first match. False positives are mitigated from the start/end time passed to the `LogChecker` object, but it is best to stay safe and use unique regexes. 46 | 6. For each stage, the `get_logs()` function is called, so be sure to account for API calls if thats how you retrieve your logs. 47 | 48 | ## Making HTTP requests programmatically 49 | Although it is preferred to make requests using the YAML format, often automated tests require making many dynamic requests. In such a case it is recommended to make use of the py.test framework in order to produce test cases that can be run as part of the whole. 50 | Generally making an HTTP request is simple: 51 | 1. create an instance of the `HttpUA()` class 52 | 2. create an instance of the `Input()` class providing whatever parameters you don\'t want to be defaulted 53 | 3. provide the instance of the input class to `HttpUA.send_request()` 54 | 55 | *For some examples see the http integration tests* 56 | 57 | -------------------------------------------------------------------------------- /ftw/ruleset.py: -------------------------------------------------------------------------------- 1 | import re 2 | import errors 3 | import urllib 4 | import urlparse 5 | 6 | 7 | class Output(object): 8 | """ 9 | This class holds the expected output from a corresponding FTW HTTP Input 10 | We are stricter in this definition by requiring at least one of status, 11 | response_contains, no_log_contains, expect_error, or log_contains 12 | """ 13 | def __init__(self, output_dict): 14 | self.STATUS = 'status' 15 | self.LOG = 'log_contains' 16 | self.NOTLOG = 'no_log_contains' 17 | self.RESPONSE = 'response_contains' 18 | self.ERROR = 'expect_error' 19 | if output_dict is None: 20 | raise errors.TestError( 21 | 'No output dictionary found', 22 | { 23 | 'function': 'ruleset.Output.__init__' 24 | } 25 | ) 26 | self.output_dict = output_dict 27 | self.status = int(output_dict[self.STATUS]) \ 28 | if self.STATUS in self.output_dict else None 29 | self.response_contains_str = self.process_regex(self.RESPONSE) 30 | self.no_log_contains_str = self.process_regex(self.NOTLOG) 31 | self.log_contains_str = self.process_regex(self.LOG) 32 | self.expect_error = bool(self.output_dict[self.ERROR]) if \ 33 | self.ERROR in self.output_dict and \ 34 | self.output_dict[self.ERROR] else None 35 | if self.status is None and self.response_contains_str is None \ 36 | and self.log_contains_str is None \ 37 | and self.no_log_contains_str is None \ 38 | and self.expect_error is None: 39 | raise errors.TestError( 40 | 'Need at least one status, response_contains ' + 41 | ', no_log_contains, or log_contains', 42 | { 43 | 'status': self.status, 44 | 'response_contains value': self.response_contains_str, 45 | 'log_contains value': self.log_contains_str, 46 | 'no_log_contains value': self.no_log_contains_str, 47 | 'expect_error value': self.expect_error, 48 | 'function': 'ruleset.Output.__init__' 49 | }) 50 | 51 | def process_regex(self, key): 52 | """ 53 | Extract the value of key from dictionary if available 54 | and process it as a python regex 55 | """ 56 | return re.compile(self.output_dict[key]) if \ 57 | key in self.output_dict else None 58 | 59 | 60 | class Input(object): 61 | """ 62 | This class holds the data associated with an HTTP Input request in FTW 63 | """ 64 | def __init__(self, raw_request=None, 65 | encoded_request=None, 66 | protocol='http', 67 | dest_addr='localhost', 68 | port=80, 69 | method='GET', 70 | uri='/', 71 | version='HTTP/1.1', 72 | headers={}, 73 | data='', 74 | save_cookie=False, 75 | stop_magic=False 76 | ): 77 | self.raw_request = raw_request 78 | self.encoded_request = encoded_request 79 | self.protocol = protocol 80 | self.dest_addr = dest_addr 81 | self.port = port 82 | self.method = method 83 | self.uri = uri 84 | self.version = version 85 | self.headers = headers 86 | self.data = data 87 | # Support data in list format and join on CRLF 88 | if isinstance(self.data, list): 89 | self.data = '\r\n'.join(self.data) 90 | self.save_cookie = save_cookie 91 | self.stop_magic = stop_magic 92 | # Check if there is any data and do defaults 93 | if self.data != '': 94 | # Default values for content length and header 95 | if 'Content-Type' not in headers.keys() and stop_magic is False: 96 | headers['Content-Type'] = 'application/x-www-form-urlencoded' 97 | # check if encoded and encode if it should be 98 | if 'Content-Type' in headers.keys(): 99 | if headers['Content-Type'] == 'application/x-www-form-urlencoded' and stop_magic is False: 100 | if urllib.unquote(self.data).decode('utf8') == self.data: 101 | query_string = urlparse.parse_qsl(self.data) 102 | if len(query_string) != 0: 103 | encoded_args = urllib.urlencode(query_string) 104 | self.data = encoded_args 105 | if 'Content-Length' not in headers.keys() and stop_magic is False: 106 | # The two is for the trailing CRLF and the one after 107 | headers['Content-Length'] = len(self.data) 108 | 109 | 110 | class Stage(object): 111 | """ 112 | This class holds information about 1 stage in a test, which contains 113 | 1 input and 1 output 114 | """ 115 | def __init__(self, stage_dict): 116 | self.stage_dict = stage_dict 117 | self.input = Input(**stage_dict['input']) 118 | self.output = Output(stage_dict['output']) 119 | 120 | 121 | class Test(object): 122 | """ 123 | This class holds information for 1 test and potentially many stages 124 | """ 125 | def __init__(self, test_dict, ruleset_meta): 126 | self.test_dict = test_dict 127 | self.ruleset_meta = ruleset_meta 128 | self.test_title = self.test_dict['test_title'] 129 | self.stages = self.build_stages() 130 | self.enabled = True 131 | if 'enabled' in self.test_dict: 132 | self.enabled = self.test_dict['enabled'] 133 | 134 | def build_stages(self): 135 | """ 136 | Processes and loads an array of stages from the test dictionary 137 | """ 138 | return map( 139 | lambda stage_dict: Stage(stage_dict['stage']), 140 | self.test_dict['stages'] 141 | ) 142 | 143 | 144 | class Ruleset(object): 145 | """ 146 | This class holds test and stage information from a YAML test file 147 | These YAML files are used to test the OWASP/Modsec CRSv3 rules 148 | """ 149 | def __init__(self, yaml_file): 150 | self.yaml_file = yaml_file 151 | self.meta = yaml_file['meta'] 152 | self.author = self.meta['author'] 153 | self.description = self.meta['description'] 154 | self.enabled = self.meta['enabled'] 155 | self.tests = self.extract_tests() if self.enabled else [] 156 | 157 | def extract_tests(self): 158 | """ 159 | Processes a loaded YAML document and 160 | creates test objects based on input 161 | """ 162 | try: 163 | return map( 164 | lambda test_dict: Test(test_dict, self.meta), 165 | self.yaml_file['tests'] 166 | ) 167 | except errors.TestError as e: 168 | e.args[1]['meta'] = self.meta 169 | raise e 170 | except Exception as e: 171 | raise Exception( 172 | 'Caught error. Message: %s on test with metadata: %s' 173 | % (str(e), str(self.meta)) 174 | ) 175 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | Copyright 2016 Chaim Sanders 181 | 182 | Licensed under the Apache License, Version 2.0 (the "License"); 183 | you may not use this file except in compliance with the License. 184 | You may obtain a copy of the License at 185 | 186 | http://www.apache.org/licenses/LICENSE-2.0 187 | 188 | Unless required by applicable law or agreed to in writing, software 189 | distributed under the License is distributed on an "AS IS" BASIS, 190 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 191 | See the License for the specific language governing permissions and 192 | limitations under the License. 193 | -------------------------------------------------------------------------------- /test/integration/test_http.py: -------------------------------------------------------------------------------- 1 | from ftw import ruleset, http, errors 2 | import pytest 3 | import sys 4 | 5 | def test_cookies1(): 6 | """Tests accessing a site that sets a cookie and then wants to resend the cookie""" 7 | http_ua = http.HttpUA() 8 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org"}) 9 | http_ua.send_request(x) 10 | with pytest.raises(KeyError): 11 | print http_ua.request_object.headers["cookie"] 12 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org"}) 13 | http_ua.send_request(x) 14 | assert(http_ua.request_object.headers["cookie"].split('=')[0] == "TS01293935") 15 | 16 | def test_cookies2(): 17 | """Test to make sure that we don't override user specified cookies""" 18 | http_ua = http.HttpUA() 19 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org"}) 20 | http_ua.send_request(x) 21 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org","cookie":"TS01293935=012f3506234413e6c5cb14e8c0d5bf890fdd02481614b01cd6cd30911c6733e3e6f79e72aa"}) 22 | http_ua.send_request(x) 23 | assert([chunk for chunk in http_ua.request_object.headers["cookie"].split(';')] == ['TS01293935=012f3506234413e6c5cb14e8c0d5bf890fdd02481614b01cd6cd30911c6733e3e6f79e72aa']) 24 | 25 | def test_cookies3(): 26 | """Test to make sure we retain cookies when user specified values are provided""" 27 | http_ua = http.HttpUA() 28 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org"}) 29 | http_ua.send_request(x) 30 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org","cookie":"TS01293935=012f3506234413e6c5cb14e8c0d5bf890fdd02481614b01cd6cd30911c6733e3e6f79e72aa; XYZ=123"}) 31 | http_ua.send_request(x) 32 | assert([chunk.split('=')[0].strip() for chunk in http_ua.request_object.headers["cookie"].split(';')] == ['XYZ', 'TS01293935']) 33 | 34 | def test_cookies4(): 35 | """Test to make sure cookies are saved when user-specified cookie is added""" 36 | http_ua = http.HttpUA() 37 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org"}) 38 | http_ua.send_request(x) 39 | x = ruleset.Input(dest_addr="ieee.org",headers={"Host":"ieee.org","cookie":"XYZ=123"}) 40 | http_ua.send_request(x) 41 | assert([chunk.split('=')[0].strip() for chunk in http_ua.request_object.headers["cookie"].split(';')] == ['XYZ', 'TS01293935']) 42 | 43 | 44 | def test_raw1(): 45 | """Test to make sure a raw request will work with \r\n replacment""" 46 | x = ruleset.Input(dest_addr="example.com",raw_request="""GET / HTTP/1.1\r\nHost: example.com\r\n\r\n""") 47 | http_ua = http.HttpUA() 48 | http_ua.send_request(x) 49 | assert http_ua.response_object.status == 200 50 | 51 | def test_raw2(): 52 | """Test to make sure a raw request will work with actual seperators""" 53 | x = ruleset.Input(dest_addr="example.com",raw_request="""GET / HTTP/1.1 54 | Host: example.com 55 | 56 | 57 | """) 58 | http_ua = http.HttpUA() 59 | http_ua.send_request(x) 60 | assert http_ua.response_object.status == 200 61 | 62 | def test_both1(): 63 | """Test to make sure that if both encoded and raw are provided there is an error""" 64 | x = ruleset.Input(dest_addr="example.com", raw_request="""GET / HTTP/1.1\r\nHost: example.com\r\n\r\n""", encoded_request="abc123==") 65 | http_ua = http.HttpUA() 66 | with pytest.raises(errors.TestError): 67 | http_ua.send_request(x) 68 | 69 | def test_encoded1(): 70 | """Test to make sure a encode request works""" 71 | x = ruleset.Input(dest_addr="example.com", encoded_request="R0VUIC8gSFRUUC8xLjFcclxuSG9zdDogZXhhbXBsZS5jb21cclxuXHJcbg==") 72 | http_ua = http.HttpUA() 73 | http_ua.send_request(x) 74 | assert http_ua.response_object.status == 200 75 | 76 | def test_error1(): 77 | """Will return mail -- not header should cause error""" 78 | x = ruleset.Input(dest_addr="Smtp.aol.com",port=25,headers={"Host":"example.com"}) 79 | http_ua = http.HttpUA() 80 | with pytest.raises(errors.TestError): 81 | http_ua.send_request(x) 82 | 83 | def test_error5(): 84 | """Invalid Header should cause error""" 85 | http_ua = http.HttpUA() 86 | with pytest.raises(errors.TestError): 87 | response = http.HttpResponse("HTTP/1.1 200 OK\r\ntest\r\n", http_ua) 88 | 89 | 90 | def test_error6(): 91 | """Valid HTTP response should process fine""" 92 | http_ua = http.HttpUA() 93 | response = http.HttpResponse("HTTP/1.1 200 OK\r\ntest: hello\r\n", http_ua) 94 | 95 | def test_error7(): 96 | """Invalid content-type should fail""" 97 | http_ua = http.HttpUA() 98 | with pytest.raises(errors.TestError): 99 | response = http.HttpResponse("HTTP/1.1 200 OK\r\nContent-Encoding: XYZ\r\n", http_ua) 100 | 101 | def test_error2(): 102 | """Invalid request should cause timeout""" 103 | x = ruleset.Input(dest_addr="example.com",port=123,headers={"Host":"example.com"}) 104 | http_ua = http.HttpUA() 105 | with pytest.raises(errors.TestError): 106 | http_ua.send_request(x) 107 | 108 | def test_error3(): 109 | """Invalid status returned in response line""" 110 | http_ua = http.HttpUA() 111 | with pytest.raises(errors.TestError): 112 | response = http.HttpResponse("HTTP1.1 test OK\r\n", http_ua) 113 | 114 | def test_error4(): 115 | """Wrong number of elements returned in response line""" 116 | with pytest.raises(errors.TestError): 117 | http_ua = http.HttpUA() 118 | response = http.HttpResponse("HTTP1.1 OK\r\n", http_ua) 119 | 120 | def test1(): 121 | """Typical request specified should be valid""" 122 | x = ruleset.Input(dest_addr="example.com",headers={"Host":"example.com"}) 123 | http_ua = http.HttpUA() 124 | http_ua.send_request(x) 125 | assert http_ua.response_object.status == 200 126 | 127 | 128 | def test2(): 129 | """Basic GET without Host on 1.1 - Expect 400""" 130 | x = ruleset.Input(dest_addr="example.com",headers={}) 131 | http_ua = http.HttpUA() 132 | http_ua.send_request(x) 133 | assert http_ua.response_object.status == 400 134 | 135 | def test3(): 136 | """Basic GET without Host on 1.0 - Expect 404 (server is VHosted)""" 137 | x = ruleset.Input(dest_addr="example.com",version="HTTP/1.0",headers={}) 138 | http_ua = http.HttpUA() 139 | http_ua.send_request(x) 140 | assert http_ua.response_object.status == 404 141 | 142 | def test4(): 143 | """Basic GET wit Host on 1.0 - Expect 200""" 144 | x = ruleset.Input(dest_addr="example.com",version="HTTP/1.0",headers={"Host":"example.com"}) 145 | http_ua = http.HttpUA() 146 | http_ua.send_request(x) 147 | assert http_ua.response_object.status == 200 148 | 149 | def test5(): 150 | """Basic GET without Host on 0.9 - Expect 505 version not supported""" 151 | x = ruleset.Input(dest_addr="example.com",version="HTTP/0.9",headers={}) 152 | http_ua = http.HttpUA() 153 | http_ua.send_request(x) 154 | assert http_ua.response_object.status == 505 155 | 156 | def test6(): 157 | """Basic GET without Host with invalid version (request line) - Expect 400 invalid""" 158 | x = ruleset.Input(dest_addr="example.com",version="HTTP/1.0 x",headers={}) 159 | http_ua = http.HttpUA() 160 | http_ua.send_request(x) 161 | assert http_ua.response_object.status == 400 162 | 163 | def test7(): 164 | """TEST method which doesn't exist - Expect 501""" 165 | x = ruleset.Input(method="TEST",dest_addr="example.com",version="HTTP/1.0",headers={}) 166 | http_ua = http.HttpUA() 167 | http_ua.send_request(x) 168 | assert http_ua.response_object.status == 501 169 | 170 | def test8(): 171 | """PROPFIND method which isn't allowed - Expect 405""" 172 | x = ruleset.Input(method="PROPFIND",dest_addr="example.com",version="HTTP/1.0",headers={}) 173 | http_ua = http.HttpUA() 174 | http_ua.send_request(x) 175 | assert http_ua.response_object.status == 405 176 | 177 | def test9(): 178 | """OPTIONS method - Expect 200""" 179 | x = ruleset.Input(method="OPTIONS",dest_addr="example.com",version="HTTP/1.0",headers={}) 180 | http_ua = http.HttpUA() 181 | http_ua.send_request(x) 182 | assert http_ua.response_object.status == 200 183 | 184 | def test10(): 185 | """HEAD method - Expect 200""" 186 | x = ruleset.Input(method="HEAD",dest_addr="example.com",version="HTTP/1.0",headers={"Host":"example.com"}) 187 | http_ua = http.HttpUA() 188 | http_ua.send_request(x) 189 | assert http_ua.response_object.status == 200 190 | 191 | def test11(): 192 | """POST method no data - Expect 411""" 193 | x = ruleset.Input(method="POST",dest_addr="example.com",version="HTTP/1.0",headers={}) 194 | http_ua = http.HttpUA() 195 | http_ua.send_request(x) 196 | assert http_ua.response_object.status == 411 197 | 198 | def test12(): 199 | """POST method no data with content length header - Expect 200""" 200 | x = ruleset.Input(method="POST",dest_addr="example.com",version="HTTP/1.0",headers={"Content-Length":"0","Host":"example.com"},data="") 201 | http_ua = http.HttpUA() 202 | http_ua.send_request(x) 203 | assert http_ua.response_object.status == 200 204 | 205 | def test13(): 206 | """Request https on port 80 (default)""" 207 | x = ruleset.Input(protocol="https",dest_addr="example.com",headers={"Host":"example.com"}) 208 | http_ua = http.HttpUA() 209 | with pytest.raises(errors.TestError): 210 | http_ua.send_request(x) 211 | 212 | def test14(): 213 | """Request https on port 443 should work""" 214 | x = ruleset.Input(protocol="https",port=443,dest_addr="example.com",headers={"Host":"example.com"}) 215 | http_ua = http.HttpUA() 216 | http_ua.send_request(x) 217 | assert http_ua.response_object.status == 200 218 | 219 | 220 | def test15(): 221 | """Request with content-type and content-length specified""" 222 | x = ruleset.Input(method="POST", protocol="http",port=80,dest_addr="example.com",headers={"Content-Type": "application/x-www-form-urlencoded","Host":"example.com","Content-Length":"7"},data="test=hi") 223 | http_ua = http.HttpUA() 224 | http_ua.send_request(x) 225 | assert http_ua.response_object.status == 200 226 | 227 | def test16(): 228 | """Post request with content-type but not content-length""" 229 | x = ruleset.Input(method="POST", protocol="http",port=80,dest_addr="example.com",headers={"Content-Type": "application/x-www-form-urlencoded","Host":"example.com"},data="test=hi") 230 | http_ua = http.HttpUA() 231 | http_ua.send_request(x) 232 | assert http_ua.response_object.status == 200 233 | 234 | def test17(): 235 | """Post request with no content-type AND no content-length""" 236 | x = ruleset.Input(method="POST", protocol="http",port=80,uri="/",dest_addr="example.com",headers={"Host":"example.com"},data="test=hi") 237 | http_ua = http.HttpUA() 238 | http_ua.send_request(x) 239 | assert http_ua.response_object.status == 200 240 | 241 | def test18(): 242 | """Send a request and check that the space is encoded automagically""" 243 | x = ruleset.Input(method="POST", protocol="http",port=80,uri="/",dest_addr="example.com",headers={"Host":"example.com"},data="test=hit f&test2=hello") 244 | http_ua = http.HttpUA() 245 | http_ua.send_request(x) 246 | assert http_ua.request_object.data == "test=hit+f&test2=hello" 247 | def test19(): 248 | """Send a raw question mark and test it is encoded automagically""" 249 | x = ruleset.Input(method="POST", protocol="http",port=80,uri="/",dest_addr="example.com",headers={"Host":"example.com"},data="test=hello?x") 250 | http_ua = http.HttpUA() 251 | http_ua.send_request(x) 252 | assert http_ua.request_object.data == "test=hello%3Fx" 253 | -------------------------------------------------------------------------------- /docs/ExtendingFTW.md: -------------------------------------------------------------------------------- 1 | Using FTW and integrating log_search with your WAF 2 | === 3 | 4 | It may be in your organization's best interest to use FTW as a library to run internal WAF tests that are not visible to the world. This can be due to sensitive information being present in WAF rules, policy and compliance or not revealing your defense tradecraft to adversaries. This is especially relevant for custom WAF rules tuned to your deployment. 5 | 6 | If that is the case, it may be best to run FTW as a library. This is accomplished through an integration with FTW and `py.test`. This tutorial will show how one uses FTW as a python library, as well as creating a custom log search using the `log_contains` directive. Logs are ideal for confirmation of a WAF rule working due to the WAF generating a success or failure log and they can provide additional context surrounding the WAF rule trigger. 7 | 8 | The rest of this tutorial will show how to setup a git project that installs ftw as a library, run a basic YAML test and finally integrate into a logging to find the presence of a WAF rule trigger. 9 | 10 | If youd like to fork FTW and write tests and integrations from there, but keep it in a private repository, you can skip to Step 3. If youd like to use FTW as a library, proceed to Step 1. 11 | 12 | Step 1 - Virtualenvironment, requirements.txt 13 | == 14 | 15 | `cd` to your directory where you want this project, then type `git init`. 16 | 17 | Next, initiate a python virtual environment using `virtualenv env`, then drop into that environment `. env/bin/activate`. 18 | 19 | Add `ftw` to `requirements.txt` 20 | 21 | `pip install -r requirements.txt` 22 | 23 | Done! 24 | 25 | 26 | A note on py.test testing environment 27 | --- 28 | 29 | `ftw` uses `py.test` as a way to do regression testing for WAFs. `py.test` is a powerful tool that helps make testing easier through a powerful command line tool interface, test parameterization and a rich API to help customize testing frameworks. You can read more about it [here](http://docs.pytest.org/en/latest/). 30 | 31 | `ftw` ships with a plugin to configure your `py.test` environment. This includes setting up informative test names in the console, command line arguments and helper functions. You can extend this further by writing your own [conftest.py](http://pytest.org/2.2.4/plugins.html) or submitting a P/R to `ftw` and change our `ftw/pytest_plugin.py`. 32 | 33 | Step 2 - Writing a test file 34 | == 35 | 36 | `touch test_foo.py`. `py.test` will be passed this file, and it will know to run any functions that start with `test_` within the file. So, for example, `def test_bar():` will be ran when the command `py.test test_foo.py` is issued. 37 | 38 | At the top of the file, import the necessary ftw modules. 39 | For those that installed `ftw` as a library, you can import `from ftw import ruleset, logchecker, testrunner`, but if you are writing a test directly in `ftw`, you need `ftw.ftw` in the `from ftw` line (for more examples of this, checkout one of the integration tests under `test/integration`) 40 | 41 | Write the following function definition underneath the import: 42 | 43 | ``` 44 | def test_bar(ruleset, test): 45 | runner = testrunner.TestRunner() 46 | for stage in test.stages: 47 | runner.run_stage(stage) 48 | ``` 49 | 50 | This is the bare minimum you need to run `ftw` tests. `py.test` hooks into the `test_bar` function, then finds two arguments, `ruleset` and `test`. These are `py.test` fixtures, and essentially load YAML rules from the commandline, parameterize them and send them to this function. 51 | 52 | Spawn a local webserver on port 80 using `sudo pythom -m SimpleHTTPServer 80`. Then, make a `yaml` directory `mkdir yaml`. Copy the following `ftw yaml` configuration into `yaml/foo.yaml` 53 | 54 | ``` 55 | --- 56 | meta: 57 | author: "Foo" 58 | enabled: true 59 | name: "foo.yaml" 60 | description: "Description" 61 | tests: 62 | - 63 | rule_id: 1234 64 | stages: 65 | - 66 | stage: 67 | input: 68 | method: "GET" 69 | port: 80 70 | headers: 71 | User-Agent: "Foo" 72 | Host: "localhost" 73 | protocol: "http" 74 | uri: "/" 75 | output: 76 | status: 200 77 | - 78 | rule_id: 1235 79 | stages: 80 | - 81 | stage: 82 | input: 83 | method: "GET" 84 | port: 80 85 | headers: 86 | User-Agent: "Foo" 87 | Host: "localhost" 88 | protocol: "http" 89 | uri: "/fail.html" 90 | output: 91 | status: 404 92 | 93 | ``` 94 | 95 | This YAML file has 2 tests with 1 stage each. The first test does a GET request to localhost on port 80 to the "/" URI, and expects a status 200. The second test does a GET request to localhost on port 80 to "/fail.html", and expects a 404 status. Each also set headers. You'll notice there is no `destaddr` directive, where you can put `localhost`, because there is a default to `localhost` in `ftw`. If you wish to go to another destination, add a `destaddr` directive under `input`. For examples, check out the integrations in `ftw`. 96 | 97 | Step 3 - Run your test! 98 | == 99 | After you configured your `test_foo.py`, `conftest.py`, `yaml/foo.yaml` and you run your local webserver, run `py.test test_foo.py -s -v --rule=yaml/foo.yaml`. 100 | 101 | The `-s -v` show stdout output and verbose test output, respectively. 102 | 103 | You should see this output 104 | 105 | ``` 106 | └─[15:01]$ py.test test_foo.py -s -v --rule=./yaml/foo.yaml 107 | ============================================================ test session starts ============================================================ 108 | platform darwin -- Python 2.7.10, pytest-2.9.2, py-1.4.31, pluggy-0.3.1 -- /Users/zallen/git/ftw_test/env/bin/python 109 | cachedir: .cache 110 | rootdir: /Users/zallen/git/ftw_test, inifile: 111 | collected 2 items 112 | 113 | test_foo.py::test_bar[ruleset0-foo.yaml_ruleid_1234] PASSED 114 | test_foo.py::test_bar[ruleset1-foo.yaml_ruleid_1235] PASSED 115 | 116 | ========================================================= 2 passed in 0.61 seconds ========================================================= 117 | ``` 118 | 119 | 2 tests were ran as specified in `foo.yaml`, and `py.test` shows verbose logging on which test was ran and what rule was ran. So we know `ruleid_1234` and `ruleid_1235` passed. If these errored out, you can investigate each rule to see what went wrong. 120 | 121 | These passed because the `output` directive in both rules specified an expected `status` directive. The webserver returned 200 and 404, respectively, and made the whole test suite passed. You can also pass in `html_contains: "^regex$"` to have FTW check for a pattern within the html_response. 122 | 123 | Step 4 - Log integration 124 | == 125 | 126 | Although `status` and `html_contains` may be useful, it could give attackers information on how your WAF interacts with certain payloads. Because of this, you might want to be more stealthy in how you respond to certain payloads, but you still need to verify that the rules are firing. This is where the `log_contains` directive can be used to check your WAF logs. 127 | 128 | If you add a `log_contains: "^regex$"` directive to the `output` field in yaml file, `ftw` will use that regex on a list of log lines to see if that pattern is present. The issue here is that you have to write the integration and send `ftw` the loglines for it to run the regex on. This makes `ftw` extendible because it can integrate with virtually any log system (API, filesystem) as long as you have programmatic access to it. 129 | 130 | Here is how you integrate a log file into `ftw`. 131 | 132 | The `logchecker` object you imported earlier is an abstract object that is passed into the TestRunner. Every `logchecker` object has an abstract function `get_logs()`. TestRunner in `ftw` will call this function everytime it sees a stage that contains `log_contains` directive. It expects the user to return an array of strings to perform the regex pattern check on. 133 | 134 | Add `import pytest` to the top of your file. 135 | 136 | Add the following class and `py.test` fixture function to your `test_foo.py` python file. 137 | 138 | ``` 139 | class FooLogChecker(logchecker.LogChecker): 140 | def get_logs(self): 141 | return [] 142 | 143 | @pytest.fixture 144 | def logchecker_obj(): 145 | """ 146 | Returns a LoggerTest Integration object 147 | """ 148 | return FooLogChecker() 149 | ``` 150 | 151 | Then update your `test_bar` function: 152 | 153 | ``` 154 | def test_bar(ruleset, test, logchecker_obj): 155 | runner = testrunner.TestRunner() 156 | for stage in test.stages: 157 | runner.run_stage(stage, logchecker_obj) 158 | ``` 159 | 160 | By adding `logchecker_obj` fixture to `test_bar`, `py.test` will call the `logchecker_obj()` fixture and return an instantiation of the `FooLogChecker` object. Now, everytime a test is ran with `log_contains`, `ftw` will call the `get_logs` function within that class. 161 | 162 | Two instance variables exist in every `LogChecker` object, `self.start` and `self.end`. When a test is ran, `ftw` records the time in epoch time it started the input stage, and marks the time in epoch when the input stage is complete. You can reference these in `FooLogChecker` by accessing `self.start` and `self.end`. Use these to pull apart a logfile or send to an API to help filter out logs that were not in the time you issued the attack. 163 | 164 | In `get_logs`, you can create a separate function that pulls apart a log file in modsecurity, or issues a search to an ELK stack that contains the logs from your WAF. 165 | 166 | Step 6 - YAML File 167 | == 168 | To confirm functionality of log_contains, return the following array of strings from `get_logs` 169 | 170 | `['GET Request from foo', 'Trigger WAF rule-id-1234']` 171 | 172 | Obviously, you would issue a search in a true deployment to wherever your logs are stored. 173 | 174 | In `yaml/foo.yaml`, edit `rule_id: 1234` `output` field and add: 175 | 176 | `log_contains: "rule-id-1234"` 177 | 178 | It should look like this: 179 | ``` 180 | --- 181 | meta: 182 | author: "Foo" 183 | enabled: true 184 | name: "foo.yaml" 185 | description: "Description" 186 | tests: 187 | - 188 | rule_id: 1234 189 | stages: 190 | - 191 | stage: 192 | input: 193 | method: "GET" 194 | port: 80 195 | headers: 196 | User-Agent: "Foo" 197 | Host: "localhost" 198 | protocol: "http" 199 | uri: "/" 200 | output: 201 | status: 200 202 | log_contains: "rule-id-1234" 203 | - 204 | rule_id: 1235 205 | stages: 206 | - 207 | stage: 208 | input: 209 | method: "GET" 210 | port: 80 211 | headers: 212 | User-Agent: "Foo" 213 | Host: "localhost" 214 | protocol: "http" 215 | uri: "/fail.html" 216 | output: 217 | status: 404 218 | ``` 219 | 220 | Run `py.test test_foo.py -s -v --rule=./yaml/foo.yaml`, you should see similar output: 221 | 222 | ``` 223 | └─[15:35]$ py.test test_foo.py -s -v --rule=./yaml/foo.yaml 224 | ============================================================ test session starts ============================================================ 225 | platform darwin -- Python 2.7.10, pytest-2.9.2, py-1.4.31, pluggy-0.3.1 -- /Users/zallen/git/ftw_test/env/bin/python 226 | cachedir: .cache 227 | rootdir: /Users/zallen/git/ftw_test, inifile: 228 | collected 2 items 229 | 230 | test_foo.py::test_bar[ruleset0-foo.yaml_ruleid_1234] PASSED 231 | test_foo.py::test_bar[ruleset1-foo.yaml_ruleid_1235] PASSED 232 | 233 | ========================================================= 2 passed in 0.62 seconds ========================================================= 234 | ``` 235 | 236 | Great! `ftw` loaded the tests, went to `ruleid_1234`, saw the `log_contains` directive, called the `get_logs()` function, was returned the list of 2 strings where one of them contained `rule-id-1234` and found that regex in your list. Now you have integration with `log_contains` working. 237 | -------------------------------------------------------------------------------- /docs/YAMLFormat.md: -------------------------------------------------------------------------------- 1 | =============== 2 | The YAML Format 3 | =============== 4 | 5 | Welcome to the the FTW YAMLFormat documentation. In this document we will explain all the possible options that can be used within the YAML format. Generally this is the preferred format for writing tests in as they don't require any programming skills in order to understand and change. If you find a bug in this format please open an issue. 6 | 7 | Metadata Parameters 8 | ================== 9 | Metadata parameters are present once per test file and are located by convention right after the start of the file. In general, this data should give a general overview of the following tests and what they apply to. An example usage is as follows: 10 | 11 | ``` 12 | --- 13 | meta: 14 | author: "csanders-git" 15 | enabled: true 16 | name: "Example_Tests" 17 | description: "This file contains example tests." 18 | ... 19 | ``` 20 | What follows are all the possible Metadata parameters that are current suported 21 | 22 | Author 23 | ------ 24 | **Description**: Lists the author(s). 25 | 26 | **Syntax:** ```author: ""``` 27 | 28 | **Example Usage:** ```author: "csanders-git"``` 29 | 30 | **Default Value:** "" 31 | 32 | **Scope:** Metadata 33 | 34 | **Added Version:** 0.1 35 | 36 | Description 37 | ----------- 38 | **Description**: A breif description of what the following tests are meant to accomplish 39 | 40 | **Syntax:** ```description: ""``` 41 | 42 | **Example Usage:** ```description: "The following is a description"``` 43 | 44 | **Default Value:** "" 45 | 46 | **Scope:** Metadata 47 | 48 | **Added Version:** 0.1 49 | 50 | Enabled 51 | ----------- 52 | **Description**: Determines if the tests in the file will be run 53 | 54 | **Syntax:** ```enabled: (true|false)``` 55 | 56 | **Example Usage:** ```enabled: false``` 57 | 58 | **Default Value:** true 59 | 60 | **Scope:** Metadata 61 | 62 | **Added Version:** 0.1 63 | 64 | Name 65 | ----------- 66 | **Description**: A name for the test file 67 | 68 | **Syntax:** ```enabled: (true|false)``` 69 | 70 | **Example Usage:** ```enabled: false``` 71 | 72 | **Default Value:** "" 73 | 74 | **Scope:** Metadata 75 | 76 | **Added Version:** 0.1 77 | 78 | *Note: The name specified here is used as part of the test execution name, in conjunction with the test_title. 79 | 80 | Tests Parameters 81 | ================== 82 | The tests area is made up of multiple tests. Each test contains Stages and an optional rule_id. Within the Stage there is additional information that is outlined in Stage Paramaters 83 | 84 | Test_title 85 | ----------- 86 | **Description**: Information about the test being performed, this will be included as the test name when run. 87 | 88 | **Syntax:** ```test_title: ""``` 89 | 90 | **Example Usage:** ```test_title: "Rule:1234/Test:1``` 91 | 92 | **Default Value:** None (Required) 93 | 94 | **Scope:** Tests 95 | 96 | **Added Version:** 0.1 97 | 98 | Stages 99 | ----------- 100 | **Description**: A parameter to encapsalate all the different stages of a give test 101 | 102 | **Syntax:** ```stages: n*``` 103 | 104 | **Example Usage:** ```stages:``` 105 | 106 | **Default Value:** TODO 107 | 108 | **Scope:** Tests 109 | 110 | **Added Version:** 0.1 111 | 112 | 113 | Stage Parameters 114 | ================== 115 | There can be multiple stages per test. Each stage represents a single request/response. Each stage paramater encapsalates an input and output parameters. 116 | 117 | Input 118 | ----------- 119 | **Description**: A container for the parameters that will be used to construct an HTTP request 120 | 121 | **Syntax:** ```input: ``` 122 | 123 | **Example Usage:** 124 | ``` 125 | stage: 126 | input: 127 | dest_addr: "localhost" 128 | port: 80 129 | ``` 130 | 131 | **Default Value:** No Default, Required with each Stage (TODO) 132 | 133 | **Scope:** Stage 134 | 135 | **Added Version:** 0.1 136 | 137 | Output 138 | ----------- 139 | **Description**: A container for parameters that will describe what do after the HTTP request is sent. 140 | 141 | **Syntax:** ```output: ``` 142 | 143 | **Example Usage:** 144 | ``` 145 | stage: 146 | output: 147 | status: 403 148 | ``` 149 | **Default Value:** "" 150 | 151 | **Scope:** Stage 152 | 153 | **Added Version:** 0.1 154 | 155 | Input Parameters 156 | ================== 157 | Items within input represent parameters that affect the construction and processing of the resulting HTTP packet. 158 | 159 | dest_addr 160 | ----------- 161 | **Description**: What address the packet should be sent to at the IP level 162 | 163 | **Syntax:** ```input: ``` 164 | 165 | **Example Usage:** ```dest_addr: "129.21.3.17"``` 166 | 167 | **Default Value:** "127.0.0.1" 168 | 169 | **Scope:** Input 170 | 171 | **Added Version:** 0.1 172 | 173 | port 174 | ----------- 175 | **Description**: What port the packet should be sent to at the IP level 176 | 177 | **Syntax:** ```input: ``` 178 | 179 | **Example Usage:** ```dest_addr: 8080``` 180 | 181 | **Default Value:** 80 182 | 183 | **Scope:** Input 184 | 185 | **Added Version:** 0.1 186 | 187 | method 188 | ------ 189 | **Description**: What HTTP method should be used within the HTTP portion of the packet 190 | 191 | **Syntax:** ```input: ``` 192 | 193 | **Example Usage:** ```dest_addr: "GET"``` 194 | 195 | **Default Value:** "GET" 196 | 197 | **Scope:** Input 198 | 199 | **Added Version:** 0.1 200 | 201 | headers 202 | ----------- 203 | **Description**: A collection that will be used to fill the header portion of the HTTP request 204 | 205 | **Syntax:** 206 | ``` 207 | headers: 208 | header1_name: "HeaderValue" 209 | header2_name: "HeaderValue" 210 | headerN_name: "HeaderValue" 211 | ``` 212 | 213 | **Example Usage:** 214 | ``` 215 | headers: 216 | User-Agent: "ModSecurity CRS 3 Tests" 217 | Host: "localhost" 218 | ``` 219 | 220 | **Default Value:** "" 221 | 222 | **Scope:** Input 223 | 224 | **Added Version:** 0.1 225 | 226 | *Note: in the future if stop_magic is enabled this will prevent automatic header values TODO* 227 | *Note: If a Content-Type is passed and a charset attribute is set, FTW will try to encode the data with that codec. It must be a valid Python codec and the default is UTF-8.* 228 | 229 | protocol 230 | ----------- 231 | **Description**: Specifies if the request should be using SSL/TLS or not 232 | 233 | **Syntax:** ```protocol: (http|https) 234 | 235 | **Example Usage:** ```protocol: http``` 236 | 237 | **Default Value:** "http" 238 | 239 | **Scope:** Input 240 | 241 | **Added Version:** 0.1 242 | 243 | Uri 244 | ----------- 245 | **Description**: The URI value that should be placed into the request-line of the HTTP request 246 | 247 | **Syntax:** ```uri: ``` 248 | 249 | **Example Usage:** ```uri: /test.php?param=value``` 250 | 251 | **Default Value:** "/" 252 | 253 | **Scope:** Input 254 | 255 | **Added Version:** 0.1 256 | 257 | Version 258 | ----------- 259 | **Description**: The version value that will be provided in the request-line of the HTTP request 260 | 261 | **Syntax:** ```version: ""``` 262 | 263 | **Example Usage:** ```version: "HTTP/0.9"``` 264 | 265 | **Default Value:** "HTTP/1.1" 266 | 267 | **Scope:** Input 268 | 269 | **Added Version:** 0.1 270 | 271 | Data 272 | ----------- 273 | **Description**: The optional data porition of the HTTP request. Typically these are provided with the content-type header. Data can be provided as a string or as a YAML list. 274 | 275 | **Syntax:** ```data: ""``` 276 | 277 | **Example Usage (string):** ```data: "xyz=123"`` 278 | 279 | **Example Usage (list):** 280 | ``` 281 | data: 282 | - "----------397236876" 283 | - "Content-Disposition: form-data; name=\"text\";" 284 | - "" 285 | - "test default" 286 | - "----------397236876" 287 | - "Content-Disposition: form-data; name=\"file1\"; filename=\"a.txt\"" 288 | - "Content-Type: text/plain" 289 | - "" 290 | - "Content of a.txt." 291 | - "" 292 | - "----------397236876--" 293 | ``` 294 | 295 | **Default Value:** "HTTP/1.1" 296 | 297 | **Scope:** Input 298 | 299 | **Added Version:** 0.1 300 | 301 | *Note: literals \r and \n will be replaced be replaced with CRLF when stop_magic is on.* 302 | *Note: if urlencoded content-type header is provided and parameters aren't in name=value form, data will be made empty, unless stop_magic is on.* 303 | 304 | 305 | Save_cookie 306 | ----------- 307 | **Description**: If there are multiple stages and save cookie is set, it will automaticlly be provided in the next stage if the site in question provides the Set-Cookie response header. 308 | 309 | **Syntax:** ```save_cookie: (true|false)``` 310 | 311 | **Example Usage:** ```save_cookie: true``` 312 | 313 | **Default Value:** false 314 | 315 | **Scope:** Input 316 | 317 | **Added Version:** 0.1 318 | 319 | Stop_magic 320 | ----------- 321 | **Description**: The framework will take care of certain things automaticlly like setting content-length, encoding, etc. When stop_magic is on, the framework will not do anything automagically. 322 | 323 | **Syntax:** ```stop_magic: (true|false)``` 324 | 325 | **Example Usage:** ```stop_magic: true``` 326 | 327 | **Default Value:** false 328 | 329 | **Scope:** Input 330 | 331 | **Added Version:** 0.1 332 | 333 | Encoded_request 334 | ----------- 335 | **Description**: This argument will take a base64 encoded string that will be decoded and sent through as the request. It will override all other settings 336 | 337 | **Syntax:** ```encoded_request: ``` 338 | 339 | **Example Usage:** ```encoded_request: "R0VUIFwgSFRUUFwxLjFcclxuClxyXG4="``` 340 | 341 | **Default Value:** None 342 | 343 | **Scope:** Input 344 | 345 | **Added Version:** 0.1 346 | 347 | *Note: literals \r and \n will be replaced be replaced with CRLF when stop_magic is on. (TODO)* 348 | 349 | Raw_request 350 | ----------- 351 | **Description**: This argument will take a unencoded string that will be sent through as the request. It will override all other settings 352 | 353 | **Syntax:** ```raw_request: ``` 354 | 355 | **Example Usage:** ```raw_request: "GET \ HTTP\r\n\r\n"``` 356 | 357 | **Default Value:** None 358 | 359 | **Scope:** Input 360 | 361 | **Added Version:** 0.1 362 | 363 | *Note: literals \r and \n will be replaced be replaced with CRLF when stop_magic is on. (TODO)* 364 | Output Parameters 365 | ================== 366 | Items within the output represent actions that should be taken as a result of the HTTP request being made. 367 | 368 | Status 369 | ----------- 370 | **Description**: Checks the response code of the response to see if it matches the provided value 371 | 372 | **Syntax:** ```status: ``` 373 | 374 | **Example Usage:** ```status: 200``` 375 | 376 | **Default Value:** None 377 | 378 | **Scope:** Output 379 | 380 | **Added Version:** 0.1 381 | 382 | html_contains 383 | ----------- 384 | **Description**: Checks the entire response against the regular expression provided. 385 | 386 | **Syntax:** ```html_contains: ""``` 387 | 388 | **Example Usage:** ```html_contains: "hello-[a-z]orld"``` 389 | 390 | **Default Value:** None 391 | 392 | **Scope:** Output 393 | 394 | **Added Version:** 0.1 395 | 396 | log_contains 397 | ----------- 398 | **Description**: Will use the provided LogChecker (must be supplied) and run the provided regex against each one of the logs provided by LogChecker.get_logs() looking for a match. If it doesn't find a match it will raise an assertion 399 | 400 | **Syntax:** ```log_contains: ""``` 401 | 402 | **Example Usage:** ```log_contains: "id:1234"``` 403 | 404 | **Default Value:** None 405 | 406 | **Scope:** Output 407 | 408 | **Added Version:** 0.1 409 | 410 | no_log_contains 411 | ----------- 412 | **Description**: Will use the provided LogChecker (must be supplied) and run the provided regex against each one of the logs provided by LogChecker.get_logs() looking for a match. However, if it finds a match it will raise an assertion. 413 | 414 | **Syntax:** ```no_log_contains: ""``` 415 | 416 | **Example Usage:** ```no_log_contains: "id:1234"``` 417 | 418 | **Default Value:** None 419 | 420 | **Scope:** Output 421 | 422 | **Added Version:** 0.1 423 | 424 | Expect_error 425 | ----------- 426 | **Description**: Sometimes it will happen that FTW cannot receive a response and will throw an error (i.e simple requests or 408s). As a result sometimes there are requests that can be sent that will always error back and are not expected to change. You can catch these 'TestErrors using Expect_error 427 | 428 | **Syntax:** ```expect_error: ``` 429 | 430 | **Example Usage:** ```expect_error: True``` 431 | 432 | **Default Value:** False 433 | 434 | **Scope:** Output 435 | 436 | **Added Version:** 0.1 437 | 438 | *Note: Setting this to false is only available for completeness it isn't ever really needed * 439 | -------------------------------------------------------------------------------- /ftw/http.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import socket 4 | import ssl 5 | import string 6 | import errno 7 | import time 8 | import StringIO 9 | import gzip 10 | import os 11 | import sys 12 | import re 13 | import base64 14 | import zlib 15 | import Cookie 16 | import encodings 17 | from IPy import IP 18 | import errors 19 | 20 | reload(sys) 21 | sys.setdefaultencoding('utf8') 22 | 23 | class HttpResponse(object): 24 | def __init__(self, http_response, user_agent): 25 | self.response = http_response 26 | # For testing purposes HTTPResponse might be called OOL 27 | try: 28 | self.dest_addr = user_agent.request_object.dest_addr 29 | except AttributeError: 30 | self.dest_addr = '127.0.0.1' 31 | self.response_line = None 32 | self.status = None 33 | self.cookiejar = user_agent.cookiejar 34 | self.status_msg = None 35 | self.version = None 36 | self.headers = None 37 | self.data = None 38 | self.CRLF = '\r\n' 39 | self.process_response() 40 | 41 | def parse_content_encoding(self, response_headers, response_data): 42 | """ 43 | Parses a response that contains Content-Encoding to retrieve 44 | response_data 45 | """ 46 | if response_headers['content-encoding'] == 'gzip': 47 | buf = StringIO.StringIO(response_data) 48 | zipbuf = gzip.GzipFile(fileobj=buf) 49 | response_data = zipbuf.read() 50 | elif response_headers['content-encoding'] == 'deflate': 51 | data = StringIO.StringIO(zlib.decompress(response_data)) 52 | response_data = data.read() 53 | else: 54 | raise errors.TestError( 55 | 'Received unknown Content-Encoding', 56 | { 57 | 'content-encoding': 58 | str(response_headers['content-encoding']), 59 | 'function': 'http.HttpResponse.parse_content_encoding' 60 | }) 61 | return response_data 62 | 63 | def check_for_cookie(self, cookie): 64 | # http://bayou.io/draft/cookie.domain.html 65 | # Check if our originDomain is an IP 66 | origin_is_ip = True 67 | try: 68 | IP(self.dest_addr) 69 | except ValueError: 70 | origin_is_ip = False 71 | for cookie_morsals in cookie.values(): 72 | # If the coverdomain is blank or the domain is an IP set the domain to be the origin 73 | if cookie_morsals['domain'] == '' or origin_is_ip: 74 | # We want to always add a domain so it's easy to parse later 75 | return (cookie, self.dest_addr) 76 | # If the coverdomain is set it can be any subdomain 77 | else: 78 | cover_domain = cookie_morsals['domain'] 79 | # strip leading dots 80 | # Find all leading dots not just first one 81 | # http://tools.ietf.org/html/rfc6265#section-4.1.2.3 82 | first_non_dot = 0 83 | for i in range(len(cover_domain)): 84 | if cover_domain[i] != '.': 85 | first_non_dot = i 86 | break 87 | cover_domain = cover_domain[first_non_dot:] 88 | # We must parse the coverDomain to make sure its not in the suffix list 89 | psl_path = os.path.dirname(__file__) + os.path.sep + \ 90 | 'util' + os.path.sep + 'public_suffix_list.dat' 91 | # Check if the public suffix list is present in the ftw dir 92 | if os.path.exists(psl_path): 93 | pass 94 | else: 95 | raise errors.TestError( 96 | 'unable to find the needed public suffix list', 97 | { 98 | 'Search_Dir': os.path.dirname(__file__), 99 | 'function': 'http.HttpResponse.check_for_cookie' 100 | }) 101 | try: 102 | with open(psl_path, 'r') as public_suffixs: 103 | for line in public_suffixs: 104 | if line[:2] == '//' or line[0] == ' ' or line[0].strip() == '': 105 | continue 106 | if cover_domain == line.strip(): 107 | return False 108 | except IOError: 109 | raise errors.TestError( 110 | 'unable to open the needed publix suffix list', 111 | { 112 | 'path': psl_path, 113 | 'function': 'http.HttpResponse.check_for_cookie' 114 | }) 115 | # Generate Origin Domain TLD 116 | i = self.dest_addr.rfind('.') 117 | o_tld = self.dest_addr[i+1:] 118 | # if our cover domain is the origin TLD we ignore 119 | # Quick sanity check 120 | if cover_domain == o_tld: 121 | return False 122 | # check if our coverdomain is a subset of our origin domain 123 | # Domain match (case insensative) 124 | if cover_domain == self.dest_addr: 125 | return (cookie, self.dest_addr) 126 | # Domain match algorithm (rfc2965) 127 | bvalue = cover_domain.lower() 128 | hdn = self.dest_addr.lower() 129 | nend = hdn.find(bvalue) 130 | if nend is not False: 131 | nvalue = hdn[0:nend] 132 | # Modern browsers don't care about dot 133 | if nvalue[-1] == '.': 134 | nvalue = nvalue[0:-1] 135 | else: 136 | # We don't have an address of the form 137 | return False 138 | if nvalue == '': 139 | return False 140 | return (cookie, self.dest_addr) 141 | 142 | def process_response(self): 143 | """ 144 | Parses an HTTP response after an HTTP request is sent 145 | """ 146 | split_response = self.response.split(self.CRLF) 147 | response_line = split_response[0] 148 | response_headers = {} 149 | response_data = None 150 | data_line = None 151 | for line_num in range(1, len(split_response[1:])): 152 | # CRLF represents the start of data 153 | if split_response[line_num] == '': 154 | data_line = line_num + 1 155 | break 156 | else: 157 | # Headers are all split by ':' 158 | header = split_response[line_num].split(':', 1) 159 | if len(header) != 2: 160 | raise errors.TestError( 161 | 'Did not receive a response with valid headers', 162 | { 163 | 'header_rcvd': str(header), 164 | 'function': 'http.HttpResponse.process_response' 165 | }) 166 | response_headers[header[0].lower()] = header[1].lstrip() 167 | if 'set-cookie' in response_headers.keys(): 168 | try: 169 | cookie = Cookie.SimpleCookie() 170 | cookie.load(response_headers['set-cookie']) 171 | except Cookie.CookieError as err: 172 | raise errors.TestError( 173 | 'Error processing the cookie content into a SimpleCookie', 174 | { 175 | 'msg': str(err), 176 | 'set_cookie': str(response_headers['set-cookie']), 177 | 'function': 'http.HttpResponse.process_response' 178 | }) 179 | # if the check_for_cookie is invalid then we don't save it 180 | if self.check_for_cookie(cookie) is False: 181 | raise errors.TestError( 182 | 'An invalid cookie was specified', 183 | { 184 | 'set_cookie': str(response_headers['set-cookie']), 185 | 'function': 'http.HttpResponse.process_response' 186 | }) 187 | else: 188 | self.cookiejar.append((cookie, self.dest_addr)) 189 | if data_line is not None and data_line < len(split_response): 190 | response_data = self.CRLF.join(split_response[data_line:]) 191 | 192 | # if the output headers say there is encoding 193 | if 'content-encoding' in response_headers.keys(): 194 | response_data = self.parse_content_encoding( 195 | response_headers, response_data) 196 | if len(response_line.split(' ', 2)) != 3: 197 | raise errors.TestError( 198 | 'The HTTP response line returned the wrong args', 199 | { 200 | 'response_line': str(response_line), 201 | 'function': 'http.HttpResponse.process_response' 202 | }) 203 | try: 204 | self.status = int(response_line.split(' ', 2)[1]) 205 | except ValueError: 206 | raise errors.TestError( 207 | 'The status num of the response line isn\'t convertable', 208 | { 209 | 'msg': 'This may be an HTTP 1.0 \'Simple Req\\Res\', it \ 210 | doesn\'t have HTTP headers and FTW will not parse these', 211 | 'response_line': str(response_line), 212 | 'function': 'http.HttpResponse.process_response' 213 | }) 214 | self.status_msg = response_line.split(' ', 2)[2] 215 | self.version = response_line.split(' ', 2)[0] 216 | self.response_line = response_line 217 | self.headers = response_headers 218 | self.data = response_data 219 | 220 | class HttpUA(object): 221 | """ 222 | Act as the User Agent for our regression testing 223 | """ 224 | def __init__(self): 225 | """ 226 | Initalize an HTTP object 227 | """ 228 | self.request_object = None 229 | self.response_object = None 230 | self.request = None 231 | self.cookiejar = [] 232 | self.sock = None 233 | self.CIPHERS = \ 234 | 'ADH-AES256-SHA:ECDHE-ECDSA-AES128-GCM-SHA256:' \ 235 | 'ECDHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:' 236 | self.CRLF = '\r\n' 237 | self.HTTP_TIMEOUT = .3 238 | self.RECEIVE_BYTES = 8192 239 | self.SOCKET_TIMEOUT = 5 240 | 241 | def send_request(self, http_request): 242 | """ 243 | Send a request and get response 244 | """ 245 | self.request_object = http_request 246 | self.build_socket() 247 | self.build_request() 248 | try: 249 | self.sock.send(self.request) 250 | except socket.error as err: 251 | raise errors.TestError( 252 | 'We were unable to send the request to the socket', 253 | { 254 | 'msg': err, 255 | 'function': 'http.HttpUA.send_request' 256 | }) 257 | finally: 258 | self.get_response() 259 | 260 | def build_socket(self): 261 | """ 262 | Generate either an HTTPS or HTTP socket 263 | """ 264 | try: 265 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 266 | self.sock.settimeout(self.SOCKET_TIMEOUT) 267 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 268 | # Check if TLS 269 | if self.request_object.protocol == 'https': 270 | self.sock = ssl.wrap_socket(self.sock, ciphers=self.CIPHERS) 271 | self.sock.connect( 272 | (self.request_object.dest_addr, self.request_object.port)) 273 | except socket.error as msg: 274 | raise errors.TestError( 275 | 'Failed to connect to server', 276 | { 277 | 'host': self.request_object.dest_addr, 278 | 'port': self.request_object.port, 279 | 'proto': self.request_object.protocol, 280 | 'message': msg, 281 | 'function': 'http.HttpUA.build_socket' 282 | }) 283 | 284 | def find_cookie(self): 285 | """ 286 | Find a list of all cookies for a given domain 287 | """ 288 | return_cookies = [] 289 | origin_domain = self.request_object.dest_addr 290 | for cookie in self.cookiejar: 291 | for cookie_morsals in cookie[0].values(): 292 | cover_domain = cookie_morsals['domain'] 293 | if cover_domain == '': 294 | if origin_domain == cookie[1]: 295 | return_cookies.append(cookie[0]) 296 | else: 297 | # Domain match algorithm 298 | bvalue = cover_domain.lower() 299 | hdn = origin_domain.lower() 300 | nend = hdn.find(bvalue) 301 | if nend is not False: 302 | return_cookies.append(cookie[0]) 303 | return return_cookies 304 | 305 | def build_request(self): 306 | request = '#method# #uri##version#%s#headers#%s#data#' % \ 307 | (self.CRLF, self.CRLF) 308 | request = string.replace( 309 | request, '#method#', self.request_object.method) 310 | # We add a space after here to account for HEAD requests with no url 311 | request = string.replace( 312 | request, '#uri#', self.request_object.uri + ' ') 313 | request = string.replace( 314 | request, '#version#', self.request_object.version) 315 | available_cookies = self.find_cookie() 316 | # If the user has requested a tracked cookie and we have one set it 317 | if available_cookies: 318 | cookie_value = '' 319 | if 'cookie' in self.request_object.headers.keys(): 320 | # Create a SimpleCookie out of our provided cookie 321 | try: 322 | provided_cookie = Cookie.SimpleCookie() 323 | provided_cookie.load(self.request_object.headers['cookie']) 324 | except Cookie.CookieError as err: 325 | raise errors.TestError( 326 | 'Error processing the existing cookie into a SimpleCookie', 327 | { 328 | 'msg': str(err), 329 | 'set_cookie': str(self.request_object.headers['cookie']), 330 | 'function': 'http.HttpResponse.build_request' 331 | }) 332 | result_cookie = {} 333 | for cookie_key, cookie_morsal in provided_cookie.iteritems(): 334 | result_cookie[cookie_key] = provided_cookie[cookie_key].value 335 | for cookie in available_cookies: 336 | for cookie_key, cookie_morsal in cookie.iteritems(): 337 | if cookie_key in result_cookie.keys(): 338 | # we don't overwrite a user specified cookie with a saved one 339 | pass 340 | else: 341 | result_cookie[cookie_key] = cookie[cookie_key].value 342 | for key, value in result_cookie.iteritems(): 343 | cookie_value += (unicode(key) + '=' + unicode(value) + '; ') 344 | # Remove the trailing semicolon 345 | cookie_value = cookie_value[:-2] 346 | self.request_object.headers['cookie'] = cookie_value 347 | else: 348 | for cookie in available_cookies: 349 | for cookie_key, cookie_morsal in cookie.iteritems(): 350 | cookie_value += (unicode(cookie_key) + '=' + unicode(cookie_morsal.coded_value) + '; ') 351 | # Remove the trailing semicolon 352 | cookie_value = cookie_value[:-2] 353 | self.request_object.headers['cookie'] = cookie_value 354 | 355 | # Expand out our headers into a string 356 | headers = '' 357 | if self.request_object.headers != {}: 358 | for hname, hvalue in self.request_object.headers.iteritems(): 359 | headers += unicode(hname) + ': ' + unicode(hvalue) + self.CRLF 360 | request = string.replace(request, '#headers#', headers) 361 | 362 | # If we have data append it 363 | if self.request_object.data != '': 364 | # Before we do that see if that is a charset 365 | encoding = "utf-8" 366 | # Check to see if we have a content type and magic is off (otherwise UTF-8) 367 | if 'Content-Type' in self.request_object.headers.keys() and self.request_object.stop_magic is False: 368 | pattern = re.compile(r'\;\s{0,1}?charset\=(.*?)(?:$|\;|\s)') 369 | m = re.search(pattern, self.request_object.headers['Content-Type']) 370 | if m: 371 | possible_choices = list(set(encodings.aliases.aliases.keys())) + list(set(encodings.aliases.aliases.values())) 372 | choice = m.group(1) 373 | # Python will allow these aliases but doesn't list them 374 | choice = choice.replace('-','_') 375 | choice = choice.lower() 376 | if choice in possible_choices: 377 | encoding = choice 378 | try: 379 | data = self.request_object.data.encode(encoding) 380 | except UnicodeEncodeError as err: 381 | raise errors.TestError( 382 | 'Error encoding the data with the charset specified', 383 | { 384 | 'msg': str(err), 385 | 'Content-Type': str(self.request_object.headers['Content-Type']), 386 | 'data': unicode(self.request_object.data), 387 | 'function': 'http.HttpResponse.build_request' 388 | }) 389 | request = string.replace(request, '#data#', data) 390 | else: 391 | request = string.replace(request, '#data#', '') 392 | # If we have a Raw Request we should use that instead 393 | if self.request_object.raw_request is not None: 394 | if self.request_object.encoded_request is not None: 395 | raise errors.TestError( 396 | 'Cannot specify both raw and encoded modes', 397 | { 398 | 'function': 'http.HttpUA.build_request' 399 | }) 400 | request = self.request_object.raw_request 401 | # We do this regardless of magic if you want to send a literal 402 | # '\' 'r' or 'n' use encoded request. 403 | request = request.decode('string_escape') 404 | if self.request_object.encoded_request is not None: 405 | request = base64.b64decode(self.request_object.encoded_request) 406 | request = request.decode('string_escape') 407 | # if we have an Encoded request we should use that 408 | self.request = request 409 | 410 | def get_response(self): 411 | """ 412 | Get the response from the socket 413 | """ 414 | self.sock.setblocking(0) 415 | our_data = [] 416 | # Beginning time 417 | begin = time.time() 418 | while True: 419 | # If we have data then if we're passed the timeout break 420 | if our_data and time.time() - begin > self.HTTP_TIMEOUT: 421 | break 422 | # If we're dataless wait just a bit 423 | elif time.time() - begin > self.HTTP_TIMEOUT * 2: 424 | break 425 | # Recv data 426 | try: 427 | data = self.sock.recv(self.RECEIVE_BYTES) 428 | if data: 429 | our_data.append(data) 430 | begin = time.time() 431 | else: 432 | # Sleep for sometime to indicate a gap 433 | time.sleep(self.HTTP_TIMEOUT) 434 | except socket.error as err: 435 | # Check if we got a timeout 436 | if err.errno == errno.EAGAIN: 437 | pass 438 | # SSL will return SSLWantRead instead of EAGAIN 439 | elif sys.platform == 'win32' and \ 440 | err.errno == errno.WSAEWOULDBLOCK: 441 | pass 442 | elif (self.request_object.protocol == 'https' and 443 | err[0] == ssl.SSL_ERROR_WANT_READ): 444 | continue 445 | # If we didn't it's an error 446 | else: 447 | raise errors.TestError( 448 | 'Failed to connect to server', 449 | { 450 | 'host': self.request_object.dest_addr, 451 | 'port': self.request_object.port, 452 | 'proto': self.request_object.protocol, 453 | 'message': err, 454 | 'function': 'http.HttpUA.get_response' 455 | }) 456 | if ''.join(our_data) == '': 457 | raise errors.TestError( 458 | 'No response from server. Request likely timed out.', 459 | { 460 | 'host': self.request_object.dest_addr, 461 | 'port': self.request_object.port, 462 | 'proto': self.request_object.protocol, 463 | 'msg': 'Please send the request and check Wireshark', 464 | 'function': 'http.HttpUA.get_response' 465 | }) 466 | self.response_object = HttpResponse(''.join(our_data), self) 467 | try: 468 | self.sock.shutdown(1) 469 | self.sock.close() 470 | except socket.error as err: 471 | raise errors.TestError( 472 | 'We were unable to close the socket as expected.', 473 | { 474 | 'msg': err, 475 | 'function': 'http.HttpUA.get_response' 476 | }) 477 | --------------------------------------------------------------------------------