├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE.md ├── README.md ├── codewars_test ├── __init__.py └── test_framework.py ├── gen-fixture.sh ├── setup.py └── tests ├── __init__.py ├── fixtures ├── expect_error_sample.expected.txt ├── expect_error_sample.py ├── multiple_groups.expected.txt ├── multiple_groups.py ├── nested_groups.expected.txt ├── nested_groups.py ├── old_group.expected.txt ├── old_group.py ├── old_top_level_assertion_fail.expected.txt ├── old_top_level_assertion_fail.py ├── old_top_level_assertion_fail_pass.expected.txt ├── old_top_level_assertion_fail_pass.py ├── old_top_level_assertion_pass.expected.txt ├── old_top_level_assertion_pass.py ├── passing_failing.expected.txt ├── passing_failing.py ├── single_failing.expected.txt ├── single_failing.py ├── single_passing.expected.txt ├── single_passing.py ├── timeout_failing.expected.txt ├── timeout_failing.py ├── timeout_passing.expected.txt └── timeout_passing.py └── test_outputs.py /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | test: 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | os: [ubuntu-latest] 10 | python-version: [3.6, 3.7, 3.8] 11 | runs-on: ${{ matrix.os }} 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Test 19 | run: python -m unittest 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Qualified Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Codewars Test Framework for Python 2 | 3 | ### Installation 4 | 5 | ```bash 6 | pip install git+https://github.com/codewars/python-test-framework.git#egg=codewars_test 7 | ``` 8 | 9 | ### Basic Example 10 | 11 | ```python 12 | import codewars_test as test 13 | from solution import add 14 | 15 | @test.describe('Example Tests') 16 | def example_tests(): 17 | 18 | @test.it('Example Test Case') 19 | def example_test_case(): 20 | test.assert_equals(add(1, 1), 2, 'Optional Message on Failure') 21 | ``` 22 | 23 | 65 | -------------------------------------------------------------------------------- /codewars_test/__init__.py: -------------------------------------------------------------------------------- 1 | from .test_framework import * 2 | -------------------------------------------------------------------------------- /codewars_test/test_framework.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | 4 | class AssertException(Exception): 5 | pass 6 | 7 | 8 | def format_message(message): 9 | return message.replace("\n", "<:LF:>") 10 | 11 | 12 | def display(type, message, label="", mode=""): 13 | print("\n<{0}:{1}:{2}>{3}".format( 14 | type.upper(), mode.upper(), label, format_message(message)) 15 | , flush=True) 16 | 17 | 18 | def expect(passed=None, message=None, allow_raise=False): 19 | if passed: 20 | display('PASSED', 'Test Passed') 21 | else: 22 | message = message or "Value is not what was expected" 23 | display('FAILED', message) 24 | if allow_raise: 25 | raise AssertException(message) 26 | 27 | 28 | def assert_equals(actual, expected, message=None, allow_raise=False): 29 | equals_msg = "{0} should equal {1}".format(repr(actual), repr(expected)) 30 | if message is None: 31 | message = equals_msg 32 | else: 33 | message += ": " + equals_msg 34 | 35 | expect(actual == expected, message, allow_raise) 36 | 37 | 38 | def assert_not_equals(actual, expected, message=None, allow_raise=False): 39 | r_actual, r_expected = repr(actual), repr(expected) 40 | equals_msg = "{0} should not equal {1}".format(r_actual, r_expected) 41 | if message is None: 42 | message = equals_msg 43 | else: 44 | message += ": " + equals_msg 45 | 46 | expect(not (actual == expected), message, allow_raise) 47 | 48 | 49 | def expect_error(message, function, exception=Exception): 50 | passed = False 51 | try: 52 | function() 53 | except exception: 54 | passed = True 55 | except Exception as e: 56 | message = "{}: {} should be {}".format(message or "Unexpected exception", repr(e), repr(exception)) 57 | expect(passed, message) 58 | 59 | 60 | def expect_no_error(message, function, exception=BaseException): 61 | try: 62 | function() 63 | except exception as e: 64 | fail("{}: {}".format(message or "Unexpected exception", repr(e))) 65 | return 66 | except: 67 | pass 68 | pass_() 69 | 70 | 71 | def pass_(): expect(True) 72 | 73 | 74 | def fail(message): expect(False, message) 75 | 76 | 77 | def assert_approx_equals( 78 | actual, expected, margin=1e-9, message=None, allow_raise=False): 79 | msg = "{0} should be close to {1} with absolute or relative margin of {2}" 80 | equals_msg = msg.format(repr(actual), repr(expected), repr(margin)) 81 | if message is None: 82 | message = equals_msg 83 | else: 84 | message += ": " + equals_msg 85 | div = max(abs(actual), abs(expected), 1) 86 | expect(abs((actual - expected) / div) < margin, message, allow_raise) 87 | 88 | 89 | ''' 90 | Usage: 91 | @describe('describe text') 92 | def describe1(): 93 | @it('it text') 94 | def it1(): 95 | # some test cases... 96 | ''' 97 | 98 | 99 | def _timed_block_factory(opening_text): 100 | from timeit import default_timer as timer 101 | from traceback import format_exception 102 | from sys import exc_info 103 | 104 | def _timed_block_decorator(s, before=None, after=None): 105 | display(opening_text, s) 106 | 107 | def wrapper(func): 108 | if callable(before): 109 | before() 110 | time = timer() 111 | try: 112 | func() 113 | except AssertionError as e: 114 | display('FAILED', str(e)) 115 | except Exception: 116 | fail('Unexpected exception raised') 117 | tb_str = ''.join(format_exception(*exc_info())) 118 | display('ERROR', tb_str) 119 | display('COMPLETEDIN', '{:.2f}'.format((timer() - time) * 1000)) 120 | if callable(after): 121 | after() 122 | return wrapper 123 | return _timed_block_decorator 124 | 125 | 126 | describe = _timed_block_factory('DESCRIBE') 127 | it = _timed_block_factory('IT') 128 | 129 | 130 | ''' 131 | Timeout utility 132 | Usage: 133 | @timeout(sec) 134 | def some_tests(): 135 | any code block... 136 | Note: Timeout value can be a float. 137 | ''' 138 | 139 | 140 | def timeout(sec): 141 | def wrapper(func): 142 | from multiprocessing import Process 143 | msg = 'Should not throw any exceptions inside timeout' 144 | 145 | def wrapped(): 146 | expect_no_error(msg, func) 147 | process = Process(target=wrapped) 148 | process.start() 149 | process.join(sec) 150 | if process.is_alive(): 151 | fail('Exceeded time limit of {:.3f} seconds'.format(sec)) 152 | process.terminate() 153 | process.join() 154 | return wrapper 155 | -------------------------------------------------------------------------------- /gen-fixture.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # ./gen-fixture.sh tests/fixtures/example.py 3 | # ./gen-fixture.sh tests/fixtures/example.py expected 4 | # ./gen-fixture.sh tests/fixtures/example.py sample 5 | # for f in $(ls tests/fixtures/*.py); do ./gen-fixture.sh "$f"; done 6 | 7 | PYTHONPATH=./ python "$1" > "${1%.py}.${2:-expected}.txt" 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="codewars_test", 5 | version="0.2.1", 6 | packages=["codewars_test"], 7 | license="MIT", 8 | description="Codewars test framework for Python", 9 | install_requires=[], 10 | url="https://github.com/codewars/python-test-framework", 11 | ) 12 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codewars/python-test-framework/3e2ceebd6e6eda7b5058792d1d23a48be8eefa6a/tests/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/expect_error_sample.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | expect_error, new version 3 | 4 | f0 raises nothing 5 | 6 | f0 did not raise any exception 7 | 8 | f0 did not raise Exception 9 | 10 | f0 did not raise ArithmeticError 11 | 12 | f0 did not raise ZeroDivisionError 13 | 14 | f0 did not raise LookupError 15 | 16 | f0 did not raise KeyError 17 | 18 | f0 did not raise OSError 19 | 20 | 0.03 21 | 22 | f1 raises Exception 23 | 24 | Test Passed 25 | 26 | Test Passed 27 | 28 | f1 did not raise ArithmeticError: Exception() should be 29 | 30 | f1 did not raise ZeroDivisionError: Exception() should be 31 | 32 | f1 did not raise LookupError: Exception() should be 33 | 34 | f1 did not raise KeyError: Exception() should be 35 | 36 | f1 did not raise OSError: Exception() should be 37 | 38 | 0.02 39 | 40 | f2 raises Exception >> ArithmeticError >> ZeroDivisionError 41 | 42 | Test Passed 43 | 44 | Test Passed 45 | 46 | Test Passed 47 | 48 | Test Passed 49 | 50 | f2 did not raise LookupError: ZeroDivisionError('integer division or modulo by zero') should be 51 | 52 | f2 did not raise KeyError: ZeroDivisionError('integer division or modulo by zero') should be 53 | 54 | f2 did not raise OSError: ZeroDivisionError('integer division or modulo by zero') should be 55 | 56 | 0.02 57 | 58 | f3 raises Exception >> LookupError >> KeyError 59 | 60 | Test Passed 61 | 62 | Test Passed 63 | 64 | f3 did not raise ArithmeticError: KeyError(1) should be 65 | 66 | f3 did not raise ZeroDivisionError: KeyError(1) should be 67 | 68 | Test Passed 69 | 70 | Test Passed 71 | 72 | f3 did not raise OSError: KeyError(1) should be 73 | 74 | 0.02 75 | 76 | 0.11 -------------------------------------------------------------------------------- /tests/fixtures/expect_error_sample.py: -------------------------------------------------------------------------------- 1 | # https://www.codewars.com/kumite/5ab735bee7093b17b2000084?sel=5ab735bee7093b17b2000084 2 | import codewars_test as test 3 | 4 | 5 | def f0(): 6 | pass 7 | 8 | 9 | # BaseException >> Exception 10 | def f1(): 11 | raise Exception() 12 | 13 | 14 | # BaseException >> Exception >> ArithmeticError >> ZeroDivisionError 15 | def f2(): 16 | return 1 // 0 17 | 18 | 19 | # BaseException >> Exception >> LookupError >> KeyError 20 | def f3(): 21 | return {}[1] 22 | 23 | 24 | excn = ( 25 | "Exception", 26 | "ArithmeticError", 27 | "ZeroDivisionError", 28 | "LookupError", 29 | "KeyError", 30 | "OSError", 31 | ) 32 | exc = (Exception, ArithmeticError, ZeroDivisionError, LookupError, KeyError, OSError) 33 | 34 | 35 | @test.describe("expect_error, new version") 36 | def d2(): 37 | @test.it("f0 raises nothing") 38 | def i0(): 39 | test.expect_error("f0 did not raise any exception", f0) 40 | for i in range(6): 41 | test.expect_error("f0 did not raise {}".format(excn[i]), f0, exc[i]) 42 | 43 | @test.it("f1 raises Exception") 44 | def i1(): 45 | test.expect_error("f1 did not raise Exception", f1) 46 | for i in range(6): 47 | test.expect_error("f1 did not raise {}".format(excn[i]), f1, exc[i]) 48 | 49 | @test.it("f2 raises Exception >> ArithmeticError >> ZeroDivisionError") 50 | def i2(): 51 | test.expect_error("f2 did not raise Exception", f2) 52 | for i in range(6): 53 | test.expect_error("f2 did not raise {}".format(excn[i]), f2, exc[i]) 54 | 55 | @test.it("f3 raises Exception >> LookupError >> KeyError") 56 | def i3(): 57 | test.expect_error("f3 did not raise Exception", f3) 58 | for i in range(6): 59 | test.expect_error("f3 did not raise {}".format(excn[i]), f3, exc[i]) 60 | 61 | -------------------------------------------------------------------------------- /tests/fixtures/multiple_groups.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | test 1 5 | 6 | 1 should equal 2 7 | 8 | 0.01 9 | 10 | 0.02 11 | 12 | group 2 13 | 14 | test 1 15 | 16 | 1 should equal 2 17 | 18 | 0.00 19 | 20 | 0.01 21 | -------------------------------------------------------------------------------- /tests/fixtures/multiple_groups.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.it("test 1") 7 | def test_1(): 8 | test.assert_equals(1, 2) 9 | 10 | 11 | @test.describe("group 2") 12 | def group_2(): 13 | @test.it("test 1") 14 | def test_1(): 15 | test.assert_equals(1, 2) 16 | -------------------------------------------------------------------------------- /tests/fixtures/nested_groups.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | group 1 1 5 | 6 | test 1 7 | 8 | 1 should equal 2 9 | 10 | 0.01 11 | 12 | 0.02 13 | 14 | 0.02 15 | -------------------------------------------------------------------------------- /tests/fixtures/nested_groups.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.describe("group 1 1") 7 | def group_1_1(): 8 | @test.it("test 1") 9 | def test_1(): 10 | test.assert_equals(1, 2) 11 | -------------------------------------------------------------------------------- /tests/fixtures/old_group.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | test 1 5 | 6 | Test Passed 7 | -------------------------------------------------------------------------------- /tests/fixtures/old_group.py: -------------------------------------------------------------------------------- 1 | # Deprecated and should not be used 2 | import codewars_test as test 3 | 4 | test.describe("group 1") 5 | test.it("test 1") 6 | test.assert_equals(1, 1) 7 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_fail.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | 1 should equal 2 3 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_fail.py: -------------------------------------------------------------------------------- 1 | # Deprecated and should not be used 2 | import codewars_test as test 3 | 4 | test.assert_equals(1, 2) 5 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_fail_pass.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | 1 should equal 2 3 | 4 | Test Passed 5 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_fail_pass.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | test.assert_equals(1, 2) 4 | test.assert_equals(1, 1) 5 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_pass.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | Test Passed 3 | -------------------------------------------------------------------------------- /tests/fixtures/old_top_level_assertion_pass.py: -------------------------------------------------------------------------------- 1 | # Deprecated and should not be used 2 | import codewars_test as test 3 | 4 | test.assert_equals(1, 1) 5 | -------------------------------------------------------------------------------- /tests/fixtures/passing_failing.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | test 1 5 | 6 | Test Passed 7 | 8 | 0.00 9 | 10 | test 2 11 | 12 | 1 should equal 2 13 | 14 | 0.00 15 | 16 | 0.02 17 | -------------------------------------------------------------------------------- /tests/fixtures/passing_failing.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.it("test 1") 7 | def test_1(): 8 | test.assert_equals(1, 1) 9 | 10 | @test.it("test 2") 11 | def test_2(): 12 | test.assert_equals(1, 2) 13 | -------------------------------------------------------------------------------- /tests/fixtures/single_failing.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | test 1 5 | 6 | 1 should equal 2 7 | 8 | 0.00 9 | 10 | 0.01 11 | -------------------------------------------------------------------------------- /tests/fixtures/single_failing.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.it("test 1") 7 | def test_1(): 8 | test.assert_equals(1, 2) 9 | -------------------------------------------------------------------------------- /tests/fixtures/single_passing.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | test 1 5 | 6 | Test Passed 7 | 8 | 0.01 9 | 10 | 0.02 11 | -------------------------------------------------------------------------------- /tests/fixtures/single_passing.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.it("test 1") 7 | def test_1(): 8 | test.assert_equals(1, 1) 9 | -------------------------------------------------------------------------------- /tests/fixtures/timeout_failing.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | Exceeded time limit of 0.010 seconds 5 | 6 | 30.41 7 | -------------------------------------------------------------------------------- /tests/fixtures/timeout_failing.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | @test.timeout(0.01) 7 | def test_1(): 8 | x = 0 9 | while x < 10 ** 9: 10 | x += 1 11 | test.pass_() 12 | -------------------------------------------------------------------------------- /tests/fixtures/timeout_passing.expected.txt: -------------------------------------------------------------------------------- 1 | 2 | group 1 3 | 4 | Test Passed 5 | 6 | Test Passed 7 | 8 | 17.19 9 | -------------------------------------------------------------------------------- /tests/fixtures/timeout_passing.py: -------------------------------------------------------------------------------- 1 | import codewars_test as test 2 | 3 | 4 | @test.describe("group 1") 5 | def group_1(): 6 | # This outputs 2 PASSED 7 | @test.timeout(0.01) 8 | def test_1(): 9 | test.assert_equals(1, 1) 10 | 11 | -------------------------------------------------------------------------------- /tests/test_outputs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import subprocess 3 | import os 4 | import re 5 | from pathlib import Path 6 | 7 | 8 | class TestOutputs(unittest.TestCase): 9 | pass 10 | 11 | 12 | def test_against_expected(test_file, expected_file, env): 13 | def test(self): 14 | # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` 15 | result = subprocess.run( 16 | ["python", test_file], 17 | env=env, 18 | stdout=subprocess.PIPE, 19 | stderr=subprocess.PIPE, 20 | ) 21 | with open(expected_file, "r", encoding="utf-8") as r: 22 | # Allow duration to change 23 | expected = re.sub(r"([()])", r"\\\1", r.read()) 24 | expected = re.sub( 25 | r"(?<=)\d+(?:\.\d+)?", r"\\d+(?:\\.\\d+)?", expected 26 | ) 27 | 28 | self.assertRegex(result.stdout.decode("utf-8"), expected) 29 | 30 | return test 31 | 32 | 33 | def get_commands(output): 34 | return re.findall(r"<(?:DESCRIBE|IT|PASSED|FAILED|ERROR|COMPLETEDIN)::>", output) 35 | 36 | 37 | def test_against_sample(test_file, sample_file, env): 38 | def test(self): 39 | # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` 40 | result = subprocess.run( 41 | ["python", test_file], 42 | env=env, 43 | stdout=subprocess.PIPE, 44 | stderr=subprocess.PIPE, 45 | ) 46 | with open(sample_file, "r", encoding="utf-8") as r: 47 | # Ensure that it contains the same output structure 48 | self.assertEqual( 49 | get_commands(result.stdout.decode("utf-8")), get_commands(r.read()) 50 | ) 51 | 52 | return test 53 | 54 | 55 | def define_tests(): 56 | fixtures_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures") 57 | package_dir = Path(fixtures_dir).parent.parent 58 | files = (f for f in os.listdir(fixtures_dir) if f.endswith(".py")) 59 | for f in files: 60 | expected_file = os.path.join(fixtures_dir, f.replace(".py", ".expected.txt")) 61 | if os.path.exists(expected_file): 62 | test_func = test_against_expected( 63 | os.path.join(fixtures_dir, f), 64 | expected_file, 65 | {"PYTHONPATH": package_dir}, 66 | ) 67 | else: 68 | # Use `.sample.txt` when testing against outputs with more variables. 69 | # This version only checks for the basic structure. 70 | test_func = test_against_sample( 71 | os.path.join(fixtures_dir, f), 72 | os.path.join(fixtures_dir, f.replace(".py", ".sample.txt")), 73 | {"PYTHONPATH": package_dir}, 74 | ) 75 | setattr(TestOutputs, "test_{0}".format(f.replace(".py", "")), test_func) 76 | 77 | 78 | define_tests() 79 | 80 | if __name__ == "__main__": 81 | unittest.main() 82 | --------------------------------------------------------------------------------