├── .github └── workflows │ └── sync-main.yml ├── .gitignore ├── LICENSE ├── README.md ├── notebooks ├── 00_Taxonomy_of_Testing_STUDENT.ipynb ├── 01_Describing_Data_STUDENT.ipynb ├── 02_Test_Strategies_STUDENT.ipynb └── 03_Putting_into_Practice_STUDENT.ipynb ├── requirements.txt ├── runtime.txt ├── setup.py ├── src └── pbt_tutorial │ ├── __init__.py │ └── basic_functions.py └── tests ├── test_settings.py ├── test_statistics.py └── test_with_target.py /.github/workflows/sync-main.yml: -------------------------------------------------------------------------------- 1 | name: Sync main branch from instructor branch 2 | 3 | on: 4 | push: 5 | branches: [ instructors ] 6 | 7 | jobs: 8 | sync-from-instructors: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Python 3.8 13 | uses: actions/setup-python@v2 14 | with: 15 | python-version: 3.8 16 | - name: Install dependencies 17 | run: python -m pip install --upgrade cogbooks 18 | - name: Build student versions 19 | run: | 20 | cogbooks -f notebooks/ 21 | rm -r notebooks/*.md 22 | rm tests/test_fuzz.py 23 | rm tests/test_manual.py 24 | rm tests/test_metamorphic.py 25 | rm tests/test_parameterized.py 26 | rm tests/test_properties.py 27 | - name: Push to main branch 28 | run: | 29 | git config --local user.email "action@github.com" 30 | git config --local user.name "GitHub Action" 31 | git add --force . 32 | git commit -m "Sync from instructors branch" 33 | git push --force origin HEAD:main 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | .venv*/ 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # Environments 86 | .env 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | # vim 108 | *.swp 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Ryan Soklaski and Zac Hatfield-Dodds 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # An Introduction to Property-Based Testing 2 | > Created by: Zac Hatfield-Dodds and Ryan Soklaski 3 | 4 | This tutorial is designed to introduce attendees to (the wonderful world of) automated testing in Python 5 | – specifically for data science applications. 6 | 7 | - [An Introduction to Property-Based Testing](#an-introduction-to-property-based-testing) 8 | - [Introduction](#introduction) 9 | - [Syllabus](#syllabus) 10 | - [Property-Based Testing 101](#property-based-testing-101) 11 | - [Describe your Data](#describe-your-data) 12 | - [Common Tests](#common-tests) 13 | - [Putting it into Practice](#putting-it-into-practice) 14 | - [Prerequisites](#prerequisites) 15 | - [Preparing a Python Environment for this Tutorial](#preparing-a-python-environment-for-this-tutorial) 16 | - [Creating The New Environment](#creating-the-new-environment) 17 | - [Installing Required Packages](#installing-required-packages) 18 | - [For Instructors](#for-instructors) 19 | 20 | ## Introduction 21 | 22 | This tutorial is designed to promote simple but powerful methods for bolstering your software-based work/research with 23 | property-based testing, and thereby help to close the aperture on “the things that could go wrong” because of your code. 24 | It is structured as four blocks, each consisting of a short talk, live-coded demo, and extensive exercises for attendees: 25 | 1. Property-Based Testing 101: core concepts and the core of the Hypothesis library 26 | 2. Describe your Data: from numbers, to arrays, to recursive and more complicated things 27 | 3. Common Tests: from "does not crash" to "write+read == noop" to 'metamorphic relations' 28 | 4. Putting it into Practice: use what you've learned to find real bugs in a real project! 29 | 30 | Each of these blocks runs for 40-60 minutes, consisting of a 5-15 minute presentation, occasional live-coded demos, and guided exercises. 31 | Along with a short break in the middle, we find this pattern balances content, practice, and focus well for most classes. 32 | 33 | [You can read through the slides here](https://docs.google.com/presentation/d/1Yv4hmaJb3CUejX8L3OkYn5Npddukyol18DWoaIxw-8I/), 34 | as a reference for later. At the end of each section, please work through the exercises before reading ahead! 35 | 36 | 37 | 38 | ## Syllabus 39 | The material will be offered as a half-day session. 40 | There will be a mixture of lectures with hands-on exercises. 41 | The following topics will be explored: 42 | 43 | ### Property-Based Testing 101 44 | *For in-person tutorials, this section is a talk rather than exercises.* 45 | - In this section we will take a tour through the taxonomy of testing: encountering manual tests, parameterized tests, and property-based tests. In the meanwhile, we will be introduced to the testing library Hypothesis, which provides us with automated test-case generation and reduction. 46 | 47 | ### Describe your Data 48 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/rsokl/testing-tutorial/HEAD?labpath=notebooks%2F01_Describing_Data_STUDENT.ipynb) 49 | - It is often the case that the process of *describing our data* is by far the heaviest burden that we must bear when writing tests. This process of assessing "what variety of values should I test?", "have I thought of all the important edge-cases?", and "how much is 'enough'?" will crop up with nearly every test that we write. [Hypothesis](https://hypothesis.readthedocs.io/) is a powerful Python library that empowers us to write a _description_ (specification, to be more precise) of the data that we want to use to exercise our test. 50 | It will then *generate* test cases that satisfy this description and will run our test on these cases. 51 | Let's witness this in its most basic form. 52 | 53 | ### Common Tests 54 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/rsokl/testing-tutorial/HEAD?labpath=notebooks%2F02_Test_Strategies_STUDENT.ipynb) 55 | - In this section, we will put Hypothesis strategies for describing data to good use and discuss some common tactics for writing effective property-based tests for our code. This will familiarize with certain patterns and properties - e.g. roundtrip pairs, equivalent functions, metamorphic relationships - to look for when crafting property-based tests for our code. 56 | 57 | ### Putting it into Practice 58 | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/rsokl/testing-tutorial/HEAD?labpath=notebooks%2F03_Putting_into_Practice_STUDENT.ipynb) 59 | - Now that we know how to write property-based tests, in this final section we will cover practical tips for using them as part of a larger project. Test-suite design patterns, use of performance and determinism settings, reproducing test failures from a CI server, etc; they're all practical topics and typically left out of shorter introductions! 60 | 61 | 62 | ## Prerequisites 63 | 64 | This tutorial is for anybody who regularly writes tests in Python, and would like an easier and more effective way to do so. 65 | We assume that you are comfortable reading, running, and writing traditional unit tests; and familar with ideas like assertions. 66 | Most attendees will have heard "given, when, then" and "arrange, act, assert". 67 | You may or may not have heard of pre- and post-conditions - we will explain what "property-based" means without reference to Haskell or anything algebraic. 68 | 69 | Attendees are expected to have access to a computer with with Python 3.7+ installed on it and should know how to run Jupyter notebooks. 70 | A complete description of how to do this is detailed in [Module 1 of Python Like You Meant It](https://www.pythonlikeyoumeanit.com/module_1.html). 71 | 72 | It is also recommended that you prepare yourself to work in your IDE of choice for Python. If you do not have an IDE that you are comfortable with yet, [Visual Studio code and PyCharm are both highly recommended](https://www.pythonlikeyoumeanit.com/Module1_GettingStartedWithPython/Getting_Started_With_IDEs_and_Notebooks.html). 73 | 74 | 75 | ## Preparing a Python Environment for this Tutorial 76 | 77 | To complete the tasks in this tutorial you will need to setup a proper python environment. 78 | These are the big steps you are going to follow: 79 | 80 | 0. Create a new virtual environment (optional but recomended) 81 | 1. Install all required external and local packages in your new virtual environment 82 | 83 | ### Creating The New Environment 84 | 85 |
86 | Create an environment using pip 87 | 88 | The instructions can be found at [create and activate a virtual environment](https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment) 89 | 90 | > Tip: do not forget to activate your environment after creating it! 91 | 92 |
93 | 94 |
95 | Create an environment using conda 96 | 97 | To create a mini-conda environment for this tutorial, [install mini-conda](https://docs.conda.io/en/latest/miniconda.html) and then, in your terminal, execute: 98 | 99 | ```shell 100 | > conda create -n test-tutorial python=3.10 pip 101 | > conda activate test-tutorial 102 | ``` 103 | 104 |
105 | 106 |   107 | ### Installing Required Packages 108 | 109 | At this step you are going to install some external packages and also install a local package located at `src/pbt_tutorial`, so the functions defined there are accessible by our notebooks. 110 | 111 | The external packages below will be installed: 112 | 113 | - numpy 114 | - notebook 115 | - pytest 116 | - hypothesis 117 | 118 | Navigate to the top level of this repo and than execute the command below to install all required packages: 119 | 120 | ```shell 121 | pip install -r requirements.txt 122 | ``` 123 | 124 |
125 | Details on the package installation 126 | 127 | The first line of the `requirements.txt` file provided is 128 | 129 | ```shell 130 | -e . 131 | ``` 132 | 133 | which actually means 134 | 135 | ```shell 136 | pip install -e . 137 | ``` 138 | 139 | This command is necessary because in this tutorial, you will be populating your own toy Python project, and will populate code under `./src/pbt_tutorial`. 140 | 141 | For this code to be available to import in our notebooks we need to make a "dev-mode" installation of this library in our environment. This is made by navigating to the top level of this repo and running: 142 | 143 | ```shell 144 | pip install -e . 145 | ``` 146 | 147 | The command will install `pbt_tutorial` so that any code that you add to it will immediately be reflected in your installation. 148 | 149 |
150 | 151 |   152 | 153 | After running the command above the code under `./src/pbt_tutorial` will be available to import. so if you add to `./src/pbt_tutorial/__init__.py` the function `def my_func(): return 1`, then you will be able to import that function: 154 | 155 | ```python 156 | >>> from pbt_tutorial import my_func 157 | >>> my_func() 158 | 1 159 | ``` 160 | 161 | ## For Instructors 162 | 163 | The source material for this tutorial is written in [jupytext-markdown](https://jupytext.readthedocs.io/en/latest/formats.html#jupytext-markdown) in lieu of native Jupyter notebooks. 164 | These are markdown files that can be opened as Jupyter notebooks. 165 | These markdown notebooks contain both the exercises and solutions. 166 | To convert these files to Jupyter notebooks to be distributed to the students, with the solutions excised, we are using the [cogbooks](https://github.com/CogWorksBWSI/Cogbooks) tool, which can be installed with: `pip install cogbooks` 167 | 168 | -------------------------------------------------------------------------------- /notebooks/00_Taxonomy_of_Testing_STUDENT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "ef4887f2", 6 | "metadata": {}, 7 | "source": [ 8 | "# Traversing the Taxonomy of Testing\n", 9 | "\n", 10 | "In this section we will take a tour through the taxonomy of testing: encountering manual tests, parameterized tests, and property-based tests.\n", 11 | "In the meanwhile, we will be introduced to the testing library Hypothesis, which provides us with automated test-case generation and reduction." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "e6d2c990", 17 | "metadata": {}, 18 | "source": [ 19 | "## Getting Started\n", 20 | "\n", 21 | "We expect your to write your to write pytest-style tests, which is exemplified by `test_count_vowels`, which can be found below at the beginning of the Manual Tests section.\n", 22 | "All this entails is defining a function that has the word \"test\" in its name, and place it in a Python script that has the word test in *its* name, in order for pytest to find-and-run that test function.\n", 23 | "\n", 24 | "You can run these tests from the terminal with:\n", 25 | "\n", 26 | "```shell\n", 27 | "pytest tests\n", 28 | "```\n", 29 | "\n", 30 | "presuming that you place your test scripts in the `tests/` directory.\n", 31 | "\n", 32 | "You can also configure IDEs to \"look for\" pytest-style tests and make individual tests and files easy to run from the IDE.\n", 33 | "The following links point to detailed instructions for configuring pytest with PyCharm and VSCode, respectively:\n", 34 | "\n", 35 | "- [Running tests in PyCharm](https://www.jetbrains.com/help/pycharm/pytest.html)\n", 36 | "- [Running tests in VSCode](https://code.visualstudio.com/docs/python/testing)\n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "id": "c9aa6f75", 42 | "metadata": {}, 43 | "source": [ 44 | "## Manual Tests\n", 45 | "\n", 46 | "A manual test will use inputs and corresponding outputs to a function that we have constructed \"by hand\", and simply assert that `func(hand_crafted_input) == expected_output`.\n", 47 | "This is the most rudimentary form of testing that one will encounter in software testing.\n", 48 | "For example, the following is a manual test of `pbt_tutorial.count_vowels`:\n", 49 | "\n", 50 | "```python\n", 51 | "from pbt_tutorial.basic_functions import count_vowels\n", 52 | "\n", 53 | "def test_count_vowels():\n", 54 | " # test basic strings with uppercase and lowercase letters\n", 55 | " assert count_vowels(\"aA bB yY\", include_y=False) == 2\n", 56 | " assert count_vowels(\"aA bB yY\", include_y=True) == 4\n", 57 | "```\n", 58 | "\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "id": "ecbab215", 64 | "metadata": {}, 65 | "source": [ 66 | "
\n", 67 | "\n", 68 | "**Exercise: Writing Manual Tests**\n", 69 | "\n", 70 | "The following tests should be placed in `tests/s0_pbt_101/test_manual.py`.\n", 71 | "\n", 72 | "Let's write some manual tests for `count_vowels` and `merge_max_mappings`, both of which are defined in `src/pbt_tutorial/basic_functions.py`.\n", 73 | "For each function:\n", 74 | " - Write a single test that includes multiple assertions, akin to `test_count_vowels` shown above.\n", 75 | " - In a comment or docstring associated with the test, document what sorts of behaviors and edge cases that you have covered with the test.\n", 76 | " - An example of an edge case might be: providing empty sequences/containers as inputs to your function\n", 77 | "\n", 78 | "(Are you finding this to be tedious? Good!)\n", 79 | "\n", 80 | " \n", 81 | "
\n" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "f0313013", 87 | "metadata": {}, 88 | "source": [ 89 | "## Parameterized Tests\n", 90 | "\n", 91 | "Looking back to both `test_count_vowels_basic` and `test_merge_max_mappings`, we see that there is a lot of redundancy within the bodies of these test functions.\n", 92 | "The assertions that we make within a given test-function share identical forms - they differ only in the parameters that we feed into our functions and their expected output.\n", 93 | "Another shortcoming of this test-structure is that a failing assertion will block subsequent assertions from being evaluated.\n", 94 | "That is, if the second assertion in `test_count_vowels_basic` fails, the third and fourth assertions will not be evaluated in that run.\n", 95 | "This precludes us from potentially seeing useful patterns among the failing assertions.\n", 96 | "\n", 97 | "pytest provides a useful tool that will allow us to eliminate these structural shortcomings by transforming our test-functions into so-called _parameterized tests_. Let's parametrize the following test:\n", 98 | "\n", 99 | "```python\n", 100 | "# a simple test with redundant assertions\n", 101 | "\n", 102 | "def test_range_length_unparameterized():\n", 103 | " assert len(range(0)) == 0\n", 104 | " assert len(range(1)) == 1\n", 105 | " assert len(range(2)) == 2\n", 106 | " assert len(range(3)) == 3\n", 107 | "```\n", 108 | "\n", 109 | "This test is checking the property `len(range(n)) == n`, where `n` is any non-negative integer.\n", 110 | "Thus, the parameter to be varied here is the \"size\" of the range-object being created.\n", 111 | "Let's treat it as such by using pytest to write a parameterized test:\n", 112 | "\n", 113 | "```python\n", 114 | "# parameterizing a test\n", 115 | "import pytest\n", 116 | "\n", 117 | "# note that this test must be run by pytest to work properly\n", 118 | "@pytest.mark.parametrize(\"size\", [0, 1, 2, 3])\n", 119 | "def test_range_length(size):\n", 120 | " assert len(range(size)) == size\n", 121 | "```\n", 122 | "\n", 123 | "\n", 124 | "Make note that a pytest-parameterized test must be run using pytest; an error will raise if we manually call `test_range_length()` from within a Python REPL.\n", 125 | "When executed, pytest will treat this parameterized test as _four separate tests_ - one for each parameter value:\n", 126 | "\n", 127 | "```\n", 128 | "test_basic_functions.py::test_range_length[0] PASSED [ 25%]\n", 129 | "test_basic_functions.py::test_range_length[1] PASSED [ 50%]\n", 130 | "test_basic_functions.py::test_range_length[2] PASSED [ 75%]\n", 131 | "test_basic_functions.py::test_range_length[3] PASSED [100%]\n", 132 | "```\n", 133 | "\n", 134 | "See that we have successfully eliminated the redundancy from `test_range_length`;\n", 135 | "the body of the function now contains only a single assertion, making obvious the property that is being tested.\n", 136 | "Furthermore, the four assertions are now being run independently from one another and thus we can potentially see patterns across multiple fail cases in concert." 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "id": "5dfb3dab", 142 | "metadata": {}, 143 | "source": [ 144 | "#### Parameterization Syntax\n", 145 | "\n", 146 | "The general form for creating a parameterizing decorator with *a single parameter*, as we formed above, is:\n", 147 | "\n", 148 | "```python\n", 149 | "@pytest.mark.parametrize(\"\", [, , ...])\n", 150 | "def test_function():\n", 151 | " ...\n", 152 | "```\n", 153 | "\n", 154 | "We will often have tests that require multiple parameters.\n", 155 | "The general form for creating the parameterization decorator for $N$ parameters,\n", 156 | "each of which assume $J$ values, is:\n", 157 | "\n", 158 | "```python\n", 159 | "@pytest.mark.parametrize(\", , [...], \", \n", 160 | " [(, , [...], ),\n", 161 | " (, , [...], ),\n", 162 | " ...\n", 163 | " (, , [...], ),\n", 164 | " ])\n", 165 | "def test_function(, , [...], ):\n", 166 | " ...\n", 167 | "```\n", 168 | "\n", 169 | "For example, let's take the following trivial test:\n", 170 | "\n", 171 | "```python\n", 172 | "def test_inequality_unparameterized():\n", 173 | " assert 1 < 2 < 3\n", 174 | " assert 4 < 5 < 6\n", 175 | " assert 7 < 8 < 9\n", 176 | " assert 10 < 11 < 12\n", 177 | "```\n", 178 | "\n", 179 | "and rewrite it in parameterized form. \n", 180 | "The decorator will have three distinct parameters, and each parameter, let's simply call them `a`, `b`, and `c`, will take on four values.\n", 181 | "\n", 182 | "```python\n", 183 | "# the parameterized form of `test_inequality_unparameterized`\n", 184 | "@pytest.mark.parametrize(\"a, b, c\", [(1, 2, 3), (4, 5, 6), (7, 8, 9), (10, 11, 12)])\n", 185 | "def test_inequality(a, b, c):\n", 186 | " assert a < b < c\n", 187 | "```" 188 | ] 189 | }, 190 | { 191 | "cell_type": "markdown", 192 | "id": "6a8ac20e", 193 | "metadata": {}, 194 | "source": [ 195 | "
\n", 196 | "\n", 197 | "**Exercise: Running a Parameterized Test**\n", 198 | "\n", 199 | "Create the file `tests/test_parameterized.py`.\n", 200 | "Add the parameterized tests `test_range_length` and `test_inequality`, which are defined above,\n", 201 | "and run them.\n", 202 | "
\n" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "id": "90b2b3b2", 208 | "metadata": {}, 209 | "source": [ 210 | "
\n", 211 | "\n", 212 | "**Exercise: Writing Parameterized Tests**\n", 213 | "\n", 214 | "Rewrite `test_count_vowels_basic` as a parameterized test with the parameters: `input_string`, `include_y`, and `expected_count`.\n", 215 | "\n", 216 | "Rewrite `test_merge_max_mappings` as a parameterized test with the parameters: `dict_a`, `dict_b`, and `expected_merged`.\n", 217 | "\n", 218 | "Before rerunning the tests in `test_basic_functions.py` predict how many distinct test cases will be reported by pytest. \n", 219 | "
\n" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "id": "da2ad1e8", 225 | "metadata": {}, 226 | "source": [ 227 | "
\n", 228 | "\n", 229 | "**Note**\n", 230 | "\n", 231 | "The formatting for multi-parameter tests can quickly become unwieldy.\n", 232 | "It isn't always obvious where one should introduce line breaks and indentations to improve readability.\n", 233 | "This is a place where the [\"black\" auto-formatter](https://black.readthedocs.io/en/stable/) really shines!\n", 234 | "Black will make all of these formatting decisions for us - we can write our parameterized tests as haphazardly as we like and simply run black to format our code. This is also a huge help with writing tests with Hypothesis, as you will see\n", 235 | "
\n" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "id": "6c0f0a51", 241 | "metadata": {}, 242 | "source": [ 243 | "## Interlude: A Brief Intro to Hypothesis\n", 244 | "\n", 245 | "It is often the case that the process of *describing our data* is by far the heaviest burden that we must bear when writing tests. This process of assessing \"what variety of values should I test?\", \"have I thought of all the important edge-cases?\", and \"how much is 'enough'?\" will crop up with nearly every test that we write.\n", 246 | "Indeed, these are questions that you may have been asking yourself when writing `test_count_vowels_basic` and `test_merge_max_mappings` in the previous sections of this module.\n", 247 | "\n", 248 | "[Hypothesis](https://hypothesis.readthedocs.io/) is a powerful Python library that empowers us to write a _description_ (specification, to be more precise) of the data that we want to use to exercise our test.\n", 249 | "It will then *generate* test cases that satisfy this description and will run our test on these cases.\n", 250 | "Let's witness this in its most basic form.\n", 251 | "\n", 252 | "The following is the basic anatomy of a Hypothesis test:\n", 253 | "\n", 254 | "```python\n", 255 | "from hypothesis import given\n", 256 | "import hypothesis.strategies as st\n", 257 | "\n", 258 | "@given(x=st.integers())\n", 259 | "def a_test_using_hypothesis(x):\n", 260 | " # x will be any integer\n", 261 | " # use `x` to test your function\n", 262 | " ...\n", 263 | "```\n", 264 | "\n", 265 | "The `@given` decorator is to be fed one or more *Hypothesis strategies* for describing data.\n", 266 | "In this example, we specified a single strategy, which describes arbitrary integer values.\n", 267 | "Executing `a_test_using_hypothesis()` will prompt Hypothesis to run the test's body _multiple times_ (100 times, by default); for run draws a new set of values from the strategies that we provided to `@given`.\n", 268 | "\n", 269 | "Let's write a simple example to help us see this behavior clearly:\n", 270 | "\n", 271 | "```python\n", 272 | "saves_numbers = []\n", 273 | "\n", 274 | "@given(x=st.integers())\n", 275 | "def a_test_using_hypothesis(x):\n", 276 | " saves_numbers.append(x)\n", 277 | "```\n", 278 | "\n", 279 | "Now, we execute the function and inspect the contents of the list `saves_numbers`, which gets mutated (appended to) each time the body of the function is run\n", 280 | "\n", 281 | "```python\n", 282 | "# unlike a pytest-parameterized test, tests using hypothesis\n", 283 | "# can be run outside of the pytest framework\n", 284 | ">>> a_test_using_hypothesis() # tells hypothesis to run the test body 100 times\n", 285 | ">>> len(saves_numbers)\n", 286 | "100\n", 287 | "\n", 288 | "# Note that statistics will vary from run-to-run\n", 289 | ">>> min(saves_numbers)\n", 290 | "-147572478458337740506626060645758832445\n", 291 | ">>> max(saves_numbers)\n", 292 | "117191644382309356634160262916607663738\n", 293 | ">>> sorted(saves_numbers)[50] # median\n", 294 | "50\n", 295 | "```\n", 296 | "\n", 297 | "There is a [whole suite of strategies](https://hypothesis.readthedocs.io/en/latest/data.html) available for describing various sorts of data with Hypothesis.\n", 298 | "We will dive into the process of describing data in rich ways using these in the next section.\n", 299 | "Prior to doing so, we will use our nascent understanding of Hypothesis to round out the taxonomy of tests, and study fuzz tests and property-based tests." 300 | ] 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "id": "6d2bf9cc", 305 | "metadata": {}, 306 | "source": [ 307 | "### Hypothesis Performs Automated Test-Case Reduction\n", 308 | "\n", 309 | "Before proceeding, there is an important feature that Hypothesis offers and that we should come to appreciate.\n", 310 | "Try running the following test:" 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "execution_count": null, 316 | "id": "8c9bcaa5", 317 | "metadata": {}, 318 | "outputs": [], 319 | "source": [ 320 | "from hypothesis import given \n", 321 | "import hypothesis.strategies as st\n", 322 | "\n", 323 | "# using `given` with multiple parameters\n", 324 | "# `x` is an integer drawn from [0, 10]\n", 325 | "# `y` is an integer drawn from [20, 30]\n", 326 | "@given(x=st.integers(0, 10), y=st.integers(20, 30))\n", 327 | "def test_demonstrating_the_given_decorator(x, y):\n", 328 | " assert 0 <= x <= 10\n", 329 | " \n", 330 | " # `y` can be any value in [20, 30]\n", 331 | " # this is a bad assertion: it should fail!\n", 332 | " assert 20 <= y <= 25" 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "execution_count": null, 338 | "id": "51783fd6", 339 | "metadata": {}, 340 | "outputs": [], 341 | "source": [ 342 | "test_demonstrating_the_given_decorator()" 343 | ] 344 | }, 345 | { 346 | "cell_type": "markdown", 347 | "id": "8b300865", 348 | "metadata": {}, 349 | "source": [ 350 | "Hypothesis should report:\n", 351 | "\n", 352 | "```\n", 353 | "Falsifying example: test_demonstrating_the_given_decorator(\n", 354 | " x=0, y=26,\n", 355 | ")\n", 356 | "```\n", 357 | "\n", 358 | "It may seem lucky that it reported the smallest value for `y` that violates the bad inequality, but it is not luck at all!\n", 359 | "This is a feature of Hypothesis known as **automated test-case reduction**.\n", 360 | "That is, `given` decorator strives to report the \"simplest\" set of input values that produce a given error.\n", 361 | "It does this through the process of **shrinking**.\n", 362 | "\n", 363 | "Each of Hypothesis' strategies has its own prescribed shrinking behavior. For the integers strategy, this means identifying the integer closest to 0 that produces the error at hand.\n", 364 | "\n", 365 | "When running the above test, Hypothesis could have encountered the following sequence of examples during its shrinking phase:\n", 366 | "\n", 367 | "```\n", 368 | "x=0 y=20 - PASSED\n", 369 | "x=0 y=20 - PASSED\n", 370 | "x=0 y=20 - PASSED\n", 371 | "x=9 y=20 - PASSED\n", 372 | "x=9 y=21 - PASSED\n", 373 | "x=3 y=20 - PASSED\n", 374 | "x=3 y=20 - PASSED\n", 375 | "x=9 y=26 - FAILED\n", 376 | "x=3 y=26 - FAILED\n", 377 | "x=6 y=26 - FAILED\n", 378 | "x=10 y=27 - FAILED\n", 379 | "x=7 y=27 - FAILED\n", 380 | "x=3 y=30 - FAILED\n", 381 | "x=3 y=23 - PASSED\n", 382 | "x=10 y=30 - FAILED\n", 383 | "x=3 y=27 - FAILED\n", 384 | "x=3 y=27 - FAILED\n", 385 | "x=2 y=27 - FAILED\n", 386 | "x=0 y=27 - FAILED\n", 387 | "x=0 y=26 - FAILED\n", 388 | "x=0 y=21 - PASSED\n", 389 | "x=0 y=25 - PASSED\n", 390 | "x=0 y=22 - PASSED\n", 391 | "x=0 y=23 - PASSED\n", 392 | "x=0 y=24 - PASSED\n", 393 | "x=0 y=26 - FAILED\n", 394 | "```\n", 395 | "\n", 396 | "See that Hypothesis has to do a semi-random search to identify the boundaries of the fail case; it doesn't know if `x` is causing the error, or if `y` is the culprit, or if it is specific combinations of x and y that causes the failure!\n", 397 | "Despite this complexity, the pairs of variables are successfully shrunk to the simplest fail case." 398 | ] 399 | }, 400 | { 401 | "cell_type": "markdown", 402 | "id": "f8f4232e", 403 | "metadata": {}, 404 | "source": [ 405 | "### Hypothesis will Save Falsifying Examples:\n", 406 | "\n", 407 | "Albeit an advanced detail, it is important to note that Hypothesis does not have to search for falsifying examples from scratch every time we run a test function. Instead, Hypothesis will save a database of falsifying examples associated with each of your project's test functions. The database is saved under .hypothesis in whatever directory your test functions were run from.\n", 408 | "\n", 409 | "This ensures that, once Hypothesis finds a falsifying example for a test, the falsifying example will be passed to your test function each time you run it, until it no longer raises an error in your code (e.g. you update your code to fix the bug that was causing the test failure)." 410 | ] 411 | }, 412 | { 413 | "cell_type": "markdown", 414 | "id": "0e30eea7", 415 | "metadata": {}, 416 | "source": [ 417 | "
\n", 418 | "\n", 419 | "**Exercise: Understanding How Hypothesis Works**\n", 420 | "\n", 421 | "Create the file `tests/test_with_hypothesis.py`.\n", 422 | "\n", 423 | "Copy the `test_demonstrating_the_given_decorator` function from above - complete with the failing assertion - and add a print statement to the body of the function, which prints out the value for `x` and `y`.\n", 424 | "\n", 425 | "Run the test once and make note of the output that is printed. Consider copying and pasting the output to a notepad for reference. Next, rerun the test multiple times and make careful note of the printed output. What do you see? Is the output different from the first run? Does it differ between subsequent runs? Try to explain this behavior.\n", 426 | "\n", 427 | "In your file browser, navigate to the directory from which you are running this test; if you are following along in a Jupyter notebook, this is simply the directory containing said notebook. You should see a `.hypothesis` directory. As noted above, this is the database that contains the falsifying examples that Hypothesis has identified. Delete the `.hypothesis` directory and try re-running your test? What do you notice about the output now? You should also see that the `.hypothesis` directory has reappeared. Explain what is going on here.\n", 428 | "
\n", 429 | "\n", 430 | "\n", 431 | "\n" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "id": "0570716e", 437 | "metadata": {}, 438 | "source": [ 439 | "
\n", 440 | "\n", 441 | "**Exercise: Fixing the Failing Test**\n", 442 | "\n", 443 | "Update the body of test_demonstrating_the_given_decorator so that it no longer fails. Run the fixed test function. How many times is the test function actually be executed when you run it?\n", 444 | "
\n", 445 | "\n", 446 | "\n", 447 | "\n" 448 | ] 449 | }, 450 | { 451 | "cell_type": "markdown", 452 | "id": "72f370ab", 453 | "metadata": {}, 454 | "source": [ 455 | "**Now back to studying different styles of testing!** \n", 456 | "\n", 457 | "## Fuzz Testing\n", 458 | "\n", 459 | "Fuzz testing is a simple, but often (embarrassingly) powerful approach to automated testing in which we feed a function a wide variety of randomly-generated inputs to see if it ever crashes.\n", 460 | "For example, the following test \"fuzzes\" the `int` type with finite float inputs, to see if `int()` ever causes a crash.\n", 461 | "\n", 462 | "```python\n", 463 | "@given(x=st.floats(allow_infinity=False, allow_nan=False))\n", 464 | "def test_fuzz_in_with_finite_floats(x):\n", 465 | " int(x) # no asserts needed!\n", 466 | "```\n", 467 | "\n", 468 | "while this test doesn't do much to assure us that `int` casts floats *correctly*, it is nonetheless a very simple and expressive test that gives us confidence that feeding a finite float will never cause `int` to crash.\n", 469 | "We should appreciate just how trivial it is to write this test.\n", 470 | "\n", 471 | "Fuzzing can be especially useful if we have our source code with internal internal `assert` statements.\n", 472 | "Seeing that these assertions hold true even when our function is being fed truly bizarre and exotic (but valid!) input data helps close the aperture on where future bugs are coming from – we can at least be confident that they aren't coming from incorrect assertions.\n", 473 | "\n", 474 | "([Seriously! Fuzzing can be very effective](https://github.com/google/oss-fuzz))" 475 | ] 476 | }, 477 | { 478 | "cell_type": "markdown", 479 | "id": "0d96bc8a", 480 | "metadata": {}, 481 | "source": [ 482 | "
\n", 483 | "\n", 484 | "**Exercise: Running a Parameterized Test**\n", 485 | "\n", 486 | "Create the file `tests/test_fuzz.py`.\n", 487 | "Use the Hypothesis strategies [text()](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.text) and [booleans()](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.booleans) to fuzz the `count_vowels` function.\n", 488 | " \n", 489 | "Consider temporarily adding a print statement to your test to get a peek at the sort of data that you are fuzzing your function with.\n", 490 | " \n", 491 | "The more inputs you feed a function in a fuzz test, the better. Take a glance at the documentation for [Hypothesis' settings](https://hypothesis.readthedocs.io/en/latest/settings.html) and increase the number of examples run in your fuzz test from 100 to 1,000.\n", 492 | "
\n", 493 | "\n", 494 | "\n", 495 | "\n" 496 | ] 497 | }, 498 | { 499 | "cell_type": "markdown", 500 | "id": "49398e32", 501 | "metadata": {}, 502 | "source": [ 503 | "Fuzzing is convenient and powerful because it enables us to run diverse inputs into our function without requiring us to figure out what the respective outputs of our function should be.\n", 504 | "That being said, testing that your function simply doesn't crash doesn't tell us much about whether or not the function is actually doing its job.\n", 505 | "We'll see that property-based testing can help us test the correctness of our code while maintaining much of the simplicity and verbosity of a fuzz-style test." 506 | ] 507 | }, 508 | { 509 | "cell_type": "markdown", 510 | "id": "0daad3b3", 511 | "metadata": {}, 512 | "source": [ 513 | "## Property-Based Tests\n", 514 | "\n", 515 | "Property-based testing (PBT) also enables us to exercise our function with a diverse set of inputs and without knowing precisely what the corresponding outputs should be. But, unlike with fuzz testing, these tests will check that our code behaves correctly beyond not-crashing.\n", 516 | "Here we will identify properties that should be satisfied by our function and we will test that each of these properties hold for a broad and diverse set of inputs to our function.\n", 517 | "\n", 518 | "Just as Hypothesis' strategies enable us to fuzz-test our code with highly diverse inputs, they will serve to drive our property-based tests as well.\n", 519 | "\n", 520 | "As an example, let's write some property-based tests for Python's `sorted` function and make sure that it properly sorts lists of integers & floats correctly along with lists of strings." 521 | ] 522 | }, 523 | { 524 | "cell_type": "markdown", 525 | "id": "78971d73", 526 | "metadata": {}, 527 | "source": [ 528 | "```python\n", 529 | "from collections import Counter\n", 530 | "\n", 531 | "from hypothesis import given\n", 532 | "import hypothesis.strategies as st\n", 533 | "\n", 534 | "\n", 535 | "@given(\n", 536 | " arg=st.one_of(\n", 537 | " st.lists(st.integers() | st.floats(allow_nan=False)), # nans aren't orderable\n", 538 | " st.lists(st.text()),\n", 539 | " )\n", 540 | ")\n", 541 | "def test_sorted_property_based(arg):\n", 542 | " result = sorted(arg)\n", 543 | " \n", 544 | " # Property 1:\n", 545 | " # Check that entries of `result` are in ascending order.\n", 546 | " for a, b in zip(result, result[1:]):\n", 547 | " assert a <= b\n", 548 | " \n", 549 | " # Property 2:\n", 550 | " # Check that `result` and `arg` contain the same elements,\n", 551 | " # disregarding their order\n", 552 | " assert Counter(result) == Counter(arg)\n", 553 | "```" 554 | ] 555 | }, 556 | { 557 | "cell_type": "markdown", 558 | "id": "d4a06d25", 559 | "metadata": {}, 560 | "source": [ 561 | "This is a simple but powerful test!\n", 562 | "Because we are using Hypothesis to generate the inputs to this test, we are exercising `sorted` with lists of varying lengths (including empty lists), of all redundant entries, containing positive and negative infinities, and so on.\n", 563 | "Furthermore, we were able to identify two simple properties that, taken together, assure that `result` is indeed a sorted list.\n", 564 | "Put another way, any manual test that we could write for `sorted` would be redundant with the assurances provided to us by `test_sorted_property_based`.\n", 565 | "Lastly, consider how much more tedious and brittle it would be to write parameterized test that exercises a comparably-diverse set of inputs and desired outputs! \n", 566 | "\n", 567 | "It is important to note that property-based testing does not require you to identify a \"complete\" set of properties that fully guarantee correctness in our code.\n", 568 | "Let's consider the following property-based test of our `count_vowels` function:\n", 569 | "\n", 570 | "```python\n", 571 | "@given(\n", 572 | " in_string=st.text(alphabet=string.printable, max_size=20),\n", 573 | " include_y=st.booleans(),\n", 574 | " num_repeat=st.integers(0, max_value=100),\n", 575 | ")\n", 576 | "def test_count_vowels_property_based(in_string: str, include_y: bool, num_repeat: int):\n", 577 | " num_vowels = count_vowels(in_string, include_y)\n", 578 | " \n", 579 | " # Property 1:\n", 580 | " # `num_vowels` must be non-negative and cannot\n", 581 | " # exceed the length of the string itself\n", 582 | " assert 0 <= num_vowels <= len(in_string)\n", 583 | "\n", 584 | " # Property 2:\n", 585 | " # `N * in_string` must have N-times as many\n", 586 | " # vowels as `in_string`\n", 587 | " assert count_vowels(num_repeat * in_string, include_y) == num_repeat * num_vowels\n", 588 | "\n", 589 | " # Property 3:\n", 590 | " # The vowel count should be invariant to the string's ordering\n", 591 | " # Note: We can use hypothesis to shuffle our string here, but\n", 592 | " # that will require use of a more advanced feature that \n", 593 | " # we will learn about later.\n", 594 | " assert count_vowels(in_string[::-1], include_y) == num_vowels\n", 595 | " assert count_vowels(\"\".join(sorted(in_string)), include_y) == num_vowels\n", 596 | "\n", 597 | " # Property 4:\n", 598 | " # Vowel count is case-insensitive\n", 599 | " assert count_vowels(in_string.upper(), include_y) == num_vowels\n", 600 | " assert count_vowels(in_string.lower(), include_y) == num_vowels\n", 601 | "```\n", 602 | "\n", 603 | "Testing these properties alone shouldn't make us confident that `count_vowels` is behaving as expected; after all, we never actually verify that the vowel count is exactly correct.\n", 604 | "But testing these properties *alongside some manual test cases* is a **very** powerful combination.\n", 605 | "\n", 606 | "Consider the following smattering of manual tests:\n", 607 | "\n", 608 | "```python\n", 609 | "@pytest.mark.parametrize(\n", 610 | " \"input_string, include_y, expected_count\",\n", 611 | " [\n", 612 | " (\"aA bB yY\", False, 2),\n", 613 | " (\"aA bB yY\", True, 4),\n", 614 | " (\"123bacediouyz\", False, 5),\n", 615 | " (\"b a c e d i o u y z\", True, 6),\n", 616 | " ],\n", 617 | ")\n", 618 | "def test_count_vowels_parameterized(\n", 619 | " input_string: str, include_y: bool, expected_count: int\n", 620 | "):\n", 621 | " assert count_vowels(input_string, include_y) == expected_count\n", 622 | "```\n", 623 | "\n", 624 | "These manual test cases are far from exhaustive, but when viewed in conjunction with the properties that we tested above, we suddenly realize that `count_vowels` is being tested quite thoroughly!\n", 625 | "Take some time to reflect on how much more expressive each of these test cases become when we have the additional assurance that they also satisfy the properties tested above.\n", 626 | "It should be clear that we aren't explicitly testing that the properties hold for these parameterized cases, but given that Hypothesis is generating *much* more diverse (and plentiful) examples in our PBT, we can be confident that the properties hold for our manual cases as well. " 627 | ] 628 | }, 629 | { 630 | "cell_type": "markdown", 631 | "id": "b511afe4", 632 | "metadata": {}, 633 | "source": [ 634 | "
\n", 635 | "\n", 636 | "**Exercise: Running a Hypothesis-Driven, Property-Based Test**\n", 637 | "\n", 638 | "Create the file `tests/test_properties.py` and add the `test_count_vowels_property_based` test to it.\n", 639 | "Are there any other properties that you might test?\n", 640 | "Run your test suite and verify that this test is run and that it passes.\n", 641 | " \n", 642 | "Revisit the parameterized tests that you wrote for `count_vowels` and consider adding some additional cases that, in conjunction with these property based tests, will help improve our confidence in `count_vowels`.\n", 643 | "\n", 644 | "\n", 645 | "
\n", 646 | "\n", 647 | "\n", 648 | "\n" 649 | ] 650 | }, 651 | { 652 | "cell_type": "markdown", 653 | "id": "d8e364d7", 654 | "metadata": {}, 655 | "source": [ 656 | "
\n", 657 | "\n", 658 | "**Exercise: Writing Your Own Property-Based Test**\n", 659 | "\n", 660 | "Add to `tests/test_properties.py` a property-based test for `merge_max_mappings`.\n", 661 | "\n", 662 | "You can use the following strategy to generate dictionaries with strings as keys and numbers as values:\n", 663 | " \n", 664 | "```python\n", 665 | "st.dictionaries(keys=st.text(), values=st.integers() | st.floats(include_nan=False))\n", 666 | "```\n", 667 | " \n", 668 | "Can you identify a minimal and \"complete\" set of properties to test here? \n", 669 | "\n", 670 | "
\n" 671 | ] 672 | }, 673 | { 674 | "cell_type": "markdown", 675 | "id": "41c5fa6d", 676 | "metadata": {}, 677 | "source": [ 678 | "
\n", 679 | "\n", 680 | "**Exercise: Hypothesis, the automated CS tutor**\n", 681 | "\n", 682 | "Suppose that you were responsible for writing tests for Python's all-important [`range()`](https://docs.python.org/3/library/functions.html#func-range).\n", 683 | "Using Hypothesis, write a test that verifies the property that `len(range(size)) == size`, where `size` is any non-negative integer.\n", 684 | "\n", 685 | "Go ahead and write/run the test...\n", 686 | "\n", 687 | ".\n", 688 | " \n", 689 | ".\n", 690 | " \n", 691 | ".\n", 692 | "\n", 693 | "You may be surprised to find that `len(range(size)) == size` does *not* pass for arbitrary non-negative integers.\n", 694 | "Instead, this test reveals that the CPython implementation of the built-in `len` function is such that it can only handle non-negative integers smaller than $2^{63}$ (i.e. it will only allocate 64 bits to represent a signed integer - one bit is used to store the sign of the number).\n", 695 | "Hypothesis should have revealed this by generating the failing test case `size=9223372036854775808`, which is exactly $2^{63}$.\n", 696 | "\n", 697 | "Hypothesis has a knack for catching these sorts of unexpected edge cases!\n", 698 | "In fact, this overflow behavior is now documented in the CPython source code because it was discovered while writing this material 😄.\n", 699 | "Thus you can think of Hypothesis as your personal CS tutor.\n", 700 | "It will tell you things about Python that you never knew that you wanted to know.\n", 701 | "
\n" 702 | ] 703 | }, 704 | { 705 | "cell_type": "markdown", 706 | "id": "4838df58", 707 | "metadata": {}, 708 | "source": [ 709 | "## Extra: Test-Driven Development\n", 710 | "\n", 711 | "\n", 712 | "We want to complete the following function:\n", 713 | "\n", 714 | "```python\n", 715 | "def leftpad(string: str, width: int, fillchar: str) -> str:\n", 716 | " \"\"\"Left-pads `string` with `fillchar` until the resulting string\n", 717 | " has length `width`.\n", 718 | " \n", 719 | " Parameters\n", 720 | " ----------\n", 721 | " string : str\n", 722 | " The input string\n", 723 | " \n", 724 | " width : int\n", 725 | " A non-negative integer specifying the minimum guaranteed\n", 726 | " width of the left-padded output string.\n", 727 | " \n", 728 | " fillchar : str\n", 729 | " The character (length-1 string) used to pad the string.\n", 730 | " \n", 731 | " Examples\n", 732 | " --------\n", 733 | " The following is the intended behaviour of this function:\n", 734 | " \n", 735 | " >>> leftpad('cat', width=5, fillchar=\"Z\")\n", 736 | " 'ZZcat'\n", 737 | " >>> leftpad('Dog', width=2, fillchar=\"Z\")\n", 738 | " 'Dog'\n", 739 | " \"\"\"\n", 740 | " assert isinstance(width, int) and width >= 0, width\n", 741 | " assert isinstance(fillchar, str) and len(fillchar) == 1, fillchar\n", 742 | " # YOUR CODE HERE\n", 743 | "```\n", 744 | "\n", 745 | "But let's use test-driven development to do so.\n", 746 | "That is, include the above function stub in `pbt_tutorial/basic_function.py` (without completing the function) and then begin writing one or more property-based tests for it in `tests/test_properties.py`.\n", 747 | "From the outset your test should fail, since the function hasn't been implemented.\n", 748 | "\n", 749 | "Once you are satisfied with your property-based test for `leftpad`, proceed to complete your implementation of the function and use your test to drive your development (e.g. rely on it to tell you if you have gotten something wrong or have missed any edge cases). " 750 | ] 751 | } 752 | ], 753 | "metadata": { 754 | "kernelspec": { 755 | "display_name": "Python 3 (ipykernel)", 756 | "language": "python", 757 | "name": "python3" 758 | } 759 | }, 760 | "nbformat": 4, 761 | "nbformat_minor": 5 762 | } 763 | -------------------------------------------------------------------------------- /notebooks/01_Describing_Data_STUDENT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "84e2858e", 6 | "metadata": {}, 7 | "source": [ 8 | "# Describing Your Data with Hypothesis" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "id": "23710e57", 14 | "metadata": {}, 15 | "source": [ 16 | "## Useful Links\n", 17 | "- [Hypothesis' core strategies](https://hypothesis.readthedocs.io/en/latest/data.html)\n", 18 | "- [The `.map` method](https://hypothesis.readthedocs.io/en/latest/data.html#mapping)\n", 19 | "- [The `.filter` method](https://hypothesis.readthedocs.io/en/latest/data.html#filtering)\n", 20 | "- [The `example` decorator](https://hypothesis.readthedocs.io/en/latest/reproducing.html#providing-explicit-examples)\n", 21 | "- [Using `data()` to draw interactively in tests](https://hypothesis.readthedocs.io/en/latest/data.html#drawing-interactively-in-tests)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "markdown", 26 | "id": "f66248a6", 27 | "metadata": {}, 28 | "source": [ 29 | "## Hypothesis \"Strategies\"\n", 30 | "\n", 31 | "As we learned in the previous section, Hypothesis provides us with so-called \"strategies\" for describing our data.\n", 32 | "These are all located in the `hypothesis.strategies` module. The official documentation for the core strategies can be found [here](https://hypothesis.readthedocs.io/en/latest/data.html).\n", 33 | "\n", 34 | "We will import this module under the alias `st` throughout this tutorial:\n", 35 | "\n", 36 | "```python\n", 37 | "import hypothesis.strategies as st\n", 38 | "```\n", 39 | "\n", 40 | "A strategy, once initialized, is an object that Hypothesis will use to generate data for our test. **The `given` decorator is responsible for drawing values from strategies and passing them as inputs to our test.**\n", 41 | "\n", 42 | "For example, let's write a toy test for which `x` should be integers between 0 and 10, and `y` should be integers between 20 and 30:" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "id": "2040317f", 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "import hypothesis.strategies as st\n", 53 | "from hypothesis import given\n", 54 | "\n", 55 | "# using `given` with multiple parameters\n", 56 | "@given(x=st.integers(0, 10), y=st.integers(20, 30))\n", 57 | "def demonstrating_the_given_decorator(x, y):\n", 58 | " assert 0 <= x <= 10\n", 59 | " assert 20 <= y <= 30" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "id": "61fbdf9b", 65 | "metadata": {}, 66 | "source": [ 67 | "Running the test will prompt Hypothesis to draw 100 random pairs of values for `x` and `y`, according to their respective strategies, and the body of the test will be executed for each such pair:" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "id": "c36afc17", 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "demonstrating_the_given_decorator()" 78 | ] 79 | }, 80 | { 81 | "cell_type": "markdown", 82 | "id": "8dc440c7", 83 | "metadata": {}, 84 | "source": [ 85 | "Note that it is always advisable to specify the parameters of `given` as keyword arguments, so that the correspondence between our strategies with the function-signature's parameters are manifestly clear." 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "id": "c46e9dfb", 91 | "metadata": {}, 92 | "source": [ 93 | "
\n", 94 | "\n", 95 | "**Exercise: Describing data with `booleans()`**\n", 96 | "\n", 97 | "Write a test that takes in a parameter `x`, that is a boolean value. Use the [`st.booleans()` strategy](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.booleans) to describe the data. In the body of the test, assert that `x` is either `True` or `False`.\n", 98 | "\n", 99 | "Note that the `booleans()` strategy shrinks to `False`. \n", 100 | " \n", 101 | "Run your test, and try mutating it to ensure that it can fail appropriately.\n", 102 | "\n", 103 | "
\n" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "id": "14121c81", 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "# Write a test that uses the `st.booleans()` strategy\n", 114 | "# STUDENT CODE HERE" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "id": "f8e788fa", 120 | "metadata": {}, 121 | "source": [ 122 | "### Viewing examples drawn from strategies\n", 123 | "\n", 124 | "Hypothesis provides a useful mechanism for developing an intuition for the data produced by a strategy. A strategy, once initialized, has a `.example()` method that will randomly draw a representative value from the strategy. For example:\n", 125 | "\n", 126 | "```python\n", 127 | "# demonstrating usage of `.example()`\n", 128 | ">>> st.integers(-1, 1).example()\n", 129 | "-1\n", 130 | "\n", 131 | ">>> st.booleans()\n", 132 | "True\n", 133 | "```\n", 134 | "\n", 135 | "**Note: the `.example()` mechanism is only meant to be used for pedagogical purposes. You should never use this in your test suite**\n", 136 | "because (among other reasons) `.example()` biases towards smaller and simpler examples than `@given`, and lacks the features to ensure any test failures are reproducible." 137 | ] 138 | }, 139 | { 140 | "cell_type": "markdown", 141 | "id": "49eb9419", 142 | "metadata": {}, 143 | "source": [ 144 | "
\n", 145 | "\n", 146 | "**Exercise: Write a `print_examples` function**\n", 147 | "\n", 148 | "Write a function called `print_examples` that takes in two arguments:\n", 149 | " - an initialized hypothesis strategy (e.g. `st.integers(0, 10)`)\n", 150 | " - `n`: the number of examples to print\n", 151 | " \n", 152 | " Have it print the strategy (simply call `print` on the strategy) and `n` examples generated from the strategy. Use a for-loop.\n", 153 | "\n", 154 | "
\n" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": null, 160 | "id": "ff310771", 161 | "metadata": {}, 162 | "outputs": [], 163 | "source": [ 164 | "# Define the `print_examples` function\n", 165 | "# STUDENT CODE HERE" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "id": "c705ebb3", 171 | "metadata": {}, 172 | "source": [ 173 | "Print five examples from the `st.integers(...)` strategy, with values ranging from -10 to 10. Then print 10 examples from the `st.booleans()` strategy." 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": null, 179 | "id": "3b2c8465", 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "# STUDENT CODE HERE" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "id": "80ede67d", 190 | "metadata": {}, 191 | "outputs": [], 192 | "source": [ 193 | "# STUDENT CODE HERE" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "id": "b0c57210", 199 | "metadata": {}, 200 | "source": [ 201 | "### The `.map` method\n", 202 | "\n", 203 | "Hypothesis strategies have the `.map` method, which permits us to perform a mapping on the data being produced by a strategy. For example, if we want to draw only even-valued integers, we can simply use the following mapped strategy:\n", 204 | "\n", 205 | "```python\n", 206 | "# we can apply mappings to strategies\n", 207 | "even_integers = st.integers().map(lambda x: 2 * x)\n", 208 | "```\n", 209 | "(`.map` can be passed any callable, it need not be a lambda)" 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "id": "eff2ba0b", 215 | "metadata": {}, 216 | "source": [ 217 | "
\n", 218 | "\n", 219 | "**Exercise: Understanding the `.map` method**\n", 220 | "\n", 221 | "Write a test that uses the afore-defined `even_integers` strategy to generate data. The body of the test should assert that the input to the test is indeed an even-valued integer. \n", 222 | "\n", 223 | "Run your test (and try adding a bad assertion to make sure that the test can actually fail!)\n", 224 | "\n", 225 | "
\n" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "id": "a113fb20", 232 | "metadata": {}, 233 | "outputs": [], 234 | "source": [ 235 | "# write the test for `even_integers`\n", 236 | "\n", 237 | "# STUDENT CODE HERE" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "id": "25d876a5", 243 | "metadata": {}, 244 | "source": [ 245 | "
\n", 246 | "\n", 247 | "**Exercise: Getting creative with the `.map` method**\n", 248 | "\n", 249 | "Construct a Hypothesis strategy that produces either the string `\"cat\"` or the string `\"dog\"`.\n", 250 | "Write it so that the strategy shrinks to `\"dog\"`.\n", 251 | "\n", 252 | "Then write a test that checks the property of this strategy and run it\n", 253 | "\n", 254 | "
\n", 255 | "\n" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": null, 261 | "id": "bb5734f3", 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "# Write the cat-or-dog strategy\n", 266 | "# STUDENT CODE HERE" 267 | ] 268 | }, 269 | { 270 | "cell_type": "markdown", 271 | "id": "4b21ecd0", 272 | "metadata": {}, 273 | "source": [ 274 | "### The `.filter` method\n", 275 | "\n", 276 | "Hypothesis strategies can also have their data filtered via the `.filter` method. `.filter` takes a callable that accepts as input the data generated by the strategy, and returns:\n", 277 | " - `True` if the data should pass through the filter\n", 278 | " - `False` if the data should be rejected by the filter\n", 279 | "\n", 280 | "Consider, for instance, that you want to generate all integers other than `0`. You can write the filtered strategy:\n", 281 | "\n", 282 | "```python\n", 283 | "non_zero_integers = st.integers().filter(lambda x: x != 0)\n", 284 | "```" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "id": "8adeb447", 290 | "metadata": {}, 291 | "source": [ 292 | "
\n", 293 | "\n", 294 | "**Exercise: Understanding the `.filter` method**\n", 295 | "\n", 296 | "Write a test that uses the afore-defined `non_zero_integers` strategy to generate data. The body of the test should assert that the input to the test is indeed a nonzero integer. \n", 297 | "\n", 298 | "Run your test (and make sure that it can fail!)\n", 299 | "\n", 300 | "
\n" 301 | ] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "execution_count": null, 306 | "id": "4ec44675", 307 | "metadata": {}, 308 | "outputs": [], 309 | "source": [ 310 | "# Write the test for `non_zero_integers`\n", 311 | "# STUDENT CODE HERE" 312 | ] 313 | }, 314 | { 315 | "cell_type": "markdown", 316 | "id": "4bc052bf", 317 | "metadata": {}, 318 | "source": [ 319 | "The `.filter` method is not magic. Hypothesis will raise an error if your strategy rejects too many values. " 320 | ] 321 | }, 322 | { 323 | "cell_type": "markdown", 324 | "id": "25544d29", 325 | "metadata": {}, 326 | "source": [ 327 | "
\n", 328 | "\n", 329 | "**Exercise: The `.filter` method is not magic**\n", 330 | "\n", 331 | "Write a strategy that applies a filter to `st.floats(allow_nan=False)` such that it only generates values on the domain `[10, 20]`.\n", 332 | "Then write and run a test that that uses data from this strategy.\n", 333 | "\n", 334 | "What is the name of the error that Hypothesis raises? How would you rewrite this strategy instead of using `.filter`?\n", 335 | "\n", 336 | "
\n" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": null, 342 | "id": "7bf2f02c", 343 | "metadata": {}, 344 | "outputs": [], 345 | "source": [ 346 | "# STUDENT CODE HERE" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "id": "db19d279", 352 | "metadata": {}, 353 | "source": [ 354 | "## The `example` Decorator\n", 355 | "\n", 356 | "As mentioned before, Hypothesis strategies will draw values (pseudo)*randomly*.\n", 357 | "Thus our test will potentially encounter different values every time it is run.\n", 358 | "There are times where we want to be sure that, in addition the values produced by a strategy, specific values will tested. \n", 359 | "These might be known edge cases, critical use cases, or regression cases (i.e. values that were representative of passed bugs).\n", 360 | "Hypothesis provides [the `example` decorator](https://hypothesis.readthedocs.io/en/latest/reproducing.html#providing-explicit-examples), which is to be used in conjunction with the `given` decorator, towards this end.\n", 361 | "\n", 362 | "Let's suppose, for example, that we want to write a test whose data are pairs of perfect-squares (e.g. 4, 16, 25, ...), and that we want to be sure that the pairs `(100, 144)`, `(16, 25)`, and `(36, 36)` are tested *every* time the test is run. \n", 363 | "Let's use `example` to guarantee this." 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": null, 369 | "id": "c4c392ae", 370 | "metadata": {}, 371 | "outputs": [], 372 | "source": [ 373 | "from hypothesis import example\n", 374 | "\n", 375 | "perfect_squares = st.integers().map(lambda x: x ** 2)\n", 376 | "\n", 377 | "\n", 378 | "def is_square(x):\n", 379 | " return int(x ** 0.5) == x ** 0.5\n", 380 | "\n", 381 | "\n", 382 | "@example(a=36, b=36)\n", 383 | "@example(a=16, b=25)\n", 384 | "@example(a=100, b=144)\n", 385 | "@given(a=perfect_squares, b=perfect_squares)\n", 386 | "def test_pairs_of_squares(a, b):\n", 387 | " assert is_square(a)\n", 388 | " assert is_square(b)\n", 389 | "\n", 390 | "\n", 391 | "test_pairs_of_squares()" 392 | ] 393 | }, 394 | { 395 | "cell_type": "markdown", 396 | "id": "68721cad", 397 | "metadata": {}, 398 | "source": [ 399 | "Executing this test runs 103 cases: the three specified examples and one hundred pairs of values drawn via `given`." 400 | ] 401 | }, 402 | { 403 | "cell_type": "markdown", 404 | "id": "36536b3a", 405 | "metadata": {}, 406 | "source": [ 407 | "## Exploring Strategies\n", 408 | "\n", 409 | "There are a number critical Hypothesis strategies for us to become familiar with. It is worthwhile to peruse through all of Hypothesis' [core strategies](https://hypothesis.readthedocs.io/en/latest/data.html#core-strategies), but we will take time to highlight a few here.\n", 410 | "\n", 411 | "### [`st.lists ()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.lists)\n", 412 | "\n", 413 | "[`st.lists`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.lists) accepts *another* strategy, which describes the elements of the lists being generated. You can also specify:\n", 414 | " - bounds on the length of the list\n", 415 | " - if we want the elements to be unique\n", 416 | " - a mechanism for defining \"uniqueness\"\n", 417 | " \n", 418 | "**`st.lists(...)` is the strategy of choice anytime we want to generate sequences of varying lengths with elements that are, themselves, described by strategies**. Recall that we can always apply the `.map` method if we want a different type of collection (e.g. a `tuple`) other than a list.\n", 419 | "\n", 420 | "This strategy shrinks towards smaller lists with simpler values.\n", 421 | "\n", 422 | "Use `print_examples` to build an intuition for this strategy." 423 | ] 424 | }, 425 | { 426 | "cell_type": "markdown", 427 | "id": "fd9b6c4e", 428 | "metadata": {}, 429 | "source": [ 430 | "
\n", 431 | "\n", 432 | "**Exercise: Describing data with `st.lists`**\n", 433 | "\n", 434 | "Write a strategy that generates unique **tuples** of even-valued integers, ranging from length-5 to length-10. \n", 435 | "\n", 436 | "Write a test that checks these properties.\n", 437 | "\n", 438 | "
\n" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "id": "dfd375c5", 445 | "metadata": {}, 446 | "outputs": [], 447 | "source": [ 448 | "# STUDENT CODE HERE" 449 | ] 450 | }, 451 | { 452 | "cell_type": "markdown", 453 | "id": "7abb8e21", 454 | "metadata": {}, 455 | "source": [ 456 | "### [`st.floats()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.floats)\n", 457 | "\n", 458 | "[`st.floats`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.floats) is a powerful strategy that generates all variety of floats, including `math.inf` and `math.nan`. You can also specify:\n", 459 | " - whether `math.inf` and `math.nan`, respectively, should be included in the data description\n", 460 | " - bounds (either inclusive or exclusive) on the floats being generated; this will naturally preclude `math.nan` from being generated\n", 461 | " - the \"width\" of the floats; e.g. if you want to generate 16-bit or 32-bit floats vs 64-bit\n", 462 | " (while Python `float`s are always 64-bit, `width=32` ensures that the generated values can\n", 463 | " always be losslessly represented in 32 bits. This is mostly useful for Numpy arrays.)\n", 464 | "\n", 465 | "This strategy shrinks towards 0." 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "id": "033369a3", 471 | "metadata": {}, 472 | "source": [ 473 | "
\n", 474 | "\n", 475 | "**Exercise: Using Hypothesis to learn about floats.. Part 1**\n", 476 | "\n", 477 | "Use the `st.floats` strategy to identify which float(s) violate the identity: `x == x`\n", 478 | "\n", 479 | "Then, revise your usage of `st.floats` such that it only describes values that satisfy the identity. \n", 480 | "
\n" 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "id": "9f38f91e", 487 | "metadata": {}, 488 | "outputs": [], 489 | "source": [ 490 | "# using `st.floats` to find value(s) that violate `x == x`\n", 491 | "# STUDENT CODE HERE" 492 | ] 493 | }, 494 | { 495 | "cell_type": "code", 496 | "execution_count": null, 497 | "id": "cd272f2c", 498 | "metadata": {}, 499 | "outputs": [], 500 | "source": [ 501 | "# updating our usage of `st.floats` to generate only values that satisfy `x == x`\n", 502 | "# STUDENT CODE HERE" 503 | ] 504 | }, 505 | { 506 | "cell_type": "markdown", 507 | "id": "eab8318d", 508 | "metadata": {}, 509 | "source": [ 510 | "
\n", 511 | "\n", 512 | "**Exercise: Using Hypothesis to learn about floats.. Part 2**\n", 513 | "\n", 514 | "Use the `st.floats` strategy to identify which **positive** float(s) violate the inequality: `x < x + 1`.\n", 515 | "\n", 516 | "To interpret your findings, it is useful to know that a double-precision (64-bit) binary floating-point number, which is representative of Pythons `float`, has a coefficient of 53 bits (including 1 implied bit), an exponent of 11 bits, and 1 sign bit. \n", 517 | "\n", 518 | "\n", 519 | "Then, revise your usage of `st.floats` such that it only describes values that satisfy the identity. **Use the `example` decorator to ensure that the identified boundary case is tested every time**.\n", 520 | "
\n" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "execution_count": null, 526 | "id": "f571a9f1", 527 | "metadata": {}, 528 | "outputs": [], 529 | "source": [ 530 | "# using `st.floats` to find value(s) that violate `x < x + 1`\n", 531 | "# STUDENT CODE HERE" 532 | ] 533 | }, 534 | { 535 | "cell_type": "code", 536 | "execution_count": null, 537 | "id": "87d8b02e", 538 | "metadata": {}, 539 | "outputs": [], 540 | "source": [ 541 | "# updating our usage of `st.floats` to generate only values that satisfy `x < x + 1`\n", 542 | "# STUDENT CODE HERE" 543 | ] 544 | }, 545 | { 546 | "cell_type": "markdown", 547 | "id": "9e0eefad", 548 | "metadata": {}, 549 | "source": [ 550 | "### [`st.tuples()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.tuples)\n", 551 | "\n", 552 | "The [st.tuples](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.tuples) strategy accepts $N$ Hypothesis strategies, and will generate length-$N$ tuples whose elements are drawn from the respective strategies that were specified as inputs.\n", 553 | "\n", 554 | "This strategy shrinks towards simpler entries.\n", 555 | "\n", 556 | "For example, the following strategy will generate length-3 tuples whose entries are: even-valued integers, booleans, and odd-valued floats:" 557 | ] 558 | }, 559 | { 560 | "cell_type": "code", 561 | "execution_count": null, 562 | "id": "af422177", 563 | "metadata": {}, 564 | "outputs": [], 565 | "source": [ 566 | "my_tuples = st.tuples(\n", 567 | " st.integers().map(lambda x: 2 * x),\n", 568 | " st.booleans(),\n", 569 | " st.integers().map(lambda x: 2 * x + 1).map(float),\n", 570 | ")\n", 571 | "print_examples(my_tuples, 4)" 572 | ] 573 | }, 574 | { 575 | "cell_type": "markdown", 576 | "id": "898aa75b", 577 | "metadata": {}, 578 | "source": [ 579 | "### [`st.just()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.just)\n", 580 | "\n", 581 | "[st.just](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.just) is a strategy that \"just\" returns the value that you fed it. This is a convenient strategy that helps us to avoid abusing the use of `.map` to concoct particular strategies." 582 | ] 583 | }, 584 | { 585 | "cell_type": "markdown", 586 | "id": "a5e47b79", 587 | "metadata": {}, 588 | "source": [ 589 | "
\n", 590 | "\n", 591 | "**Exercise: Describing the shape of an array of 2D vectors**\n", 592 | "\n", 593 | "Write a strategy that describes the shape of an array (i.e. a tuple of integers) that contains 1-to-20 two-dimensional vectors.\n", 594 | "E.g. `(5, 2)` is the shape of the array containing five two-dimensional vectors.\n", 595 | "Avoid using `.map()` in your solution. \n", 596 | "\n", 597 | " \n", 598 | "Use `print_examples` to examine the outputs.\n", 599 | "
\n" 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": null, 605 | "id": "3711f911", 606 | "metadata": {}, 607 | "outputs": [], 608 | "source": [ 609 | "# STUDENT CODE HERE" 610 | ] 611 | }, 612 | { 613 | "cell_type": "markdown", 614 | "id": "2155131e", 615 | "metadata": {}, 616 | "source": [ 617 | "### [`st.one_of()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.one_of)\n", 618 | "\n", 619 | "The `st.one_of` allows us to specify a collection of strategies and any given datum will be drawn from \"one of\" them. E.g.\n", 620 | "\n", 621 | "```python\n", 622 | "# demonstrating st.one_of()\n", 623 | "st.one_of(st.integers(), st.lists(st.integers()))\n", 624 | "```\n", 625 | "\n", 626 | "will draw values that are either integers or list of integers. " 627 | ] 628 | }, 629 | { 630 | "cell_type": "markdown", 631 | "id": "7826873f", 632 | "metadata": {}, 633 | "source": [ 634 | "Note that the \"pipe\" operator is overloaded by Hypothesis strategies as a shorthand for `one_of`; e.g.\n", 635 | "\n", 636 | "```python\n", 637 | "st.integers() | st.floats() | st.booleans()\n", 638 | "```\n", 639 | "\n", 640 | "is equivalent to:\n", 641 | "\n", 642 | "\n", 643 | "```python\n", 644 | "st.one_of(st.integers(), st.floats(), st.booleans())\n", 645 | "```\n", 646 | "\n", 647 | "This strategy shrinks with preference for the left-most strategy." 648 | ] 649 | }, 650 | { 651 | "cell_type": "markdown", 652 | "id": "a2537c7c", 653 | "metadata": {}, 654 | "source": [ 655 | "
\n", 656 | "\n", 657 | "**Exercise: Stitching together strategies for rich behavior**\n", 658 | "\n", 659 | "Write a strategy that draws a tuple of two perfect squares (integers) or three perfect cubes (integers)\n", 660 | "\n", 661 | "Use `print_examples` to examine the behavior of your strategy.\n", 662 | "
\n" 663 | ] 664 | }, 665 | { 666 | "cell_type": "code", 667 | "execution_count": null, 668 | "id": "b61247e9", 669 | "metadata": {}, 670 | "outputs": [], 671 | "source": [ 672 | "# STUDENT CODE HERE" 673 | ] 674 | }, 675 | { 676 | "cell_type": "markdown", 677 | "id": "ef779c28", 678 | "metadata": {}, 679 | "source": [ 680 | "### [`st.text()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.text)\n", 681 | "\n", 682 | "\n", 683 | "The `st.text` accepts an \"alphabet\" – a collection of string-characters – from which it will construct strings of varying lengths, whose bounds can be specified by the user.\n", 684 | "This strategy shrinks towards shorter strings.\n", 685 | "\n", 686 | "For example, the following strategy will strings of lowercase vowels from length 2 to length 10:\n", 687 | "\n", 688 | "```python\n", 689 | ">>> st.text(\"aeiouy\", min_size=2, max_size=10).example()\n", 690 | "'oouoyoye'\n", 691 | "```" 692 | ] 693 | }, 694 | { 695 | "cell_type": "markdown", 696 | "id": "8f65c7c1", 697 | "metadata": {}, 698 | "source": [ 699 | "### [`st.fixed_dictionaries()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.fixed_dictionaries)\n", 700 | "\n", 701 | "`st.fixed_dictionaries` takes a mapping of the form `key -> strategy` and returns a strategy according to that mapping. E.g.\n", 702 | "\n", 703 | "```python\n", 704 | "# demonstrating st.fixed_dictionaries()\n", 705 | ">>> mapping = dict(age=st.integers(0, 89), height=st.floats(3, 7)\n", 706 | ">>> st.fixed_dictionaries(mapping).example()\n", 707 | "{'age': 7, 'height': 3.5}\n", 708 | "```\n", 709 | "\n", 710 | "will draw values that are either integers or list of integers. \n", 711 | "\n", 712 | "This strategy shrinks towards simpler keys and values." 713 | ] 714 | }, 715 | { 716 | "cell_type": "markdown", 717 | "id": "cdf6ac56", 718 | "metadata": {}, 719 | "source": [ 720 | "### [`st.sampled_from`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.sampled_from)\n", 721 | "\n", 722 | "[`st.sampled_from`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.sampled_from) accepts a collection of objects. The strategy will return a value that is sampled from this collection.\n", 723 | "\n", 724 | "For example, the following strategy will sample a value `0`, `\"a\"`, or `(2, 2)` from a list:\n", 725 | "\n", 726 | "```python\n", 727 | ">>> st.sampled_from([0, \"a\", (2, 2)]).example()\n", 728 | "'a'\n", 729 | "```\n", 730 | "\n", 731 | "This strategy shrinks towards the first element among the samples." 732 | ] 733 | }, 734 | { 735 | "cell_type": "markdown", 736 | "id": "99e38b42", 737 | "metadata": {}, 738 | "source": [ 739 | "
\n", 740 | "\n", 741 | "**Exercise: Describing objects that evaluate to `False`**\n", 742 | "\n", 743 | "Write a strategy that can return the boolean, integer, float, string, list, tuple, or dictionary that evaluates to `False` (when called on by `bool`)\n", 744 | "\n", 745 | "Use `print_examples` to examine the behavior of your strategy.\n", 746 | "
\n" 747 | ] 748 | }, 749 | { 750 | "cell_type": "code", 751 | "execution_count": null, 752 | "id": "ed6dabd2", 753 | "metadata": {}, 754 | "outputs": [], 755 | "source": [ 756 | "# STUDENT CODE HERE" 757 | ] 758 | }, 759 | { 760 | "cell_type": "markdown", 761 | "id": "1dbd127c", 762 | "metadata": {}, 763 | "source": [ 764 | "### [`st.from_type()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.from_type)\n", 765 | "\n", 766 | "`st.from_type` looks up a strategy associated with a given type or type-annotation\n", 767 | "\n", 768 | "```python\n", 769 | ">>> st.from_type(int)\n", 770 | "integers()\n", 771 | "\n", 772 | ">>> from typing import List, Dict\n", 773 | ">>> st.from_type(List[int])\n", 774 | "lists(integers())\n", 775 | "\n", 776 | ">>> st.from_type(Dict[str, float])\n", 777 | "dictionaries(keys=text(), values=floats())\n", 778 | "```\n", 779 | "\n", 780 | "You can use `st.register_type_strategy` to add or override type -> strategy mappings used by `st.from_type`:\n", 781 | "\n", 782 | "```python\n", 783 | "import math\n", 784 | "\n", 785 | "class ComplexUnitCircle(complex): \n", 786 | " pass\n", 787 | "\n", 788 | "st.register_type_strategy(\n", 789 | " ComplexUnitCircle,\n", 790 | " st.floats(0, 2 * math.pi).map(lambda x: math.cos(x) + 1j * math.sin(x)),\n", 791 | ")\n", 792 | "```\n", 793 | "\n", 794 | "```python\n", 795 | ">>> st.from_type(ComplexUnitCircle).example()\n", 796 | "(0.7599735905063035-0.6499539535482165j)\n", 797 | "```\n", 798 | "\n", 799 | "`...` (i.e. the `Ellipsis` object) can be used within `@given` to indicate that an argument's values should be inferred from the annotation from the test's signature.\n", 800 | "For example:\n", 801 | "\n", 802 | "```python\n", 803 | "@given(x=...) # uses `from_type(ComplexUnitCircle)`\n", 804 | "def test_with_infer(x: ComplexUnitCircle):\n", 805 | " assert isinstance(x, complex)\n", 806 | " assert math.isclose(abs(x), 1)\n", 807 | "```" 808 | ] 809 | }, 810 | { 811 | "cell_type": "markdown", 812 | "id": "a88df1ad", 813 | "metadata": {}, 814 | "source": [ 815 | "### [`st.builds()`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.builds)\n", 816 | "\n", 817 | "`st.builds` is will \"build\" (call/instantiate) some target callable. `builds` is capable of inferring strategies based on the target's annotated signature (it uses `st.from_type` to do so), otherwise we can explicitly specify arguments for it to use. \n", 818 | "\n", 819 | "```python\n", 820 | "# Defining a target for `st.builds`\n", 821 | "from dataclasses import dataclass\n", 822 | "\n", 823 | "@dataclass\n", 824 | "class C:\n", 825 | " x: int\n", 826 | " y: str\n", 827 | "```\n", 828 | "\n", 829 | "`builds` can either infer strategies to use, based on `C`'s type annotations:\n", 830 | "\n", 831 | "```python\n", 832 | ">>> st.builds(C).example() # infers x->`st.integers()`, y->`st.text()`\n", 833 | "C(x=57, y='\\U00108b39W\\U000a55f5ÈwC\\U0009db0a')\n", 834 | "```\n", 835 | "\n", 836 | "or we can explicitly specify one or more of the strategies for describing the target's parameters\n", 837 | "\n", 838 | "```python\n", 839 | ">>> st.builds(C, x=st.just(-1111)).example()\n", 840 | "C(x=-1111, y='A')\n", 841 | "```" 842 | ] 843 | }, 844 | { 845 | "cell_type": "markdown", 846 | "id": "ac3d5798", 847 | "metadata": {}, 848 | "source": [ 849 | "
\n", 850 | "\n", 851 | "**Exercise: Registering a Strategy for a Type**\n", 852 | "\n", 853 | "Given the following type\n", 854 | " \n", 855 | "```python\n", 856 | "from dataclasses import dataclass\n", 857 | "\n", 858 | "@dataclass\n", 859 | "class Student:\n", 860 | " age: int\n", 861 | " letter_grade: str\n", 862 | "```\n", 863 | "\n", 864 | "register a Hypothesis strategy in association with it (via `register_type_strategy`) that draws ages from the domain `[10, 18]` and letter grades from A-F (no pluses or minuses).\n", 865 | "Next, write a test that draws a \"class of students\" (i.e. `List[Student]`), and use `hypothesis.infer` so that you don't have to write the strategy by-hand.\n", 866 | "In the body of the test, simply print the values generated to describe the class of students.\n", 867 | "
\n" 868 | ] 869 | }, 870 | { 871 | "cell_type": "code", 872 | "execution_count": null, 873 | "id": "aaa63a55", 874 | "metadata": {}, 875 | "outputs": [], 876 | "source": [ 877 | "# STUDENT CODE HERE" 878 | ] 879 | }, 880 | { 881 | "cell_type": "markdown", 882 | "id": "8fbe2639", 883 | "metadata": {}, 884 | "source": [ 885 | "
\n", 886 | "\n", 887 | "**Exercise: Exploring additional strategies**\n", 888 | "\n", 889 | "Consult [Hypothesis' documentation](https://hypothesis.readthedocs.io/en/latest/data.html) \n", 890 | "
\n" 891 | ] 892 | }, 893 | { 894 | "cell_type": "markdown", 895 | "id": "2c6ddb1d", 896 | "metadata": {}, 897 | "source": [ 898 | "## Drawing From Strategies Within a Test\n", 899 | "\n", 900 | "We will often need to draw from a Hypothesis strategy in a context-dependent manner within our test. Suppose, for example, that we want to describe two lists of integers, but we want to be sure that the second list is longer than the first. [We can use the `st.data()` strategy to use strategies \"interactively\"](https://hypothesis.readthedocs.io/en/latest/data.html#drawing-interactively-in-tests) in this sort of way.\n", 901 | "\n", 902 | "Let's see it in action:" 903 | ] 904 | }, 905 | { 906 | "cell_type": "code", 907 | "execution_count": null, 908 | "id": "41950f20", 909 | "metadata": {}, 910 | "outputs": [], 911 | "source": [ 912 | "# We want two lists of integers, `x` and `y`, where we want `y` to be longer than `x`.\n", 913 | "\n", 914 | "@given(x=st.lists(st.integers()), data=st.data())\n", 915 | "def test_two_constrained_lists(x, data: st.DataObject):\n", 916 | " y = data.draw(st.lists(st.integers(), min_size=len(x) + 1), label=\"y\")\n", 917 | "\n", 918 | " assert len(x) < len(y)\n", 919 | "\n", 920 | "\n", 921 | "test_two_constrained_lists()" 922 | ] 923 | }, 924 | { 925 | "cell_type": "markdown", 926 | "id": "f4bfa655", 927 | "metadata": {}, 928 | "source": [ 929 | "The `given` operator is told to pass two things to our test: \n", 930 | "\n", 931 | " - `x`, which is a list of integers drawn from strategies\n", 932 | " - `data`, which is an instance of the `st.DataObject` class; this instance is what gets drawn from the `st.data()` strategy\n", 933 | "\n", 934 | "The only thing that you need to know about `st.DataObject` is that it's `draw` method expects a hypothesis search strategy, and that it will immediately draw a value from said strategy during the test. You can also, optionally, pass a string to `label` argument to the `draw` method. This simply permits you to provide a name for the item that was drawn, so that any stack-trace that your test produces is easy to interpret." 935 | ] 936 | }, 937 | { 938 | "cell_type": "markdown", 939 | "id": "c680bd0b", 940 | "metadata": {}, 941 | "source": [ 942 | "
\n", 943 | "\n", 944 | "**Exercise: Drawing from a strategy interactively**\n", 945 | "\n", 946 | "Write a test that is fed a list (of varying length) of non-negative integers. Then, draw a **set** (i.e. a unique collection) of non-negative integers whose sum is at least as large as the sum of the list.\n", 947 | "The strategy [`st.sets`](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.sets) will be useful for this.\n", 948 | "\n", 949 | "
\n" 950 | ] 951 | }, 952 | { 953 | "cell_type": "code", 954 | "execution_count": null, 955 | "id": "d801bae9", 956 | "metadata": {}, 957 | "outputs": [], 958 | "source": [ 959 | "# STUDENT CODE HERE" 960 | ] 961 | }, 962 | { 963 | "cell_type": "markdown", 964 | "id": "87082dbe", 965 | "metadata": {}, 966 | "source": [ 967 | "### `.flatmap`\n", 968 | "\n", 969 | "`flatmap` is a method enables us to define a strategy based on a value drawn from a previous strategy.\n", 970 | "\n", 971 | "For example, the following strategy produces lists of integers where each list is guaranteed to have a length that is a perfect square (e.g. a length of 0, 1, 4, 16, 25, ...)" 972 | ] 973 | }, 974 | { 975 | "cell_type": "code", 976 | "execution_count": null, 977 | "id": "32756c1c", 978 | "metadata": {}, 979 | "outputs": [], 980 | "source": [ 981 | "perfect_squares = st.integers(min_value=0, max_value=6).map(lambda x: x ** 2)\n", 982 | "\n", 983 | "sqr_len_lists = perfect_squares.flatmap(\n", 984 | " lambda x: st.lists(st.integers(), min_size=x, max_size=x)\n", 985 | ")" 986 | ] 987 | }, 988 | { 989 | "cell_type": "markdown", 990 | "id": "d2ac136e", 991 | "metadata": {}, 992 | "source": [ 993 | "## Writing Your Own Hypothesis Strategies with @composite\n", 994 | "\n", 995 | "Hypothesis provides the [@composite](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.composite) decorator, which permits us to form our own strategies for describing data by composing Hypothesis' built-in strategies. Let's see this in action by writing a strategy that will produce the bounds for a 1D interval." 996 | ] 997 | }, 998 | { 999 | "cell_type": "code", 1000 | "execution_count": null, 1001 | "id": "11f6e3aa", 1002 | "metadata": {}, 1003 | "outputs": [], 1004 | "source": [ 1005 | "from typing import Any, Callable, Optional, Tuple\n", 1006 | "from hypothesis.strategies import composite\n", 1007 | "import hypothesis.strategies as st\n", 1008 | "from math import inf\n", 1009 | "\n", 1010 | "\n", 1011 | "@composite\n", 1012 | "def interval_bounds(draw, left_bnd=-inf, right_bnd=inf, min_size=0.0):\n", 1013 | " \"\"\"A Hypothesis data strategy for generating ordered bounds on the real number line.\n", 1014 | " \n", 1015 | " Note: The `draw` parameter it reserved by Hypothesis and is not exposed by\n", 1016 | " the function signature.\n", 1017 | " \n", 1018 | " Parameters\n", 1019 | " ----------\n", 1020 | " left_bnd : float, optional (default=-inf)\n", 1021 | " If specified, the smallest value that the left-bound can take on.\n", 1022 | " \n", 1023 | " right_bnd : float, optional (default=inf)\n", 1024 | " If specified, the largest value that the right-bound can take on.\n", 1025 | " \n", 1026 | " min_size : float, optional (default=0.0)\n", 1027 | " The guaranteed minimum separateion of the bounds.\n", 1028 | " \n", 1029 | " Returns\n", 1030 | " -------\n", 1031 | " st.SearchStrategy[Tuple[float, float]]\n", 1032 | " \"\"\"\n", 1033 | " if right_bnd < left_bnd + min_size:\n", 1034 | " raise ValueError(\n", 1035 | " f\"Unsatisfiable bounds: [left_bnd={left_bnd}, right_bnd={right_bnd}], \"\n", 1036 | " f\"min-interval size: {min_size}\"\n", 1037 | " )\n", 1038 | "\n", 1039 | " # `drawn_left` is a float\n", 1040 | " drawn_left = draw(\n", 1041 | " st.floats(\n", 1042 | " min_value=left_bnd,\n", 1043 | " max_value=(None if right_bnd is None else right_bnd - min_size),\n", 1044 | " allow_nan=False,\n", 1045 | " )\n", 1046 | " )\n", 1047 | "\n", 1048 | " # `drawn_right` is a float\n", 1049 | " drawn_right = draw(\n", 1050 | " st.floats(min_value=drawn_left + min_size, max_value=right_bnd, allow_nan=False)\n", 1051 | " )\n", 1052 | "\n", 1053 | " # ensure that strategy behaves as-promised\n", 1054 | " assert left_bnd <= drawn_left <= right_bnd\n", 1055 | " assert left_bnd <= drawn_right <= right_bnd\n", 1056 | " assert drawn_left + min_size <= drawn_right\n", 1057 | "\n", 1058 | " # Note that a composite strategy definition should return the drawn values,\n", 1059 | " # and *not* Hypothesis strategies.\n", 1060 | " #\n", 1061 | " # E.g. here we return a tuple of floating point numbers, and\n", 1062 | " # *not* `st.tuples(st.floats(), st.floats())`\n", 1063 | " return (drawn_left, drawn_right)" 1064 | ] 1065 | }, 1066 | { 1067 | "cell_type": "markdown", 1068 | "id": "64712164", 1069 | "metadata": {}, 1070 | "source": [ 1071 | "The first argument, `draw`, is required by the `@composite` decorator. It is a function that is used by Hypothesis to draw values from strategies in order to generate data from our composite strategy. Each draw simply produces a value from that strategy. Thus `drawn_left` and `drawn_right` are simply floating point numbers. We then simply return a tuple of these floats, as expected from this strategy.\n", 1072 | "\n", 1073 | "Note that, even though the resulting function `interval_bounds()` is a Hypothesis search strategy, the return statement in its definition *specifies what values are to be returned*. It does *not* return strategy-instances.\n", 1074 | "\n", 1075 | "As mentioned in the docstring, the `draw` parameter is not actually exposed in the function signature of `interval_bounds`, once defined. Print the docstring for `interval_bounds` and see that the `draw` parameter is absent from the signature:" 1076 | ] 1077 | }, 1078 | { 1079 | "cell_type": "markdown", 1080 | "id": "a65de159", 1081 | "metadata": {}, 1082 | "source": [ 1083 | "Experiment with this strategy and see that it behaves as-expected. Start by calling `.example()` on it. Be sure to provide different arguments to the strategy." 1084 | ] 1085 | }, 1086 | { 1087 | "cell_type": "code", 1088 | "execution_count": null, 1089 | "id": "74877acf", 1090 | "metadata": {}, 1091 | "outputs": [], 1092 | "source": [ 1093 | "print(f\"interval_bounds().example(): {interval_bounds().example()}\")\n", 1094 | "print(f\"interval_bounds(-1, 2, min_size=1).example(): {interval_bounds(-1, 2, min_size=1).example()}\")" 1095 | ] 1096 | }, 1097 | { 1098 | "cell_type": "markdown", 1099 | "id": "e87ceea5", 1100 | "metadata": {}, 1101 | "source": [ 1102 | "Lastly, note the presence of the assertions within the composite strategies.\n", 1103 | "We will presumably be using this strategy to describe data for tests. Writing a buggy strategy - one that generates incorrect or unexpected data - is a terrible thing; this will, at best lead, to a headache. At worst, it mask bugs in the code that we are testing. \n", 1104 | "\n", 1105 | "**If there is ever a time to be fastidious with type/value checking and correctness assertions, it is at the interfaces of a custom Hypothesis strategy!**\n", 1106 | "It is also sensible to write tests for your strategies if they are sufficiently sophisticated." 1107 | ] 1108 | }, 1109 | { 1110 | "cell_type": "markdown", 1111 | "id": "0939c10e", 1112 | "metadata": {}, 1113 | "source": [ 1114 | "
\n", 1115 | "\n", 1116 | "**Exercise: Write a \"quadrilateral corners\" strategy**\n", 1117 | "\n", 1118 | "Use the `@composite` decorator to write the `quad_corners` strategy. Here is the docstring for this strategy:\n", 1119 | "\n", 1120 | "```python\n", 1121 | "\"\"\"\n", 1122 | "A Hypothesis strategy for four corners of a quadrilateral.\n", 1123 | "\n", 1124 | "The corners are guaranteed to have counter-clockwise (\"right-handed\") ordering.\n", 1125 | "\n", 1126 | "Parameters\n", 1127 | "----------\n", 1128 | "corner_magnitude : float, optional (default=1e6)\n", 1129 | " The maximum size - in magnitude - that any of the coordinates\n", 1130 | " can take on.\n", 1131 | "\n", 1132 | "min_separation : float, optional (default=1e-2)\n", 1133 | " The smallest guaranteed margin between two consecutive corners along a\n", 1134 | " single dimension.\n", 1135 | "\n", 1136 | "Returns\n", 1137 | "-------\n", 1138 | "SearchStrategy[np.ndarray]\n", 1139 | " shape-(4, 2) array of four ordered pairs (float-64)\n", 1140 | "\"\"\"\n", 1141 | "```\n", 1142 | "\n", 1143 | "
\n", 1144 | "\n", 1145 | "Feel free to work with a neighbor on this. How general is your strategy?\n", 1146 | "Did you bake in any underlying assumptions about the structure or ordering of the data that isn't explicitly part of the strategy's description?\n", 1147 | "\n", 1148 | "If you do sport some unintentional structure to the data you are generating, and which you need to randomize, *do not reach for the `random` module to mix things up*.\n", 1149 | "Rather, find a Hypothesis strategy that can do the stirring for you.\n", 1150 | "\n", 1151 | "While Hypothesis does track/control random seeds so that it can replay tests accurately, it will not be able to shrink your custom strategy effectively if you seek randomness from outside of Hypothesis' strategies." 1152 | ] 1153 | }, 1154 | { 1155 | "cell_type": "code", 1156 | "execution_count": null, 1157 | "id": "e81e8568", 1158 | "metadata": {}, 1159 | "outputs": [], 1160 | "source": [ 1161 | "# Define the `quad_corners` strategy\n", 1162 | "# STUDENT CODE HERE" 1163 | ] 1164 | }, 1165 | { 1166 | "cell_type": "markdown", 1167 | "id": "15c434fc", 1168 | "metadata": {}, 1169 | "source": [ 1170 | "## Extra: Recipes with Strategies\n", 1171 | "\n", 1172 | "It can be surprising to see some of the rich descriptions of data that we can produce by combining these primitive strategies in creative ways.\n", 1173 | "\n", 1174 | "Here is one interesting recipe\n", 1175 | "\n", 1176 | "```python\n", 1177 | "from typing import Any, Tuple, Type, Union\n", 1178 | "\n", 1179 | "def everything_except(\n", 1180 | " excluded_types: Union[Type[type], Tuple[Type[type], ...]]\n", 1181 | ") -> st.SearchStrategy[Any]:\n", 1182 | " return (\n", 1183 | " st.from_type(type)\n", 1184 | " .flatmap(st.from_type)\n", 1185 | " .filter(lambda x: not isinstance(x, excluded_types))\n", 1186 | " )\n", 1187 | "```\n", 1188 | "\n", 1189 | "This strategy will draw values from *any* strategy associated with *any* type that has been registered with `st.register_type_strategy()`, except for values that belong to `excluded_types`.\n", 1190 | "\n", 1191 | "```python\n", 1192 | ">>> [everything_except(int).example() for _ in range(5)]\n", 1193 | "00:00:00\n", 1194 | "\n", 1195 | "None\n", 1196 | "set()\n", 1197 | "2183-03-08\n", 1198 | "```\n", 1199 | "\n", 1200 | "How does this work? `st.from_type(type)` returns a strategy that draws *types* (e.g. `int`, `str`).\n", 1201 | "Then this type gets fed to `st.from_type(...)` via `.flatmap`, and thus a strategy is returned for drawing instances of that type.\n", 1202 | "Lastly, we filter any values that belong to the excluded type.\n", 1203 | "This is a pretty neat strategy for testing the strength of your code's input validation!\n", 1204 | "\n", 1205 | "Another interesting takeaway from this recipe is the fact that you don't necessarily need to use `st.composite` to write your own strategy!\n", 1206 | "Here we simply wrote a function that returns our specific strategy." 1207 | ] 1208 | } 1209 | ], 1210 | "metadata": { 1211 | "kernelspec": { 1212 | "display_name": "Python 3 (ipykernel)", 1213 | "language": "python", 1214 | "name": "python3" 1215 | } 1216 | }, 1217 | "nbformat": 4, 1218 | "nbformat_minor": 5 1219 | } 1220 | -------------------------------------------------------------------------------- /notebooks/02_Test_Strategies_STUDENT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "f3e4b826", 6 | "metadata": {}, 7 | "source": [ 8 | "# Common Test Tactics\n", 9 | "\n", 10 | "In the previous section we are familiarized ourselves with Hypothesis' core strategies for describing data, along with the various methods for augmenting them and combining thing.\n", 11 | "Now we will put these to good use and discuss some common tactics for writing effective property-based tests for our code." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "id": "294e12f0", 17 | "metadata": {}, 18 | "source": [ 19 | "## Simple Properties to Test\n", 20 | "\n", 21 | "### Valid Inputs Don't Crash The Code (i.e. \"Fuzzing\")\n", 22 | "\n", 23 | "We already encountered fuzzing in the first section of this tutorial, but it is worth reiterating that *simply calling your code with diverse, valid inputs - to see if your code crashes - works shockingly well*.\n", 24 | "This is especially true if your code contains internal assertions to guarantee that specific assumptions made about the code's internal logic are never violated.\n", 25 | "\n", 26 | "Consider the following function:\n", 27 | "\n", 28 | "\n", 29 | "```python\n", 30 | "from typing import Any\n", 31 | "\n", 32 | "def safe_name(obj: Any, repr_allowed: bool=True) -> str:\n", 33 | " \"\"\"Tries to get a descriptive name for an object. Returns '`\n", 34 | " instead of raising - useful for writing descriptive/safe error messages.\"\"\"\n", 35 | " if hasattr(obj, \"__qualname__\"):\n", 36 | " return obj.__qualname__\n", 37 | "\n", 38 | " if hasattr(obj, \"__name__\"):\n", 39 | " return obj.__name__\n", 40 | "\n", 41 | " if repr_allowed and hasattr(obj, \"__repr__\"):\n", 42 | " return repr(obj)\n", 43 | "\n", 44 | " return \"\"\n", 45 | "```\n", 46 | "\n", 47 | "This is a function that is meant to be used when writing error messages; we never want *it* to raise an error.\n", 48 | "While we will want to be sure to test that this returns correct/descriptive names, we certainly want to make sure that this function never crashes." 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "id": "180731e5", 54 | "metadata": {}, 55 | "source": [ 56 | "
\n", 57 | "\n", 58 | "**Exercise: An object by any other name (would cause a bug)**\n", 59 | "\n", 60 | "Add `safe_name` to `pbt_tutorial/basic_functions.py`.\n", 61 | "Next, in `test_fuzz.py`, add a test that fuzzes `safe_name` with a diverse set of inputs.\n", 62 | "Consider using the `st.from_type(type)` and `st.from_type(type).flatmap(st.from_type)` \"recipes\" for generating a wide-variety of types and values for this test. \n", 63 | "\n", 64 | "You can also use the `@example` decorator to add specific edge cases that you want to test against.\n", 65 | "
" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "id": "d1eb6772", 71 | "metadata": {}, 72 | "source": [ 73 | "### Roundtrip Pairs\n", 74 | "\n", 75 | "Saving and loading, encoding and decoding, sending and receiving, to-yaml from-yaml: these are all examples of pairs of functions that form \"roundtrip\" relationships.\n", 76 | "\n", 77 | "```python\n", 78 | "# f and g form a \"roundtrip\" (i.e. f is g's inverse)\n", 79 | "\n", 80 | "g(f(x)) == x\n", 81 | "```" 82 | ] 83 | }, 84 | { 85 | "cell_type": "markdown", 86 | "id": "23da0c80", 87 | "metadata": {}, 88 | "source": [ 89 | "The \"roundtrip\" property is a wonderful thing to test; such a test is simple to write, permits very flexible/complicated inputs, and tests for correctness in a meaningful way.\n", 90 | "\n", 91 | "Note that we can often choose the direction of the roundtrip that we test; e.g. `f(g(x)) == x` vs `g(f(x)) == x`.\n", 92 | "You might pick the roundtrip direction by considering which function of the pair - `f` of `g` - has an input that is easier to describe using Hypothesis' strategies. " 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "94055a96", 98 | "metadata": {}, 99 | "source": [ 100 | "
\n", 101 | "\n", 102 | "**Exercise: Roundtripping Run-Length Encoding**\n", 103 | "\n", 104 | "Run-length encoding is a simple method for compressing data that contains long sequences of repeated characters.\n", 105 | "\n", 106 | "In this compression algorithm:\n", 107 | "\n", 108 | "1. A standalone character will be unchanged. E.g \"a\" $\\rightarrow$ [\"a\"].\n", 109 | "2. A run of a character, c, repeated N times will be compressed to [\"c\", \"c\", N]. E.g. \"bbbb\" $\\rightarrow$ ['b', 'b', 4].\n", 110 | " \n", 111 | "These two rules are all that you need to perform run-length encoding.\n", 112 | "\n", 113 | "Let's look at a few examples of run-length-encoding:\n", 114 | "\n", 115 | "- \"abcd\" $\\rightarrow$ ['a', 'b', 'c', 'd']\n", 116 | "- \"abbbba\" $\\rightarrow$ ['a', 'b', 'b', 4, 'a']\n", 117 | "- \"aaaabbcccd\" $\\rightarrow$ ['a', 'a', 4, 'b', 'b', 2, 'c', 'c', 3, 'd']\n", 118 | "- \"\" $\\rightarrow$ []\n", 119 | "- \"1\" $\\rightarrow$ [\"1\"]\n", 120 | "\n", 121 | "The decompression algorithm, run-length decoding, simply reverses this process:\n", 122 | "\n", 123 | "- ['q', 'a', 'a', 4, 'b', 'b', 2, 'c', 'c', 3, 'd'] $\\rightarrow$ 'qaaaabbcccd'\n", 124 | "\n", 125 | "Here is an implementation of the encoder for this compressions scheme:\n", 126 | " \n", 127 | "```python\n", 128 | "from itertools import groupby\n", 129 | "from typing import List, Union\n", 130 | "\n", 131 | "def run_length_encoder(in_string: str) -> List[Union[str, int]]:\n", 132 | " \"\"\"\n", 133 | " >>> run_length_encoder(\"aaaaabbcbc\")\n", 134 | " ['a', 'a', 5, 'b', 'b', 2, 'c', 'b', 'c']\n", 135 | " \"\"\"\n", 136 | " assert isinstance(in_string, str)\n", 137 | " out = []\n", 138 | " for item, group in groupby(in_string):\n", 139 | " cnt = sum(1 for x in group)\n", 140 | " if cnt == 1:\n", 141 | " out.append(item)\n", 142 | " else:\n", 143 | " out.extend((item, item, cnt))\n", 144 | " assert isinstance(out, list)\n", 145 | " assert all(isinstance(x, (str, int)) for x in out)\n", 146 | " return out\n", 147 | "```\n", 148 | " \n", 149 | "Add this function to the `basic_functions.py` file.\n", 150 | "Next, begin writing the corresponding decoder\n", 151 | " \n", 152 | "```python\n", 153 | "def run_length_decoder(in_list: List[Union[str, int]]) -> str:\n", 154 | " \"\"\"\n", 155 | " >>> run_length_decoder(['a', 'a', 5, 'b', 'b', 2, 'c', 'b', 'c'])\n", 156 | " \"aaaaabbcbc\"\n", 157 | " \"\"\"\n", 158 | " # YOUR CODE HERE\n", 159 | "```\n", 160 | "\n", 161 | "But write this function using test-driven development.\n", 162 | "Consider which direction of the roundtrip is easier to test.\n", 163 | "\n", 164 | "Also, consider combining strategies via `st.one_of` to help ensure that your roundtrip is being exercised with a diverse set of inputs as well as a diverse set of patterns.\n", 165 | "That is, if you you draw highly-varied string inputs to your encoder, then you might not often encounter intricate or lengthy patterns of repetition.\n", 166 | "There is a tradeoff to be managed here.\n", 167 | "\n", 168 | "
" 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "id": "bce276de", 174 | "metadata": {}, 175 | "source": [ 176 | "
\n", 177 | "\n", 178 | "**Exercise: Ghostwriting a simple roundtrip**\n", 179 | " \n", 180 | "Run the following command in your terminal, to test [the `gzip` module](https://docs.python.org/3/library/gzip.html)\n", 181 | "\n", 182 | "```shell\n", 183 | "hypothesis write gzip.compress\n", 184 | "```\n", 185 | " \n", 186 | "and inspect the output. What changes might you make to improve this test? For example:\n", 187 | " \n", 188 | "- improving input strategies\n", 189 | "- removing or updating comments\n", 190 | "- using descriptive variable names\n", 191 | " \n", 192 | "Run your improved test to confirm that it works as you expect.\n", 193 | " \n", 194 | "Bonus round: the `mtime` argument was added to `gzip.compress()` in Python 3.8. Could you write version-independent tests, or think of extra properties that it enables?" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "id": "9c9162b0", 201 | "metadata": {}, 202 | "outputs": [], 203 | "source": [ 204 | "# Execute this cell to see the CLI output\n", 205 | "! hypothesis write gzip.compress" 206 | ] 207 | }, 208 | { 209 | "cell_type": "markdown", 210 | "id": "22360e44", 211 | "metadata": {}, 212 | "source": [ 213 | "
\n", 214 | "\n", 215 | "**Exercise: Ghostwriting a more complex roundtrip**\n", 216 | " \n", 217 | "Run the following command to test [the `json` module](https://docs.python.org/3/library/json.html)\n", 218 | "\n", 219 | "```shell\n", 220 | "hypothesis write json.dumps\n", 221 | "```\n", 222 | " \n", 223 | "and inspect the output. What changes might you make to improve this test? For example:\n", 224 | "\n", 225 | "- [defining a `st.recursive()` strategy](https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.recursive) to generate [JSON objects](https://www.json.org/) (for the `obj` argument)\n", 226 | "- improving other input strategies, or removing those for `loads()`\n", 227 | "- removing or updating comments\n", 228 | "- using descriptive variable names\n", 229 | " \n", 230 | "Extension: use `assume(obj == obj)` instead of excluding `nan` from your JSON strategy. What happens?" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "id": "864707a0", 237 | "metadata": {}, 238 | "outputs": [], 239 | "source": [ 240 | "# Execute this cell to see the CLI output\n", 241 | "! hypothesis write json.dumps" 242 | ] 243 | }, 244 | { 245 | "cell_type": "markdown", 246 | "id": "c2a7adee", 247 | "metadata": {}, 248 | "source": [ 249 | "### Equivalent Functions\n", 250 | "\n", 251 | "There are times where we are fortunate enough to have access to two distinct functions that are meant to exhibit the same behavior.\n", 252 | "Often times this comes in the form of a slow function (e.g. single-threaded) vs a faster one (multi-threaded), or \"cousin implementations\" of functions (e.g. NumPy's and PyTorch's respective implementations of `matmul`).\n", 253 | "\n", 254 | "- `f_old()` vs `f_new`\n", 255 | "- `f_singlethread()` vs `f_multithread()`\n", 256 | "- `func()` vs `numba.njit(func)()`\n", 257 | "- `numpy.matmul()` vs `torch.matmul()`\n", 258 | "- `numpy.einsum(..., optimize=False)` vs `numpy.einsum(..., optimize=True)`\n", 259 | "\n", 260 | "We can also use function which are equivalent for *some* of their possible inputs. For example, `numpy.asarray` converts the input to a `ndarray`, while `numpy.asanyarray` passes array-like types (such as sparse arrays, xarray DataArrays, etc) through unchanged. We can use the ghostwriter to check that *if passed an `ndarray`*, these functions are equivalent:" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "id": "9a255b37", 267 | "metadata": {}, 268 | "outputs": [], 269 | "source": [ 270 | "! hypothesis write --equivalent numpy.asarray numpy.asanyarray" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "id": "6d5affe7", 276 | "metadata": {}, 277 | "source": [ 278 | "### Metamorphic Relationships\n", 279 | "\n", 280 | "A metamorphic relationship is one in which a known transformation made to the input of a function has a *known* and *necessary* effect of the function's output.\n", 281 | "We already saw an example of this when we tested our `count_vowels` function:\n", 282 | "\n", 283 | "```python\n", 284 | "assert count_vowels(n * input_string) == n * count_vowels(input_string)\n", 285 | "```\n", 286 | "\n", 287 | "That is, if we replicate the input-string `n` times, then the number of vowels counted in the string should be scaled by a factor of `n` as well.\n", 288 | "Note that we had to evaluate our function twice to check its metamorphic property.\n", 289 | "**All metamorphic tests will require multiple evaluations of the function of interest.**\n", 290 | "\n", 291 | "Metamorphic testing is often a *highly* effective method of testing, which enables us to exercise our code and test for correctness under a wider range of inputs, without our needing to concoct a sophisticated \"oracle\" for validating the code's exact behavior.\n", 292 | "Basically, these tests give you a lot of bang for your buck!\n", 293 | "\n", 294 | "Let's consider some common metamorphic relationships that crop up in functions:\n", 295 | "\n", 296 | "**Linearity**\n", 297 | "\n", 298 | "```\n", 299 | "f(a * x) = a * f(x)\n", 300 | "```\n", 301 | "\n", 302 | "The example involving `count_vowels` is a demonstration of a \"linear\" metamorphic relationship.\n", 303 | "\n", 304 | "```\n", 305 | "mag = abs(x)\n", 306 | "assert a * mag == abs(a * x)\n", 307 | "```\n", 308 | "\n", 309 | "**Monotonicity**\n", 310 | "\n", 311 | "```\n", 312 | "f(x) <= f(x + |δ|)\n", 313 | "\n", 314 | "or\n", 315 | "\n", 316 | "f(x) >= f(x + |δ|)\n", 317 | "```\n", 318 | "\n", 319 | "A function is monotonic if transforming the input of th function leads an \"unwavering\" change - only-increasing or only-decreasing - in the function's output.\n", 320 | "Consider, for example, a database that returns some number of results for a query;\n", 321 | "making the query more precise *should not increase the number of results*\n", 322 | "\n", 323 | "```\n", 324 | "len(db.query(query_a)) >= len(db.query(query_a & query_b)) \n", 325 | "```\n", 326 | "\n", 327 | "**Fixed-Point Location**\n", 328 | "\n", 329 | "```python\n", 330 | "y = f(x)\n", 331 | "assert y == f(y)\n", 332 | "```\n", 333 | "\n", 334 | "A fixed point of a function `f` is any value `y` such that `f(y) -> y`.\n", 335 | "It might be surprising to see just how many functions always return fixed-points of themselves:\n", 336 | "\n", 337 | "```python\n", 338 | "y = sort(x)\n", 339 | "assert y == sort(y)\n", 340 | "\n", 341 | "sanitized = sanitize_input(x)\n", 342 | "assert sanitized == sanitize_input(sanitized)\n", 343 | "\n", 344 | "formatted_code = black_formatter(code)\n", 345 | "assert formatted_code == black_formatter(formatted_code)\n", 346 | "\n", 347 | "normed_vec = l2_normalize(vec)\n", 348 | "assert normed_vec == l2_normalize(vec)\n", 349 | "\n", 350 | "result = find_minimum(f, starting_point=x, err_tol=delta)\n", 351 | "assert result == find_minimum(f, starting_point=result, err_tol=delta)\n", 352 | "\n", 353 | "padded = left_pad(string=x, width=4, fillchar=\"a\")\n", 354 | "assert padded == left_pad(string=padded, width=4, fillchar=\"a\")\n", 355 | "```\n", 356 | "\n", 357 | "**Invariance Under Transformation**\n", 358 | "\n", 359 | "If `T(x)` is a function such that the following holds:\n", 360 | "\n", 361 | "```\n", 362 | "f(x) = f(T(x))\n", 363 | "```\n", 364 | "\n", 365 | "This might be an invariance to scaling (`f(x) == f(a * x)`), translation (`f(x) == f(x + a)`), permutation (`f(coll) == f(shuffle(coll))`).\n", 366 | "\n", 367 | "In the case of a computer-vision algorithm, like an image classifier, this could be an invariance under a change of brightness in the image or a horizontal flip of the image (e.g. these things shouldn't change if the model sees a cat in the image)." 368 | ] 369 | }, 370 | { 371 | "cell_type": "markdown", 372 | "id": "d6553d66", 373 | "metadata": {}, 374 | "source": [ 375 | "
\n", 376 | "\n", 377 | "**Exercise: Metamorphic testing**\n", 378 | "\n", 379 | "Create the file `tests/test_metamorphic.py`.\n", 380 | "For each of the following functions identify one or more metamorphic relationships that are exhibited by the function and write tests that exercise them.\n", 381 | " \n", 382 | "- [`numpy.clip`](https://numpy.org/doc/stable/reference/generated/numpy.clip.html)\n", 383 | "- `sorted` (don't test the fixed-point relationship; identify a different metamorphic relationship)\n", 384 | "- `merge_max_mappings`\n", 385 | "- `pairwise_dists` (defined below.. add this to `basic_functions.py`)\n", 386 | "\n", 387 | "```python\n", 388 | "import numpy as np\n", 389 | "\n", 390 | "def pairwise_dists(x, y):\n", 391 | " \"\"\" Computing pairwise Euclidean distance between the respective\n", 392 | " row-vectors of `x` and `y`\n", 393 | "\n", 394 | " Parameters\n", 395 | " ----------\n", 396 | " x : numpy.ndarray, shape=(M, D)\n", 397 | " y : numpy.ndarray, shape=(N, D)\n", 398 | "\n", 399 | " Returns\n", 400 | " -------\n", 401 | " numpy.ndarray, shape=(M, N)\n", 402 | " The Euclidean distance between each pair of\n", 403 | " rows between `x` and `y`.\"\"\"\n", 404 | " sqr_dists = -2 * np.matmul(x, y.T)\n", 405 | " sqr_dists += np.sum(x**2, axis=1)[:, np.newaxis]\n", 406 | " sqr_dists += np.sum(y**2, axis=1)\n", 407 | " return np.sqrt(np.clip(sqr_dists, a_min=0, a_max=None))\n", 408 | "```\n", 409 | "\n", 410 | "
" 411 | ] 412 | }, 413 | { 414 | "cell_type": "markdown", 415 | "id": "70be26fe", 416 | "metadata": {}, 417 | "source": [ 418 | "
\n", 419 | "\n", 420 | "**Exercise: Testing the softmax function**\n", 421 | "\n", 422 | "The so-called \"softmax\" function is a generalised argmax, used to normalize a set of numbers such that **they will have the properties of a probability distribution**. I.e., post-softmax, each number will reside in $[0, 1]$ and the resulting numbers will sum to $1$.\n", 423 | "\n", 424 | "The softmax of a set of $M$ numbers is:\n", 425 | "\n", 426 | "\\begin{equation}\n", 427 | "softmax([s_{k} ]_{k=1}^{M}) = \\Bigl [ \\frac{e^{s_k}}{\\sum_{i=1}^{M}{e^{s_i}}} \\Bigr ]_{k=1}^{M}\n", 428 | "\\end{equation}\n", 429 | "\n", 430 | "```python\n", 431 | ">>> softmax([10., 10., 10.])\n", 432 | "array([0.33333333, 0.33333333, 0.33333333])\n", 433 | "\n", 434 | ">>> softmax([0., 10000., 0.])\n", 435 | "array([0., 1., 0.])\n", 436 | "\n", 437 | ">>> softmax([-100., 0., -100.])\n", 438 | "array([3.72007598e-44, 1.00000000e+00, 3.72007598e-44])\n", 439 | "```\n", 440 | "\n", 441 | "Write an implementation of `softmax` in `basic_functions.py` and test the two properties of `softmax` that we described above.\n", 442 | "Note: you should use `math.isclose` when checking if two floats are approximately equal.\n", 443 | "\n", 444 | "\n", 445 | "If you implemented `softmax` in a straight-forward way (i.e. you implemented the function based on the exact equation above) then you property-based test should fail.\n", 446 | "This is due to the use of the exponential function in `softmax`, which quickly creates a numerical instability.\n", 447 | "\n", 448 | "We can fix the numerical instability by recognizing a metamorphic relationship that is satisfied by the softmax equation: it exhibits translational invariance:\n", 449 | "\n", 450 | "\\begin{align}\n", 451 | "softmax([s_{k} - a]_{k=1}^{M}) &= \\Bigl [ \\frac{e^{s_k - a}}{\\sum_{i=1}^{M}{e^{s_i - a}}} \\Bigr ]_{k=1}^{M}\\\\\n", 452 | "&= \\Bigl [ \\frac{e^{-a}e^{s_k}}{e^{-a}\\sum_{i=1}^{M}{e^{s_i}}} \\Bigr ]_{k=1}^{M}\\\\\n", 453 | "&= \\Bigl [ \\frac{e^{s_k}}{\\sum_{i=1}^{M}{e^{s_i}}} \\Bigr ]_{k=1}^{M}\\\\\n", 454 | "&= softmax([s_{k}]_{k=1}^{M})\n", 455 | "\\end{align}\n", 456 | "\n", 457 | "Thus we can address this instability by finding the max value in our vector of number, subtracting that number from each of the values in the vector, and *then* compute the softmax.\n", 458 | "Update your definition of `softmax` and see that your property-based tests now pass.\n", 459 | "\n", 460 | "Reflect on the fact that using Hypothesis to drive a property-based test lead us to identify a subtle-but-critical oversight in our function.\n", 461 | "Had we simply manually tested our function with known small inputs and outputs, we might not have discovered this issue.\n", 462 | " \n", 463 | "Are there any other properties that we could test here?\n", 464 | "Consider how to set that the respective ordering of the input and output are the same (e.g. `numpy.argsort` can get at this).\n", 465 | "
" 466 | ] 467 | } 468 | ], 469 | "metadata": { 470 | "kernelspec": { 471 | "display_name": "Python 3 (ipykernel)", 472 | "language": "python", 473 | "name": "python3" 474 | } 475 | }, 476 | "nbformat": 4, 477 | "nbformat_minor": 5 478 | } 479 | -------------------------------------------------------------------------------- /notebooks/03_Putting_into_Practice_STUDENT.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "cbc50cb9", 6 | "metadata": {}, 7 | "source": [ 8 | "# Putting it all into Practice\n", 9 | "\n", 10 | "In this final section we'll practice the practical configuration tips that will help you apply property-based testing in the real world.\n", 11 | "\n", 12 | "Now that we know how to write property-based tests, in this final section we will cover practical tips for using them as part of a larger project. Test-suite design patterns, use of performance and determinism settings, reproducing test failures from a CI server, etc; they're all practical topics and typically left out of shorter introductions!" 13 | ] 14 | }, 15 | { 16 | "cell_type": "markdown", 17 | "id": "9b6cd0c4", 18 | "metadata": {}, 19 | "source": [ 20 | "### The joy of `--hypothesis-show-statistics`\n", 21 | "\n", 22 | "Hypothesis has [great support for showing test statistics](https://hypothesis.readthedocs.io/en/latest/details.html#test-statistics), including better-than-`print()` debugging with `note()`, custom `event()`s you can add to the summary, and a variety of performance details.\n", 23 | "\n", 24 | "Let's explore those now: run `pytest --hypothesis-show-statistics test_statistics.py`. You should see\n", 25 | "\n", 26 | "- a lot of output from `printing`... if you actually want to see every example Hypothesis generates, use the `verbosity` setting instead! You can even set it from the command-line with `--hypothesis-verbosity=verbose`.\n", 27 | "- **one** line of output from `note()`\n", 28 | "- statistics on the generate phase for both tests, and the shrink phase for the failing test. If you re-run the tests, you'll see a `reuse` phase where notable examples from previous runs are replayed.\n", 29 | "\n", 30 | "Useful, isn't it!" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "id": "805d759e", 36 | "metadata": {}, 37 | "source": [ 38 | "### Settings for Performance\n", 39 | "\n", 40 | "Hypothesis is designed to behave sensibly by default, but sometimes you have something\n", 41 | "more specific in mind. At those times, [`hypothesis.settings`](https://hypothesis.readthedocs.io/en/latest/settings.html)\n", 42 | "is here to help.\n", 43 | "\n", 44 | "The main performance-related settings to know are:\n", 45 | "\n", 46 | "- `max_examples` - the number of valid examples Hypothesis will run. Defaults to 100; turning it up or down makes your testing proportionally more or less rigorous... and also proportionally slower or faster, respectively!\n", 47 | "- `deadline` - if an input takes longer than this to run, we'll treat that as an error. Useful to detect weird performance issues; but can be flaky if VM performance gets weird." 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "id": "f4609773", 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "from time import sleep\n", 58 | "from hypothesis import given, settings, strategies as st\n", 59 | "\n", 60 | "# TODO: add a settings decorator which reduces max_examples (for speed)\n", 61 | "# and increases or disables the deadline so the test passes.\n", 62 | "\n", 63 | "@given(st.floats(min_value=0.1, max_value=0.3))\n", 64 | "def test_really_slow(delay):\n", 65 | " sleep(delay)\n", 66 | "\n", 67 | "test_really_slow()" 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "id": "1bab6558", 73 | "metadata": {}, 74 | "source": [ 75 | "### The `phases` setting\n", 76 | "\n", 77 | "The phases setting allows you to individually enable or disable [Hypothesis' six phases](https://hypothesis.readthedocs.io/en/latest/settings.html#controlling-what-runs), and has two main uses:\n", 78 | "\n", 79 | "- Disabling all but the `explicit` phase, reducing Hypothesis to parametrized tests ([e.g. here](https://github.com/python/cpython/pull/22863))\n", 80 | "- Enabling the `explain` phase, accepting some overhead to report additional feedback on failures\n", 81 | "\n", 82 | "Other use-cases tend to be esoteric, but are supported if you think of one." 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": null, 88 | "id": "d9e04b44", 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "# See `tests/test_settings.py` for this exercise." 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "7e84920c", 98 | "metadata": {}, 99 | "source": [ 100 | "### Dealing with a PRNG\n", 101 | "\n", 102 | "If you have test behaviour that depends on a psudeo-random number generator, and it's not being seeded between inputs, you're going to have some flaky tests. [`hypothesis.register_random()` to the rescue!](https://hypothesis.readthedocs.io/en/latest/details.html#making-random-code-deterministic)\n", 103 | "\n", 104 | "Try running this test a few times - you'll see the `Flaky` error - and then un-comment `hypothesis.register_random(r)`. Instant determinism!" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "id": "586378fe", 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "import random\n", 115 | "import hypothesis\n", 116 | "from hypothesis.strategies import integers\n", 117 | "\n", 118 | "r = random.Random()\n", 119 | "\n", 120 | "# hypothesis.register_random(r)\n", 121 | "\n", 122 | "@hypothesis.given(integers(0, 100))\n", 123 | "def test_sometimes_flaky(x):\n", 124 | " y = r.randint(0, 100)\n", 125 | " assert x <= y\n", 126 | "\n", 127 | "test_sometimes_flaky()" 128 | ] 129 | }, 130 | { 131 | "cell_type": "markdown", 132 | "id": "f9359f06", 133 | "metadata": {}, 134 | "source": [ 135 | "### `target()`ed property-based testing\n", 136 | "\n", 137 | "Random search works well... but [guided search with `hypothesis.target()`](https://hypothesis.readthedocs.io/en/latest/details.html#targeted-example-generation)\n", 138 | "is even better. Targeted search can help\n", 139 | "\n", 140 | "- find rare bugs ([e.g.](https://github.com/astropy/astropy/pull/10373))\n", 141 | "- understand bugs, by mitigating [the \"threshold problem\"](https://hypothesis.works/articles/threshold-problem/) (where shrinking makes severe bugs look marginal)" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "id": "bbf1c9a2", 148 | "metadata": {}, 149 | "outputs": [], 150 | "source": [ 151 | "# See `tests/test_settings.py` for this exercise." 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "id": "25ca56d3", 157 | "metadata": {}, 158 | "source": [ 159 | "### Hooks for external fuzzers\n", 160 | "\n", 161 | "If you're on Linux or OSX, you may want to [experiment with external fuzzers](https://hypothesis.readthedocs.io/en/latest/details.html#use-with-external-fuzzers).\n", 162 | "For example, [here's a fuzz-test for the Black autoformatter](https://github.com/psf/black/blob/3ef339b2e75468a09d617e6aa74bc920c317bce6/fuzz.py#L75-L85)\n", 163 | "using Atheris as the fuzzing engine.\n", 164 | "\n", 165 | "We can mock this up with our own very simple fuzzer:" 166 | ] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "id": "603fc32e", 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "from secrets import token_bytes as get_bytes_from_fuzzer\n", 176 | "from hypothesis import given, strategies as st\n", 177 | "\n", 178 | "\n", 179 | "@given(st.nothing())\n", 180 | "def test(_):\n", 181 | " pass \n", 182 | "\n", 183 | "\n", 184 | "# And now for the fuzzer:\n", 185 | "for _ in range(1000):\n", 186 | " payload = get_bytes_from_fuzzer(1000)\n", 187 | " test.hypothesis.fuzz_one_input(payload)" 188 | ] 189 | } 190 | ], 191 | "metadata": { 192 | "kernelspec": { 193 | "display_name": "Python 3 (ipykernel)", 194 | "language": "python", 195 | "name": "python3" 196 | } 197 | }, 198 | "nbformat": 4, 199 | "nbformat_minor": 5 200 | } 201 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -e . 2 | notebook 3 | numpy >= 1.11 4 | pytest >= 7.1 5 | hypothesis >= 6.45 6 | -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.10 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="pbt_tutorial", 5 | description="Library code for property-based testing tutorial", 6 | packages=find_packages(where="src", exclude=["tests*"]), 7 | package_dir={"": "src"}, 8 | version="1.1.0", 9 | python_requires=">=3.7", 10 | install_requires=["numpy>=1.11"], 11 | tests_requires=["pytest >= 7.1", "hypothesis >= 6.45"], 12 | ) 13 | -------------------------------------------------------------------------------- /src/pbt_tutorial/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rsokl/testing-tutorial/a02f55971d73b7102ee99d30c86a328dbee4e490/src/pbt_tutorial/__init__.py -------------------------------------------------------------------------------- /src/pbt_tutorial/basic_functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from itertools import groupby 3 | from typing import Any, Dict, List, Union 4 | 5 | __all__ = ["count_vowels", "merge_max_mappings"] 6 | 7 | 8 | def count_vowels(x: str, include_y: bool = False) -> int: 9 | """Returns the number of vowels contained in `x`. 10 | 11 | The vowel 'y' is included optionally. 12 | 13 | Parameters 14 | ---------- 15 | x : str 16 | The input string 17 | 18 | include_y : bool, optional (default=False) 19 | If `True` count y's as vowels 20 | 21 | Returns 22 | ------- 23 | vowel_count: int 24 | 25 | Examples 26 | -------- 27 | >>> count_vowels("happy") 28 | 1 29 | >>> count_vowels("happy", include_y=True) 30 | 2 31 | """ 32 | vowels = set("aeiouAEIOU") 33 | 34 | if include_y: 35 | vowels.update("yY") 36 | 37 | return sum(char in vowels for char in x) 38 | 39 | 40 | def merge_max_mappings( 41 | dict1: Dict[str, float], dict2: Dict[str, float] 42 | ) -> Dict[str, float]: 43 | """Merges two dictionaries based on the largest value 44 | in a given mapping. 45 | 46 | The keys of the dictionaries are presumed to be floats. 47 | 48 | Parameters 49 | ---------- 50 | dict1 : Dict[str, float] 51 | dict2 : Dict[str, float] 52 | 53 | Returns 54 | ------- 55 | merged : Dict[str, float] 56 | The dictionary containing all of the keys common 57 | between `dict1` and `dict2`, retaining the largest 58 | value from common mappings. 59 | 60 | Examples 61 | -------- 62 | >>> x = {"a": 1, "b": 2} 63 | >>> y = {"b": 100, "c": -1} 64 | >>> merge_max_mappings(x, y) 65 | {'a': 1, 'b': 100, 'c': -1} 66 | """ 67 | # `dict(dict1)` makes a copy of `dict1`. We do this 68 | # so that updating `merged` doesn't also update `dict1` 69 | merged = dict(dict1) 70 | for key, value in dict2.items(): 71 | if key not in merged or value > merged[key]: 72 | merged[key] = value 73 | return merged 74 | 75 | 76 | # EXTRA: Test-drive development 77 | 78 | # SOLUTION 79 | def leftpad(string: str, width: int, fillchar: str) -> str: 80 | """Left-pads `string` with `fillchar` until the resulting string 81 | has length `width`. 82 | 83 | Parameters 84 | ---------- 85 | string : str 86 | The input string 87 | 88 | width : int 89 | A non-negative integer specifying the minimum guaranteed 90 | width of the left-padded output string. 91 | 92 | fillchar : str 93 | The character (length-1 string) used to pad the string. 94 | 95 | Examples 96 | -------- 97 | The following is the intended behaviour of this function: 98 | 99 | >>> leftpad('cat', width=5, fillchar="Z") 100 | 'ZZcat' 101 | >>> leftpad('Dog', width=2, fillchar="Z") 102 | 'Dog' 103 | """ 104 | assert isinstance(width, int) and width >= 0, width 105 | assert isinstance(fillchar, str) and len(fillchar) == 1, fillchar 106 | margin = max(width - len(string), 0) 107 | return margin * fillchar + string 108 | 109 | 110 | def safe_name(obj: Any, repr_allowed: bool=True) -> str: 111 | """Tries to get a descriptive name for an object. Returns '` 112 | instead of raising - useful for writing descriptive/safe error messages.""" 113 | if hasattr(obj, "__qualname__"): 114 | return obj.__qualname__ 115 | 116 | if hasattr(obj, "__name__"): 117 | return obj.__name__ 118 | 119 | if repr_allowed and hasattr(obj, "__repr__"): 120 | return repr(obj) 121 | 122 | return "" 123 | 124 | 125 | def run_length_encoder(in_string: str) -> List[Union[str, int]]: 126 | """ 127 | >>> run_length_encoder("aaaaabbcbc") 128 | ['a', 'a', 5, 'b', 'b', 2, 'c', 'b', 'c'] 129 | """ 130 | assert isinstance(in_string, str) 131 | out = [] 132 | for item, group in groupby(in_string): 133 | cnt = sum(1 for x in group) 134 | if cnt == 1: 135 | out.append(item) 136 | else: 137 | out.extend((item, item, cnt)) 138 | assert isinstance(out, list) 139 | assert all(isinstance(x, (str, int)) for x in out) 140 | return out 141 | 142 | 143 | # SOLUTION 144 | def run_length_decoder(in_list: List[Union[str, int]]) -> str: 145 | """ 146 | >>> run_length_decoder(['a', 'a', 5, 'b', 'b', 2, 'c', 'b', 'c']) 147 | "aaaaabbcbc" 148 | """ 149 | out: str = "" 150 | for n, item in enumerate(in_list): 151 | if isinstance(item, int): 152 | char = in_list[n - 1] 153 | assert isinstance(char, str) 154 | out += char * (item - 2) 155 | else: 156 | out += item 157 | return out 158 | 159 | 160 | def pairwise_dists(x, y): 161 | """ Computing pairwise Euclidean distance between the respective 162 | row-vectors of `x` and `y` 163 | 164 | Parameters 165 | ---------- 166 | x : numpy.ndarray, shape=(M, D) 167 | y : numpy.ndarray, shape=(N, D) 168 | 169 | Returns 170 | ------- 171 | numpy.ndarray, shape=(M, N) 172 | The Euclidean distance between each pair of 173 | rows between `x` and `y`.""" 174 | sqr_dists = -2 * np.matmul(x, y.T) 175 | sqr_dists += np.sum(x**2, axis=1)[:, np.newaxis] 176 | sqr_dists += np.sum(y**2, axis=1) 177 | return np.sqrt(np.clip(sqr_dists, a_min=0, a_max=None)) 178 | 179 | 180 | def softmax(x): 181 | x = x - x.max() 182 | return np.exp(x) / np.exp(x).sum() 183 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from hypothesis import Phase, given, settings 4 | from hypothesis import strategies as st 5 | 6 | # Running 7 | # pytest --hypothesis-show-statistics tests/test_settings.py 8 | # 9 | # will report lines which were always and only run by failing examples. 10 | # How useful is the report in this case? Would it be useful on your 11 | # code? Does it report the same lines if you re-run the tests? 12 | 13 | 14 | @settings(phases=tuple(Phase)) # Activates the `explain` phase! 15 | @given( 16 | allow_nan=st.booleans(), 17 | obj=st.recursive( 18 | st.none() | st.booleans() | st.floats() | st.text(), 19 | extend=lambda x: st.lists(x) | st.dictionaries(st.text(), x), 20 | ), 21 | ) 22 | def test_roundtrip_dumps_loads(allow_nan, obj): 23 | encoded = json.dumps(obj=obj, allow_nan=allow_nan) 24 | decoded = json.loads(s=encoded) 25 | assert obj == decoded 26 | -------------------------------------------------------------------------------- /tests/test_statistics.py: -------------------------------------------------------------------------------- 1 | from hypothesis import event, given, note 2 | from hypothesis import strategies as st 3 | 4 | 5 | @given(st.integers()) 6 | def test_demonstrating_note(x): 7 | note(f"noting {x}") 8 | print(f"printing {x}") 9 | assert x < 3 10 | 11 | 12 | @given(st.integers().filter(lambda x: x % 2 == 0)) 13 | def test_even_integers(i): 14 | event(f"i mod 3 = {i%3}") 15 | -------------------------------------------------------------------------------- /tests/test_with_target.py: -------------------------------------------------------------------------------- 1 | from hypothesis import given 2 | from hypothesis import strategies as st 3 | from hypothesis import target 4 | 5 | # OK, what's going on here? 6 | # 7 | # If you run this test, it'll fail with x=1, y=-1 ... and you might wonder 8 | # if we have some kind of comparison bug involving the sign bit. 9 | # Uncomment the `target()` though, and you'll see a new line of output: 10 | # 11 | # Highest target score: ______ (label='difference between x and y') 12 | # 13 | # and that large score should make it obvious that the bug is not small! 14 | 15 | 16 | @given(st.integers(), st.integers()) 17 | def test_positive_and_negative_integers_are_equal(x, y): 18 | if x and y: 19 | # target(abs(x - y), label="difference between x and y") 20 | assert x == y 21 | --------------------------------------------------------------------------------