├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ ├── build-docs.yml │ ├── prevent-stable-merges.yml │ ├── release.yml │ └── run-tests.yml ├── .gitignore ├── CHANGELOG.md ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.MD ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── demo ├── README.md ├── fibonacci │ ├── fibonacci.ipynb │ ├── fibonacci_dyn.pkl │ ├── fibonacci_map.pkl │ ├── fibonacci_no_recurse.pkl │ ├── index.ipynb │ └── submissions │ │ ├── subm01.ipynb │ │ ├── subm02.ipynb │ │ └── subm03.ipynb ├── goldbach │ ├── goldbach-ref.pkl │ ├── goldbach.ipynb │ ├── index.ipynb │ └── submissions │ │ ├── 1.ipynb │ │ ├── 1.py │ │ ├── 2.ipynb │ │ ├── 2.py │ │ ├── 3.ipynb │ │ ├── 3.py │ │ ├── 4.ipynb │ │ ├── 4.py │ │ ├── 5.ipynb │ │ ├── 5.py │ │ ├── 6.ipynb │ │ └── 6.py └── median │ ├── index.ipynb │ ├── median.ipynb │ ├── median.pkl │ └── submissions │ ├── subm01.ipynb │ ├── subm02.ipynb │ ├── subm03.ipynb │ ├── subm04.ipynb │ ├── subm05.ipynb │ └── subm06.ipynb ├── docs ├── .nojekyll ├── Makefile ├── _static │ └── images │ │ └── pybryt_goals.png ├── annotations │ ├── collections.rst │ ├── complexity_annotations.rst │ ├── import_annotations.rst │ ├── index.rst │ ├── relational_annotations.rst │ ├── type_annotations.rst │ └── value_annotations │ │ ├── index.rst │ │ ├── initial_conditions.rst │ │ ├── invariants.rst │ │ └── structural_patterns.rst ├── api_reference.rst ├── cli_reference.rst ├── complexity.rst ├── conf.py ├── getting_started.ipynb ├── github_action.rst ├── index.html ├── index.rst ├── installation.rst ├── make.bat ├── reference_implementations │ ├── debugging.rst │ └── index.rst ├── requirements.txt ├── resources.rst ├── strassen.py └── student_implementations.rst ├── environment.yml ├── images ├── pybryt-goals.PNG └── pybryt-logo.PNG ├── pybryt ├── __init__.py ├── __main__.py ├── annotations │ ├── __init__.py │ ├── annotation.py │ ├── collection.py │ ├── complexity │ │ ├── __init__.py │ │ ├── annotation.py │ │ └── complexities.py │ ├── import_.py │ ├── initial_condition.py │ ├── invariants.py │ ├── relation.py │ ├── structural.py │ ├── type_.py │ └── value.py ├── cli.py ├── complexities.py ├── complexity.py ├── debug.py ├── execution │ ├── __init__.py │ ├── complexity.py │ ├── memory_footprint.py │ ├── tracing.py │ └── utils.py ├── integrations │ ├── __init__.py │ └── otter.py ├── invariants.py ├── plagiarism.py ├── preprocessors │ ├── __init__.py │ ├── abstract_preprocessor.py │ ├── imports.py │ └── intermediate_variables.py ├── reference.py ├── student.py ├── utils.py └── version.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── annotations ├── __init__.py ├── structural_helpers.py ├── test_annotation.py ├── test_collection.py ├── test_complexity.py ├── test_import_.py ├── test_initial_condition.py ├── test_invariants.py ├── test_relation.py ├── test_structural.py ├── test_type_.py ├── test_value.py └── utils.py ├── execution ├── __init__.py ├── test_complexity.py ├── test_memory_footprint.py ├── test_notebook_execution.py ├── test_tracing.py └── utils.py ├── files ├── expected_ref.pkl ├── expected_ref2.pkl └── expected_result.json ├── integrations ├── __init__.py └── test_otter.py ├── preprocessors ├── __init__.py ├── test_imports.py └── test_intermediate_variables.py ├── test_cli.py ├── test_complexity.py ├── test_debug.py ├── test_reference.py ├── test_student.py ├── test_utils.py └── utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = multiprocessing 3 | source = pybryt 4 | 5 | [report] 6 | exclude_lines = 7 | # Have to re-enable the standard pragma 8 | pragma: no cover 9 | 10 | # Don't complain if tests don't hit defensive assertion code: 11 | raise NotImplementedError 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/build-docs.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Build documentation 5 | 6 | on: 7 | push: 8 | branches: [ stable ] 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | 19 | - name: Set up Python 3.8 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: 3.8 23 | 24 | - name: Checkout to stable 25 | uses: actions/checkout@v2 26 | with: 27 | ref: stable 28 | 29 | - name: Checkout to gh-pages 30 | uses: actions/checkout@v2 31 | with: 32 | ref: gh-pages 33 | 34 | - name: Merge stable -> gh-pages 35 | run: | 36 | git config --global user.name "github-actions" 37 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" 38 | git merge origin/stable -Xtheirs --allow-unrelated-histories 39 | git push origin gh-pages 40 | 41 | - name: Install dependencies 42 | run: | 43 | $CONDA/bin/conda env update -f environment.yml -n base 44 | $CONDA/bin/pip install -r docs/requirements.txt 45 | sudo apt-get install -y pandoc 46 | 47 | - name: Build documentation 48 | working-directory: ./docs 49 | run: | 50 | make html SPHINXBUILD=$CONDA/bin/sphinx-build 51 | 52 | - name: Commit documentation 53 | run: | 54 | git add -f docs/html 55 | git commit -m "build docs for ${{ github.sha }}" || : 56 | git push 57 | 58 | -------------------------------------------------------------------------------- /.github/workflows/prevent-stable-merges.yml: -------------------------------------------------------------------------------- 1 | name: Only allow main -> stable 2 | 3 | on: 4 | pull_request: 5 | branches: [ stable ] 6 | 7 | jobs: 8 | check-branch: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Only allow main -> stable merges 12 | run: | 13 | # *Only* main can be merged into stable 14 | echo "Merging ${GITHUB_HEAD_REF} into ${GITHUB_BASE_REF}" 15 | python3 -c 'import os, sys; sys.exit(not (os.environ["GITHUB_HEAD_REF"] == "main" if os.environ["GITHUB_BASE_REF"] == "stable" else True))' 16 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # follow instructions here: https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 2 | 3 | name: Publish release 4 | 5 | on: 6 | workflow_dispatch: 7 | inputs: 8 | version: 9 | description: 'New version number' 10 | required: true 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | 19 | - uses: actions/checkout@v2 20 | 21 | - name: Set up Python 3.8 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: 3.8 25 | 26 | - name: Install dependencies 27 | run: | 28 | python3 -m pip install -r requirements.txt twine wheel setuptools 29 | 30 | - name: Checkout to stable 31 | uses: actions/checkout@v2 32 | with: 33 | ref: stable 34 | 35 | - name: Merge main -> stable 36 | run: | 37 | git config --global user.name "github-actions" 38 | git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" 39 | git merge origin/main -Xtheirs --allow-unrelated-histories 40 | 41 | - name: Run release script 42 | env: 43 | TWINE_USERNAME: __token__ 44 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 45 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 46 | run: | 47 | make release VERSION="${{ github.event.inputs.version }}" 48 | 49 | - name: Push updates 50 | run: | 51 | git push origin stable 52 | 53 | - name: PR stable back into main 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | run: | 57 | hub pull-request -b microsoft:main -h microsoft:stable -l "release" \ 58 | -m "Release v${{ github.event.inputs.version }}"$'\n'$'\n'"Updates from the release of version ${{ github.event.inputs.version }}" 59 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Run tests 5 | 6 | on: 7 | push: 8 | branches: [ main ] 9 | pull_request: 10 | branches: [ main ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | 20 | - name: Set up Python 3.8 21 | uses: actions/setup-python@v2 22 | with: 23 | python-version: 3.8 24 | 25 | - name: Install dependencies 26 | run: | 27 | $CONDA/bin/conda env update -f environment.yml -n base 28 | 29 | - name: Run pytest 30 | run: | 31 | make testcov COVERAGE=$CONDA/bin/coverage 32 | 33 | - name: Upload coverage 34 | run: | 35 | $CONDA/bin/codecov 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .vscode/settings.json 132 | .doctrees/ 133 | .OTTER_LOG 134 | /*.ipynb 135 | /*.pkl 136 | /notebooks 137 | .pybryt_cache 138 | 139 | # ignore docs build 140 | /docs/html 141 | 142 | # ignore extraneous files from the getting started guide 143 | /docs/*.pkl 144 | /docs/hashes.json 145 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Stott" 5 | given-names: "Lee" 6 | orcid: "https://orcid.org/0000-0002-3715-0892" 7 | - family-names: "Gilad-Bachrach" 8 | given-names: "Ran" 9 | orcid: "https://orcid.org/0000-0002-4001-8307" 10 | - family-names: Pyles 11 | given-names: Christopher 12 | orcid: "https://orcid.org/0000-0001-8520-7593" 13 | - family-names: "Beg" 14 | given-names: "Marijan" 15 | orcid: "https://orcid.org/0000-0002-6670-3994" 16 | - family-names: "Levy" 17 | given-names: "Nir" 18 | orcid: "https://orcid.org/0000-0002-4256-4934" 19 | - family-names: "Gorman" 20 | given-names: "Gerard John" 21 | orcid: "https://orcid.org/0000-0003-0563-3678" 22 | - family-names: "Percival" 23 | given-names: "James Robert" 24 | orcid: "https://orcid.org/0000-0002-6556-0055" 25 | - family-names: "Rhodri" 26 | given-names: "Nelson" 27 | orcid: "https://orcid.org/0000-0003-2768-5735" 28 | title: "Pybryt - Python library for pedagogical auto-assessment" 29 | version: 0.7.0 30 | date-released: 2022-04-28 31 | url: "https://github.com/microsoft/pybryt" 32 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.MD: -------------------------------------------------------------------------------- 1 | 2 | ## Contributing 3 | 4 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 5 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 6 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 7 | 8 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 9 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 10 | provided by the bot. You will only need to do this once across all repos using our CLA. 11 | 12 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 13 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 14 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for PyBryt 2 | # ------------------- 3 | # To generate a release, use `make release` with the `VERSION` argument: 4 | # $ make release VERSION=0.0.1 5 | # 6 | # To run tests, use `make test` with the `TESTPATH` and/or `PYTESTOPTS` arguments: 7 | # $ make test 8 | # 9 | # The `testcov` target can be used to build a local copy of the code coverage in HTML: 10 | # $ make testcov 11 | # 12 | # To build the docs, use `make docs`: 13 | # $ make docs 14 | 15 | PYTEST = pytest 16 | TESTPATH = tests 17 | PYTESTOPTS = -v 18 | COVERAGE = coverage 19 | DATE := $(shell date "+%F") 20 | 21 | release: 22 | rm dist/* || : 23 | echo '__version__ = "$(VERSION)"' > pybryt/version.py 24 | sed -i "s/date-released: [0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}/date-released: $(DATE)/" CITATION.cff 25 | sed -i "s/^version: [0-9]\{1,\}\.[0-9]\{1,\}\.[0-9]\{1,\}/version: $(VERSION)/" CITATION.cff 26 | git add pybryt/version.py 27 | git add CITATION.cff 28 | git commit -m "update version info for v$(VERSION)" 29 | python3 setup.py sdist bdist_wheel 30 | hub release create -a dist/*.tar.gz -a dist/*.whl -m 'v$(VERSION)' $(VERSION) 31 | python3 -m twine upload dist/* 32 | 33 | test: 34 | $(PYTEST) $(TESTPATH) $(PYTESTOPTS) 35 | 36 | testcov: 37 | $(COVERAGE) run -m pytest $(TESTPATH) $(PYTESTOPTS) 38 | $(COVERAGE) combine 39 | 40 | htmlcov: testcov 41 | $(COVERAGE) html 42 | 43 | .PHONY: docs 44 | docs: 45 | $(MAKE) -C docs html 46 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## How to file issues and get help 4 | 5 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 6 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 7 | feature request as a new Issue. 8 | 9 | For help and questions about using this project, please open an issue. 10 | 11 | ## Microsoft Support Policy 12 | 13 | Support for this project is limited to the resources listed above. 14 | -------------------------------------------------------------------------------- /demo/fibonacci/fibonacci_dyn.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/demo/fibonacci/fibonacci_dyn.pkl -------------------------------------------------------------------------------- /demo/fibonacci/fibonacci_map.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/demo/fibonacci/fibonacci_map.pkl -------------------------------------------------------------------------------- /demo/fibonacci/fibonacci_no_recurse.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/demo/fibonacci/fibonacci_no_recurse.pkl -------------------------------------------------------------------------------- /demo/fibonacci/submissions/subm02.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# The Fibonacci Sequence" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 5, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import pybryt" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "nbgrader": { 23 | "grade": false, 24 | "grade_id": "intro", 25 | "locked": true, 26 | "schema_version": 2, 27 | "solution": false 28 | } 29 | }, 30 | "source": [ 31 | "In this exercise you will implement a function that returns the $n^\\text{th}$ Fibonacci sequence. Recall that the Fibonacci sequence is defined by the relation $i_n = i_{n-1} + i_{n-2}$ with $i_0 = 0$ and $i_1 = 1$." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 6, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import numpy as np\n", 41 | "\n", 42 | "def fibonacci(n):\n", 43 | " \"\"\"\n", 44 | " Return the n^th number in the Fibonacci sequence, n >= 0.\n", 45 | "\n", 46 | " Uses the dynamic programming implementation of the Fibonacci sequence algorithm.\n", 47 | " \n", 48 | " Parameters\n", 49 | " ----------\n", 50 | " n : integer\n", 51 | " the index of the desired Fibonacci number\n", 52 | "\n", 53 | " Returns\n", 54 | " -------\n", 55 | " integer\n", 56 | " the n^th Fibonacci number\n", 57 | " \"\"\"\n", 58 | " fibs = np.zeros(n + 1, dtype=int)\n", 59 | " if n > 0:\n", 60 | " fibs[1] = 1\n", 61 | " for i in range(2, n + 1):\n", 62 | " fibs[i] = fibs[i-1] + fibs[i-2]\n", 63 | " return fibs[n]" 64 | ] 65 | }, 66 | { 67 | "cell_type": "markdown", 68 | "metadata": {}, 69 | "source": [ 70 | "Let's test your `fibonacci` function by looking at the first 10 Fibonacci numbers. We'll also check the time complexity of your implementation." 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 7, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "name": "stdout", 80 | "output_type": "stream", 81 | "text": [ 82 | "0 1 1 2 3 5 8 13 21 34 " 83 | ] 84 | } 85 | ], 86 | "source": [ 87 | "for n in range(10):\n", 88 | " print(fibonacci(n), end=\" \")" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 8, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "name": "stdout", 98 | "output_type": "stream", 99 | "text": [ 100 | "CPU times: user 19.4 ms, sys: 730 µs, total: 20.1 ms\n", 101 | "Wall time: 20.3 ms\n" 102 | ] 103 | } 104 | ], 105 | "source": [ 106 | "%%time\n", 107 | "checker = pybryt.TimeComplexityChecker(\"fib_runtime\")\n", 108 | "for n in range(30):\n", 109 | " with checker(n):\n", 110 | " fibonacci(n)\n", 111 | "checker.determine_complexity()" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "metadata": {}, 117 | "source": [ 118 | " " 119 | ] 120 | } 121 | ], 122 | "metadata": { 123 | "@webio": { 124 | "lastCommId": null, 125 | "lastKernelId": null 126 | }, 127 | "_change_revision": 0, 128 | "_is_fork": false, 129 | "celltoolbar": "Create Assignment", 130 | "kernelspec": { 131 | "display_name": "Python 3 (ipykernel)", 132 | "language": "python", 133 | "name": "python3" 134 | }, 135 | "language_info": { 136 | "codemirror_mode": { 137 | "name": "ipython", 138 | "version": 3 139 | }, 140 | "file_extension": ".py", 141 | "mimetype": "text/x-python", 142 | "name": "python", 143 | "nbconvert_exporter": "python", 144 | "pygments_lexer": "ipython3", 145 | "version": "3.7.7" 146 | }, 147 | "toc": { 148 | "colors": { 149 | "hover_highlight": "#DAA520", 150 | "navigate_num": "#000000", 151 | "navigate_text": "#333333", 152 | "running_highlight": "#FF0000", 153 | "selected_highlight": "#FFD700", 154 | "sidebar_border": "#EEEEEE", 155 | "wrapper_background": "#FFFFFF" 156 | }, 157 | "moveMenuLeft": true, 158 | "nav_menu": { 159 | "height": "301px", 160 | "width": "252px" 161 | }, 162 | "navigate_menu": true, 163 | "number_sections": true, 164 | "sideBar": true, 165 | "threshold": 4, 166 | "toc_cell": false, 167 | "toc_section_display": "block", 168 | "toc_window_display": false, 169 | "widenNotebook": false 170 | } 171 | }, 172 | "nbformat": 4, 173 | "nbformat_minor": 4 174 | } 175 | -------------------------------------------------------------------------------- /demo/fibonacci/submissions/subm03.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# The Fibonacci Sequence" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 8, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import pybryt" 17 | ] 18 | }, 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "nbgrader": { 23 | "grade": false, 24 | "grade_id": "intro", 25 | "locked": true, 26 | "schema_version": 2, 27 | "solution": false 28 | } 29 | }, 30 | "source": [ 31 | "In this exercise you will implement a function that returns the $n^\\text{th}$ Fibonacci sequence. Recall that the Fibonacci sequence is defined by the relation $i_n = i_{n-1} + i_{n-2}$ with $i_0 = 0$ and $i_1 = 1$." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 9, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import numpy as np\n", 41 | "\n", 42 | "def fibonacci(n):\n", 43 | " \"\"\"\n", 44 | " Return the n^th number in the Fibonacci sequence, n >= 0.\n", 45 | "\n", 46 | " Uses the dynamic programming implementation of the Fibonacci sequence algorithm.\n", 47 | " \n", 48 | " Parameters\n", 49 | " ----------\n", 50 | " n : integer\n", 51 | " the index of the desired Fibonacci number\n", 52 | "\n", 53 | " Returns\n", 54 | " -------\n", 55 | " integer\n", 56 | " the n^th Fibonacci number\n", 57 | " \"\"\"\n", 58 | " if n == 0:\n", 59 | " return 0\n", 60 | " if n == 1:\n", 61 | " return 1\n", 62 | " return fibonacci(n-1) + fibonacci(n-2)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": {}, 68 | "source": [ 69 | "Let's test your `fibonacci` function by looking at the first 10 Fibonacci numbers. We'll also check the time complexity of your implementation." 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 7, 75 | "metadata": {}, 76 | "outputs": [ 77 | { 78 | "name": "stdout", 79 | "output_type": "stream", 80 | "text": [ 81 | "0 1 1 2 3 5 8 13 21 34 " 82 | ] 83 | } 84 | ], 85 | "source": [ 86 | "for n in range(10):\n", 87 | " print(fibonacci(n), end=\" \")" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 8, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "CPU times: user 19.4 ms, sys: 730 µs, total: 20.1 ms\n", 100 | "Wall time: 20.3 ms\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "%%time\n", 106 | "checker = pybryt.TimeComplexityChecker(\"fib_runtime\")\n", 107 | "for n in range(30):\n", 108 | " with checker(n):\n", 109 | " fibonacci(n)\n", 110 | "checker.determine_complexity()" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | " " 118 | ] 119 | } 120 | ], 121 | "metadata": { 122 | "@webio": { 123 | "lastCommId": null, 124 | "lastKernelId": null 125 | }, 126 | "_change_revision": 0, 127 | "_is_fork": false, 128 | "celltoolbar": "Create Assignment", 129 | "kernelspec": { 130 | "display_name": "Python 3 (ipykernel)", 131 | "language": "python", 132 | "name": "python3" 133 | }, 134 | "language_info": { 135 | "codemirror_mode": { 136 | "name": "ipython", 137 | "version": 3 138 | }, 139 | "file_extension": ".py", 140 | "mimetype": "text/x-python", 141 | "name": "python", 142 | "nbconvert_exporter": "python", 143 | "pygments_lexer": "ipython3", 144 | "version": "3.7.7" 145 | }, 146 | "toc": { 147 | "colors": { 148 | "hover_highlight": "#DAA520", 149 | "navigate_num": "#000000", 150 | "navigate_text": "#333333", 151 | "running_highlight": "#FF0000", 152 | "selected_highlight": "#FFD700", 153 | "sidebar_border": "#EEEEEE", 154 | "wrapper_background": "#FFFFFF" 155 | }, 156 | "moveMenuLeft": true, 157 | "nav_menu": { 158 | "height": "301px", 159 | "width": "252px" 160 | }, 161 | "navigate_menu": true, 162 | "number_sections": true, 163 | "sideBar": true, 164 | "threshold": 4, 165 | "toc_cell": false, 166 | "toc_section_display": "block", 167 | "toc_window_display": false, 168 | "widenNotebook": false 169 | } 170 | }, 171 | "nbformat": 4, 172 | "nbformat_minor": 4 173 | } 174 | -------------------------------------------------------------------------------- /demo/goldbach/goldbach-ref.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/demo/goldbach/goldbach-ref.pkl -------------------------------------------------------------------------------- /demo/goldbach/submissions/1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "def check_goldbach_for_num(n, primes_set):\n", 10 | " primes_set = sorted(primes_set)\n", 11 | " left = 0\n", 12 | " right = len(primes_set) - 1\n", 13 | " index_s = None\n", 14 | " index_b = None\n", 15 | " while left <= right:\n", 16 | " mid = (left + right) // 2\n", 17 | " if n == primes_set[mid]:\n", 18 | " return \"number is prime\"\n", 19 | " elif right == left:\n", 20 | " if n > mid:\n", 21 | " index_s = mid\n", 22 | " break\n", 23 | " else:\n", 24 | " index_b = mid\n", 25 | " elif n < primes_set[mid]:\n", 26 | " if n > primes_set[mid - 1]:\n", 27 | " index_b = mid\n", 28 | " index_s = mid - 1\n", 29 | " break\n", 30 | " right = mid - 1\n", 31 | " else:\n", 32 | " if n < primes_set[mid + 1]:\n", 33 | " index_b = mid + 1\n", 34 | " index_s = mid\n", 35 | " break\n", 36 | " left = mid + 1\n", 37 | " for i in range(0, len(primes_set) - 1):\n", 38 | " if n-primes_set[i] in primes_set:\n", 39 | " return True\n", 40 | " return False\n", 41 | "\n" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "from sympy.ntheory.generate import primerange\n", 51 | "primes = set(primerange(50,1000))\n", 52 | "check_goldbach_for_num(116, primes)" 53 | ] 54 | } 55 | ], 56 | "metadata": {}, 57 | "nbformat": 4, 58 | "nbformat_minor": 4 59 | } 60 | -------------------------------------------------------------------------------- /demo/goldbach/submissions/1.py: -------------------------------------------------------------------------------- 1 | def check_goldbach_for_num(n, primes_set): 2 | primes_set = sorted(primes_set) 3 | left = 0 4 | right = len(primes_set) - 1 5 | index_s = None 6 | index_b = None 7 | while left <= right: 8 | mid = (left + right) // 2 9 | if n == primes_set[mid]: 10 | return "number is prime" 11 | elif right == left: 12 | if n > mid: 13 | index_s = mid 14 | break 15 | else: 16 | index_b = mid 17 | elif n < primes_set[mid]: 18 | if n > primes_set[mid - 1]: 19 | index_b = mid 20 | index_s = mid - 1 21 | break 22 | right = mid - 1 23 | else: 24 | if n < primes_set[mid + 1]: 25 | index_b = mid + 1 26 | index_s = mid 27 | break 28 | left = mid + 1 29 | for i in range(0, len(primes_set) - 1): 30 | if n-primes_set[i] in primes_set: 31 | return True 32 | return False 33 | 34 | -------------------------------------------------------------------------------- /demo/goldbach/submissions/2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "def check_goldbach_for_num(n,primes_set) :\n", 10 | " '''gets an even integer- n, and a set of primes- primes_set. Returns whether there're two primes which their sum is n'''\n", 11 | " \n", 12 | " relevant_primes_set={p for p in primes_set if p` annotation, which collects a series 8 | of other annotations into a group. Collections can be instantiated by passing zero or more 9 | annotations to the constructor, along with the other usual keyword arguments for other annotations. 10 | To add additional annotations to the collection, use 11 | :py:meth:`Collection.add`. 12 | 13 | .. code-block:: python 14 | 15 | l = [3, 2, 1, 4, -3, -3, 3, 4] 16 | 17 | collection = pybryt.Collection(pybryt.Value(l)) 18 | 19 | # calculate the partial products of a list 20 | prod = 1 21 | for i in range(len(l)): 22 | prod *= l[i] 23 | l[i] = prod 24 | collection.add(pybryt.Value(l)) 25 | 26 | Annotations can be removed from the collection using 27 | :py:meth:`Collection.remove`: 28 | 29 | .. code-block:: python 30 | 31 | val = pybryt.Value(l) 32 | collection = pybryt.Collection(val) 33 | collection.remove(val) 34 | 35 | Collection annotations can be used to simplify the creation of temporal annotations. Rather than 36 | reassigning variables and using the ``before`` method over and over again, you can simply create 37 | a collection and set ``enforce_order=True`` in the constructor. This will mean that the collection 38 | is only satisfied if the satisfying timestamps of its annotations occur in non-decreasing order based 39 | on the order the annotations were added in. For the example above, this can be done with: 40 | 41 | .. code-block:: python 42 | 43 | l = [3, 2, 1, 4, -3, -3, 3, 4] 44 | 45 | collection = pybryt.Collection(pybryt.Value(l), enforce_order=True) # this is the only difference! 46 | 47 | # calculate the partial products of a list 48 | prod = 1 49 | for i in range(len(l)): 50 | prod *= l[i] 51 | l[i] = prod 52 | collection.add(pybryt.Value(l)) 53 | 54 | Collections can also be used to simplify the management of success and failure messages by linking 55 | them to a specific group of annotations: 56 | 57 | .. code-block:: python 58 | 59 | l = [3, 2, 1, 4, -3, -3, 3, 4] 60 | 61 | collection = pybryt.Collection(pybryt.Value(l), enforce_order=True, 62 | failure_message="Take a look at your partial product algorithm.") 63 | 64 | # calculate the partial products of a list 65 | prod = 1 66 | for i in range(len(l)): 67 | prod *= l[i] 68 | l[i] = prod 69 | collection.add(pybryt.Value(l)) 70 | -------------------------------------------------------------------------------- /docs/annotations/import_annotations.rst: -------------------------------------------------------------------------------- 1 | .. _import: 2 | 3 | Import Annotations 4 | ================== 5 | 6 | Import annotations can be used to either require or forbid the use of specific libraries. When PyBryt 7 | executes a notebook, it parses the AST of each code cell to determine any libraries imported in the 8 | code. It also captures the modules that any functions or classes it finds belong to when it is 9 | tracing through the notebook's code. These are collected into a set of library names, which are 10 | included in the memory footprint. 11 | 12 | All import annotations are subclasses of the abstract 13 | :py:class:`ImportAnnotation` class. When these 14 | annotations are instantiated, they attempt to import the module to ensure that the string passed 15 | for the module name is valid. This means that any modules used in import annotations must be installed 16 | in the environment in which the reference implementation is compiled. 17 | 18 | 19 | Require Import 20 | -------------- 21 | 22 | To require an import, use the :py:class:`RequireImport` 23 | annotation, which takes the name of the library as a string (along with the usual set of arguments 24 | that annotation constructors accept). 25 | 26 | .. code-block:: python 27 | 28 | pybryt.RequireImport("pandas") 29 | 30 | 31 | Forbid Import 32 | ------------- 33 | 34 | To forbid an import, use the :py:class:`ForbidImport` 35 | annotation, which, like the :py:class:`RequireImport` 36 | annotation, takes the name of the libary as its only positional argument. 37 | 38 | .. code-block:: python 39 | 40 | pybryt.ForbidImport("numpy") 41 | -------------------------------------------------------------------------------- /docs/annotations/index.rst: -------------------------------------------------------------------------------- 1 | .. _annotations: 2 | 3 | Annotations 4 | =========== 5 | 6 | .. toctree:: 7 | :maxdepth: 3 8 | :hidden: 9 | 10 | value_annotations/index 11 | relational_annotations 12 | complexity_annotations 13 | type_annotations 14 | import_annotations 15 | collections 16 | 17 | Annotations are the basic building blocks, out of which reference 18 | implementations are constructed. The annotations represent a single condition 19 | that a student's implementation should meet, and they define a set of behaviors 20 | for responding to the passing or failing of those conditions. Annotations 21 | provide conditions not only for expecting a specific value but also for 22 | combining those expectations to form conditions on the structure of the 23 | student's code, including the temporal relationship of values and complex 24 | boolean logic surrounding the presence or absence of those values. 25 | 26 | All annotations are created by instantiating subclasses of the abstract 27 | :py:class:`Annotation` class. There 28 | are five main types of annotations: 29 | 30 | * :ref:`value annotations` 31 | * :ref:`relational annotations` 32 | * :ref:`complexity annotations` 33 | * :ref:`type annotations` 34 | * :ref:`import annotations` 35 | * :ref:`annotation collections` 36 | 37 | 38 | Annotation Arguments 39 | -------------------- 40 | 41 | All annotations contain some core configurations that can be set using keyword arguments in the 42 | constructor or by accessing the instance variables of that object. The table below lists these 43 | arguments. 44 | 45 | .. list-table:: 46 | :widths: 10 10 10 70 47 | 48 | * - Field Name 49 | - Keyword Argument 50 | - Type 51 | - Description 52 | * - ``name`` 53 | - ``name`` 54 | - ``str`` 55 | - The name of the annotation; used to process ``limit``; automatically set if unspecified 56 | * - ``limit`` 57 | - ``limit`` 58 | - ``str`` 59 | - The maximum number of annotations with a given ``name`` to track 60 | * - ``group`` 61 | - ``group`` 62 | - ``str`` 63 | - The name of a group to which this annotation belongs; for running specific groups of annotations 64 | from a reference implementation 65 | * - ``success_message`` 66 | - ``success_message`` 67 | - ``str`` 68 | - A message to be displayed to the student if the annotation is satisfied 69 | * - ``failure_message`` 70 | - ``failure_message`` 71 | - ``str`` 72 | - A message to be displayed to the student if the annotation is *not* satisfied 73 | -------------------------------------------------------------------------------- /docs/annotations/relational_annotations.rst: -------------------------------------------------------------------------------- 1 | .. _relational: 2 | 3 | Relational Annotations 4 | ====================== 5 | 6 | Relational annotations define some kind of relationship between two or more 7 | annotations. Currently, PyBryt supports two types of relational annotations: 8 | 9 | * temporal annotations and 10 | * boolean annotations. 11 | 12 | All relational annotations are subclasses of the abstract 13 | :py:class:`RelationalAnnotation` 14 | class, which defines some helpful defaults for working with annotations that 15 | have child annotations. 16 | 17 | 18 | Temporal Annotations 19 | -------------------- 20 | 21 | Temporal annotations describe *when* variables should appear in student's code 22 | relative to one another. For example, let us consider the problem of a dynamic 23 | programming algorithm to compute the Fibonacci sequence: the array containing 24 | :math:`n-1` first Fibonacci numbers should appear in memory before the array 25 | with :math:`n` first Fibonacci numbers. To enforce such a constraint, the 26 | :py:class:`Annotation` class defines a ``before`` method that 27 | asserts that one annotation occurs before another: 28 | 29 | .. code-block:: python 30 | 31 | def fib(n): 32 | """ 33 | Compute and return an array of the first n Fibonacci numbers using dynamic programming. 34 | 35 | Args: 36 | n (``int``): the number of Fibonacci numbers to return 37 | 38 | Returns: 39 | ``np.ndarray``: the first ``n`` Fibonacci numbers 40 | 41 | """ 42 | fibs = np.zeros(n, dtype=int) 43 | 44 | fibs[0] = 0 45 | curr_val = pybryt.Value(fibs) 46 | if n == 1: 47 | return fibs 48 | 49 | fibs[1] = 1 50 | v = pybryt.Value(fibs) 51 | curr_val.before(v) # we expect curr_val to appear in memory before v 52 | curr_val = v 53 | if n == 2: 54 | return fibs 55 | 56 | for i in range(2, n): 57 | fibs[i] = fibs[i-1] + fibs[i-2] 58 | 59 | v = pybryt.Value(fibs) # array of first n Fibonacci numbers 60 | curr_val.before(v) # check that first n-1 appear before first n Fibonacci numbers 61 | curr_val = v # update curr_val for next iteration 62 | 63 | return fibs 64 | 65 | In the example above, updating ``curr_val`` in the loop allows us to create a 66 | ``before`` condition to ensure the student followed the correct dynamic 67 | programming algorithm by checking each update to the ``fibs`` array. 68 | 69 | Temporal annotations are satisfied when the student's code satisfies all of the 70 | child :py:class:`Value` annotations and when the first annotation 71 | (the one calling :py:meth:`Annotation.before`) has a 72 | timestamp greater than or equal to the timestamp of the second annotation. 73 | 74 | Note that :py:meth:`Annotation.before` returns an 75 | instance of the 76 | :py:class:`BeforeAnnotation` 77 | class, which is itself a subclass of :py:class:`Annotation` 78 | and supports all of the same operations. 79 | :py:class:`Annotation` also provides 80 | :py:meth:`Annotation.after`, which also returns an 81 | instance of the 82 | :py:class:`BeforeAnnotation` 83 | class, but with the operands switched. 84 | 85 | 86 | Boolean Annotations 87 | ------------------- 88 | 89 | Boolean annotations define conditions on the presence of different values. For 90 | example, in solving an exercise, students may be able to take two different 91 | paths, and this logic can be enforced using a 92 | :py:class:`XorAnnotation` to ensure 93 | that only one of the two possible values is present. 94 | 95 | Relational annotations can be created either by instantiating the classes 96 | directly using the constructor or, as it is more recommended, by using Python's 97 | bitwise logical operators, ``&``, ``|``, ``^``, and ``~``, on annotations. The 98 | special (dunder) methods for these operators have been overridden in 99 | :py:class:`Annotation` class, and return the 100 | :py:class:`RelationalAnnotation` 101 | subclass instance corresponding to the logical operator used. 102 | 103 | To create the XOR example from two values ``v1`` and ``v2``, we write 104 | 105 | .. code-block:: python 106 | 107 | v1 ^ v2 108 | 109 | To assert that a student should *not* have a specific value ``v`` in their code, 110 | we use 111 | 112 | .. code-block:: python 113 | 114 | ~v 115 | -------------------------------------------------------------------------------- /docs/annotations/type_annotations.rst: -------------------------------------------------------------------------------- 1 | .. _type: 2 | 3 | Type Annotations 4 | ================ 5 | 6 | Type annotations are somewhat similar to value annotations, although they operate on types 7 | insted of individual values or objects. They can be used to check the presence, or lack thereof, of 8 | objects of a specific type in a memory footprint, or conditions on that presence. 9 | 10 | 11 | Forbid Type 12 | ----------- 13 | 14 | The :py:class:`ForbidType` annotation asserts that a student's 15 | memory footprint has no values of a specific type in it. For example, if students should be implementing 16 | an algorithm without vectorization, it is possible to create an annotation forbidding the use of 17 | arrays: 18 | 19 | .. code-block:: python 20 | 21 | import numpy as np 22 | 23 | pybryt.ForbidType(np.ndarray, failure_message="Arrays are not allowed!") 24 | 25 | Checking for values is performed using Python's built-in ``isinstance`` function. Types passed to 26 | the constructor must be pickleable with the ``dill`` library. 27 | -------------------------------------------------------------------------------- /docs/annotations/value_annotations/initial_conditions.rst: -------------------------------------------------------------------------------- 1 | Initial Conditions 2 | ================== 3 | 4 | For some problems, it is necessary for students to choose some configurations or initial conditions 5 | that can vary from student to student; handling all possible values of these initial conditions 6 | would require writing quite a few references. Instead, PyBryt offers the 7 | :py:class:`InitialCondition` class to 8 | represent a value that will be set when the student's submission is executed for use in writing 9 | annotations. 10 | 11 | 12 | Writing a Reference with Initial Conditions 13 | ------------------------------------------- 14 | 15 | Using initial conditions in your references is pretty simple. Each 16 | :py:class:`InitialCondition` has a name, 17 | the first argument to its constructor. This name is how the value in the student submission will be 18 | identified (but more on that later). 19 | 20 | Once you've instantiated an 21 | :py:class:`InitialCondition`, you can use it 22 | as normal in value and other types of annotations (except attribute annotations). 23 | 24 | .. code-block:: python 25 | 26 | ic = pybryt.InitialCondition("foo") 27 | pybryt.Value(ic) 28 | 29 | Sometimes, however, you may want to look for a value derived from an initial condition. For this 30 | reason, :py:class:`InitialCondition` 31 | supports all of Python's arithmetic operators, and you can also apply transformations by writing 32 | functions: 33 | 34 | .. code-block:: python 35 | 36 | pybryt.Value(ic + 2) 37 | pybryt.Value(2 * ic - 3) 38 | 39 | pybryt.Value(ic.apply(np.transpose).apply(np.linalg.norm)) 40 | pybryt.Value(ic.apply(lambda v: pow(v, 10, 73))) 41 | 42 | Each statement inside the :py:class:`Value` constructors above 43 | returns an :py:class:`InitialCondition`. 44 | Initial conditions maintain a list of transformations that need to be applied to reach the final 45 | value for the annotation, and when a value is supplied from the submission, the transformations 46 | are applied in sequence to determine the value that the value annotations should look for. 47 | 48 | 49 | Collecting Initial Conditions in Submissions 50 | -------------------------------------------- 51 | 52 | In order to link the initial conditions in the reference implementation to the values in the 53 | submission, you must call 54 | :py:func:`pybryt.set_initial_conditions` in the 55 | submission. This function accepts a dictionary as its only argument mapping strings corresponding 56 | to the names of initial conditions to the values of those initial conditions. When PyBryt is not 57 | actively tracing, it has no effect; but, when called while PyBryt executes the submission, it tells 58 | PyBryt to store the initial conditions passed to it in the memory footprint for later use by the 59 | annotations. 60 | 61 | For example, continuing the example from above, you would need to call 62 | 63 | .. code-block:: python 64 | 65 | pybryt.set_initial_conditions({ 66 | "foo": 2, 67 | }) 68 | 69 | to set the initial condition. 70 | 71 | This function can be called multiple times, but the memory footprint only retains a single store of 72 | initial conditions, so calling it multiple times with the same key will overwrite any old values 73 | of that key. (For example, calling it with ``{"foo": 1, "bar": 3}`` and then ``{"foo": 2}`` 74 | will result in initial conditions ``{"foo": 2, "bar": 3}``). 75 | -------------------------------------------------------------------------------- /docs/annotations/value_annotations/invariants.rst: -------------------------------------------------------------------------------- 1 | .. _invariants: 2 | 3 | Invariants 4 | ========== 5 | 6 | Invariants provide logic for determining whether a value satisfies a value annotation. They can be 7 | used to ensure that annotations can still be satisfied independent of the different ways students 8 | format their answers. 9 | 10 | 11 | Invariant Structure 12 | ------------------- 13 | 14 | Invariants are subclasses of the abstract base class 15 | :py:class:`invariants.invariant`. All subclasses implement 16 | the static :py:meth:`run` method, which takes in a list of 17 | objects and returns a transformed list of objects that contains all versions of every object in the 18 | input list which are considered equal. 19 | 20 | Consider an invariant for string capitalization. The ``run`` method of this invariant takes in a list 21 | of objects and returns the same list but with every string lowercased. For a matrix transposition 22 | invariant, the ``run`` method might return a list with every 2D array's transpose included, as well 23 | as the original array. 24 | 25 | Invariants have a custom ``__new__`` method that calls the ``run`` method, so that they function as 26 | callables rather than classes. 27 | 28 | PyBryt supports custom invariants. To create your own invariant, subclass 29 | :py:class:`invariants.invariant` and implement the ``run`` 30 | method. 31 | 32 | 33 | Built-In Invariants 34 | ------------------- 35 | 36 | A list of built-in invariants can be found :ref:`here`. 37 | -------------------------------------------------------------------------------- /docs/annotations/value_annotations/structural_patterns.rst: -------------------------------------------------------------------------------- 1 | Structural Pattern Matching 2 | =========================== 3 | 4 | PyBryt supports structural pattern matching in order to allow you to create annotations that check 5 | the structure of objects instead of using an ``==`` check. Structural patterns can be created by 6 | accessing attributes of the singleton 7 | :py:obj:`pybryt.structural` and calling them 8 | with attribute-value pairs as arguments. For example, if you're matching an instance of 9 | ``mypackage.Foo`` with attribute ``bar`` set to ``2``; a structural pattern for this could be 10 | created with 11 | 12 | .. code-block:: python 13 | 14 | pybryt.structural.mypackage.Foo(bar=2) 15 | 16 | If there are attributes you want to look for without a specific name, you can pass these as 17 | positional arguments: 18 | 19 | .. code-block:: python 20 | 21 | pybryt.structural.mypackage.Foo(3, bar=2) 22 | 23 | To determine whether an object matches the structural pattern, PyBryt imports the package and 24 | retrieves the specified class. In the examples above, this would look like 25 | 26 | .. code-block:: python 27 | 28 | getattr(importlib.import_module("mypackage"), "Foo") 29 | 30 | If the provided object is an instance of this class and has the specified attributes, the object 31 | matches. You can determine if an object matches a structural pattern using an ``==`` comparison. 32 | 33 | If no package is specified for the class, the pattern just checks that the name of the class 34 | matches the name of the class in the structural pattern, without importing any modules. For 35 | example: 36 | 37 | .. code-block:: python 38 | 39 | df_pattern = pybryt.structural.DataFrame() 40 | df_pattern == pd.DataFrame() # returns True 41 | 42 | class DataFrame: 43 | pass 44 | 45 | df_pattern == DataFrame() # returns True 46 | 47 | Attribute values are matched using the same algorithm as 48 | :py:class:`Value` annotations. If you would like to make use of 49 | the options available to :py:class:`Value` annotations, you can 50 | also pass an annotation as an attribute value: 51 | 52 | .. code-block:: python 53 | 54 | pybryt.structural.mypackage.Foo(pi=pybryt.Value(np.pi, atol=1e-5)) 55 | 56 | For checking whether an object contains specific members (determined via the use of Python's 57 | ``in`` operator), use the ``contains_`` method: 58 | 59 | .. code-block:: python 60 | 61 | pybryt.structural.mypackage.MyList().contains_(1, 2, 3) 62 | 63 | To use structural patterns, pass them as values to :py:class:`Value` 64 | annotations. When a value annotation is checking for a structural pattern, it uses the pattern's 65 | ``==`` check to determine whether any object in the memory footprint matches. 66 | 67 | .. code-block:: python 68 | 69 | pybryt.Value(pybryt.structural.mypackage.MyList()) 70 | -------------------------------------------------------------------------------- /docs/api_reference.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API Reference 4 | ============= 5 | 6 | 7 | Annotations 8 | ----------- 9 | 10 | .. autoclass:: pybryt.annotations.annotation.Annotation 11 | :members: 12 | :undoc-members: 13 | 14 | 15 | Value Annotations 16 | +++++++++++++++++ 17 | 18 | .. automodule:: pybryt.annotations.value 19 | :members: 20 | :undoc-members: 21 | 22 | 23 | Structural Patterns 24 | ??????????????????? 25 | 26 | .. autodata:: pybryt.annotations.structural.structural 27 | :annotation: 28 | 29 | 30 | .. _invariants_ref: 31 | 32 | Invariants 33 | ?????????? 34 | 35 | .. automodule:: pybryt.annotations.invariants 36 | :members: 37 | :undoc-members: 38 | 39 | 40 | Relational Annotations 41 | ++++++++++++++++++++++ 42 | 43 | .. automodule:: pybryt.annotations.relation 44 | :members: 45 | :undoc-members: 46 | 47 | 48 | Complexity Annotations 49 | ++++++++++++++++++++++ 50 | 51 | .. automodule:: pybryt.annotations.complexity.annotation 52 | :members: 53 | :undoc-members: 54 | 55 | 56 | .. _complexities: 57 | 58 | Complexity Classes 59 | ?????????????????? 60 | 61 | .. automodule:: pybryt.annotations.complexity.complexities 62 | :members: 63 | :undoc-members: 64 | 65 | 66 | Type Annotations 67 | ++++++++++++++++ 68 | 69 | .. automodule:: pybryt.annotations.type_ 70 | :members: 71 | :undoc-members: 72 | 73 | 74 | Import Annotations 75 | ++++++++++++++++++ 76 | 77 | .. automodule:: pybryt.annotations.import_ 78 | :members: 79 | :undoc-members: 80 | 81 | 82 | Annotation Collections 83 | ++++++++++++++++++++++ 84 | 85 | .. automodule:: pybryt.annotations.collection 86 | :members: 87 | :undoc-members: 88 | 89 | 90 | Initial Conditions 91 | ++++++++++++++++++ 92 | 93 | .. automodule:: pybryt.annotations.initial_condition 94 | :members: 95 | :undoc-members: 96 | 97 | 98 | Annotation Results 99 | ++++++++++++++++++ 100 | 101 | .. autoclass:: pybryt.annotations.annotation.AnnotationResult 102 | :members: 103 | :undoc-members: 104 | 105 | 106 | Reference Implementations 107 | ------------------------- 108 | 109 | .. autoclass:: pybryt.reference.ReferenceImplementation 110 | :members: 111 | :undoc-members: 112 | 113 | 114 | Reference Results 115 | +++++++++++++++++ 116 | 117 | .. autoclass:: pybryt.reference.ReferenceResult 118 | :members: 119 | :undoc-members: 120 | 121 | .. autofunction:: pybryt.reference.generate_report 122 | 123 | 124 | Debug mode 125 | ++++++++++ 126 | 127 | .. automodule:: pybryt.debug 128 | :members: 129 | :undoc-members: 130 | 131 | 132 | Student Implementations 133 | ----------------------- 134 | 135 | .. automodule:: pybryt.student 136 | :members: 137 | :undoc-members: 138 | 139 | 140 | Execution 141 | +++++++++ 142 | 143 | .. automodule:: pybryt.execution 144 | :members: 145 | :undoc-members: 146 | 147 | 148 | Complexity Analysis 149 | ------------------- 150 | 151 | .. automodule:: pybryt.complexity 152 | :members: 153 | :undoc-members: 154 | -------------------------------------------------------------------------------- /docs/cli_reference.rst: -------------------------------------------------------------------------------- 1 | CLI Reference 2 | ============= 3 | 4 | PyBryt includes a small command-line interface for performing quick tasks using its functionality. 5 | The main tasks that the CLI currently supports are: 6 | 7 | - compiling reference implementations and saving them to a file 8 | - executing student implementations and saving them to a file 9 | - checking reference implementations against student implementations (as notebooks or pickled objects) 10 | and saving/echoing the results 11 | 12 | Each of the commands is detailed in the :ref:`refererence` below, but a short summary 13 | of each is provided here. The CLI can be invoked via the command ``pybryt`` or by calling PyBryt as a 14 | Python module: ``python3 -m pybryt``. For simplicity, the convention of the former is used here. 15 | 16 | 17 | CLI Sub-commands 18 | ---------------- 19 | 20 | 21 | ``check`` 22 | +++++++++ 23 | 24 | ``pybryt check`` is used to check a student implementation against a reference implementation. It 25 | takes two position arguments corresponding to the path to a reference implementation and the path to 26 | a student implementation. Both paths can lead either to notebooks (which will be executed/compiled 27 | if provided) or to pickle files. 28 | 29 | The output of this command comes in three forms: 30 | 31 | - a pickled :py:class:`ReferenceResult` object, written to a file 32 | - a JSON file with a text-based representation of the ``ReferenceResult`` object 33 | - a report echoed to the console 34 | 35 | To set the output type, use the ``-t`` flag. If a file is written, use the ``-o`` flag to set the 36 | output path (defaults to ``{stu.stem}_results.[pkl|json]``). 37 | 38 | .. code-block:: console 39 | 40 | $ ls 41 | reference.ipynb subm.ipynb 42 | $ pybryt check reference.ipynb subm.ipynb 43 | $ ls 44 | reference.ipynb subm.ipynb subm_results.pkl 45 | $ pybryt check reference.pkl subm.pkl -t json 46 | $ ls 47 | reference.ipynb subm.ipynb subm_results.json subm_results.pkl 48 | $ pybryt check reference.ipynb subm.pkl -t report 49 | REFERENCE: median 50 | SATISFIED: True 51 | MESSAGES: 52 | ... 53 | 54 | 55 | ``compile`` 56 | +++++++++++ 57 | 58 | ``pybryt compile`` is used to compile a reference implementation to a file. It takes a single positional 59 | argument, the path to the notebook to be compiled. To set the destination path, use the ``-d`` flag. 60 | If a name is needed for the reference implementation, it can be provided with the ``-n`` flag. 61 | 62 | .. code-block:: console 63 | 64 | $ pybryt compile reference.ipynb 65 | $ pybryt compile reference.ipynb -d ref.pkl -n foo 66 | 67 | 68 | ``execute`` 69 | +++++++++++ 70 | 71 | ``pybryte execute`` is used to execute one or more student implementations and write the memory 72 | footprints to files for futher processing. All paths passed as position arguments to this command 73 | are paths to student implementation notebooks. 74 | 75 | Because execution can be time-consuming, ``pybryt execute`` supports parallelism using Python's 76 | ``multiprocessing`` library. To enable parallelism, use the ``-p`` flag. 77 | 78 | .. code-block:: console 79 | 80 | $ pybryt execute submissions/*.ipynb 81 | $ pybryt execute submissions/*.ipynb -p 82 | 83 | 84 | .. _cli_reference: 85 | 86 | Reference 87 | --------- 88 | 89 | This section is a short reference for all of the commands and options for PyBryt's CLI. 90 | 91 | .. click:: pybryt.cli:click_cli 92 | :prog: pybryt 93 | :nested: full 94 | -------------------------------------------------------------------------------- /docs/complexity.rst: -------------------------------------------------------------------------------- 1 | Complexity Analysis 2 | =================== 3 | 4 | PyBryt exposes its code complexity analysis tools so that they can be used to analyze blocks of code 5 | without necessitating the use of the annotation framework. These tools can be used to give students 6 | a way to test the complexity of their implementations, or just to analyze code complexity in a manner 7 | that is more reliable than using a timer. 8 | 9 | These tools use PyBryt's internal tracing mechanisms, meaning that the data used to determine the 10 | complexity is based on a step counter rather than the system clock, making it deterministic and more 11 | accurate for analyzing the runtime of solutions. 12 | 13 | To make use of these analysis tools, use the 14 | :py:class:`TimeComplexityChecker` class. This class, once 15 | instantiated, can be used as a context manager to track the runtime data for a block of code given 16 | an input length. The context manager functions similarly to :ref:`PyBryt's context manager for 17 | complexity annotations` in that you provide it with the input length (or 18 | the input itself if it supports ``len``) and use it multiple times to gather observations for each 19 | input length. 20 | 21 | Once the data has been gathered, use the 22 | :py:meth:`TimeComplexityChecker.determine_complexity` 23 | method to determine the complexity class. This uses PyBryt's complexity analysis framework to find 24 | the complexity and returns the corresponding complexity class from 25 | :py:mod:`pybryt.complexities`. For more information on 26 | how the complexity class is determined, see :ref:`complexity`. 27 | 28 | .. code-block:: python 29 | 30 | checker = pybryt.TimeComplexityChecker() 31 | for exp in range(8): 32 | n = 10 ** exp 33 | with checker(n): 34 | some_function(n) 35 | 36 | checker.determine_complexity() 37 | 38 | Note that, unlike :py:class:`check_time_complexity`, 39 | you must use the same instance of the class for each entry into the context so that the observations 40 | can be gathered together. 41 | 42 | 43 | Working with Annotations 44 | ------------------------ 45 | 46 | The :py:class:`TimeComplexityChecker` uses 47 | :py:class:`check_time_complexity` under the hood, 48 | so it is also compatible with PyBryt's annotation framework. To use a checker that will also track 49 | results for satisfying annotations, pass the name of the annotation to the constructor, just like 50 | with :py:class:`check_time_complexity`: 51 | 52 | .. code-block:: python 53 | 54 | checker = pybryt.TimeComplexityChecker("fib_runtime") 55 | for exp in range(30): 56 | with checker(n): 57 | fib(n) 58 | 59 | This is a good alternative to :py:class:`check_time_complexity` 60 | when you want students to be able to check the complexity of their code easily, without 61 | having to run PyBryt on the submission each time. 62 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('..')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'PyBryt' 21 | copyright = '2021, Microsoft' 22 | author = 'Chris Pyles' 23 | 24 | 25 | # -- General configuration --------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | 'sphinx.ext.autodoc', 32 | 'sphinx.ext.napoleon', 33 | 'sphinxarg.ext', 34 | 'IPython.sphinxext.ipython_console_highlighting', 35 | 'IPython.sphinxext.ipython_directive', 36 | 'sphinx_click', 37 | 'nbsphinx', 38 | ] 39 | 40 | # extension configurations 41 | napoleon_google_docstring = True 42 | napoleon_numpy_docstring = False 43 | # napoleon_use_param = True 44 | 45 | apidoc_module_dir = '../pybryt' 46 | apidoc_output_dir = '.' 47 | apidoc_excluded_paths = [] 48 | 49 | # add_module_names = False 50 | 51 | autosummary_generate = False 52 | 53 | # The master toctree document. 54 | master_doc = 'index' 55 | 56 | github_doc_root = 'https://github.com/microsoft/pybryt/tree/main/docs/' 57 | 58 | # The language for content autogenerated by Sphinx. Refer to documentation 59 | # for a list of supported languages. 60 | # 61 | # This is also used if you do content translation via gettext catalogs. 62 | # Usually you set "language" from the command line for these cases. 63 | language = 'en' 64 | 65 | # Add any paths that contain templates here, relative to this directory. 66 | templates_path = ['_templates'] 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This pattern also affects html_static_path and html_extra_path. 71 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'modules.rst'] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = 'sphinx' 75 | 76 | 77 | # -- Options for HTML output ------------------------------------------------- 78 | 79 | # The theme to use for HTML and HTML Help pages. See the documentation for 80 | # a list of builtin themes. 81 | # 82 | html_theme = 'furo' 83 | # html_logo = '' 84 | 85 | html_theme_options = { 86 | # 'github_url': 'https://github.com/microsoft/pybryt', 87 | # 'repository_url': 'https://github.com/microsoft/pybryt', 88 | } 89 | 90 | # github_url = 'https://github.com/microsoft' 91 | 92 | # Add any paths that contain custom static files (such as style sheets) here, 93 | # relative to this directory. They are copied after the builtin static files, 94 | # so a file named "default.css" will overwrite the builtin "default.css". 95 | html_static_path = ['_static'] 96 | 97 | nbsphinx_execute = 'never' 98 | -------------------------------------------------------------------------------- /docs/github_action.rst: -------------------------------------------------------------------------------- 1 | .. _github_action: 2 | 3 | PyBryt GitHub Action 4 | ==================== 5 | 6 | `GitHub Actions`_ allow you to automate workflows for repositories for tasks like CI/CD. With GitHub 7 | Actions, or indeed any CI/CD service, you can automate PyBryt to run on repositories of student 8 | code. 9 | 10 | PyBryt has a pre-made action that can be used in GitHub Actions workflows called 11 | |microsoft/pybryt-action|_. With this action, you can set up PyBryt to run automatically on 12 | students' submissions whenever they push to their repositories. 13 | 14 | Let's consider an example assignment with student repositories structured like: 15 | 16 | .. code-block:: 17 | 18 | . 19 | ├── refs 20 | │ ├── ref-1.pkl 21 | │ └── ref-2.pkl 22 | └── submission.ipynb 23 | 24 | To automate PyBryt for the submission ``submission.ipynb``, you would add a new GitHub Actions 25 | workflow file in ``.github/workflows``: 26 | 27 | .. code-block:: 28 | 29 | . 30 | ├── .github 31 | │ └── workflows 32 | │ └── run-pybryt.yml 33 | ├── refs 34 | │ ├── ref-1.pkl 35 | │ └── ref-2.pkl 36 | └── submission.ipynb 37 | 38 | Inside this file, define the following action configuration: 39 | 40 | .. code-block:: yaml 41 | 42 | name: Run PyBryt 43 | 44 | on: 45 | push: 46 | branches: [ main ] 47 | workflow_dispatch: 48 | 49 | jobs: 50 | build: 51 | runs-on: ubuntu-latest 52 | steps: 53 | - name: Checkout 54 | uses: actions/checkout@v3 55 | 56 | - name: Run PyBryt 57 | uses: microsoft/pybryt-action@v0.1.1 58 | with: 59 | submission-path: submission.ipynb 60 | references: | 61 | refs/ref-1.pkl 62 | refs/ref-2.pkl 63 | 64 | This configuration tells the PyBryt action that it should execute the student implementation 65 | ``submission.ipynb`` against the references listed in the ``references`` key. (Elements of the 66 | ``references`` key can be either paths to files or URLs.) It is set up to run whenever a commit is 67 | pushed to the ``main`` branch or when triggered manually by the user. 68 | 69 | When this action runs, PyBryt will execute the submission specified to generate a student 70 | implementation and then it will run that implementation against the provided references. It will 71 | then print a report to the console containing the details of the passing and failing of the 72 | implementation for each reference as well as any messages generated by the reference. 73 | 74 | In some cases, you may want a record of the objects PyBryt generates, like the report and student 75 | implementation and results objects. The PyBryt action stores these artifacts in files that can be 76 | found by looking at the outputs of the step that runs PyBryt. 77 | 78 | For example, to save all of these artifacts as files in a ``results`` directory, you could use a 79 | configuration like this: 80 | 81 | .. code-block:: yaml 82 | 83 | name: Run PyBryt 84 | 85 | on: 86 | push: 87 | branches: [ main ] 88 | workflow_dispatch: 89 | 90 | jobs: 91 | build: 92 | runs-on: ubuntu-latest 93 | steps: 94 | - name: Checkout 95 | uses: actions/checkout@v3 96 | 97 | - name: Run PyBryt 98 | id: pybryt 99 | uses: microsoft/pybryt-action@v0.1.1 100 | with: 101 | submission-path: submission.ipynb 102 | references: | 103 | refs/ref-1.pkl 104 | refs/ref-2.pkl 105 | 106 | - name: Save, commit, and push results 107 | run: | 108 | mkdir -p results 109 | cp ${{ steps.pybryt.outputs.report-path }} results/report.txt 110 | cp ${{ steps.pybryt.outputs.results-path }} results/results.pkl 111 | cp ${{ steps.pybryt.outputs.student-implementation-path }} results/student-implementation.pkl 112 | git add results 113 | git commit -m "PyBryt results for ${{ github.sha }}" 114 | git push 115 | 116 | The second step of the action above copies the files from the paths listed in the outputs from the 117 | PyBryt action into files in the ``results`` directory before committing and pushing them. The two 118 | ``.pkl`` files can be unpickled with the ``dill`` library for further use. 119 | 120 | For more information about the PyBryt action and how to use it, see its `documentation 121 | `_. 122 | 123 | 124 | .. _GitHub Actions: https://docs.github.com/en/actions 125 | .. |microsoft/pybryt-action| replace:: ``microsoft/pybryt-action`` 126 | .. _microsoft/pybryt-action: https://github.com/microsoft/pybryt-action 127 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. PyBryt documentation master file, created by 2 | sphinx-quickstart on Sun Feb 21 11:51:56 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | PyBryt Documentation 7 | ==================== 8 | 9 | .. toctree:: 10 | :maxdepth: 3 11 | :hidden: 12 | 13 | installation 14 | getting_started 15 | annotations/index 16 | reference_implementations/index 17 | student_implementations 18 | complexity 19 | github_action 20 | cli_reference 21 | api_reference 22 | resources 23 | Changelog 24 | 25 | PyBryt is an open source Python auto-assessment library for teaching and 26 | learning. Its goal is to empower students and educators to learn about 27 | technology through fun, guided, and hands-on content aimed at specific learning 28 | objectives. PyBryt is designed to work with existing auto-grading solutions and 29 | workflows, such as `Otter Grader`_, `OkPy`_, and `Autolab`_. 30 | 31 | .. image:: _static/images/pybryt_goals.png 32 | 33 | Educators and institutions can leverage PyBryt to integrate auto-assessment and 34 | reference models into hands-on lab exercises and assessments. Some of the PyBryt 35 | benefits are: 36 | 37 | - Educators do not have to enforce the structure of the solution 38 | - Learners practice algorithm design, code design, and solution implementation 39 | - Learners receive quick and meaningful pedagogical feedback, which substantially contributes 40 | to the learning experience 41 | - Complexity of the learner's solution can be analyzed 42 | - Plagiarism detection and support for reference implementations 43 | - Easy integration into existing organizational or institutional grading infrastructure 44 | 45 | PyBryt's core auto-assessment behavior operates by comparing a **student's 46 | implementation** of a programming problem to a series of reference 47 | implementations provided by an instructor. A **reference implementation** 48 | defines a pattern of values and conditions on those values expected 49 | to be present in students' implementation. By comparing student's and reference 50 | implementations, PyBryt provides tailored feedback and advice to the student 51 | on how to bring their implementation closer to the reference one. PyBryt, 52 | instead of only comparing the output of their solution to the reference one, 53 | enables students to reconsider the design choices they made and improve their 54 | implementation in incremental steps. 55 | 56 | A reference implementation is created by annotating the code written or found by an 57 | instructor and executing this code to create a 58 | :py:class:`ReferenceImplementation` object. 59 | Annotations are created by creating instances of subclasses of the abstract 60 | :py:class:`Annotation` class. 61 | 62 | .. _Otter Grader: https://otter-grader.readthedocs.io 63 | .. _OkPy: https://okpy.org 64 | .. _Autolab: https://autolab.readthedocs.io/ 65 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | PyBryt can be installed with ``pip``, in a Python virtual environment, or using line magic in a 5 | Jupyter Notebook. 6 | 7 | 8 | Installing with ``pip`` 9 | ----------------------- 10 | 11 | Before installing, make sure you have a recent version of `Python 3 installed 12 | `_. Then, you can install PyBryt with ``pip``: 13 | 14 | .. code-block:: console 15 | 16 | pip install pybryt 17 | 18 | This method also works for installing PyBryt in a Conda environment, as long as you have activated 19 | the environment in which you want PyBryt installed. You can also use: 20 | 21 | .. code-block:: console 22 | 23 | conda run -n ENVIRONMENT_NAME pip install pybryt 24 | 25 | 26 | Installing in a Python Virtual Environment 27 | ------------------------------------------ 28 | 29 | To install PyBryt in a Python virtual environment, update ``pip`` and install ``pybryt``. 30 | 31 | 32 | Windows PowerShell 33 | ++++++++++++++++++ 34 | 35 | .. code-block:: console 36 | 37 | py -3 -m venv .venv 38 | .\.venv\scripts\activate.ps1 39 | pip install -U pip 40 | pip install pybryt 41 | 42 | 43 | macOS or Linux command-line 44 | +++++++++++++++++++++++++++ 45 | 46 | .. code-block:: console 47 | 48 | python3 -m venv .venv 49 | source .venv/bin/activate 50 | pip install -U pip 51 | pip install pybryt 52 | 53 | 54 | Installing within a Jupyter Notebook 55 | ------------------------------------ 56 | 57 | You can add a line magic command to a Jupyter notebook to install the requirements for a ``pip`` 58 | environment: 59 | 60 | .. code-block:: python 61 | 62 | %pip install pybryt 63 | import pybryt 64 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | cp -r %BUILDDIR%/html html 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/reference_implementations/debugging.rst: -------------------------------------------------------------------------------- 1 | .. _debugging: 2 | 3 | Debugging References 4 | ==================== 5 | 6 | To assist in debugging your reference implementations, PyBryt comes with a debug mode that can be 7 | enabled programmatically. To enable debug mode, use 8 | :py:func:`pybryt.enable_debug_mode`. To disable debug mode, use 9 | :py:func:`pybryt.disable_debug_mode`. Alternatively, you can 10 | enable debug mode in a ``with`` block uses the context manager 11 | :py:obj:`pybryt.debug_mode`. 12 | 13 | .. code-block:: python 14 | 15 | pybryt.enable_debug_mode() 16 | # debug your reference 17 | pybryt.disable_debug_mode() 18 | 19 | # or... 20 | 21 | with pybryt.debug_mode(): 22 | # debug your reference 23 | 24 | In debug mode, PyBryt will raise exceptions instead of ignoring conditions that could cause 25 | unexpected behavior. Currently, conditions that raise exceptions in debug mode are: 26 | 27 | * when a custom equivalence function is passed to a :py:class:`Value` 28 | annotation along with ``atol`` and/or ``rtol`` 29 | * when a custom equivalence function raises an exception 30 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | furo 2 | sphinxcontrib-apidoc 3 | sphinx-argparse 4 | sphinx-click 5 | nbsphinx 6 | -------------------------------------------------------------------------------- /docs/resources.rst: -------------------------------------------------------------------------------- 1 | Resources 2 | ========= 3 | 4 | Take a look at the resources below to learn more about PyBryt and its applications. 5 | 6 | * `Introduction to PyBryt Microsoft Learn Module 7 | `_: A Microsoft Learn module that 8 | covers the basics of PyBryt, including annotations, annotation messages, and creating reference 9 | implementations. 10 | * `Advanced PyBryt Microsoft Learn Module 11 | `_: A Microsoft Learn module that 12 | covers more advanced PyBryt topics than the intro module, including things like annotation 13 | options, relational and complexity annotations, and grading pipelines. 14 | * `PyBryt for Pedagogical Auto-assessment 15 | `_: 16 | This is a short blog post that explains the core idea behind and rationale for PyBryt. It includes 17 | a brief discussion of why PyBryt was created, an outline of how assignments work with PyBryt, and 18 | testimonials from a pilot of PyBryt at Imperial College London. 19 | * `Demos folder `_: This folder contains some 20 | short demos of PyBryt assessing submissions for common algorithms. Each includes a notebook that 21 | goes through the entire assessment process, from compiling references to generating reports. 22 | * |microsoft/pybryt-action|_: This GitHub Action can be used to run PyBryt as CI in student repos. 23 | You can also learn more about this action :ref:`on this page`. 24 | 25 | .. |microsoft/pybryt-action| replace:: ``microsoft/pybryt-action`` 26 | .. _microsoft/pybryt-action: https://github.com/microsoft/pybryt-action 27 | -------------------------------------------------------------------------------- /docs/strassen.py: -------------------------------------------------------------------------------- 1 | """Pythonic matrix multiplcation using Strassen's algorithm""" 2 | 3 | # This implementation is adapted from https://martin-thoma.com/strassen-algorithm-in-python-java-cpp/ 4 | 5 | from math import ceil, log 6 | 7 | 8 | LEAF_SIZE = 8 9 | 10 | 11 | def naive_matmul(A, B): 12 | n = len(A) 13 | C = [[0 for i in range(n)] for j in range(n)] 14 | for i in range(n): 15 | for k in range(n): 16 | for j in range(n): 17 | C[i][j] += A[i][k] * B[k][j] 18 | return C 19 | 20 | 21 | def add(A, B): 22 | n = len(A) 23 | C = [[0 for j in range(0, n)] for i in range(0, n)] 24 | for i in range(0, n): 25 | for j in range(0, n): 26 | C[i][j] = A[i][j] + B[i][j] 27 | return C 28 | 29 | 30 | def subtract(A, B): 31 | n = len(A) 32 | C = [[0 for j in range(0, n)] for i in range(0, n)] 33 | for i in range(0, n): 34 | for j in range(0, n): 35 | C[i][j] = A[i][j] - B[i][j] 36 | return C 37 | 38 | 39 | def strassen(A, B): 40 | n = len(A) 41 | 42 | if n <= LEAF_SIZE: 43 | return naive_matmul(A, B) 44 | 45 | else: 46 | # initializing the new sub-matrices 47 | new_size = n // 2 48 | a11 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 49 | a12 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 50 | a21 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 51 | a22 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 52 | 53 | b11 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 54 | b12 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 55 | b21 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 56 | b22 = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 57 | 58 | A_result = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 59 | B_result = [[0 for j in range(0, new_size)] for i in range(0, new_size)] 60 | 61 | # dividing the matrices in 4 sub-matrices: 62 | for i in range(0, new_size): 63 | for j in range(0, new_size): 64 | a11[i][j] = A[i][j] # top left 65 | a12[i][j] = A[i][j + new_size] # top right 66 | a21[i][j] = A[i + new_size][j] # bottom left 67 | a22[i][j] = A[i + new_size][j + new_size] # bottom right 68 | 69 | b11[i][j] = B[i][j] # top left 70 | b12[i][j] = B[i][j + new_size] # top right 71 | b21[i][j] = B[i + new_size][j] # bottom left 72 | b22[i][j] = B[i + new_size][j + new_size] # bottom right 73 | 74 | # Calculating p1 to p7: 75 | A_result = add(a11, a22) 76 | B_result = add(b11, b22) 77 | p1 = strassen(A_result, B_result) # p1 = (a11+a22) * (b11+b22) 78 | 79 | A_result = add(a21, a22) # a21 + a22 80 | p2 = strassen(A_result, b11) # p2 = (a21+a22) * (b11) 81 | 82 | B_result = subtract(b12, b22) # b12 - b22 83 | p3 = strassen(a11, B_result) # p3 = (a11) * (b12 - b22) 84 | 85 | B_result = subtract(b21, b11) # b21 - b11 86 | p4 = strassen(a22, B_result) # p4 = (a22) * (b21 - b11) 87 | 88 | A_result = add(a11, a12) # a11 + a12 89 | p5 = strassen(A_result, b22) # p5 = (a11+a12) * (b22) 90 | 91 | A_result = subtract(a21, a11) # a21 - a11 92 | B_result = add(b11, b12) # b11 + b12 93 | p6 = strassen(A_result, B_result) # p6 = (a21-a11) * (b11+b12) 94 | 95 | A_result = subtract(a12, a22) # a12 - a22 96 | B_result = add(b21, b22) # b21 + b22 97 | p7 = strassen(A_result, B_result) # p7 = (a12-a22) * (b21+b22) 98 | 99 | # calculating c21, c21, c11 e c22: 100 | c12 = add(p3, p5) # c12 = p3 + p5 101 | c21 = add(p2, p4) # c21 = p2 + p4 102 | 103 | A_result = add(p1, p4) # p1 + p4 104 | B_result = add(A_result, p7) # p1 + p4 + p7 105 | c11 = subtract(B_result, p5) # c11 = p1 + p4 - p5 + p7 106 | 107 | A_result = add(p1, p3) # p1 + p3 108 | B_result = add(A_result, p6) # p1 + p3 + p6 109 | c22 = subtract(B_result, p2) # c22 = p1 + p3 - p2 + p6 110 | 111 | # Grouping the results obtained in a single matrix: 112 | C = [[0 for j in range(0, n)] for i in range(0, n)] 113 | for i in range(0, new_size): 114 | for j in range(0, new_size): 115 | C[i][j] = c11[i][j] 116 | C[i][j + new_size] = c12[i][j] 117 | C[i + new_size][j] = c21[i][j] 118 | C[i + new_size][j + new_size] = c22[i][j] 119 | 120 | return C 121 | 122 | 123 | def matmul(A, B): 124 | assert type(A) == list and type(B) == list 125 | assert len(A) == len(A[0]) == len(B) == len(B[0]) 126 | 127 | # Make the matrices bigger so that you can apply the strassen 128 | # algorithm recursively without having to deal with odd 129 | # matrix sizes 130 | next_power_of_two = lambda n: 2 ** int(ceil(log(n, 2))) 131 | n = len(A) 132 | m = next_power_of_two(n) 133 | A_prep = [[0 for i in range(m)] for j in range(m)] 134 | B_prep = [[0 for i in range(m)] for j in range(m)] 135 | for i in range(n): 136 | for j in range(n): 137 | A_prep[i][j] = A[i][j] 138 | B_prep[i][j] = B[i][j] 139 | 140 | C_prep = strassen(A_prep, B_prep) 141 | C = [[0 for i in range(n)] for j in range(n)] 142 | for i in range(n): 143 | for j in range(n): 144 | C[i][j] = C_prep[i][j] 145 | 146 | return C 147 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - python=3.8 3 | - pip 4 | - pip: 5 | - -r requirements.txt 6 | - pytest 7 | - codecov 8 | - matplotlib 9 | - otter-grader 10 | - -r docs/requirements.txt 11 | 12 | -------------------------------------------------------------------------------- /images/pybryt-goals.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/images/pybryt-goals.PNG -------------------------------------------------------------------------------- /images/pybryt-logo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/images/pybryt-logo.PNG -------------------------------------------------------------------------------- /pybryt/__init__.py: -------------------------------------------------------------------------------- 1 | """Auto-assessment library for Python""" 2 | 3 | from . import utils 4 | from .annotations import * 5 | from .complexity import TimeComplexityChecker 6 | from .debug import * 7 | from .execution import * 8 | from .reference import * 9 | from .student import * 10 | from .version import __version__ 11 | -------------------------------------------------------------------------------- /pybryt/__main__.py: -------------------------------------------------------------------------------- 1 | """Main module for PyBryt CLI""" 2 | 3 | from .cli import cli 4 | 5 | 6 | if __name__ == "__main__": 7 | cli() 8 | -------------------------------------------------------------------------------- /pybryt/annotations/__init__.py: -------------------------------------------------------------------------------- 1 | """Annotations for marking-up reference implementations""" 2 | 3 | from . import invariants 4 | from .annotation import * 5 | from .collection import * 6 | from .complexity import * 7 | from .import_ import * 8 | from .initial_condition import * 9 | from .relation import * 10 | from .structural import * 11 | from .type_ import * 12 | from .value import * 13 | -------------------------------------------------------------------------------- /pybryt/annotations/collection.py: -------------------------------------------------------------------------------- 1 | """Class for gathering and operating on collections of annotations""" 2 | 3 | __all__ = ["Collection"] 4 | 5 | from typing import Any, Dict, List, Tuple 6 | 7 | from .annotation import Annotation, AnnotationResult 8 | 9 | from ..execution import MemoryFootprint 10 | 11 | 12 | class Collection(Annotation): 13 | """ 14 | A class for collecting and operating on multiple annotations. 15 | 16 | If ``enforce_order`` is true, this collection will only be satisfied if all of its children are 17 | satisfied *and* the satisfying timestamps are in non-decreasing order (for those that have them). 18 | 19 | Args: 20 | *annotations (:py:class:`Annotation`): the child 21 | annotations being operated on 22 | enforce_order (``bool``, optional): whether to enforce the ordering of annotations as added 23 | to this collection 24 | **kwargs: additional keyword arguments passed to the 25 | :py:class:`Annotation` constructor 26 | """ 27 | 28 | _annotations: List[Annotation] 29 | """the child annotations that this annotation operates on""" 30 | 31 | enforce_order: bool 32 | """whether to enforce the ordering of annotations as added to this collection""" 33 | 34 | def __init__(self, *annotations: Annotation, enforce_order: bool = False, **kwargs): 35 | self._annotations = list(annotations) 36 | for ann in self._annotations: 37 | if not isinstance(ann, Annotation): 38 | raise ValueError("One of the arguments is not an annotation") 39 | 40 | self.enforce_order = enforce_order 41 | 42 | super().__init__(**kwargs) 43 | 44 | @property 45 | def children(self) -> List[Annotation]: 46 | """ 47 | ``list[Annotation]``: the child annotations that this annotation operates on 48 | """ 49 | return self._annotations 50 | 51 | def __eq__(self, other: Any) -> bool: 52 | """ 53 | Checks whether this annotation is equal to another object. 54 | 55 | For an object to equal a relational annotation, it must also be a relational annotation of 56 | the same type and have the same child annotations. 57 | 58 | Args: 59 | other (``object``): the object to compare to 60 | 61 | Returns: 62 | ``bool``: whether the objects are equal 63 | """ 64 | return super().__eq__(other) and self.children == other.children and \ 65 | self.enforce_order == other.enforce_order 66 | 67 | def check(self, footprint: MemoryFootprint) -> AnnotationResult: 68 | """ 69 | Checks that all child annotations are satisfied by the values in the memory footprint, and 70 | that the timestamps of the satisfying values occur in non-decreasing order if 71 | ``self.enforce_order`` is true. 72 | 73 | Args: 74 | footprint (:py:class:`pybryt.execution.memory_footprint.MemoryFootprint`): the 75 | memory footprint to check against 76 | 77 | Returns: 78 | :py:class:`AnnotationResult`: the results of this annotation against ``footprint`` 79 | """ 80 | results = [] 81 | for ann in self.children: 82 | results.append(ann.check(footprint)) 83 | 84 | if self.enforce_order and all(res.satisfied for res in results): 85 | before = [] 86 | with_timestamp = [res for res in results if res.satisfied_at != -1] 87 | for i in range(len(with_timestamp) - 1): 88 | before.append(with_timestamp[i].satisfied_at < with_timestamp[i + 1].satisfied_at) 89 | 90 | return AnnotationResult(all(before), self, children = results) 91 | 92 | else: 93 | return AnnotationResult(None, self, children = results) 94 | 95 | def to_dict(self) -> Dict[str, Any]: 96 | """ 97 | Converts this annotation's details to a JSON-friendly dictionary format. 98 | 99 | Returns: 100 | ``dict[str, object]``: the dictionary representation of this annotation 101 | """ 102 | d = super().to_dict() 103 | d.update({ 104 | "enforce_order": self.enforce_order, 105 | }) 106 | return d 107 | 108 | def add(self, annotation: Annotation) -> None: 109 | """ 110 | Adds an annotation to this collection. 111 | 112 | Args: 113 | annotation (:py:class:`Annotation`): the 114 | annotation to add 115 | """ 116 | if not isinstance(annotation, Annotation): 117 | raise TypeError(f"{annotation} is not an annotation") 118 | 119 | self._annotations.append(annotation) 120 | try: 121 | self.get_tracked_annotations().remove(annotation) 122 | except ValueError: # pragma: no cover 123 | pass 124 | 125 | def remove(self, annotation: Annotation) -> None: 126 | """ 127 | Removes an annotation from this collection. 128 | 129 | Args: 130 | annotation (:py:class:`Annotation`): the 131 | annotation to remove 132 | """ 133 | if not isinstance(annotation, Annotation): 134 | raise TypeError(f"{annotation} is not an annotation") 135 | 136 | if annotation not in self._annotations: 137 | raise ValueError(f"The specified annotation is not part of this collection") 138 | 139 | self._annotations.remove(annotation) 140 | -------------------------------------------------------------------------------- /pybryt/annotations/complexity/__init__.py: -------------------------------------------------------------------------------- 1 | """Annotation scaffold for asserting the complexity of code""" 2 | 3 | from . import complexities 4 | from .annotation import * 5 | -------------------------------------------------------------------------------- /pybryt/annotations/import_.py: -------------------------------------------------------------------------------- 1 | """Annotations for checking imported modules within memory footprints""" 2 | 3 | __all__ = ["ForbidImport", "ImportAnnotation", "RequireImport"] 4 | 5 | import importlib 6 | import sys 7 | 8 | from abc import abstractmethod 9 | from typing import Any, Dict, List 10 | 11 | from .annotation import Annotation, AnnotationResult 12 | 13 | from ..execution import MemoryFootprint 14 | 15 | 16 | class ImportAnnotation(Annotation): 17 | """ 18 | Annotation class for asserting conditions on the set of imported modules 19 | 20 | The set of modules imported in the student implementation is determined using ``sys.modules``, 21 | meaning that subclasses count any modules imported by third-party libaries, not just directly 22 | by the student's code. 23 | 24 | Args: 25 | module (``str``): the module name to forbid in importable form 26 | **kwargs: additional keyword arguments passed to the 27 | :py:class:`Annotation` constructor 28 | """ 29 | 30 | module: str 31 | """the module name to forbid""" 32 | 33 | def __init__(self, module, **kwargs): 34 | if not isinstance(module, str): 35 | raise TypeError(f"{module} is not a string") 36 | 37 | try: 38 | already_imported = module in sys.modules 39 | importlib.import_module(module) 40 | if not already_imported: # clean up sys.modules if needed 41 | sys.modules.pop(module) 42 | except Exception as e: 43 | raise ValueError(f"{module} is not importable") 44 | 45 | self.module = module 46 | 47 | super().__init__(**kwargs) 48 | 49 | @property 50 | def children(self) -> List[Annotation]: 51 | return [] 52 | 53 | @abstractmethod 54 | def check(self, footprint: MemoryFootprint) -> AnnotationResult: 55 | ... # pragma: no cover 56 | 57 | def __eq__(self, other: Any) -> bool: 58 | """ 59 | Checks whether this annotation is equal to another object. 60 | 61 | To be equal to a ``ForbidImport``, the other object must also be a ``ForbidImport`` object, 62 | have the same instance variables, and the same forbidden module. 63 | 64 | Args: 65 | other (``object``): the object to compare to 66 | 67 | Returns: 68 | ``bool``: whether the objects are equal 69 | """ 70 | return super().__eq__(other) and self.module == other.module 71 | 72 | def to_dict(self) -> Dict[str, Any]: 73 | """ 74 | Converts this annotation's details to a JSON-friendly dictionary format. 75 | 76 | Output dictionary contains the annotation's name, group, limit number, success message, and 77 | failure message, as well as the module name. 78 | 79 | Returns: 80 | ``dict[str, object]``: the dictionary representation of this annotation 81 | """ 82 | d = super().to_dict() 83 | d.update({ 84 | "module": str(self.module), 85 | }) 86 | return d 87 | 88 | 89 | class RequireImport(ImportAnnotation): 90 | """ 91 | Annotation class for asserting that a specific module was imported. 92 | 93 | Args: 94 | module (``str``): the module name to forbid in importable form 95 | **kwargs: additional keyword arguments passed to the 96 | :py:class:`Annotation` constructor 97 | """ 98 | 99 | def check(self, footprint: MemoryFootprint) -> AnnotationResult: 100 | """ 101 | Checks that the memory footprint's imports contain the specified module. 102 | 103 | Args: 104 | footprint (:py:class:`pybryt.execution.memory_footprint.MemoryFootprint`): the 105 | memory footprint to check against 106 | 107 | Returns: 108 | :py:class:`AnnotationResult`: the results of this annotation against ``footprint`` 109 | """ 110 | return AnnotationResult(self.module in footprint.imports, self) 111 | 112 | 113 | class ForbidImport(ImportAnnotation): 114 | """ 115 | Annotation class for asserting that a specific module was not imported. 116 | 117 | Args: 118 | module (``str``): the module name to forbid in importable form 119 | **kwargs: additional keyword arguments passed to the 120 | :py:class:`Annotation` constructor 121 | """ 122 | 123 | def check(self, footprint: MemoryFootprint) -> AnnotationResult: 124 | """ 125 | Checks that the memory footprint's imports don't contain the specified module. 126 | 127 | Args: 128 | footprint (:py:class:`pybryt.execution.memory_footprint.MemoryFootprint`): the 129 | memory footprint to check against 130 | 131 | Returns: 132 | :py:class:`AnnotationResult`: the results of this annotation against ``footprint`` 133 | """ 134 | return AnnotationResult(self.module not in footprint.imports, self) 135 | -------------------------------------------------------------------------------- /pybryt/annotations/invariants.py: -------------------------------------------------------------------------------- 1 | """Invariants for value annotations""" 2 | 3 | import numpy as np 4 | 5 | from abc import ABC, abstractmethod 6 | from collections.abc import Iterable 7 | from typing import Any, List, Optional, Union 8 | 9 | 10 | class invariant(ABC): 11 | """ 12 | Abstract base class for invariants. 13 | 14 | All subclasses should implement the :py:meth:`run` static method for generating 15 | values that this invariant accepts as "correct". Invariants have a custom ``__new__`` method 16 | that returns the value of calling the :py:meth:`run` method, making them 17 | function as callables. 18 | """ 19 | 20 | @staticmethod 21 | def __new__(cls, *args, **kwargs): 22 | return cls.run(*args, **kwargs) 23 | 24 | @staticmethod 25 | @abstractmethod 26 | def run(values: List[Any], **kwargs) -> List[Any]: 27 | """ 28 | Returns a list of values that this invariant accepts as correct. 29 | 30 | Takes in a list of acceptable values from a :py:class:`Value` annotation and 31 | returns a list of values that would evaluate as "the same" under the conditions of this 32 | invariant. 33 | 34 | For example, if ``values`` as a list with a single element, a numpy matrix, and the 35 | invariant was matrix transposition, this method would return a length-2 list containing the 36 | original matrix and its transpose. 37 | 38 | Args: 39 | values (``list[object]``): acceptable values, either from the initial constructor call 40 | of the annotation or from the results of other invariants 41 | kwargs: additional keyword arguments 42 | 43 | Returns: 44 | ``list[object]``: the values that would evaluate as "the same" under the conditions 45 | of this invariant 46 | """ 47 | ... # pragma: no cover 48 | 49 | 50 | class string_capitalization(invariant): 51 | """ 52 | An invariant that compares strings ignoring the capitalization of letters. 53 | """ 54 | 55 | @staticmethod 56 | def run(values: List[Any]) -> List[Any]: 57 | """ 58 | Returns a list of values in which all strings have been lowercased. 59 | 60 | Args: 61 | values (``list[object]``): acceptable values, either from the initial constructor call 62 | of the annotation or from the results of other invariants 63 | 64 | Returns: 65 | ``list[object]``: the transformed values 66 | """ 67 | ret = [] 68 | for v in values: 69 | if not isinstance(v, str): 70 | ret.append(v) 71 | else: 72 | ret.append(v.lower()) 73 | return ret 74 | 75 | 76 | class matrix_transpose(invariant): 77 | """ 78 | An invariant that compares 2-dimensional arrays ignoring transposition. 79 | """ 80 | 81 | @staticmethod 82 | def run(values: List[Any]) -> List[Any]: 83 | """ 84 | Returns a list of values in which all 2D iterables have been converted to NumPy arrays and 85 | have had their transpose added 86 | 87 | Args: 88 | values (``list[object]``): acceptable values, either from the initial constructor call 89 | of the annotation or from the results of other invariants 90 | 91 | Returns: 92 | ``list[object]``: the transformed values 93 | """ 94 | ret = [] 95 | for v in values: 96 | if isinstance(v, np.ndarray): 97 | ret.append(v) 98 | ret.append(v.T) 99 | elif isinstance(v, Iterable) and not isinstance(v, str): 100 | try: 101 | arr = np.array(v) 102 | ret.append(arr) 103 | ret.append(arr.T) 104 | except: 105 | ret.append(v) 106 | else: 107 | ret.append(v) 108 | return ret 109 | 110 | 111 | class list_permutation(invariant): 112 | """ 113 | An invariant that compares iterables (except strings) ignoring ordering, using ``sorted``. 114 | """ 115 | 116 | @staticmethod 117 | def run(values: List[Any]) -> List[Any]: 118 | """ 119 | Returns a list of values in which all iterables have been sorted. 120 | 121 | Args: 122 | values (``list[object]``): acceptable values, either from the initial constructor call 123 | of the annotation or from the results of other invariants 124 | 125 | Returns: 126 | ``list[object]``: the transformed values 127 | """ 128 | ret = [] 129 | for v in values: 130 | if isinstance(v, np.ndarray): 131 | ret.append(np.sort(v)) 132 | elif isinstance(v, Iterable) and not isinstance(v, str): 133 | ret.append(sorted(v)) 134 | else: 135 | ret.append(v) 136 | return ret 137 | -------------------------------------------------------------------------------- /pybryt/annotations/type_.py: -------------------------------------------------------------------------------- 1 | """Annotations for checking type presence, or lack thereof, within memory footprints""" 2 | 3 | __all__ = ["ForbidType"] 4 | 5 | import dill 6 | 7 | from typing import Any, Dict, List, Tuple 8 | 9 | from .annotation import Annotation, AnnotationResult 10 | 11 | from ..execution import MemoryFootprint 12 | 13 | 14 | class ForbidType(Annotation): 15 | """ 16 | Annotation class for asserting there are no objects of a specific type in the memory footprint. 17 | 18 | Indicates that the student's memory footprint should not have any values of a certain type. Uses 19 | the ``isinstance`` function to determine if any values match the specified type. The type itself 20 | must be pickleable by ``dill``. 21 | 22 | Args: 23 | type\\_ (``type``): the type to forbid 24 | **kwargs: additional keyword arguments passed to the 25 | :py:class:`Annotation` constructor 26 | """ 27 | 28 | type_: type 29 | """the type to forbid""" 30 | 31 | def __init__(self, type_, **kwargs): 32 | if not isinstance(type_, type): 33 | raise TypeError(f"{type_} is not a type") 34 | 35 | try: 36 | dill.dumps(type_) 37 | except Exception as e: 38 | raise ValueError(f"Types must be serializable but the following error was thrown during serialization:\n{e}") 39 | 40 | self.type_ = type_ 41 | 42 | super().__init__(**kwargs) 43 | 44 | @property 45 | def children(self) -> List[Annotation]: 46 | return [] 47 | 48 | def check(self, footprint: MemoryFootprint) -> AnnotationResult: 49 | """ 50 | Checks that there are no values of type ``self.type_`` in the memory footprint. 51 | 52 | Args: 53 | footprint (:py:class:`pybryt.execution.memory_footprint.MemoryFootprint`): the 54 | memory footprint to check against 55 | 56 | Returns: 57 | :py:class:`AnnotationResult`: the results of this annotation against ``footprint`` 58 | """ 59 | for mfp_val in footprint: 60 | if isinstance(mfp_val.value, self.type_): 61 | return AnnotationResult(False, self) 62 | return AnnotationResult(True, self) 63 | 64 | def __eq__(self, other: Any) -> bool: 65 | """ 66 | Checks whether this annotation is equal to another object. 67 | 68 | To be equal to a ``ForbidType``, the other object must also be a ``Forbidtype`` object, have 69 | the same instance variables, and the same forbidden type. 70 | 71 | Args: 72 | other (``object``): the object to compare to 73 | 74 | Returns: 75 | ``bool``: whether the objects are equal 76 | """ 77 | return super().__eq__(other) and self.type_ == other.type_ 78 | 79 | def to_dict(self) -> Dict[str, Any]: 80 | """ 81 | Converts this annotation's details to a JSON-friendly dictionary format. 82 | 83 | Output dictionary contains the annotation's name, group, limit number, success message, and 84 | failure message, as well as the type as a string. 85 | 86 | Returns: 87 | ``dict[str, object]``: the dictionary representation of this annotation 88 | """ 89 | d = super().to_dict() 90 | d.update({ 91 | "type_": str(self.type_), 92 | }) 93 | return d 94 | -------------------------------------------------------------------------------- /pybryt/cli.py: -------------------------------------------------------------------------------- 1 | """Command-line interface for PyBryt""" 2 | 3 | import os 4 | import dill 5 | import json 6 | import click 7 | 8 | from . import ( 9 | generate_report, generate_student_impls, ReferenceImplementation, StudentImplementation, __version__ 10 | ) 11 | from .utils import get_stem 12 | 13 | 14 | @click.group() 15 | @click.version_option(__version__) 16 | def click_cli(): 17 | """ 18 | A command-line interface for PyBryt. See https://microsoft.github.io/pybryt for more information. 19 | """ 20 | pass 21 | 22 | 23 | @click_cli.command("compile") 24 | @click.option("-d", "--dest", default=None, type=click.Path(dir_okay=False), 25 | help="Path at which to write the pickled reference implementation") 26 | @click.option("-n", "--name", default=None, type=click.STRING, 27 | help="Optional name for the reference implementation") 28 | @click.argument("src", type=click.Path(exists=True, dir_okay=False)) 29 | def compile_reference(src, dest, name): 30 | """ 31 | Compile the reference implementation SRC. 32 | 33 | SRC must be a path to a Jupyter Notebook file which defines a reference implementation. 34 | """ 35 | ref = ReferenceImplementation.compile(src, name=name) 36 | ref.dump(dest) 37 | 38 | 39 | @click_cli.command() 40 | @click.option("--output", default=None, type=click.Path(dir_okay=False), 41 | help="Path at which to write the output notebook from executing the student submission") 42 | @click.option("-d", "--dest", default=None, type=click.Path(dir_okay=False), 43 | help="Path at which to write the results of the check") 44 | @click.option("-t", "--type", "output_type", default="pickle", show_default=True, 45 | type=click.Choice(["pickle", "json", "report"]), help="Type of output to write") 46 | @click.argument("ref", type=click.Path(exists=True, dir_okay=False)) 47 | @click.argument("stu", type=click.Path(exists=True, dir_okay=False)) 48 | def check(ref, stu, output, dest, output_type): 49 | """ 50 | Run a student submission against a reference implementation. 51 | 52 | REF can be a path to a pickled reference implementation or to a notebook to be compiled on-the-fly. 53 | STU can be a path to a pickled student implementation or to a notebook to be executed. 54 | 55 | If TYPE is "pickle" or "json", the output is a file. If TYPE is "report", a report is echoed to 56 | the console and OUTPUT is ignored. 57 | """ 58 | if os.path.splitext(ref)[1] == ".ipynb": 59 | ref = ReferenceImplementation.compile(ref) 60 | else: 61 | try: 62 | ref = ReferenceImplementation.load(ref) 63 | except: 64 | raise RuntimeError(f"Could not load the reference implementation {ref}") 65 | 66 | if dest is None: 67 | dest = get_stem(stu) + "_results" + (".pkl", ".json")[output_type == "json"] 68 | 69 | if os.path.splitext(stu)[1] == ".ipynb": 70 | stu = StudentImplementation(stu, output=output) 71 | else: 72 | try: 73 | stu = StudentImplementation.load(stu) 74 | except: 75 | raise RuntimeError(f"Could not load the student implementation {stu}") 76 | 77 | res = stu.check(ref) 78 | 79 | if output_type == "pickle": 80 | if isinstance(res, list): 81 | with open(dest, "wb+") as f: 82 | dill.dump(res, f) 83 | else: 84 | res.dump(dest) 85 | elif output_type == "json": 86 | if isinstance(res, list): 87 | d = [r.to_dict() for r in res] 88 | else: 89 | d = res.to_dict() 90 | with open(dest, "w+") as f: 91 | json.dump(d, f, indent=2) 92 | elif output_type == "report": 93 | report = generate_report(res) 94 | click.echo(report) 95 | 96 | 97 | @click_cli.command() 98 | @click.option("-p", "--parallel", is_flag=True, default=False, 99 | help="Execute notebooks in parallel using the multiprocessing library") 100 | @click.option("-d", "--dest", default=None, type=click.Path(), 101 | help="Path at which to write the pickled student implementation") 102 | @click.option("--timeout", default=1200, type=click.INT, 103 | help="Timeout for notebook execution in seconds") 104 | @click.argument("subm", nargs=-1, type=click.Path(exists=True, dir_okay=False)) 105 | def execute(subm, parallel, dest, timeout): 106 | """ 107 | Execute student submissions to generate memory footprints. 108 | 109 | Executes the student submission(s) SUBM at writes the pickled objects to some output file. If 110 | DEST is unspecified, this defaults to "./{SUBM.stem}.pkl" (e.g. for SUBM 111 | "submissions/subm01.ipynb", this is "./subm01.pkl"). 112 | """ 113 | if len(subm) == 0: 114 | raise ValueError("You must specify at least one notebook to execute") 115 | 116 | stus = generate_student_impls(subm, parallel=parallel, timeout=timeout) 117 | 118 | if dest is None: 119 | dest = "./" 120 | 121 | if len(subm) == 1: 122 | if os.path.isdir(dest): 123 | stem = get_stem(subm[0]) 124 | output = os.path.join(dest, stem + ".pkl") 125 | 126 | stus[0].dump(output) 127 | 128 | else: 129 | if not os.path.isdir(dest): 130 | raise ValueError(f"Destination directory {dest} does not exist or is not a directory") 131 | 132 | for s, stu in zip(subm, stus): 133 | stem = get_stem(s) 134 | path = os.path.join(dest, stem + ".pkl") 135 | stu.dump(path) 136 | 137 | 138 | def cli(*args, **kwargs): 139 | """ 140 | Wrapper for the click CLI that sets the prog name. 141 | """ 142 | prog_name = kwargs.pop("prog_name", "pybryt") 143 | return click_cli(*args, prog_name=prog_name, **kwargs) 144 | -------------------------------------------------------------------------------- /pybryt/complexities.py: -------------------------------------------------------------------------------- 1 | """Alias for ``pybryt.annotations.complexity.complexities``""" 2 | 3 | from .annotations.complexity.complexities import * 4 | -------------------------------------------------------------------------------- /pybryt/complexity.py: -------------------------------------------------------------------------------- 1 | """Time complexity analysis utilities""" 2 | 3 | import inspect 4 | 5 | from collections.abc import Sized 6 | from itertools import chain 7 | from typing import List, Optional, Union 8 | 9 | from pybryt.execution.memory_footprint import MemoryFootprintValue 10 | 11 | from .annotations import complexities as cplx, TimeComplexity 12 | from .execution import ( 13 | check_time_complexity, 14 | get_tracing_frame, 15 | FrameTracer, 16 | MemoryFootprint, 17 | TimeComplexityResult, 18 | ) 19 | 20 | 21 | ANNOTATION_NAME = "__time_complexity_checker__" 22 | 23 | 24 | class TimeComplexityChecker: 25 | """ 26 | A utility class for using PyBryt's tracing functionality to check the time complexity of a block 27 | of code. 28 | 29 | Uses PyBryt's tracing internals to set a trace function that counts the number of steps taken 30 | to execute a block of code, and then uses the complexity annotation framework to determine the 31 | best-matched complexity class of the block. 32 | """ 33 | 34 | name: str 35 | """the name to use for the annotation""" 36 | 37 | results: List[TimeComplexityResult] 38 | """the result objects holding the step data for each input length""" 39 | 40 | def __init__(self, name: Optional[str] = None) -> None: 41 | self.name = name if name is not None else ANNOTATION_NAME 42 | self.results = [] 43 | 44 | def __call__(self, n: Union[int, float, Sized]) -> "_check_time_complexity_wrapper": 45 | """ 46 | Create a wrapper for :py:class:`pybryt.execution.complexity.check_time_complexity` that enables 47 | tracing and collects the results object produced by that context manager. 48 | 49 | Args: 50 | n (``Union[int, float, Sized]``): the input length or the input itself if it supports ``len`` 51 | 52 | Returns: 53 | :py:class:`_check_time_complexity_wrapper`: the initialized context manager that wraps 54 | ``check_time_complexity`` 55 | """ 56 | return _check_time_complexity_wrapper(self, n) 57 | 58 | def add_result(self, result: TimeComplexityResult) -> None: 59 | """ 60 | Add a time complexity result to the collection of results. 61 | 62 | Args: 63 | result (``TimeComplexityResult``): the result object 64 | """ 65 | self.results.append(result) 66 | 67 | def determine_complexity(self) -> cplx.complexity: 68 | """ 69 | Determine the best-matched complexity class based on the results collected. 70 | 71 | Returns: 72 | :py:class:`pybryt.annotations.complexity.complexities.complexity`: the complexity class 73 | corresponding to the best-matched complexity 74 | """ 75 | annot = TimeComplexity(cplx.constant, name=self.name) 76 | footprint = MemoryFootprint.from_values( 77 | *[MemoryFootprintValue(tcr, i, None) for i, tcr in enumerate(self.results)]) 78 | result = annot.check(footprint) 79 | return result.value 80 | 81 | 82 | class _check_time_complexity_wrapper: 83 | """ 84 | A wrapper for :py:class:`pybryt.execution.complexity.check_time_complexity` that enables tracing 85 | for and collects the results from that context manager. 86 | 87 | Args: 88 | checker (:py:class:`TimeComplexityChecker`): the complexity checker that is using this wrapper 89 | n (``Union[int, float, Sized]``): the input length or the input itself if it supports ``len`` 90 | """ 91 | 92 | checker: TimeComplexityChecker 93 | """the complexity checker that is using this wrapper""" 94 | 95 | check_context: check_time_complexity 96 | """the time complexity context being wrapped""" 97 | 98 | frame_tracer: FrameTracer 99 | """the frame tracer being used to manage execution tracing""" 100 | 101 | n: Union[int, float, Sized] 102 | """the input length or the input itself if it supports ``len``""" 103 | 104 | def __init__(self, checker: TimeComplexityChecker, n: Union[int, float, Sized]) -> None: 105 | self.checker = checker 106 | self.n = n 107 | self.check_context = None 108 | self.frame_tracer = None 109 | 110 | def __enter__(self) -> None: 111 | self.frame_tracer = FrameTracer(inspect.currentframe().f_back) 112 | self.frame_tracer.start_trace() 113 | self.check_context = check_time_complexity(self.checker.name, self.n) 114 | self.check_context.__enter__() 115 | 116 | def __exit__(self, exc_type, exc_value, traceback) -> bool: 117 | self.check_context.__exit__(exc_type, exc_value, traceback) 118 | self.frame_tracer.end_trace() 119 | 120 | result = None 121 | for mfp_val in self.frame_tracer.get_footprint(): 122 | if isinstance(mfp_val.value, TimeComplexityResult): 123 | result = mfp_val.value 124 | 125 | if result: 126 | self.checker.add_result(result) 127 | 128 | return False 129 | -------------------------------------------------------------------------------- /pybryt/debug.py: -------------------------------------------------------------------------------- 1 | """Debug mode""" 2 | 3 | __all__ = ["debug_mode", "disable_debug_mode", "enable_debug_mode"] 4 | 5 | from contextlib import contextmanager 6 | from typing import NoReturn 7 | 8 | 9 | _DEBUG_MODE_ENABLED = False 10 | 11 | 12 | def _debug_mode_enabled() -> bool: 13 | """ 14 | Return whether debug mode is currently enabled. 15 | 16 | Returns: 17 | ``bool``: whether debug mode is enabled 18 | """ 19 | return _DEBUG_MODE_ENABLED 20 | 21 | 22 | def enable_debug_mode() -> None: 23 | """ 24 | Enable PyBryt's debug mode. 25 | """ 26 | global _DEBUG_MODE_ENABLED 27 | _DEBUG_MODE_ENABLED = True 28 | 29 | 30 | def disable_debug_mode() -> None: 31 | """ 32 | Disable PyBryt's debug mode. 33 | """ 34 | global _DEBUG_MODE_ENABLED 35 | _DEBUG_MODE_ENABLED = False 36 | 37 | 38 | @contextmanager 39 | def debug_mode() -> None: 40 | """ 41 | A context in which debug mode is enabled. 42 | 43 | When the context exits, debug mode is disabled globally. This means debug mode will be disabled 44 | even if it was enabled before the context was entered. 45 | """ 46 | enable_debug_mode() 47 | yield 48 | disable_debug_mode() 49 | -------------------------------------------------------------------------------- /pybryt/execution/__init__.py: -------------------------------------------------------------------------------- 1 | """Submission execution internals for PyBryt""" 2 | 3 | __all__ = [ 4 | "check_time_complexity", 5 | "MemoryFootprint", 6 | "no_tracing", 7 | "set_initial_conditions", 8 | "TimeComplexityResult", 9 | ] 10 | 11 | import os 12 | import dill 13 | import nbformat 14 | 15 | from nbconvert.preprocessors import ExecutePreprocessor 16 | from copy import deepcopy 17 | from tempfile import mkstemp 18 | from typing import List, Optional 19 | from textwrap import dedent 20 | 21 | from .complexity import check_time_complexity, is_complexity_tracing_enabled, TimeComplexityResult 22 | from .memory_footprint import Event, MemoryFootprint, MemoryFootprintValue 23 | from .tracing import ( 24 | create_collector, 25 | FrameTracer, 26 | get_tracing_frame, 27 | no_tracing, 28 | set_initial_conditions, 29 | tracing_off, 30 | tracing_on, 31 | ) 32 | 33 | from ..preprocessors import NotebookPreprocessor 34 | from ..utils import make_secret 35 | 36 | 37 | NBFORMAT_VERSION = 4 38 | 39 | 40 | def execute_notebook( 41 | nb: nbformat.NotebookNode, 42 | nb_path: str, 43 | addl_filenames: List[str] = [], 44 | timeout: Optional[int] = 1200 45 | ) -> MemoryFootprint: 46 | """ 47 | Executes a submission using ``nbconvert`` and returns the memory footprint. 48 | 49 | Takes in a notebook object and preprocesses it before running it through the 50 | ``nbconvert.ExecutePreprocessor`` to execute it. The notebook writes the memory footprint, a 51 | list of observed values and their timestamps, to a file, which is loaded using ``dill`` by this 52 | function. Errors during execution are ignored, and the executed notebook can be written to a 53 | file using the ``output`` argument. 54 | 55 | Args: 56 | nb (``nbformat.NotebookNode``): the notebook to be executed 57 | nb_path (``str``): path to the notebook ``nb`` 58 | addl_filenames (``list[str]``, optional): a list of additional files to trace inside 59 | output (``str``, optional): a file path at which to write the executed notebook 60 | timeout (``int``, optional): number of seconds to allow for notebook execution; set to 61 | ``None`` for no time limit 62 | 63 | Returns: 64 | :py:class:`pybryt.execution.memory_footprint.MemoryFootprint`: the memory footprint 65 | """ 66 | nb = deepcopy(nb) 67 | preprocessor = NotebookPreprocessor() 68 | nb = preprocessor.preprocess(nb) 69 | 70 | _, footprint_fp = mkstemp() 71 | nb_dir = os.path.abspath(os.path.split(nb_path)[0]) 72 | 73 | secret = make_secret() 74 | frame_tracer_varname = f"frame_tracer_{secret}" 75 | 76 | first_cell = nbformat.v4.new_code_cell(dedent(f"""\ 77 | import inspect 78 | import sys 79 | from pybryt.execution import FrameTracer 80 | {frame_tracer_varname} = FrameTracer(inspect.currentframe()) 81 | {frame_tracer_varname}.start_trace(addl_filenames={addl_filenames}) 82 | %cd {nb_dir} 83 | """)) 84 | 85 | last_cell = nbformat.v4.new_code_cell(dedent(f"""\ 86 | {frame_tracer_varname}.end_trace() 87 | footprint = {frame_tracer_varname}.get_footprint() 88 | footprint.filter_out_unpickleable_values() 89 | import dill 90 | with open("{footprint_fp}", "wb+") as f: 91 | dill.dump(footprint, f) 92 | """)) 93 | 94 | nb['cells'].insert(0, first_cell) 95 | nb['cells'].append(last_cell) 96 | 97 | ep = ExecutePreprocessor(timeout=timeout, allow_errors=True) 98 | ep.preprocess(nb) 99 | 100 | with open(footprint_fp, "rb") as f: 101 | footprint: MemoryFootprint = dill.load(f) 102 | 103 | os.remove(footprint_fp) 104 | 105 | footprint.add_imports(*preprocessor.get_imports()) 106 | footprint.set_executed_notebook(nb) 107 | 108 | return footprint 109 | -------------------------------------------------------------------------------- /pybryt/execution/complexity.py: -------------------------------------------------------------------------------- 1 | """Complexity analysis internals""" 2 | 3 | from collections.abc import Sized 4 | from typing import Optional, Union 5 | 6 | from .memory_footprint import MemoryFootprint 7 | 8 | 9 | COMPLEXITY_TRACING_ENABLED = False 10 | 11 | 12 | class TimeComplexityResult: 13 | """ 14 | A class for collecting the results of time complexity check blocks. 15 | 16 | Args: 17 | name (``str``): the name of the block 18 | n (``int``): the input length 19 | start (``int``): the step counter value at the start of the block 20 | stop (``int``): the step counter value at the end of the block 21 | """ 22 | 23 | name: str 24 | """the name of the block""" 25 | 26 | n: int 27 | """the input length or the input itself""" 28 | 29 | start: int 30 | """the step counter value at the start of the block""" 31 | 32 | stop: int 33 | """the step counter value at the end of the block""" 34 | 35 | def __init__(self, name, n, start, stop): 36 | self.name = name 37 | self.n = n 38 | self.start = start 39 | self.stop = stop 40 | 41 | 42 | class check_time_complexity: 43 | """ 44 | A context manager for checking the time complexity of student code. 45 | 46 | Halts tracking of values in memory and sets the trace function to only increment the step 47 | counter. When the block exits, the step counter is checked an a ``TimeComplexityResult`` object 48 | is appended to the student's memory footprint. 49 | 50 | If the current call stack is not being traced by PyBryt, no action is taken. 51 | 52 | Args: 53 | name (``str``): the name of the check; should match with the name of an annotation in the 54 | reference implementation 55 | n (``Union[int, float, Sized]``): the input length or the input itself if it supports ``len`` 56 | """ 57 | 58 | name: str 59 | 60 | n: int 61 | 62 | start_steps: Optional[int] 63 | 64 | footprint: Optional[MemoryFootprint] 65 | 66 | def __init__(self, name: str, n: Union[int, float, Sized]): 67 | if isinstance(n, float): 68 | n = int(n) 69 | if isinstance(n, Sized): 70 | n = len(n) 71 | if not isinstance(n, int): 72 | try: 73 | n = int(n) 74 | except: 75 | raise TypeError(f"n has invalid type {type(n)}") 76 | 77 | self.name = name 78 | self.n = n 79 | self.start_steps, self.footprint = None, None 80 | 81 | def __enter__(self): 82 | global COMPLEXITY_TRACING_ENABLED 83 | 84 | if get_active_footprint() is not None: 85 | self.footprint = get_active_footprint() 86 | self.start_steps = self.footprint.counter.get_value() 87 | 88 | COMPLEXITY_TRACING_ENABLED = True 89 | 90 | def __exit__(self, exc_type, exc_value, traceback): 91 | global COMPLEXITY_TRACING_ENABLED 92 | 93 | COMPLEXITY_TRACING_ENABLED = False 94 | 95 | if self.start_steps is not None: 96 | end_steps = self.footprint.counter.get_value() 97 | self.footprint.add_value(TimeComplexityResult( 98 | self.name, self.n, self.start_steps, end_steps), allow_duplicates=True) 99 | 100 | return False 101 | 102 | 103 | def is_complexity_tracing_enabled() -> bool: 104 | """ 105 | Return whether complexity tracing is currently enabled. 106 | 107 | Returns: 108 | ``bool``: whether complexity tracing is currently enabled 109 | """ 110 | return COMPLEXITY_TRACING_ENABLED 111 | 112 | 113 | from .tracing import get_active_footprint 114 | -------------------------------------------------------------------------------- /pybryt/execution/utils.py: -------------------------------------------------------------------------------- 1 | """Execution utilities""" 2 | 3 | import os 4 | 5 | from types import FrameType 6 | 7 | 8 | def is_ipython_frame(frame: FrameType) -> bool: 9 | """ 10 | Determine whether a frame is being executed by IPython. 11 | 12 | Args: 13 | frame (``types.FrameType``): the frame to examine 14 | 15 | Returns: 16 | ``bool``: whether the frame is an IPython frame 17 | """ 18 | filename = frame.f_code.co_filename 19 | parent_dir = os.path.split(os.path.split(filename)[0])[1] 20 | return filename.startswith(" List[ReferenceImplementation]: 14 | """ 15 | """ 16 | if seed is not None: 17 | random.seed(seed) 18 | 19 | # TODO: this doesn't work with the new MemoryFootprint API 20 | values = [[t[0] for t in stu.values if t[0] is not None] for stu in student_impls] 21 | 22 | k = int(min([len(v) for v in values]) * frac) 23 | 24 | refs = [] 25 | for stu, vals in zip(student_impls, values): 26 | ref_values = random.sample(vals, k) 27 | refs.append(ReferenceImplementation([Value(v) for v in ref_values])) 28 | 29 | return refs 30 | 31 | 32 | def get_impl_results(ref_impl: ReferenceImplementation, student_impls: List["StudentImplementation"], 33 | arr=True, **kwargs) -> Union[List[ReferenceResult], np.ndarray]: 34 | """ 35 | Returns matrix where rows are student impls and cols represent whether each value was satisfied 36 | by the student impl 37 | """ 38 | # results = [] 39 | # for ref in ref_impls: 40 | # ref_results = [] 41 | # for stu in student_impls: 42 | # ref_results.append(stu.check(ref)) 43 | # results.append(ref_results) 44 | 45 | results = [] 46 | for stu in student_impls: 47 | results.append(stu.check(ref_impl)) 48 | 49 | if arr: 50 | return np.array([r.to_array() for r in results]) 51 | else: 52 | return results 53 | 54 | 55 | def compare_implementations(student_impls: List["StudentImplementation"], **kwargs) -> \ 56 | List[Union[List[ReferenceResult], np.ndarray]]: 57 | """ 58 | """ 59 | refs = create_references(student_impls, **kwargs) 60 | results = [] 61 | for ref in refs: 62 | results.append(get_impl_results(ref, student_impls, **kwargs)) 63 | return results 64 | 65 | 66 | from .student import StudentImplementation 67 | -------------------------------------------------------------------------------- /pybryt/preprocessors/__init__.py: -------------------------------------------------------------------------------- 1 | """Submission preprocessors for PyBryt""" 2 | 3 | import nbformat 4 | 5 | from typing import List, Set 6 | 7 | from .abstract_preprocessor import AbstractPreprocessor 8 | from .imports import ImportFindingPreprocessor 9 | from .intermediate_variables import IntermediateVariablePreprocessor 10 | 11 | 12 | PREPROCESSORS = [ 13 | ImportFindingPreprocessor, 14 | IntermediateVariablePreprocessor 15 | ] 16 | 17 | 18 | class NotebookPreprocessor(AbstractPreprocessor): 19 | """ 20 | A class for applying a series of preprocessors to a notebook. 21 | """ 22 | 23 | preprocessors: List[AbstractPreprocessor] 24 | 25 | def __init__(self) -> None: 26 | self.preprocessors = [] 27 | for preprocessor_class in PREPROCESSORS: 28 | self.preprocessors.append(preprocessor_class()) 29 | 30 | def preprocess(self, nb: nbformat.NotebookNode) -> nbformat.NotebookNode: 31 | for preprocessor in self.preprocessors: 32 | nb = preprocessor.preprocess(nb) 33 | 34 | return nb 35 | 36 | def get_imports(self) -> Set[str]: 37 | """ 38 | Get the set of imports from the :py:class:`ImportFinder None: 17 | self.transformer_manager = TransformerManager() 18 | 19 | @abstractmethod 20 | def preprocess(self, nb: nbformat.NotebookNode) -> nbformat.NotebookNode: 21 | """ 22 | Preprocesses a notebook for execution. 23 | 24 | Args: 25 | nb (``nbformat.NotebookNode``): the notebook to be preprocessed 26 | 27 | Returns: 28 | ``nbformat.NotebookNode``: the updated notebook 29 | """ 30 | ... # pragma: no cover 31 | -------------------------------------------------------------------------------- /pybryt/preprocessors/imports.py: -------------------------------------------------------------------------------- 1 | """Preprocessor for collecting the set of imported modules""" 2 | 3 | import ast 4 | import nbformat 5 | 6 | from typing import Set 7 | 8 | from .abstract_preprocessor import AbstractPreprocessor 9 | 10 | 11 | class ImportFinder(ast.NodeVisitor): 12 | """ 13 | An AST walker for collecting imported modules. 14 | """ 15 | 16 | def __init__(self) -> None: 17 | self.imports = set() 18 | 19 | def visit_Import(self, node: ast.AST) -> None: 20 | """ 21 | Visit an import statement and collect the names of the imported modules 22 | """ 23 | for alias in node.names: 24 | self.imports.add(alias.name.split(".")[0]) 25 | 26 | def visit_ImportFrom(self, node: ast.AST) -> None: 27 | """ 28 | Visit an import from statement and collect the names of the imported modules 29 | """ 30 | self.imports.add(node.module) 31 | 32 | 33 | class ImportFindingPreprocessor(AbstractPreprocessor): 34 | """ 35 | A notebook preprocessor that collects the set of modules import in the notebook. 36 | """ 37 | 38 | imports: Set[str] 39 | 40 | def __init__(self) -> None: 41 | self.imports = set() 42 | super().__init__() 43 | 44 | def preprocess(self, nb: nbformat.NotebookNode) -> nbformat.NotebookNode: 45 | """ 46 | Populate ``self.imports`` by walking the AST of each code cell. 47 | 48 | Args: 49 | nb (``nbformat.NotebookNode``): the notebook to be preprocessed 50 | 51 | Returns: 52 | ``nbformat.NotebookNode``: the notebook, unchanged 53 | """ 54 | for cell in nb['cells']: 55 | if cell['cell_type'] == 'code': 56 | code = cell['source'] 57 | code = self.transformer_manager.transform_cell(code) 58 | tree = ast.parse(code) 59 | import_finder = ImportFinder() 60 | import_finder.visit(tree) 61 | self.imports.update(import_finder.imports) 62 | 63 | return nb 64 | -------------------------------------------------------------------------------- /pybryt/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.7.0" 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | IPython 2 | nbconvert 3 | nbformat 4 | numpy 5 | pandas 6 | dill 7 | click 8 | astunparse 9 | Cython 10 | ipykernel 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import sys 3 | 4 | with open("README.md", "r") as fh: 5 | long_description = fh.read() 6 | 7 | with open("requirements.txt") as f: 8 | install_requires = f.readlines() 9 | 10 | # add dataclasses backport for python 3.6 11 | if sys.version_info.minor < 7: 12 | install_requires.append("dataclasses") 13 | 14 | # get version 15 | env = {} 16 | with open("pybryt/version.py") as f: 17 | exec(f.read(), env) 18 | version = env["__version__"] 19 | 20 | setuptools.setup( 21 | name = "pybryt", 22 | version = version, 23 | author = "Chris Pyles", 24 | author_email = "pybryt-support@microsoft.com", 25 | description = "Python auto-assessment library", 26 | long_description = long_description, 27 | long_description_content_type = "text/markdown", 28 | url = "https://github.com/microsoft/pybryt", 29 | license = "MIT", 30 | packages = setuptools.find_packages(exclude=["test"]), 31 | classifiers = [ 32 | "Programming Language :: Python :: 3", 33 | "License :: OSI Approved :: MIT License", 34 | "Operating System :: OS Independent", 35 | ], 36 | install_requires=install_requires, 37 | entry_points={ 38 | "console_scripts": ["pybryt=pybryt.cli:cli"] 39 | }, 40 | ) 41 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tests/annotations/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for PyBryt's various annotations""" -------------------------------------------------------------------------------- /tests/annotations/structural_helpers.py: -------------------------------------------------------------------------------- 1 | """Helpers for structural pattern tests""" 2 | 3 | from typing import Any, Dict, List 4 | 5 | 6 | class AttrContainer: 7 | 8 | attrs: Dict[str, Any] 9 | 10 | def __init__(self, **kwargs) -> None: 11 | self.attrs = {**kwargs} 12 | 13 | def __setattr__(self, name: str, value: Any) -> None: 14 | if name == "attrs": 15 | self.__dict__["attrs"] = value 16 | else: 17 | self.attrs[name] = value 18 | 19 | def __getattr__(self, name: str) -> Any: 20 | if name in {"__getstate__", "__slots__", "__setstate__"} or name not in self.attrs: 21 | raise AttributeError 22 | 23 | return self.attrs[name] 24 | 25 | def __dir__(self) -> List[str]: 26 | return sorted(list(self.attrs.keys()) + ["attrs"]) 27 | 28 | def __eq__(self, other): 29 | return isinstance(other, type(self)) and self.attrs == other.attrs 30 | 31 | 32 | class Container(AttrContainer): 33 | 34 | def __init__(self, *elements: Any) -> None: 35 | super().__init__() 36 | self.elements = list(elements) 37 | 38 | def __contains__(self, element: Any) -> bool: 39 | return element in self.elements 40 | -------------------------------------------------------------------------------- /tests/annotations/test_annotation.py: -------------------------------------------------------------------------------- 1 | """Tests for abstract annotations and annotation results""" 2 | 3 | import pybryt 4 | from pybryt.execution.memory_footprint import MemoryFootprint 5 | 6 | from .utils import assert_object_attrs, generate_memory_footprint 7 | 8 | 9 | def test_name_group_limit(): 10 | """ 11 | """ 12 | footprint = generate_memory_footprint() 13 | val = footprint.get_value(2) 14 | 15 | pybryt.Annotation.reset_tracked_annotations() 16 | vs = [] 17 | for _ in range(100): 18 | vs.append(pybryt.Value(val, name="foo", limit=11)) 19 | 20 | tracked = pybryt.Annotation.get_tracked_annotations() 21 | assert len(tracked) == 100 22 | assert tracked == vs, "Wrong tracked annotations" 23 | 24 | res = vs[-1].check(footprint) 25 | assert_object_attrs(res, { 26 | "name": "foo", 27 | "group": None, 28 | }) 29 | 30 | v1 = pybryt.Value(footprint.get_value(0).value, group="bar") 31 | v2 = pybryt.Value(footprint.get_value(1).value, group="bar") 32 | assert_object_attrs(v1, {"group": "bar"}) 33 | assert_object_attrs(v2, {"group": "bar"}) 34 | 35 | res = v1.check(footprint) 36 | print(res.name) 37 | assert_object_attrs(res, { 38 | "name": "Annotation 101", 39 | "group": "bar", 40 | }) 41 | 42 | 43 | def test_get_reset_tracked_annotations(): 44 | """ 45 | """ 46 | tracked = pybryt.Annotation.get_tracked_annotations() 47 | pybryt.Annotation.reset_tracked_annotations() 48 | assert len(tracked) == 0 49 | 50 | v1 = pybryt.Value(1) 51 | v2 = pybryt.Value(2) 52 | assert len(tracked) == 2 53 | 54 | v3 = pybryt.Value(3) 55 | assert len(tracked) == 3 56 | 57 | v4 = v3.before(v2) 58 | assert len(tracked) == 2 59 | 60 | assert v1 in tracked 61 | assert v2 not in tracked 62 | assert v3 not in tracked 63 | assert v4 in tracked 64 | 65 | pybryt.Annotation.reset_tracked_annotations() 66 | assert len(tracked) == 0 67 | 68 | 69 | def test_messages(): 70 | """ 71 | """ 72 | footprint = generate_memory_footprint() 73 | pybryt.Annotation.reset_tracked_annotations() 74 | 75 | val1 = footprint.get_value(0).value 76 | val2 = footprint.get_value(1).value 77 | 78 | v1 = pybryt.Value(val1, success_message="m1", failure_message="m2") 79 | v2 = pybryt.Value(val2) 80 | 81 | v = v1.before(v2) 82 | res = v.check(footprint) 83 | 84 | assert len(res.messages) == 3 85 | assert_object_attrs(res.messages[0], {"message": "m1", "satisfied": True}) 86 | 87 | v.failure_message = "m3" 88 | res = v.check(footprint) 89 | 90 | assert len(res.messages) == 3 91 | 92 | res = v.check(MemoryFootprint.from_values()) 93 | 94 | assert len(res.messages) == 3 95 | assert_object_attrs(res.messages[0], {"message": "m2", "satisfied": False}) 96 | assert_object_attrs(res.messages[1], {"message": None, "satisfied": False}) 97 | assert_object_attrs(res.messages[2], {"message": "m3", "satisfied": False}) 98 | 99 | v2.name = "v2" 100 | v2.failure_message = "m4" 101 | res = v.check(MemoryFootprint.from_values()) 102 | 103 | assert len(res.messages) == 3 104 | assert_object_attrs(res.messages[1], {"message": "m4", "name": "v2", "satisfied": False}) 105 | 106 | 107 | def test_bitwise_ops(): 108 | """ 109 | """ 110 | a1, a2 = pybryt.Value(1), pybryt.Value(2) 111 | 112 | assert isinstance(a1 & a2, pybryt.AndAnnotation), "& operator returns wrong type" 113 | assert isinstance(a1 | a2, pybryt.OrAnnotation), "| operator returns wrong type" 114 | assert isinstance(a1 ^ a2, pybryt.XorAnnotation), "^ operator returns wrong type" 115 | assert isinstance(~a1, pybryt.NotAnnotation), "~ operator returns wrong type" 116 | -------------------------------------------------------------------------------- /tests/annotations/test_collection.py: -------------------------------------------------------------------------------- 1 | """Tests for annotation collections""" 2 | 3 | import pytest 4 | 5 | import pybryt 6 | 7 | from .utils import * 8 | 9 | 10 | def test_collection(): 11 | """ 12 | """ 13 | footprint = generate_memory_footprint() 14 | pybryt.Annotation.reset_tracked_annotations() 15 | 16 | v1, v2 = pybryt.Value(footprint.get_value(0).value), pybryt.Value(footprint.get_value(2).value) 17 | a = pybryt.Collection(v1, v2, success_message="foo") 18 | 19 | res = a.check(footprint) 20 | assert_object_attrs(a, {"children__len": 2}) 21 | assert_object_attrs(res, { 22 | "children__len": 2, 23 | "satisfied": True, 24 | "_satisfied": None, 25 | "annotation": a, 26 | "timestamp": -1, 27 | "satisfied_at": 2, 28 | }) 29 | 30 | assert a.to_dict() == { 31 | "name": "Annotation 3", 32 | "children": [v1.to_dict(), v2.to_dict()], 33 | "success_message": "foo", 34 | "failure_message": None, 35 | "limit": None, 36 | "group": None, 37 | "type": "collection", 38 | "enforce_order": False, 39 | } 40 | 41 | a = pybryt.Collection(v1, v2, enforce_order=True, success_message="foo") 42 | res = a.check(footprint) 43 | assert res.satisfied 44 | 45 | v3 = pybryt.Value(footprint.get_value(1).value) 46 | a.add(v3) 47 | res = a.check(footprint) 48 | assert not res.satisfied 49 | 50 | a.remove(v3) 51 | res = a.check(footprint) 52 | assert res.satisfied 53 | 54 | a = pybryt.Collection(v2, v1, enforce_order=True, success_message="foo") 55 | res = a.check(footprint) 56 | assert not res.satisfied 57 | 58 | # test errors 59 | with pytest.raises(ValueError, match="One of the arguments is not an annotation"): 60 | pybryt.Collection(v1, 1, v2) 61 | 62 | with pytest.raises(TypeError, match="1 is not an annotation"): 63 | a.add(1) 64 | -------------------------------------------------------------------------------- /tests/annotations/test_complexity.py: -------------------------------------------------------------------------------- 1 | """Tests for complexity annotations""" 2 | 3 | from typing import Type 4 | import numpy as np 5 | from numpy.lib.arraysetops import isin 6 | import pytest 7 | 8 | import pybryt 9 | import pybryt.complexities as cplx 10 | 11 | from pybryt.execution import MemoryFootprintValue 12 | 13 | from .utils import assert_object_attrs 14 | 15 | 16 | def generate_complexity_footprint(name, t_transform, max_exp=8): 17 | values = [] 18 | for i, e in enumerate(range(1, max_exp + 1)): 19 | n = 10 ** e 20 | t = t_transform(n) 21 | values.append(MemoryFootprintValue(pybryt.TimeComplexityResult(name, n, 0, t), i, None)) 22 | return pybryt.MemoryFootprint.from_values(*values) 23 | 24 | 25 | def test_complexity_abc(): 26 | """ 27 | """ 28 | pybryt.Annotation.reset_tracked_annotations() 29 | 30 | a = pybryt.TimeComplexity(cplx.constant, name="foo") 31 | assert_object_attrs(a, { 32 | "name": "foo", 33 | "complexity": cplx.constant, 34 | }) 35 | 36 | b = pybryt.TimeComplexity(cplx.constant, name="foo") 37 | assert a == b 38 | 39 | b.complexity = cplx.linear 40 | assert a != b 41 | 42 | b.name = "bar" 43 | assert a != b 44 | 45 | b.complexity = cplx.constant 46 | assert a != b 47 | 48 | # test constructor errors 49 | with pytest.raises(ValueError, match="Complexity annotations require a 'name' kwarg"): 50 | pybryt.TimeComplexity(cplx.constant) 51 | 52 | with pytest.raises(ValueError, match="Invalid valid for argument 'complexity': 1"): 53 | pybryt.TimeComplexity(1, name="foo") 54 | 55 | 56 | def test_time_complexity(): 57 | pybryt.Annotation.reset_tracked_annotations() 58 | 59 | a = pybryt.TimeComplexity(cplx.constant, name="foo") 60 | 61 | footprint = generate_complexity_footprint("foo", lambda v: 1012) 62 | res = a.check(footprint) 63 | assert res.satisfied 64 | assert res.value == cplx.constant 65 | 66 | footprint.add_value(np.random.uniform(size=100), 9) 67 | footprint.add_value( pybryt.TimeComplexityResult("bar", 10, 0, 10 ** 3), 10) 68 | res = a.check(footprint) 69 | assert res.satisfied 70 | assert res.value == cplx.constant 71 | 72 | footprint = generate_complexity_footprint("foo", np.log2) 73 | res = a.check(footprint) 74 | assert not res.satisfied 75 | assert res.value == cplx.logarithmic 76 | 77 | footprint = generate_complexity_footprint("foo", lambda v: v * np.log2(v)) 78 | res = a.check(footprint) 79 | assert not res.satisfied 80 | assert res.value == cplx.linearithmic 81 | 82 | a.complexity = cplx.exponential 83 | res = a.check(footprint) 84 | assert not res.satisfied 85 | assert res.value == cplx.linearithmic 86 | 87 | 88 | def test_alias(): 89 | from pybryt.annotations.complexity import complexities as cplx2 90 | assert cplx.complexity_classes is cplx2.complexity_classes 91 | 92 | 93 | def test_complexity_union(): 94 | """ 95 | Tests for complexity unions. 96 | """ 97 | union = cplx.logarithmic | cplx.linear 98 | assert isinstance(union, cplx.ComplexityUnion) 99 | assert set(union.get_complexities()) == {cplx.logarithmic, cplx.linear} 100 | 101 | union2 = union | cplx.constant 102 | assert isinstance(union2, cplx.ComplexityUnion) 103 | assert set(union2.get_complexities()) == {cplx.logarithmic, cplx.linear, cplx.constant} 104 | 105 | union3 = cplx.constant | union 106 | assert isinstance(union3, cplx.ComplexityUnion) 107 | assert set(union3.get_complexities()) == {cplx.logarithmic, cplx.linear, cplx.constant} 108 | 109 | union4 = union2 | union 110 | assert isinstance(union4, cplx.ComplexityUnion) 111 | assert set(union4.get_complexities()) == {cplx.logarithmic, cplx.linear, cplx.constant} 112 | 113 | union.add_complexity(cplx.linearithmic) 114 | assert set(union.get_complexities()) == {cplx.logarithmic, cplx.linear, cplx.linearithmic} 115 | 116 | assert union == cplx.logarithmic | cplx.linear | cplx.linearithmic 117 | assert union != cplx.logarithmic | cplx.linear 118 | 119 | # test whether the union works with annotations 120 | a = pybryt.TimeComplexity(cplx.constant | cplx.logarithmic, name="foo") 121 | footprint = generate_complexity_footprint("foo", lambda v: 1012) 122 | res = a.check(footprint) 123 | assert res.satisfied 124 | assert res.value == cplx.constant 125 | 126 | footprint = generate_complexity_footprint("foo", lambda v: v) 127 | res = a.check(footprint) 128 | assert not res.satisfied 129 | assert res.value == cplx.linear 130 | 131 | # test errors 132 | with pytest.raises(TypeError): 133 | cplx.constant | 1 134 | 135 | with pytest.raises(TypeError): 136 | cplx.ComplexityUnion.from_or(1, cplx.constant) 137 | -------------------------------------------------------------------------------- /tests/annotations/test_import_.py: -------------------------------------------------------------------------------- 1 | """Tests for import annotations""" 2 | 3 | import pytest 4 | 5 | from unittest import mock 6 | 7 | import pybryt 8 | from pybryt.execution.memory_footprint import MemoryFootprint 9 | 10 | from .utils import assert_object_attrs 11 | 12 | 13 | IMPORTED_MODULES = ["numpy", "pandas", "matplotlib"] 14 | 15 | 16 | def generate_import_mfp(): 17 | footprint = MemoryFootprint() 18 | footprint.add_imports(*IMPORTED_MODULES) 19 | return footprint 20 | 21 | 22 | @mock.patch.multiple(pybryt.ImportAnnotation, __abstractmethods__=frozenset()) 23 | def test_import_init_behaviors(): 24 | """ 25 | """ 26 | with mock.patch("pybryt.annotations.import_.importlib") as mocked_importlib, \ 27 | mock.patch("pybryt.annotations.import_.sys") as mocked_sys: 28 | pybryt.ImportAnnotation("foo") 29 | mocked_importlib.import_module.assert_called_with("foo") 30 | mocked_sys.modules.pop.assert_called_with("foo") 31 | 32 | mocked_sys.modules = {"foo": None} 33 | pybryt.ImportAnnotation("foo") 34 | assert mocked_sys.modules == {"foo": None} 35 | 36 | with pytest.raises(TypeError): 37 | pybryt.ImportAnnotation(1) 38 | 39 | mocked_importlib.import_module.side_effect = ImportError() 40 | with pytest.raises(ValueError): 41 | pybryt.ImportAnnotation("foo") 42 | 43 | 44 | def test_require_import(): 45 | """ 46 | """ 47 | footprint = generate_import_mfp() 48 | pybryt.Annotation.reset_tracked_annotations() 49 | 50 | with mock.patch("pybryt.annotations.import_.importlib") as mocked_importlib, \ 51 | mock.patch("pybryt.annotations.import_.sys") as mocked_sys: 52 | a = pybryt.RequireImport(IMPORTED_MODULES[0]) 53 | b = pybryt.RequireImport("foo") 54 | 55 | res1 = a.check(footprint) 56 | 57 | assert_object_attrs(a, {"children__len": 0}) 58 | assert_object_attrs(res1, { 59 | "children": [], 60 | "satisfied": True, 61 | "_satisfied": True, 62 | "annotation": a, 63 | "timestamp": -1, 64 | "value": None, 65 | }) 66 | 67 | assert a.to_dict() == { 68 | "name": "Annotation 1", 69 | "children": [], 70 | "success_message": None, 71 | "failure_message": None, 72 | "limit": None, 73 | "group": None, 74 | "type": "requireimport", 75 | "module": IMPORTED_MODULES[0], 76 | } 77 | 78 | res2 = b.check(footprint) 79 | assert not res2.satisfied 80 | 81 | 82 | 83 | def test_forbid_import(): 84 | """ 85 | """ 86 | footprint = generate_import_mfp() 87 | pybryt.Annotation.reset_tracked_annotations() 88 | 89 | with mock.patch("pybryt.annotations.import_.importlib") as mocked_importlib, \ 90 | mock.patch("pybryt.annotations.import_.sys") as mocked_sys: 91 | a = pybryt.ForbidImport("foo") 92 | b = pybryt.ForbidImport(IMPORTED_MODULES[0]) 93 | 94 | res1 = a.check(footprint) 95 | 96 | assert_object_attrs(a, {"children__len": 0}) 97 | assert_object_attrs(res1, { 98 | "children": [], 99 | "satisfied": True, 100 | "_satisfied": True, 101 | "annotation": a, 102 | "timestamp": -1, 103 | "value": None, 104 | }) 105 | 106 | assert a.to_dict() == { 107 | "name": "Annotation 1", 108 | "children": [], 109 | "success_message": None, 110 | "failure_message": None, 111 | "limit": None, 112 | "group": None, 113 | "type": "forbidimport", 114 | "module": "foo", 115 | } 116 | 117 | res2 = b.check(footprint) 118 | assert not res2.satisfied 119 | -------------------------------------------------------------------------------- /tests/annotations/test_initial_condition.py: -------------------------------------------------------------------------------- 1 | """Tests for initial conditions""" 2 | 3 | import numpy as np 4 | import operator 5 | 6 | from unittest import mock 7 | 8 | import pytest 9 | 10 | from pybryt import InitialCondition 11 | from pybryt.execution.memory_footprint import MemoryFootprint, MemoryFootprintValue 12 | 13 | from .utils import assert_object_attrs 14 | 15 | 16 | def test_constructor(): 17 | """ 18 | Tests for the ``InitialCondition`` constructor. 19 | """ 20 | name = "foo" 21 | transforms = [lambda x: 2 * x, lambda x: np.cos(x)] 22 | ic = InitialCondition(name, transforms) 23 | assert_object_attrs(ic, { 24 | "name": name, 25 | "transforms": transforms, 26 | }) 27 | 28 | # test errors 29 | with pytest.raises(TypeError): 30 | InitialCondition(2) 31 | 32 | 33 | def test_apply(): 34 | """ 35 | Tests for ``InitialCondition.apply``. 36 | """ 37 | name = "foo" 38 | transforms = [lambda x: 2 * x, lambda x: np.cos(x)] 39 | ic = InitialCondition(name, transforms) 40 | 41 | new_transform = lambda x: np.sin(x) 42 | ic2 = ic.apply(new_transform) 43 | 44 | assert isinstance(ic2, InitialCondition) and ic2 is not ic 45 | assert_object_attrs(ic2, { 46 | "name": name, 47 | "transforms": transforms + [new_transform], 48 | }) 49 | 50 | 51 | def test_supply_value(): 52 | """ 53 | Tests for ``InitialCondition.supply_value``. 54 | """ 55 | name = "foo" 56 | transforms = [mock.MagicMock() for _ in range(10)] 57 | ic = InitialCondition(name, transforms) 58 | 59 | val = 2 60 | res = ic.supply_value(val) 61 | 62 | expected_arg = val 63 | for t in transforms: 64 | t.assert_called_once_with(expected_arg) 65 | expected_arg = t.return_value 66 | 67 | assert res == transforms[-1].return_value 68 | 69 | 70 | def test_supply_footprint_and_supply_values(): 71 | """ 72 | Tests for ``InitialCondition.supply_footprint`` and ``InitialCondition.supply_values``. 73 | """ 74 | name = "foo" 75 | transforms = [mock.MagicMock() for _ in range(10)] 76 | ic = InitialCondition(name, transforms) 77 | 78 | val = 2 79 | mocked_fp = mock.MagicMock() 80 | mocked_fp.get_initial_conditions.return_value = {name: val} 81 | 82 | res = ic.supply_footprint(mocked_fp) 83 | 84 | mocked_fp.get_initial_conditions.assert_called_once() 85 | 86 | expected_arg = val 87 | for t in transforms: 88 | t.assert_called_once_with(expected_arg) 89 | expected_arg = t.return_value 90 | 91 | assert res == transforms[-1].return_value 92 | 93 | # test nested initial conditions 94 | n1, n2, v1, v2 = "foo", "bar", 2, 3 95 | ic1, ic2 = InitialCondition(n1), InitialCondition(n2) 96 | vals = {n1: v1, n2: v2} 97 | ic3 = ic1 + ic2 98 | assert ic3.supply_values(vals) == v1 + v2 99 | 100 | # test errors 101 | with pytest.raises(ValueError, match=f"The provided values do not have key '{name}'"): 102 | ic.supply_values({"bar": 1}) 103 | 104 | 105 | def test_eq(): 106 | """ 107 | Tests for ``==`` comparisons on ``InitialCondition``s. 108 | """ 109 | ic1, ic2, ic3 = InitialCondition("foo"), InitialCondition("foo"), InitialCondition("bar") 110 | 111 | assert ic1 == ic2 112 | assert ic1 != ic3 113 | 114 | t = lambda v: 2 * v 115 | ic2 = ic2.apply(t) 116 | assert ic1 != ic2 117 | 118 | ic1 = ic1.apply(t) 119 | assert ic1 == ic2 120 | 121 | 122 | def test_operators(): 123 | """ 124 | Tests for the use of operators on ``InitialCondition``s. 125 | """ 126 | ic = InitialCondition("foo") 127 | l, r, m = 2, 3, 4 128 | 129 | def test_op(op, right, use_m=False): 130 | if right: 131 | if use_m: 132 | op_ic = op(l, ic, m) 133 | else: 134 | op_ic = op(l, ic) 135 | 136 | else: 137 | if use_m: 138 | op_ic = op(ic, r, m) 139 | else: 140 | op_ic = op(ic, r) 141 | 142 | if use_m: 143 | exp = op(l, r, m) 144 | else: 145 | exp = op(l, r) 146 | 147 | assert isinstance(op_ic, InitialCondition) 148 | 149 | eq = op_ic.supply_value(r if right else l) == exp 150 | if isinstance(exp, np.ndarray): 151 | assert eq.all() 152 | else: 153 | assert eq 154 | 155 | # test binary ops 156 | bin_ops = [ 157 | operator.add, operator.sub, operator.mul, operator.truediv, operator.floordiv, operator.mod,\ 158 | operator.pow, operator.lshift, operator.rshift, operator.and_, operator.xor, operator.or_, 159 | divmod, 160 | ] 161 | for op in bin_ops: 162 | test_op(op, False) 163 | test_op(op, True) 164 | 165 | # test ternary pow 166 | test_op(pow, False, use_m=True) 167 | 168 | # test unary ops 169 | unary_ops = [operator.neg, operator.pos, operator.abs, operator.invert] 170 | for op in unary_ops: 171 | op_ic = op(ic) 172 | assert isinstance(op_ic, InitialCondition) 173 | assert op_ic.supply_value(l) == op(l) 174 | 175 | # test matmul 176 | l, r = mock.MagicMock(), mock.MagicMock() 177 | test_op(operator.matmul, False) 178 | l.__matmul__.assert_called_with(r) 179 | 180 | l = mock.Mock() 181 | test_op(operator.matmul, True) 182 | r.__rmatmul__.assert_called_with(l) 183 | -------------------------------------------------------------------------------- /tests/annotations/test_invariants.py: -------------------------------------------------------------------------------- 1 | """Tests for value annotation invariants""" 2 | 3 | import numpy as np 4 | 5 | from unittest import mock 6 | 7 | import pybryt.invariants as inv 8 | 9 | 10 | def test_call_structure(): 11 | with mock.patch.object(inv.invariant, "run") as mocked_run: 12 | inv.invariant() 13 | mocked_run.assert_called() 14 | 15 | assert inv.invariant([]) is None 16 | 17 | 18 | def test_string_capitalization(): 19 | np.random.seed(42) 20 | values = ["someString", "some_otherO!I(I*$NDdnnfkf", 120484, ["hi", "there"], np.random.uniform(size=100)] 21 | expected_values = [s.lower() if isinstance(s, str) else s for s in values] 22 | np.testing.assert_equal(inv.string_capitalization(values), expected_values) 23 | 24 | 25 | def test_matrix_transpose(): 26 | np.random.seed(42) 27 | arr = np.random.uniform(size=(10,10)) 28 | values = ["someString", "some_otherO!I(I*$NDdnnfkf", 120484, ["hi", "there"], arr, [[1, 2, 3], [4, 5, 6]]] 29 | print(inv.matrix_transpose(values)) 30 | expected_values = ["someString", "some_otherO!I(I*$NDdnnfkf", 120484, ["hi", "there"], ["hi", "there"], \ 31 | arr, arr.T, [[1, 2, 3], [4, 5, 6]], [[1, 4], [2, 5], [3, 6]]] 32 | np.testing.assert_equal(inv.matrix_transpose(values), expected_values) 33 | 34 | 35 | def test_list_permutation(): 36 | np.random.seed(42) 37 | arr = np.random.uniform(size=(10,10)) 38 | values = ["someString", "some_otherO!I(I*$NDdnnfkf", 120484, ["there", "hi", "z"], arr, [[1, 2, 3], [4, 5, 6]]] 39 | expected_values = ["someString", "some_otherO!I(I*$NDdnnfkf", 120484, ["hi", "there", "z"], np.sort(arr), \ 40 | [[1, 2, 3], [4, 5, 6]]] 41 | np.testing.assert_equal(inv.list_permutation(values), expected_values) 42 | 43 | 44 | def test_alias(): 45 | import pybryt.annotations.invariants as inv2 46 | assert inv2.string_capitalization is inv.string_capitalization 47 | -------------------------------------------------------------------------------- /tests/annotations/test_structural.py: -------------------------------------------------------------------------------- 1 | """Tests for structural patterns""" 2 | 3 | import pandas as pd 4 | import pytest 5 | 6 | from unittest import mock 7 | 8 | from pybryt import structural 9 | from pybryt.annotations.structural import _StructuralPattern 10 | 11 | from .structural_helpers import AttrContainer, Container 12 | 13 | 14 | def test__getattr__(): 15 | """ 16 | Tests for ``_StructuralPattern.__getattr__``. 17 | """ 18 | assert isinstance(structural.tests, _StructuralPattern) 19 | 20 | pat = structural.tests.annotations.structural_helpers.AttrContainer() 21 | assert isinstance(pat, _StructuralPattern) 22 | 23 | # test errors to make sure dill works 24 | with pytest.raises(AttributeError): 25 | pat.__getstate__ 26 | 27 | with pytest.raises(AttributeError): 28 | pat.__setstate__ 29 | 30 | with pytest.raises(AttributeError): 31 | pat.__slots__ 32 | 33 | 34 | def test_named_attribute_checking(): 35 | """ 36 | Tests for named attribute checking. 37 | """ 38 | expected_attrs = dict(a=1, b=2, c=3) 39 | pat = structural.tests.annotations.structural_helpers.AttrContainer(**expected_attrs) 40 | 41 | obj = AttrContainer(**expected_attrs) 42 | assert pat == obj 43 | 44 | obj = AttrContainer(**expected_attrs, d=1) 45 | assert pat == obj 46 | 47 | obj = AttrContainer(**{**expected_attrs, "c": 4}) 48 | assert pat != obj 49 | 50 | expected_attrs.pop("b") 51 | obj = AttrContainer(**expected_attrs) 52 | assert pat != obj 53 | 54 | expected_attrs["df"] = structural.pandas.DataFrame(shape=(2, 1)) 55 | pat = structural.tests.annotations.structural_helpers.AttrContainer(**expected_attrs) 56 | obj = AttrContainer(**{**expected_attrs, "df": pd.DataFrame({"a": [1, 2]})}) 57 | assert pat == obj 58 | 59 | obj = AttrContainer(**{**expected_attrs, "df": pd.DataFrame({"a": [1, 2], "b": [3, 4]})}) 60 | assert pat != obj 61 | 62 | 63 | def test_unnamed_attribute_checking(): 64 | """ 65 | Tests for unnamed attribute checking. 66 | """ 67 | expected_attrs = dict(a=1, b=2, c=3) 68 | pat = structural.tests.annotations.structural_helpers.AttrContainer(*expected_attrs.values()) 69 | 70 | obj = AttrContainer(**expected_attrs) 71 | assert pat == obj 72 | 73 | obj = AttrContainer(**expected_attrs, d=1) 74 | assert pat == obj 75 | 76 | obj = AttrContainer(**{**expected_attrs, "c": 4}) 77 | assert pat != obj 78 | 79 | expected_attrs.pop("b") 80 | obj = AttrContainer(**expected_attrs) 81 | assert pat != obj 82 | 83 | 84 | def test_contains(): 85 | """ 86 | Tests for patterns checking whether an object is contained in a matching object. 87 | """ 88 | expected_elems = [1, 2, 3] 89 | pat = structural.tests.annotations.structural_helpers.Container().contains_(*expected_elems) 90 | 91 | obj = Container(*expected_elems) 92 | assert pat == obj 93 | 94 | obj = Container(*expected_elems, 4) 95 | assert pat == obj 96 | 97 | obj = Container(*expected_elems[:-1]) 98 | assert pat != obj 99 | 100 | with mock.patch.object(Container, "__contains__") as mocked_contains: 101 | mocked_contains.side_effect = TypeError() 102 | obj = Container(*expected_elems) 103 | assert pat != obj 104 | 105 | 106 | def test___repr__(): 107 | """ 108 | Tests for ``_StructuralPattern.__repr__``. 109 | """ 110 | expected_attrs = dict(a=1, b=2, c=3) 111 | pat = structural.tests.annotations.structural_helpers.AttrContainer(**expected_attrs) 112 | pat_repr = repr(pat) 113 | assert pat_repr.startswith("pybryt.structural.tests.annotations.structural_helpers.AttrContainer(") 114 | for k, v in expected_attrs.items(): 115 | assert f"{k}={v}" in pat_repr 116 | 117 | -------------------------------------------------------------------------------- /tests/annotations/test_type_.py: -------------------------------------------------------------------------------- 1 | """Tests for type annotations""" 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from unittest import mock 7 | 8 | import pybryt 9 | 10 | from .utils import assert_object_attrs, generate_memory_footprint 11 | 12 | 13 | def test_forbid_type(): 14 | """ 15 | """ 16 | footprint = generate_memory_footprint() 17 | pybryt.Annotation.reset_tracked_annotations() 18 | 19 | a = pybryt.ForbidType(bool) 20 | res = a.check(footprint) 21 | 22 | assert_object_attrs(a, {"children__len": 0}) 23 | assert_object_attrs(res, { 24 | "children": [], 25 | "satisfied": True, 26 | "_satisfied": True, 27 | "annotation": a, 28 | "timestamp": -1, 29 | "value": None, 30 | }) 31 | 32 | assert a.to_dict() == { 33 | "name": "Annotation 1", 34 | "children": [], 35 | "success_message": None, 36 | "failure_message": None, 37 | "limit": None, 38 | "group": None, 39 | "type": "forbidtype", 40 | "type_": "", 41 | } 42 | 43 | a = pybryt.ForbidType(np.ndarray) 44 | res = a.check(footprint) 45 | assert_object_attrs(res, { 46 | "children": [], 47 | "satisfied": False, 48 | "_satisfied": False, 49 | "annotation": a, 50 | "timestamp": -1, 51 | "value": None, 52 | }) 53 | 54 | # check constructor errors 55 | with pytest.raises(TypeError, match=f"1 is not a type"): 56 | pybryt.ForbidType(1) 57 | 58 | with mock.patch("pybryt.annotations.type_.dill") as mocked_dill: 59 | mocked_dill.dumps.side_effect = Exception() 60 | 61 | with pytest.raises(ValueError, match="Types must be serializable but the following error was thrown during serialization:\n"): 62 | pybryt.ForbidType(np.ndarray) 63 | -------------------------------------------------------------------------------- /tests/annotations/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for PyBryt annotation tests""" 2 | 3 | import numpy as np 4 | 5 | from functools import lru_cache 6 | from itertools import chain 7 | 8 | from pybryt.execution import MemoryFootprint, MemoryFootprintValue 9 | 10 | 11 | def assert_object_attrs(obj, attrs): 12 | """ 13 | """ 14 | for k, v in attrs.items(): 15 | if k.endswith("__len"): 16 | assert len(getattr(obj, k[:-5])) == v, \ 17 | f"Attr '{k}' length is wrong: expected {v} but got {len(getattr(obj, k[:-5]))}" 18 | else: 19 | attr = getattr(obj, k) 20 | if isinstance(attr, (np.ndarray, float, np.generic, int)): 21 | is_eq = np.allclose(attr, v) 22 | else: 23 | is_eq = attr == v 24 | if isinstance(is_eq, np.ndarray): 25 | assert is_eq.all(), f"Attr '{k}' is wrong: expected {v} but got {getattr(obj, k)}" 26 | else: 27 | assert is_eq, f"Attr '{k}' is wrong: expected {v} but got {getattr(obj, k)}" 28 | 29 | 30 | @lru_cache(1) 31 | def generate_memory_footprint() -> MemoryFootprint: 32 | """ 33 | """ 34 | np.random.seed(42) 35 | objs = [ 36 | np.random.uniform(-100, 100, size=(100, 100)), 37 | 4.0, 38 | list(range(100))[::-1], 39 | 1, 40 | np.e, 41 | None, 42 | None, 43 | np.random.normal(size=102), 44 | 4.0, 45 | "some CasE insenSITIve StrINg!", 46 | ] 47 | return MemoryFootprint.from_values(*[MemoryFootprintValue(o, i, None) for i, o in enumerate(objs)]) 48 | -------------------------------------------------------------------------------- /tests/execution/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/tests/execution/__init__.py -------------------------------------------------------------------------------- /tests/execution/test_complexity.py: -------------------------------------------------------------------------------- 1 | """Tests for complexity checking internals""" 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | import pybryt 7 | import pybryt.execution 8 | 9 | from .utils import generate_mocked_frame 10 | 11 | 12 | def test_time_complexity(): 13 | def do_work(n, transform): 14 | n = int(transform(n)) 15 | for _ in range(n): 16 | 2 * 2 17 | return n 18 | 19 | footprint, cir = pybryt.execution.create_collector() 20 | 21 | for e in range(1, 9): 22 | n = 10 ** e 23 | with pybryt.check_time_complexity("foo", n): 24 | do_work(n, lambda v: v) 25 | 26 | assert len(footprint) == 8 27 | assert all(isinstance(mfp_val.value, pybryt.TimeComplexityResult) for mfp_val in footprint) 28 | 29 | footprint.clear() 30 | 31 | for e in range(1, 9): 32 | n = float(10 ** e) 33 | with pybryt.check_time_complexity("foo", n): 34 | do_work(n, np.log2) 35 | 36 | n = np.random.uniform(size=int(n)) 37 | with pybryt.check_time_complexity("bar", n): 38 | do_work(n, len) 39 | 40 | assert len(footprint) == 16 41 | assert all(isinstance(mfp_val.value, pybryt.TimeComplexityResult) for mfp_val in footprint) 42 | assert [mfp_val.value.name for mfp_val in footprint] == ["foo", "bar"] * 8 43 | 44 | # test error 45 | n = lambda v: v 46 | with pytest.raises(TypeError, match=f"n has invalid type {type(n)}"): 47 | with pybryt.check_time_complexity("foo", n): 48 | pass 49 | 50 | # check that tracking is disabled 51 | frame = generate_mocked_frame("", "bar", 1) 52 | with pybryt.check_time_complexity("foo", 10): 53 | cir(frame, "return", 10) 54 | assert len(footprint) == 16 55 | -------------------------------------------------------------------------------- /tests/execution/test_notebook_execution.py: -------------------------------------------------------------------------------- 1 | """Tests for PyBryt execution internals""" 2 | 3 | import dill 4 | import nbformat 5 | import numpy as np 6 | import pathlib 7 | import random 8 | import tempfile 9 | 10 | from unittest import mock 11 | 12 | import pybryt.execution 13 | 14 | 15 | def generate_test_notebook(): 16 | """ 17 | """ 18 | nb = nbformat.v4.new_notebook() 19 | nb.cells.append(nbformat.v4.new_code_cell( 20 | "import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline" 21 | )) 22 | nb.cells.append(nbformat.v4.new_code_cell( 23 | "np.random.seed(42)\nx = np.random.uniform(size=1000)\ny = np.random.normal(size=1000)" 24 | )) 25 | nb.cells.append(nbformat.v4.new_code_cell("df = pd.DataFrame({'x': x, 'y': y})")) 26 | return nb 27 | 28 | 29 | def test_notebook_execution(): 30 | """ 31 | """ 32 | random.seed(42) 33 | nb = generate_test_notebook() 34 | 35 | with tempfile.NamedTemporaryFile(delete=False) as observed_ntf: 36 | with mock.patch("pybryt.execution.mkstemp") as mocked_tempfile: 37 | mocked_tempfile.return_value = (None, observed_ntf.name) 38 | 39 | footprint = pybryt.execution.execute_notebook(nb, "") 40 | assert len(footprint) > 0 41 | assert all(i in footprint.imports for i in ["pandas", "numpy", "matplotlib"]) 42 | assert len(footprint.calls) > 0 43 | -------------------------------------------------------------------------------- /tests/execution/utils.py: -------------------------------------------------------------------------------- 1 | """""" 2 | 3 | from ..utils import AttrDict 4 | 5 | 6 | def generate_mocked_frame( 7 | co_filename, co_name, f_lineno, f_globals={}, f_locals={}, f_trace=None, f_back=None 8 | ): 9 | """ 10 | Factory for generating objects with the instance variables of an ``inspect`` frame that are 11 | needed by PyBryt's trace function. 12 | """ 13 | code = AttrDict({ 14 | "co_filename": co_filename, 15 | "co_name": co_name, 16 | }) 17 | return AttrDict({ 18 | "f_lineno": f_lineno, 19 | "f_globals": f_globals, 20 | "f_locals": f_locals, 21 | "f_trace": f_trace, 22 | "f_back": f_back, 23 | "f_code": code, 24 | }) -------------------------------------------------------------------------------- /tests/files/expected_ref.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/tests/files/expected_ref.pkl -------------------------------------------------------------------------------- /tests/files/expected_ref2.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/tests/files/expected_ref2.pkl -------------------------------------------------------------------------------- /tests/integrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/tests/integrations/__init__.py -------------------------------------------------------------------------------- /tests/preprocessors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/pybryt/1e87fbe06e3e190bab075dab1064cfe275044f75/tests/preprocessors/__init__.py -------------------------------------------------------------------------------- /tests/preprocessors/test_imports.py: -------------------------------------------------------------------------------- 1 | """Tests for the imports preprocessor""" 2 | 3 | import nbformat 4 | 5 | from textwrap import dedent 6 | 7 | from pybryt.preprocessors import ImportFindingPreprocessor 8 | 9 | 10 | def test_preprocessor(): 11 | """ 12 | """ 13 | nb = nbformat.v4.new_notebook() 14 | nb.cells.append(nbformat.v4.new_code_cell(dedent("""\ 15 | import numpy 16 | import pandas as pd 17 | from random import choice 18 | from itertools import chain as c 19 | import statsmodels.api 20 | exec("import tqdm") 21 | """))) 22 | 23 | ifp = ImportFindingPreprocessor() 24 | 25 | nb2 = ifp.preprocess(nb) 26 | assert nb2 is nb 27 | assert ifp.imports == {"numpy", "pandas", "random", "itertools", "statsmodels"} 28 | -------------------------------------------------------------------------------- /tests/preprocessors/test_intermediate_variables.py: -------------------------------------------------------------------------------- 1 | """""" 2 | 3 | import random 4 | import nbformat 5 | 6 | from textwrap import dedent 7 | 8 | from pybryt.preprocessors import IntermediateVariablePreprocessor 9 | 10 | 11 | def test_preprocessor(): 12 | """ 13 | """ 14 | nb = nbformat.v4.new_notebook() 15 | nb.cells.append(nbformat.v4.new_code_cell(dedent("""\ 16 | a = True 17 | b = False 18 | f = lambda x: not x 19 | 20 | g = f(a) + f(b) 21 | 22 | if f(a) and f(b): 23 | print("hi") 24 | 25 | if f(a) or f(b): 26 | print("hi") 27 | 28 | if a or b: 29 | print("bye") 30 | 31 | l = [f(i) for i in [a, b]] 32 | 33 | f = lambda x: [not i for i in l] 34 | l = [a, b] 35 | if all(f(l)): 36 | print("ok") 37 | else: 38 | l = any(f(l)) 39 | """))) 40 | 41 | ivp = IntermediateVariablePreprocessor() 42 | 43 | random.seed(42) 44 | nb = ivp.preprocess(nb) 45 | print(nb.cells[0].source) 46 | assert len(nb.cells) == 1 47 | assert nb.cells[0].source.strip() == dedent("""\ 48 | a = True 49 | b = False 50 | f = (lambda x: (not x)) 51 | var_HBRPOI = f(a) 52 | var_G8F1CB = f(b) 53 | g = (var_HBRPOI + var_G8F1CB) 54 | var_FNO6B9 = f(a) 55 | if (var_FNO6B9): 56 | var_M80O2R = f(b) 57 | if (var_FNO6B9 and var_M80O2R): 58 | var_AK1VRJ = print('hi') 59 | var_AK1VRJ 60 | var_NVGFYG = f(a) 61 | if (not (var_NVGFYG)): 62 | var_WWQC38 = f(b) 63 | if (var_NVGFYG or var_WWQC38): 64 | var_HYF9SX = print('hi') 65 | var_HYF9SX 66 | if (a or b): 67 | var_MECOSF = print('bye') 68 | var_MECOSF 69 | l = [f(i) for i in [a, b]] 70 | f = (lambda x: [(not i) for i in l]) 71 | l = [a, b] 72 | var_KXWNRE = f(l) 73 | var_K8PK3Y = all(var_KXWNRE) 74 | if var_K8PK3Y: 75 | var_R9OUDO = print('ok') 76 | var_R9OUDO 77 | else: 78 | var_CUZREN = f(l) 79 | l = any(var_CUZREN) 80 | """).strip() 81 | -------------------------------------------------------------------------------- /tests/test_complexity.py: -------------------------------------------------------------------------------- 1 | """Tests for ``pybryt.complexity``""" 2 | 3 | from unittest import mock 4 | 5 | from pybryt import complexities as cplx 6 | from pybryt.complexity import ANNOTATION_NAME, _check_time_complexity_wrapper, TimeComplexityChecker 7 | from pybryt.execution import MemoryFootprintValue, TimeComplexityResult 8 | 9 | 10 | def test_time_complexity_checker(): 11 | """ 12 | Tests for ``pybryt.complexity.TimeComplexityChecker``. 13 | """ 14 | checker = TimeComplexityChecker() 15 | 16 | with mock.patch("pybryt.complexity._check_time_complexity_wrapper") as mocked_wrapper: 17 | n = 10 18 | checker(n) 19 | mocked_wrapper.assert_called_once_with(checker, n) 20 | 21 | res = TimeComplexityResult("foo", n, 0, 10) 22 | checker.add_result(res) 23 | assert checker.results == [res] 24 | 25 | with mock.patch("pybryt.complexity.TimeComplexity") as mocked_annot: 26 | ret = checker.determine_complexity() 27 | mocked_annot.assert_called_once_with(cplx.constant, name=ANNOTATION_NAME) 28 | mocked_annot.return_value.check.assert_called() 29 | assert ret == mocked_annot.return_value.check.return_value.value 30 | 31 | 32 | def test_wrapper(): 33 | """ 34 | Tests for ``pybryt.complexity._check_time_complexity_wrapper``. 35 | """ 36 | n = 10 37 | checker = TimeComplexityChecker() 38 | res = TimeComplexityResult(ANNOTATION_NAME, n, 0, 10) 39 | with mock.patch.object(checker, "add_result") as mocked_add, \ 40 | mock.patch("pybryt.complexity.FrameTracer") as mocked_tracer, \ 41 | mock.patch("pybryt.complexity.inspect") as mocked_inspect, \ 42 | mock.patch("pybryt.complexity.check_time_complexity") as mocked_cm: 43 | mocked_tracer.return_value.get_footprint.return_value.__iter__ = \ 44 | mock.Mock(return_value=iter([MemoryFootprintValue(res, 0, None)])) 45 | wrapper = _check_time_complexity_wrapper(checker, n) 46 | with wrapper: 47 | mocked_tracer.assert_called_once_with(mocked_inspect.currentframe.return_value.f_back) 48 | mocked_tracer.return_value.start_trace.assert_called_once() 49 | mocked_cm.assert_called_once_with(ANNOTATION_NAME, n) 50 | mocked_cm.return_value.__enter__.assert_called_once() 51 | 52 | mocked_cm.return_value.__exit__.assert_called_once() 53 | mocked_tracer.return_value.end_trace.assert_called_once() 54 | mocked_tracer.return_value.get_footprint.assert_called_once() 55 | mocked_add.assert_called_once_with(res) 56 | -------------------------------------------------------------------------------- /tests/test_debug.py: -------------------------------------------------------------------------------- 1 | """Tests for debug mode""" 2 | 3 | from pybryt import debug_mode, disable_debug_mode, enable_debug_mode 4 | from pybryt.debug import _debug_mode_enabled 5 | 6 | 7 | def test_debug_mode(): 8 | """ 9 | Test that the debug mode controllers correctly enable and disable debug mode. 10 | """ 11 | enable_debug_mode() 12 | assert _debug_mode_enabled() 13 | disable_debug_mode() 14 | assert not _debug_mode_enabled() 15 | 16 | with debug_mode(): 17 | assert _debug_mode_enabled() 18 | assert not _debug_mode_enabled() 19 | 20 | enable_debug_mode() 21 | with debug_mode(): 22 | assert _debug_mode_enabled() 23 | assert not _debug_mode_enabled() 24 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | """""" 2 | 3 | import pytest 4 | import random 5 | import tempfile 6 | 7 | from textwrap import dedent 8 | from unittest import mock 9 | 10 | from pybryt.utils import * 11 | 12 | from .test_reference import generate_reference_notebook 13 | 14 | 15 | def test_filter_pickleable_list(): 16 | """ 17 | """ 18 | l = [1, 2, 3] 19 | filter_pickleable_list(l) 20 | assert len(l) == 3 21 | 22 | with mock.patch("dill.dumps") as mocked_dill: 23 | mocked_dill.side_effect = Exception() 24 | filter_pickleable_list(l) 25 | assert len(l) == 0 26 | 27 | 28 | def test_notebook_to_string(): 29 | """ 30 | """ 31 | ref = generate_reference_notebook() 32 | s = notebook_to_string(ref) 33 | assert s.strip() == dedent("""\ 34 | import pybryt 35 | 36 | def median(S): 37 | sorted_S = sorted(S) 38 | pybryt.Value(sorted_S, name="sorted", group="median", limit=5, success_message="SUCCESS: Sorted the sample correctly", 39 | failure_message="ERROR: The sample was not sorted") 40 | 41 | size_of_set = len(S) 42 | pybryt.Value(size_of_set, name="size", group="median", success_message = "SUCCESS: Computed the size of the sample", 43 | failure_message="ERROR: Did not capture the size of the set to determine if it is odd or even") 44 | 45 | middle = size_of_set // 2 46 | is_set_size_even = (size_of_set % 2) == 0 47 | 48 | if is_set_size_even: 49 | return (sorted_S[middle-1] + sorted_S[middle]) / 2 50 | else: 51 | return sorted_S[middle] 52 | 53 | import numpy as np 54 | np.random.seed(42) 55 | for _ in range(10): 56 | vals = [np.random.randint(-1000, 1000) for _ in range(np.random.randint(1, 1000))] 57 | val = median(vals) 58 | pybryt.Value(val, name="median", group="median", success_message="SUCCESS: computed the correct median", 59 | failure_message="ERROR: failed to compute the median") 60 | """).strip() 61 | 62 | with pytest.raises(TypeError, match="invalid notebook type"): 63 | notebook_to_string(1) 64 | 65 | 66 | def test_make_secret(): 67 | """ 68 | """ 69 | random.seed(42) 70 | s = make_secret() 71 | assert s == "HBRPOI" 72 | 73 | 74 | def test_save_notebook(): 75 | """ 76 | """ 77 | with mock.patch("pybryt.utils.get_ipython") as mocked_get: 78 | with mock.patch("pybryt.utils.publish_display_data") as mocked_pub: 79 | mocked_get.return_value = True 80 | with tempfile.NamedTemporaryFile(suffix=".ipynb") as ntf: 81 | v = save_notebook(ntf.name, timeout=1) 82 | mocked_pub.assert_called() 83 | assert not v 84 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | """""" 2 | 3 | 4 | class AttrDict(dict): 5 | 6 | def __getattr__(self, attr): 7 | return self[attr] 8 | --------------------------------------------------------------------------------