├── .github
    └── workflows
    │   └── main.yml
├── .gitignore
├── .readthedocs.yml
├── LICENSE
├── MANIFEST.in
├── README.md
├── check50
    ├── __init__.py
    ├── __main__.py
    ├── _api.py
    ├── _exceptions.py
    ├── _simple.py
    ├── c.py
    ├── contextmanagers.py
    ├── flask.py
    ├── internal.py
    ├── py.py
    ├── regex.py
    ├── renderer
    │   ├── __init__.py
    │   ├── _renderers.py
    │   └── templates
    │   │   └── results.html
    └── runner.py
├── docs
    ├── Makefile
    ├── requirements.txt
    └── source
    │   ├── ansi_output.png
    │   ├── api.rst
    │   ├── check50_user.rst
    │   ├── check_writer.rst
    │   ├── commit.png
    │   ├── conf.py
    │   ├── extension_writer.rst
    │   ├── html_output.png
    │   ├── index.rst
    │   ├── json_specification.rst
    │   ├── new_file.png
    │   ├── new_yaml.png
    │   └── repo.png
├── setup.cfg
├── setup.py
└── tests
    ├── __init__.py
    ├── __main__.py
    ├── api_tests.py
    ├── c_tests.py
    ├── check50_tests.py
    ├── checks
        ├── compile_exit
        │   └── .cs50.yaml
        ├── compile_prompt
        │   └── .cs50.yaml
        ├── compile_std
        │   └── .cs50.yaml
        ├── exists
        │   ├── .cs50.yaml
        │   └── check.py
        ├── exit_code
        │   ├── error
        │   │   ├── .cs50.yaml
        │   │   └── check.py
        │   ├── failure
        │   │   ├── .cs50.yaml
        │   │   └── __init__.py
        │   └── success
        │   │   ├── .cs50.yaml
        │   │   └── __init__.py
        ├── exit_py
        │   ├── .cs50.yaml
        │   └── check.py
        ├── hidden
        │   ├── .cs50.yaml
        │   └── __init__.py
        ├── internal_directories
        │   ├── .cs50.yaml
        │   └── __init__.py
        ├── output
        │   ├── .cs50.yaml
        │   └── __init__.py
        ├── payload
        │   ├── .cs50.yaml
        │   └── __init__.py
        ├── remote_exception_no_traceback
        │   ├── .cs50.yaml
        │   └── check.py
        ├── remote_exception_traceback
        │   ├── .cs50.yaml
        │   └── check.py
        ├── stdin_human_readable_py
        │   ├── .cs50.yaml
        │   └── check.py
        ├── stdin_multiline
        │   ├── .cs50.yaml
        │   └── check.py
        ├── stdin_prompt_py
        │   ├── .cs50.yaml
        │   └── check.py
        ├── stdin_py
        │   ├── .cs50.yaml
        │   └── check.py
        ├── stdout_py
        │   ├── .cs50.yaml
        │   └── check.py
        ├── target
        │   ├── .cs50.yaml
        │   └── __init__.py
        └── unpicklable_attribute
        │   ├── .cs50.yaml
        │   └── __init__.py
    ├── flask_tests.py
    ├── internal_tests.py
    ├── py_tests.py
    ├── runner_tests.py
    └── simple_tests.py


/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | on: push
 2 | jobs:
 3 |   test-and-deploy:
 4 |     runs-on: ubuntu-latest
 5 |     steps:
 6 |       - uses: actions/checkout@v4
 7 | 
 8 |       - uses: actions/setup-python@v5
 9 |         with:
10 |           python-version: "3.13"
11 | 
12 |       - name: Run tests
13 |         run: |
14 |           pip install babel flask
15 |           pip install .
16 |           python -m tests
17 | 
18 |       - name: Install pypa/build
19 |         run: python -m pip install build --user
20 | 
21 |       - name: Build a binary wheel and a source tarball
22 |         run: python -m build --sdist --wheel --outdir dist/ .
23 | 
24 |       - name: Deploy to PyPI
25 |         if: ${{ github.ref == 'refs/heads/main' }}
26 |         uses: pypa/gh-action-pypi-publish@release/v1
27 |         with:
28 |           user: __token__
29 |           password: ${{ secrets.PYPI_API_TOKEN }}
30 | 
31 |       - name: Extract program version
32 |         id: program_version
33 |         run: |
34 |           echo ::set-output name=version::$(check50 --version | cut --delimiter ' ' --fields 2)
35 | 
36 |       - name: Create Release
37 |         if: ${{ github.ref == 'refs/heads/main' }}
38 |         uses: actions/github-script@v7
39 |         with:
40 |           github-token: ${{ github.token }}
41 |           script: |
42 |             github.rest.repos.createRelease({
43 |               owner: context.repo.owner,
44 |               repo: context.repo.repo,
45 |               tag_name: "v${{ steps.program_version.outputs.version }}",
46 |               tag_commitish: "${{ github.sha }}"
47 |             })
48 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | .DS_store
  2 | 
  3 | # Byte-compiled / optimized / DLL files
  4 | __pycache__/
  5 | *.py[cod]
  6 | *$py.class
  7 | 
  8 | # C extensions
  9 | *.so
 10 | 
 11 | # Distribution / packaging
 12 | .Python
 13 | build/
 14 | develop-eggs/
 15 | dist/
 16 | downloads/
 17 | eggs/
 18 | .eggs/
 19 | lib/
 20 | lib64/
 21 | parts/
 22 | sdist/
 23 | var/
 24 | wheels/
 25 | *.egg-info/
 26 | .installed.cfg
 27 | *.egg
 28 | MANIFEST
 29 | 
 30 | # PyInstaller
 31 | #  Usually these files are written by a python script from a template
 32 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 33 | *.manifest
 34 | *.spec
 35 | 
 36 | # Installer logs
 37 | pip-log.txt
 38 | pip-delete-this-directory.txt
 39 | 
 40 | # Unit test / coverage reports
 41 | htmlcov/
 42 | .tox/
 43 | .coverage
 44 | .coverage.*
 45 | .cache
 46 | nosetests.xml
 47 | coverage.xml
 48 | *.cover
 49 | .hypothesis/
 50 | .pytest_cache/
 51 | 
 52 | # Translations
 53 | *.mo
 54 | *.pot
 55 | 
 56 | # Django stuff:
 57 | *.log
 58 | local_settings.py
 59 | db.sqlite3
 60 | 
 61 | # Flask stuff:
 62 | instance/
 63 | .webassets-cache
 64 | 
 65 | # Scrapy stuff:
 66 | .scrapy
 67 | 
 68 | # Sphinx documentation
 69 | docs/build/
 70 | 
 71 | # Jupyter Notebook
 72 | .ipynb_checkpoints
 73 | 
 74 | # pyenv
 75 | .python-version
 76 | 
 77 | # celery beat schedule file
 78 | celerybeat-schedule
 79 | 
 80 | # SageMath parsed files
 81 | *.sage.py
 82 | 
 83 | # Environments
 84 | .env
 85 | .venv
 86 | env/
 87 | venv/
 88 | ENV/
 89 | env.bak/
 90 | venv.bak/
 91 | 
 92 | # Spyder project settings
 93 | .spyderproject
 94 | .spyproject
 95 | 
 96 | # Rope project settings
 97 | .ropeproject
 98 | 
 99 | # mkdocs documentation
100 | /site
101 | 
102 | # mypy
103 | .mypy_cache/
104 | 


--------------------------------------------------------------------------------
/.readthedocs.yml:
--------------------------------------------------------------------------------
 1 | version: 2
 2 | build:
 3 |   os: "ubuntu-22.04"
 4 |   tools:
 5 |     python: "3.7"
 6 | 
 7 | python:
 8 |   install:
 9 |     - requirements: docs/requirements.txt
10 | 
11 | sphinx:
12 |   builder: dirhtml
13 | 
14 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | recursive-include check50/locale *
2 | graft check50/renderer/templates
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # check50
2 | 
3 | check50 is a testing tool for checking student code. As a student you can use check50 to check your CS50 problem sets or any other Problem sets for which check50 checks exist. check50 allows teachers to automatically grade code on correctness and to provide automatic feedback while students are coding.
4 | 
5 | You can find documentation and instructions for writing your own checks at https://cs50.readthedocs.io/projects/check50/.
6 | 
7 | 


--------------------------------------------------------------------------------
/check50/__init__.py:
--------------------------------------------------------------------------------
 1 | def _set_version():
 2 |     """Set check50 __version__"""
 3 |     global __version__
 4 |     from importlib.metadata import PackageNotFoundError, version
 5 |     import os
 6 |     # https://stackoverflow.com/questions/17583443/what-is-the-correct-way-to-share-package-version-with-setup-py-and-the-package
 7 |     try:
 8 |         __version__ = version("check50")
 9 |     except PackageNotFoundError:
10 |         __version__ = "UNKNOWN"
11 | 
12 | 
13 | def _setup_translation():
14 |     import gettext
15 |     from importlib.resources import files
16 |     global _translation
17 |     _translation = gettext.translation(
18 |         "check50", str(files("check50").joinpath("locale")), fallback=True)
19 |     _translation.install()
20 | 
21 | 
22 | 
23 | # Encapsulated inside a function so their local variables/imports aren't seen by autocompleters
24 | _set_version()
25 | _setup_translation()
26 | 
27 | from ._api import (
28 |     import_checks,
29 |     data, _data,
30 |     exists,
31 |     hash,
32 |     include,
33 |     run,
34 |     log, _log,
35 |     hidden,
36 |     Failure, Mismatch, Missing
37 | )
38 | 
39 | 
40 | from . import regex
41 | from .runner import check
42 | from pexpect import EOF
43 | 
44 | __all__ = ["import_checks", "data", "exists", "hash", "include", "regex",
45 |            "run", "log", "Failure", "Mismatch", "Missing", "check", "EOF"]
46 | 


--------------------------------------------------------------------------------
/check50/__main__.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import contextlib
  3 | import enum
  4 | import gettext
  5 | import importlib
  6 | import inspect
  7 | import itertools
  8 | from json import JSONDecodeError
  9 | import logging
 10 | import os
 11 | import platform
 12 | import site
 13 | from pathlib import Path
 14 | import shutil
 15 | import signal
 16 | import subprocess
 17 | import sys
 18 | import tempfile
 19 | import time
 20 | 
 21 | import attr
 22 | import lib50
 23 | import requests
 24 | import termcolor
 25 | 
 26 | from . import _exceptions, internal, renderer, __version__
 27 | from .contextmanagers import nullcontext
 28 | from .runner import CheckRunner
 29 | 
 30 | LOGGER = logging.getLogger("check50")
 31 | 
 32 | lib50.set_local_path(os.environ.get("CHECK50_PATH", "~/.local/share/check50"))
 33 | 
 34 | 
 35 | class LogLevel(enum.IntEnum):
 36 |     DEBUG = logging.DEBUG
 37 |     INFO = logging.INFO
 38 |     WARNING = logging.WARNING
 39 |     ERROR = logging.ERROR
 40 | 
 41 | 
 42 | class ColoredFormatter(logging.Formatter):
 43 |     COLORS = {
 44 |         "ERROR": "red",
 45 |         "WARNING": "yellow",
 46 |         "DEBUG": "cyan",
 47 |         "INFO": "magenta",
 48 |     }
 49 | 
 50 |     def __init__(self, fmt, use_color=True):
 51 |         super().__init__(fmt=fmt)
 52 |         self.use_color = use_color
 53 | 
 54 |     def format(self, record):
 55 |         msg = super().format(record)
 56 |         return msg if not self.use_color else termcolor.colored(msg, getattr(record, "color", self.COLORS.get(record.levelname)))
 57 | 
 58 | 
 59 | _exceptions.ExceptHook.initialize()
 60 | 
 61 | 
 62 | def install_dependencies(dependencies):
 63 |     """Install all packages in dependency list via pip."""
 64 |     if not dependencies:
 65 |         return
 66 | 
 67 |     with tempfile.TemporaryDirectory() as req_dir:
 68 |         req_file = Path(req_dir) / "requirements.txt"
 69 | 
 70 |         with open(req_file, "w") as f:
 71 |             for dependency in dependencies:
 72 |                 f.write(f"{dependency}\n")
 73 | 
 74 |         pip = [sys.executable or "python3", "-m", "pip", "install", "-r", req_file]
 75 |         # Unless we are in a virtualenv, we need --user
 76 |         if sys.base_prefix == sys.prefix and not hasattr(sys, "real_prefix"):
 77 |             pip.append("--user")
 78 | 
 79 |         try:
 80 |             output = subprocess.check_output(pip, stderr=subprocess.STDOUT)
 81 |         except subprocess.CalledProcessError:
 82 |             raise _exceptions.Error(_("failed to install dependencies"))
 83 | 
 84 |         LOGGER.info(output)
 85 | 
 86 |         # Reload sys.path, to find recently installed packages
 87 |         importlib.reload(site)
 88 | 
 89 | 
 90 | def install_translations(config):
 91 |     """Add check translations according to ``config`` as a fallback to existing translations"""
 92 | 
 93 |     if not config:
 94 |         return
 95 | 
 96 |     from . import _translation
 97 |     checks_translation = gettext.translation(domain=config["domain"],
 98 |                                              localedir=internal.check_dir / config["localedir"],
 99 |                                              fallback=True)
100 |     _translation.add_fallback(checks_translation)
101 | 
102 | 
103 | def compile_checks(checks, prompt=False):
104 |     # Prompt to replace __init__.py (compile destination)
105 |     if prompt and os.path.exists(internal.check_dir / "__init__.py"):
106 |         if not internal._yes_no_prompt("check50 will compile the YAML checks to __init__.py, are you sure you want to overwrite its contents?"):
107 |             raise Error("Aborting: could not overwrite to __init__.py")
108 | 
109 |     # Compile simple checks
110 |     with open(internal.check_dir / "__init__.py", "w") as f:
111 |         f.write(simple.compile(checks))
112 | 
113 |     return "__init__.py"
114 | 
115 | 
116 | def setup_logging(level):
117 |     """
118 |     Sets up logging for lib50.
119 |     level 'info' logs all git commands run to stderr
120 |     level 'debug' logs all git commands and their output to stderr
121 |     """
122 | 
123 |     for logger in (logging.getLogger("lib50"), LOGGER):
124 |         # Set verbosity level on the lib50 logger
125 |         logger.setLevel(level.upper())
126 | 
127 |         handler = logging.StreamHandler(sys.stderr)
128 |         handler.setFormatter(ColoredFormatter("(%(levelname)s) %(message)s", use_color=sys.stderr.isatty()))
129 | 
130 |         # Direct all logs to sys.stderr
131 |         logger.addHandler(handler)
132 | 
133 |     # Don't animate the progressbar if LogLevel is either info or debug
134 |     lib50.ProgressBar.DISABLED = logger.level < LogLevel.WARNING
135 | 
136 | 
137 | def await_results(commit_hash, slug, pings=45, sleep=2):
138 |     """
139 |     Ping {url} until it returns a results payload, timing out after
140 |     {pings} pings and waiting {sleep} seconds between pings.
141 |     """
142 | 
143 |     try:
144 |         for _i in range(pings):
145 |             # Query for check results.
146 |             res = requests.get(f"https://submit.cs50.io/api/results/check50", params={"commit_hash": commit_hash, "slug": slug})
147 |             results = res.json()
148 | 
149 |             if res.status_code not in [404, 200]:
150 |                 raise _exceptions.RemoteCheckError(results)
151 | 
152 |             if res.status_code == 200 and results["received_at"] is not None:
153 |                 break
154 |             time.sleep(sleep)
155 |         else:
156 |             # Terminate if no response
157 |             raise _exceptions.Error(
158 |                 _("check50 is taking longer than normal!\n"
159 |                 "See https://submit.cs50.io/check50/{} for more detail").format(commit_hash))
160 | 
161 |     except JSONDecodeError:
162 |         # Invalid JSON object received from submit.cs50.io
163 |         raise _exceptions.Error(
164 |             _("Sorry, something's wrong, please try again.\n"
165 |             "If the problem persists, please visit our status page https://cs50.statuspage.io for more information."))
166 | 
167 |     if not results["check50"]:
168 |         raise _exceptions.RemoteCheckError(results)
169 | 
170 |     if "error" in results["check50"]:
171 |         raise _exceptions.RemoteCheckError(results["check50"])
172 | 
173 |     # TODO: Should probably check payload["version"] here to make sure major version is same as __version__
174 |     # (otherwise we may not be able to parse results)
175 |     return results["tag_hash"], {
176 |         "slug": results["check50"]["slug"],
177 |         "results": results["check50"]["results"],
178 |         "version": results["check50"]["version"]
179 |     }
180 | 
181 | 
182 | class LogoutAction(argparse.Action):
183 |     """Hook into argparse to allow a logout flag"""
184 | 
185 |     def __init__(self, option_strings, dest=argparse.SUPPRESS, default=argparse.SUPPRESS, help=_("logout of check50")):
186 |         super().__init__(option_strings, dest=dest, nargs=0, default=default, help=help)
187 | 
188 |     def __call__(self, parser, namespace, values, option_string=None):
189 |         try:
190 |             lib50.logout()
191 |         except lib50.Error:
192 |             raise _exceptions.Error(_("failed to logout"))
193 |         else:
194 |             termcolor.cprint(_("logged out successfully"), "green")
195 |         parser.exit()
196 | 
197 | 
198 | def raise_invalid_slug(slug, offline=False):
199 |     """Raise an error signalling slug is invalid for check50."""
200 |     msg = _("Could not find checks for {}.").format(slug)
201 | 
202 |     similar_slugs = lib50.get_local_slugs("check50", similar_to=slug)[:3]
203 |     if similar_slugs:
204 |         msg += _(" Did you mean:")
205 |         for similar_slug in similar_slugs:
206 |             msg += f"\n    {similar_slug}"
207 |         msg += _("\nDo refer back to the problem specification if unsure.")
208 | 
209 |     if offline:
210 |         msg += _("\nIf you are confident the slug is correct and you have an internet connection," \
211 |                 " try running without --offline.")
212 | 
213 |     raise _exceptions.Error(msg)
214 | 
215 | 
216 | def process_args(args):
217 |     """Validate arguments and apply defaults that are dependent on other arguments"""
218 | 
219 |     # dev implies offline, verbose, and log level "INFO" if not overwritten
220 |     if args.dev:
221 |         args.offline = True
222 |         if "ansi" in args.output:
223 |             args.ansi_log = True
224 | 
225 |         if not args.log_level:
226 |             args.log_level = "info"
227 | 
228 |     # offline implies local
229 |     if args.offline:
230 |         args.no_install_dependencies = True
231 |         args.no_download_checks = True
232 |         args.local = True
233 | 
234 |     if not args.log_level:
235 |         args.log_level = "warning"
236 | 
237 |     # Setup logging for lib50
238 |     setup_logging(args.log_level)
239 | 
240 |     # Warning in case of running remotely with no_download_checks or no_install_dependencies set
241 |     if not args.local:
242 |         useless_args = []
243 |         if args.no_download_checks:
244 |             useless_args.append("--no-downloads-checks")
245 |         if args.no_install_dependencies:
246 |             useless_args.append("--no-install-dependencies")
247 | 
248 |         if useless_args:
249 |             LOGGER.warning(_("You should always use --local when using: {}").format(", ".join(useless_args)))
250 | 
251 |     # Filter out any duplicates from args.output
252 |     seen_output = []
253 |     for output in args.output:
254 |         if output in seen_output:
255 |             LOGGER.warning(_("Duplicate output format specified: {}").format(output))
256 |         else:
257 |             seen_output.append(output)
258 | 
259 |     args.output = seen_output
260 | 
261 |     if args.ansi_log and "ansi" not in seen_output:
262 |         LOGGER.warning(_("--ansi-log has no effect when ansi is not among the output formats"))
263 | 
264 | 
265 | class LoggerWriter:
266 |     def __init__(self, logger, level):
267 |         self.logger = logger
268 |         self.level = level
269 | 
270 |     def write(self, message):
271 |         if message != "\n":
272 |             self.logger.log(self.level, message)
273 | 
274 |     def flush(self):
275 |         pass
276 | 
277 | 
278 | def main():
279 |     parser = argparse.ArgumentParser(prog="check50", formatter_class=argparse.RawTextHelpFormatter)
280 | 
281 |     parser.add_argument("slug", help=_("prescribed identifier of work to check"))
282 |     parser.add_argument("-d", "--dev",
283 |                         action="store_true",
284 |                         help=_("run check50 in development mode (implies --offline, and --log-level info).\n"
285 |                                "causes slug to be interpreted as a literal path to a checks package."))
286 |     parser.add_argument("--offline",
287 |                         action="store_true",
288 |                         help=_("run checks completely offline (implies --local, --no-download-checks and --no-install-dependencies)"))
289 |     parser.add_argument("-l", "--local",
290 |                         action="store_true",
291 |                         help=_("run checks locally instead of uploading to cs50"))
292 |     parser.add_argument("-o", "--output",
293 |                         action="store",
294 |                         nargs="+",
295 |                         default=["ansi", "html"],
296 |                         choices=["ansi", "json", "html"],
297 |                         help=_("format of check results"))
298 |     parser.add_argument("--target",
299 |                         action="store",
300 |                         nargs="+",
301 |                         help=_("target specific checks to run"))
302 |     parser.add_argument("--output-file",
303 |                         action="store",
304 |                         metavar="FILE",
305 |                         help=_("file to write output to"))
306 |     parser.add_argument("--log-level",
307 |                         action="store",
308 |                         choices=[level.name.lower() for level in LogLevel],
309 |                         type=str.lower,
310 |                         help=_('warning: displays usage warnings.'
311 |                                '\ninfo: adds all commands run, any locally installed dependencies and print messages.'
312 |                                '\ndebug: adds the output of all commands run.'))
313 |     parser.add_argument("--ansi-log",
314 |                         action="store_true",
315 |                         help=_("display log in ansi output mode"))
316 |     parser.add_argument("--no-download-checks",
317 |                         action="store_true",
318 |                         help=_("do not download checks, but use previously downloaded checks instead (only works with --local)"))
319 |     parser.add_argument("--no-install-dependencies",
320 |                         action="store_true",
321 |                         help=_("do not install dependencies (only works with --local)"))
322 |     parser.add_argument("-V", "--version",
323 |                         action="version",
324 |                         version=f"%(prog)s {__version__}")
325 |     parser.add_argument("--logout", action=LogoutAction)
326 | 
327 |     args = parser.parse_args()
328 | 
329 |     internal.slug = args.slug
330 | 
331 |     # Validate arguments and apply defaults
332 |     process_args(args)
333 | 
334 |     # Set excepthook
335 |     _exceptions.ExceptHook.initialize(args.output, args.output_file)
336 | 
337 |     # If remote, push files to GitHub and await results
338 |     if not args.local:
339 |         commit_hash = lib50.push("check50", internal.slug, internal.CONFIG_LOADER, data={"check50": True})[1]
340 |         with lib50.ProgressBar("Waiting for results") if "ansi" in args.output else nullcontext():
341 |             tag_hash, results = await_results(commit_hash, internal.slug)
342 | 
343 |     # Otherwise run checks locally
344 |     else:
345 |         with lib50.ProgressBar("Checking") if "ansi" in args.output else nullcontext():
346 |             # If developing, assume slug is a path to check_dir
347 |             if args.dev:
348 |                 internal.check_dir = Path(internal.slug).expanduser().resolve()
349 |                 if not internal.check_dir.is_dir():
350 |                     raise _exceptions.Error(_("{} is not a directory").format(internal.check_dir))
351 |             # Otherwise have lib50 create a local copy of slug
352 |             else:
353 |                 try:
354 |                     internal.check_dir = lib50.local(internal.slug, offline=args.no_download_checks)
355 |                 except lib50.ConnectionError:
356 |                     raise _exceptions.Error(_("check50 could not retrieve checks from GitHub. Try running check50 again with --offline.").format(internal.slug))
357 |                 except lib50.InvalidSlugError:
358 |                     raise_invalid_slug(internal.slug, offline=args.no_download_checks)
359 | 
360 |             # Load config
361 |             config = internal.load_config(internal.check_dir)
362 | 
363 |             # Compile local checks if necessary
364 |             if isinstance(config["checks"], dict):
365 |                 config["checks"] = internal.compile_checks(config["checks"], prompt=args.dev)
366 | 
367 |             install_translations(config["translations"])
368 | 
369 |             if not args.no_install_dependencies:
370 |                 install_dependencies(config["dependencies"])
371 | 
372 |             checks_file = (internal.check_dir / config["checks"]).resolve()
373 | 
374 |             # Have lib50 decide which files to include
375 |             included_files = lib50.files(config.get("files"))[0]
376 | 
377 |             # Create a working_area (temp dir) named - with all included student files
378 |             with CheckRunner(checks_file, included_files) as check_runner, \
379 |                     contextlib.redirect_stdout(LoggerWriter(LOGGER, logging.NOTSET)), \
380 |                     contextlib.redirect_stderr(LoggerWriter(LOGGER, logging.NOTSET)):
381 | 
382 |                 check_results = check_runner.run(args.target)
383 |                 results = {
384 |                     "slug": internal.slug,
385 |                     "results": [attr.asdict(result) for result in check_results],
386 |                     "version": __version__
387 |                 }
388 | 
389 |     LOGGER.debug(results)
390 | 
391 |     # Render output
392 |     file_manager = open(args.output_file, "w") if args.output_file else nullcontext(sys.stdout)
393 |     with file_manager as output_file:
394 |         for output in args.output:
395 |             if output == "json":
396 |                 output_file.write(renderer.to_json(**results))
397 |                 output_file.write("\n")
398 |             elif output == "ansi":
399 |                 output_file.write(renderer.to_ansi(**results, _log=args.ansi_log))
400 |                 output_file.write("\n")
401 |             elif output == "html":
402 |                 if os.environ.get("CS50_IDE_TYPE") and args.local:
403 |                     html = renderer.to_html(**results)
404 |                     subprocess.check_call(["c9", "exec", "renderresults", "check50", html])
405 |                 else:
406 |                     if args.local:
407 |                         html = renderer.to_html(**results)
408 |                         with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html") as html_file:
409 |                             html_file.write(html)
410 | 
411 |                         if "microsoft-standard" in platform.uname().release:
412 |                             stream = os.popen(f"wslpath -m {html_file.name}")
413 |                             wsl_path = stream.read().strip()
414 |                             url = f"file://{wsl_path}"
415 |                         else:
416 |                             url = f"file://{html_file.name}"
417 |                     else:
418 |                         url = f"https://submit.cs50.io/check50/{tag_hash}"
419 | 
420 |                     termcolor.cprint(_("To see more detailed results go to {}").format(url), "white", attrs=["bold"])
421 | 
422 |     sys.exit(should_fail(results))
423 | 
424 | def should_fail(results):
425 |     return "error" in results or any(not result["passed"] for result in results["results"])
426 | 
427 | 
428 | if __name__ == "__main__":
429 |     main()
430 | 


--------------------------------------------------------------------------------
/check50/_api.py:
--------------------------------------------------------------------------------
  1 | import hashlib
  2 | import functools
  3 | import numbers
  4 | import os
  5 | import re
  6 | import shlex
  7 | import shutil
  8 | import signal
  9 | import sys
 10 | import time
 11 | 
 12 | import pexpect
 13 | from pexpect.exceptions import EOF, TIMEOUT
 14 | 
 15 | from . import internal, regex
 16 | 
 17 | _log = []
 18 | internal.register.before_every(_log.clear)
 19 | 
 20 | 
 21 | def log(line):
 22 |     """
 23 |     Add to check log
 24 | 
 25 |     :param line: line to be added to the check log
 26 |     :type line: str
 27 | 
 28 |     The check log is student-visible via the ``--log`` flag to ``check50``.
 29 |     """
 30 |     _log.append(line.replace("\n", "\\n"))
 31 | 
 32 | 
 33 | _data = {}
 34 | internal.register.before_every(_data.clear)
 35 | 
 36 | 
 37 | def data(**kwargs):
 38 |     """
 39 |     Add data to the check payload
 40 | 
 41 |     :params kwargs: key/value mappings to be added to the check payload
 42 | 
 43 |     Example usage::
 44 | 
 45 |         check50.data(time=7.3, mem=23)
 46 | 
 47 |     """
 48 | 
 49 |     _data.update(kwargs)
 50 | 
 51 | 
 52 | def include(*paths):
 53 |     """
 54 |     Copy files/directories from the check directory (:data:`check50.internal.check_dir`),
 55 |     to the current directory
 56 | 
 57 |     :params paths: files/directories to be copied
 58 | 
 59 |     Example usage::
 60 | 
 61 |         check50.include("foo.txt", "bar.txt")
 62 |         assert os.path.exists("foo.txt") and os.path.exists("bar.txt")
 63 | 
 64 |     """
 65 |     cwd = os.getcwd()
 66 |     for path in paths:
 67 |         _copy((internal.check_dir / path).resolve(), cwd)
 68 | 
 69 | 
 70 | def hash(file):
 71 |     """
 72 |     Hashes file using SHA-256.
 73 | 
 74 |     :param file: name of file to be hashed
 75 |     :type file: str
 76 |     :rtype: str
 77 |     :raises check50.Failure: if ``file`` does not exist
 78 | 
 79 |     """
 80 | 
 81 |     exists(file)
 82 |     log(_("hashing {}...").format(file))
 83 | 
 84 |     # https://stackoverflow.com/a/22058673
 85 |     with open(file, "rb") as f:
 86 |         sha256 = hashlib.sha256()
 87 |         for block in iter(lambda: f.read(65536), b""):
 88 |             sha256.update(block)
 89 |         return sha256.hexdigest()
 90 | 
 91 | 
 92 | def exists(*paths):
 93 |     """
 94 |     Assert that all given paths exist.
 95 | 
 96 |     :params paths: files/directories to be checked for existence
 97 |     :raises check50.Failure: if any ``path in paths`` does not exist
 98 | 
 99 |     Example usage::
100 | 
101 |         check50.exists("foo.c", "foo.h")
102 | 
103 |     """
104 |     for path in paths:
105 |         log(_("checking that {} exists...").format(path))
106 |         if not os.path.exists(path):
107 |             raise Failure(_("{} not found").format(path))
108 | 
109 | 
110 | def import_checks(path):
111 |     """
112 |     Import checks module given relative path.
113 | 
114 |     :param path: relative path from which to import checks module
115 |     :type path: str
116 |     :returns: the imported module
117 |     :raises FileNotFoundError: if ``path / .check50.yaml`` does not exist
118 |     :raises yaml.YAMLError: if ``path / .check50.yaml`` is not a valid YAML file
119 | 
120 |     This function is particularly useful when a set of checks logically extends
121 |     another, as is often the case in CS50's own problems that have a "less comfy"
122 |     and "more comfy" version. The "more comfy" version can include all of the
123 |     "less comfy" checks like so::
124 | 
125 |         less = check50.import_checks("../less")
126 |         from less import *
127 | 
128 |     .. note::
129 |         the ``__name__`` of the imported module is given by the basename
130 |         of the specified path (``less`` in the above example).
131 | 
132 |     """
133 |     dir = internal.check_dir / path
134 |     file = internal.load_config(dir)["checks"]
135 |     mod = internal.import_file(dir.name, (dir / file).resolve())
136 |     sys.modules[dir.name] = mod
137 |     return mod
138 | 
139 | 
140 | 
141 | class run:
142 |     """
143 |     Run a command.
144 | 
145 |     :param command: command to be run
146 |     :param env: environment in which to run command
147 |     :type command: str
148 |     :type env: dict
149 | 
150 |     By default, the command will be run using the same environment as ``check50``,
151 |     these mappings may be overridden via the ``env`` parameter::
152 | 
153 |         check50.run("./foo").stdin("foo").stdout("bar").exit(0)
154 |         check50.run("./foo", env={ "HOME": "/" }).stdin("foo").stdout("bar").exit(0)
155 | 
156 |     """
157 | 
158 |     def __init__(self, command, env={}):
159 |         log(_("running {}...").format(command))
160 | 
161 |         full_env = os.environ.copy()
162 |         full_env.update(env)
163 | 
164 |         # Workaround for OSX pexpect bug http://pexpect.readthedocs.io/en/stable/commonissues.html#truncated-output-just-before-child-exits
165 |         # Workaround from https://github.com/pexpect/pexpect/issues/373
166 |         command = "bash -c {}".format(shlex.quote(command))
167 |         self.process = pexpect.spawn(command, encoding="utf-8", echo=False, env=full_env)
168 | 
169 |     def stdin(self, line, str_line=None, prompt=True, timeout=3):
170 |         """
171 |         Send line to stdin, optionally expect a prompt.
172 | 
173 |         :param line: line to be send to stdin
174 |         :type line: str
175 |         :param str_line: what will be displayed as the delivered input, a human \
176 |                            readable form of ``line``
177 |         :type str_line: str
178 |         :param prompt: boolean indicating whether a prompt is expected, if True absorbs \
179 |                        all of stdout before inserting line into stdin and raises \
180 |                        :class:`check50.Failure` if stdout is empty
181 |         :type prompt: bool
182 |         :param timeout: maximum number of seconds to wait for prompt
183 |         :type timeout: int / float
184 |         :raises check50.Failure: if ``prompt`` is set to True and no prompt is given
185 | 
186 |         """
187 |         if str_line is None:
188 |             str_line = line
189 | 
190 |         if line == EOF:
191 |             log("sending EOF...")
192 |         else:
193 |             log(_("sending input {}...").format(str_line))
194 | 
195 |         if prompt:
196 |             try:
197 |                 self.process.expect(".+", timeout=timeout)
198 |             except (TIMEOUT, EOF):
199 |                 raise Failure(_("expected prompt for input, found none"))
200 |             except UnicodeDecodeError:
201 |                 raise Failure(_("output not valid ASCII text"))
202 | 
203 |             # Consume everything on the output buffer
204 |             try:
205 |                 for _i in range(int(timeout * 10)):
206 |                     self.process.expect(".+", timeout=0.1)
207 |             except (TIMEOUT, EOF):
208 |                 pass
209 | 
210 |         try:
211 |             if line == EOF:
212 |                 self.process.sendeof()
213 |             else:
214 |                 self.process.sendline(line)
215 |         except OSError:
216 |             pass
217 |         return self
218 | 
219 |     def stdout(self, output=None, str_output=None, regex=True, timeout=3, show_timeout=False):
220 |         """
221 |         Retrieve all output from stdout until timeout (3 sec by default). If ``output``
222 |         is None, ``stdout`` returns all of the stdout outputted by the process, else
223 |         it returns ``self``.
224 | 
225 |         :param output: optional output to be expected from stdout, raises \
226 |                        :class:`check50.Failure` if no match \
227 |                        In case output is a float or int, the check50.number_regex \
228 |                        is used to match just that number". \
229 |                        In case output is a stream its contents are used via output.read().
230 |         :type output: str, int, float, stream
231 |         :param str_output: what will be displayed as expected output, a human \
232 |                            readable form of ``output``
233 |         :type str_output: str
234 |         :param regex: flag indicating whether ``output`` should be treated as a regex
235 |         :type regex: bool
236 |         :param timeout: maximum number of seconds to wait for ``output``
237 |         :type timeout: int / float
238 |         :param show_timeout: flag indicating whether the timeout in seconds \
239 |                                   should be displayed when a timeout occurs
240 |         :type show_timeout: bool
241 |         :raises check50.Mismatch: if ``output`` is specified and nothing that the \
242 |                                   process outputs matches it
243 |         :raises check50.Failure: if process times out or if it outputs invalid UTF-8 text.
244 | 
245 |         Example usage::
246 | 
247 |             check50.run("./hello").stdout("[Hh]ello, world!?", "hello, world").exit()
248 | 
249 |             output = check50.run("./hello").stdout()
250 |             if not re.match("[Hh]ello, world!?", output):
251 |                 raise check50.Mismatch("hello, world", output)
252 |         """
253 |         if output is None:
254 |             self._wait(timeout)
255 |             return self.process.before.replace("\r\n", "\n").lstrip("\n")
256 | 
257 |         # In case output is a stream (file-like object), read from it
258 |         try:
259 |             output = output.read()
260 |         except AttributeError:
261 |             pass
262 | 
263 |         if str_output is None:
264 |             str_output = str(output)
265 | 
266 |         # In case output is an int/float, use a regex to match exactly that int/float
267 |         if isinstance(output, numbers.Number):
268 |             regex = True
269 |             output = globals()["regex"].decimal(output)
270 | 
271 |         expect = self.process.expect if regex else self.process.expect_exact
272 | 
273 |         if output == EOF:
274 |             log(_("checking for EOF..."))
275 |         else:
276 |             output = str(output).replace("\n", "\r\n")
277 |             log(_("checking for output \"{}\"...").format(str_output))
278 | 
279 |         try:
280 |             expect(output, timeout=timeout)
281 |         except EOF:
282 |             result = self.process.before + self.process.buffer
283 |             if self.process.after != EOF:
284 |                 result += self.process.after
285 |             raise Mismatch(str_output, result.replace("\r\n", "\n"))
286 |         except TIMEOUT:
287 |             if show_timeout:
288 |                 raise Missing(str_output, self.process.before,
289 |                               help=_("check50 waited {} seconds for the output of the program").format(timeout))
290 |             raise Missing(str_output, self.process.before)
291 |         except UnicodeDecodeError:
292 |             raise Failure(_("output not valid ASCII text"))
293 |         except Exception:
294 |             raise Failure(_("check50 could not verify output"))
295 | 
296 |         # If we expected EOF and we still got output, report an error.
297 |         if output == EOF and self.process.before:
298 |             raise Mismatch(EOF, self.process.before.replace("\r\n", "\n"))
299 | 
300 |         return self
301 | 
302 |     def reject(self, timeout=1):
303 |         """
304 |         Check that the process survives for timeout. Useful for checking whether program is waiting on input.
305 | 
306 |         :param timeout: number of seconds to wait
307 |         :type timeout: int / float
308 |         :raises check50.Failure: if process ends before ``timeout``
309 | 
310 |         """
311 |         log(_("checking that input was rejected..."))
312 |         try:
313 |             self._wait(timeout)
314 |         except Failure as e:
315 |             if not isinstance(e.__cause__, TIMEOUT):
316 |                 raise
317 |         else:
318 |             raise Failure(_("expected program to reject input, but it did not"))
319 |         return self
320 | 
321 |     def exit(self, code=None, timeout=5):
322 |         """
323 |         Wait for process to exit or until timeout (5 sec by default) and asserts
324 |         that process exits with ``code``. If ``code`` is ``None``, returns the code
325 |         the process exited with.
326 | 
327 |         ..note:: In order to ensure that spawned child processes do not outlive the check that spawned them, it is good practice to call either method (with no arguments if the exit code doesn't matter) or ``.kill()`` on every spawned process.
328 | 
329 |         :param code: code to assert process exits with
330 |         :type code: int
331 |         :param timeout: maximum number of seconds to wait for the program to end
332 |         :type timeout: int / float
333 |         :raises check50.Failure: if ``code`` is given and does not match the actual exitcode within ``timeout``
334 | 
335 |         Example usage::
336 | 
337 |             check50.run("./hello").exit(0)
338 | 
339 |             code = check50.run("./hello").exit()
340 |             if code != 0:
341 |                 raise check50.Failure(f"expected exit code 0, not {code}")
342 | 
343 | 
344 |         """
345 |         self._wait(timeout)
346 | 
347 |         if code is None:
348 |             return self.exitcode
349 | 
350 |         log(_("checking that program exited with status {}...").format(code))
351 |         if self.exitcode != code:
352 |             raise Failure(_("expected exit code {}, not {}").format(code, self.exitcode))
353 |         return self
354 | 
355 |     def kill(self):
356 |         """Kill the process.
357 | 
358 |         Child will first be sent a ``SIGHUP``, followed by a ``SIGINT`` and
359 |         finally a ``SIGKILL`` if it ignores the first two."""
360 |         self.process.close(force=True)
361 |         return self
362 | 
363 |     def _wait(self, timeout=5):
364 |         try:
365 |             self.process.expect(EOF, timeout=timeout)
366 |         except TIMEOUT:
367 |             raise Failure(_("timed out while waiting for program to exit")) from TIMEOUT(timeout)
368 |         except UnicodeDecodeError:
369 |             raise Failure(_("output not valid ASCII text"))
370 | 
371 |         self.kill()
372 | 
373 |         if self.process.signalstatus == signal.SIGSEGV:
374 |             raise Failure(_("failed to execute program due to segmentation fault"))
375 | 
376 |         self.exitcode = self.process.exitstatus
377 |         return self
378 | 
379 | 
380 | class Failure(Exception):
381 |     """
382 |     Exception signifying check failure.
383 | 
384 |     :param rationale: message to be displayed capturing why the check failed
385 |     :type rationale: str
386 |     :param help: optional help message to be displayed
387 |     :type help: str
388 | 
389 |     Example usage::
390 | 
391 |         out = check50.run("./cash").stdin("4.2").stdout()
392 |         if 10 not in out:
393 |             help = None
394 |             if 11 in out:
395 |                 help = "did you forget to round your result?"
396 |             raise check50.Failure("Expected a different result", help=help)
397 |     """
398 | 
399 |     def __init__(self, rationale, help=None):
400 |         self.payload = {"rationale": rationale, "help": help}
401 | 
402 |     def __str__(self):
403 |         return self.payload["rationale"]
404 | 
405 | 
406 | class Missing(Failure):
407 |     """
408 |     Exception signifying check failure due to an item missing from a collection.
409 |     This is typically a specific substring in a longer string, for instance the contents of stdout.
410 | 
411 |     :param item: the expected item / substring
412 |     :param collection: the collection / string
413 |     :param help: optional help message to be displayed
414 |     :type help: str
415 | 
416 |     Example usage::
417 | 
418 |         actual = check50.run("./fibonacci 5").stdout()
419 | 
420 |         if "5" not in actual and "3" in actual:
421 |             help = "Be sure to start the sequence at 1"
422 |             raise check50.Missing("5", actual, help=help)
423 | 
424 |     """
425 | 
426 |     def __init__(self, missing_item, collection, help=None):
427 |         super().__init__(rationale=_("Did not find {} in {}").format(_raw(missing_item), _raw(collection)), help=help)
428 | 
429 |         if missing_item == EOF:
430 |             missing_item = "EOF"
431 | 
432 |         self.payload.update({"missing_item": str(missing_item), "collection": str(collection)})
433 | 
434 | 
435 | class Mismatch(Failure):
436 |     """
437 |     Exception signifying check failure due to a mismatch in expected and actual outputs.
438 | 
439 |     :param expected: the expected value
440 |     :param actual: the actual value
441 |     :param help: optional help message to be displayed
442 |     :type help: str
443 | 
444 |     Example usage::
445 | 
446 |         from re import match
447 |         expected = "[Hh]ello, world!?\\n"
448 |         actual = check50.run("./hello").stdout()
449 |         if not match(expected, actual):
450 |             help = None
451 |             if match(expected[:-1], actual):
452 |                 help = r"did you forget a newline ('\\n') at the end of your printf string?"
453 |             raise check50.Mismatch("hello, world\\n", actual, help=help)
454 | 
455 |     """
456 | 
457 |     def __init__(self, expected, actual, help=None):
458 |         super().__init__(rationale=_("expected {}, not {}").format(_raw(expected), _raw(actual)), help=help)
459 | 
460 |         if expected == EOF:
461 |             expected = "EOF"
462 | 
463 |         if actual == EOF:
464 |             actual = "EOF"
465 | 
466 |         self.payload.update({"expected": expected, "actual": actual})
467 | 
468 | 
469 | def hidden(failure_rationale):
470 |     """
471 |     Decorator that marks a check as a 'hidden' check. This will suppress the log
472 |     accumulated throughout the check and will catch any :class:`check50.Failure`s thrown
473 |     during the check, and reraising a new :class:`check50.Failure` with the given ``failure_rationale``.
474 | 
475 |     :param failure_rationale: the rationale that will be displayed to the student if the check fails
476 |     :type failure_rationale: str
477 | 
478 |     Example usage::
479 | 
480 |         @check50.check()
481 |         @check50.hidden("Your program isn't returning the expected result. Try running it on some sample inputs.")
482 |         def hidden_check():
483 |             check50.run("./foo").stdin("bar").stdout("baz").exit()
484 | 
485 |     """
486 |     def decorator(f):
487 |         @functools.wraps(f)
488 |         def wrapper(*args, **kwargs):
489 |             try:
490 |                 return f(*args, **kwargs)
491 |             except Failure:
492 |                 raise Failure(failure_rationale)
493 |             finally:
494 |                 _log.clear()
495 |         return wrapper
496 |     return decorator
497 | 
498 | 
499 | def _raw(s):
500 |     """Get raw representation of s, truncating if too long."""
501 | 
502 |     if isinstance(s, list):
503 |         s = "\n".join(_raw(item) for item in s)
504 | 
505 |     if s == EOF:
506 |         return "EOF"
507 | 
508 |     s = f'"{repr(str(s))[1:-1]}"'
509 |     if len(s) > 15:
510 |         s = s[:15] + "...\""  # Truncate if too long
511 |     return s
512 | 
513 | 
514 | def _copy(src, dst):
515 |     """Copy src to dst, copying recursively if src is a directory."""
516 |     try:
517 |         shutil.copy(src, dst)
518 |     except (FileNotFoundError, IsADirectoryError):
519 |         if os.path.isdir(dst):
520 |             dst = os.path.join(dst, os.path.basename(src))
521 |         shutil.copytree(src, dst)
522 | 


--------------------------------------------------------------------------------
/check50/_exceptions.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import sys
 3 | import traceback
 4 | 
 5 | import lib50
 6 | import termcolor
 7 | 
 8 | from . import internal, __version__
 9 | from .contextmanagers import nullcontext
10 | 
11 | class Error(Exception):
12 |     """Exception for internal check50 errors."""
13 |     pass
14 | 
15 | 
16 | class RemoteCheckError(Error):
17 |     """An exception for errors that happen in check50's remote operation."""
18 |     def __init__(self, remote_json):
19 |         super().__init__("check50 ran into an error while running checks! Please visit our status page https://cs50.statuspage.io for more information.")
20 |         self.payload = {"remote_json": remote_json}
21 | 
22 | 
23 | class ExceptHook:
24 |     def __init__(self, outputs=("ansi",), output_file=None):
25 |         self.outputs = outputs
26 |         self.output_file = output_file
27 | 
28 |     def __call__(self, cls, exc, tb):
29 |         # If an error happened remotely, grab its traceback and message
30 |         if issubclass(cls, RemoteCheckError) and "error" in exc.payload["remote_json"]:
31 |             formatted_traceback = exc.payload["remote_json"]["error"]["traceback"]
32 |             show_traceback = exc.payload["remote_json"]["error"]["actions"]["show_traceback"]
33 |             message = exc.payload["remote_json"]["error"]["actions"]["message"]
34 |         # Otherwise, create the traceback and message from this error
35 |         else:
36 |             formatted_traceback = traceback.format_exception(cls, exc, tb)
37 |             show_traceback = False
38 | 
39 |             if (issubclass(cls, Error) or issubclass(cls, lib50.Error)) and exc.args:
40 |                 message = str(exc)
41 |             elif issubclass(cls, FileNotFoundError):
42 |                 message = _("{} not found").format(exc.filename)
43 |             elif issubclass(cls, KeyboardInterrupt):
44 |                 message = _("check cancelled")
45 |             elif not issubclass(cls, Exception):
46 |                 # Class is some other BaseException, better just let it go
47 |                 return
48 |             else:
49 |                 show_traceback = True
50 |                 message = _("Sorry, something is wrong! check50 ran into an error, please try again.\n" \
51 |                             "If the problem persists, please visit our status page https://cs50.statuspage.io for more information.")
52 | 
53 |         # Output exception as json
54 |         if "json" in self.outputs:
55 |             ctxmanager = open(self.output_file, "w") if self.output_file else nullcontext(sys.stdout)
56 |             with ctxmanager as output_file:
57 |                 json.dump({
58 |                     "slug": internal.slug,
59 |                     "error": {
60 |                         "type": cls.__name__,
61 |                         "value": str(exc),
62 |                         "traceback": formatted_traceback,
63 |                         "actions": {
64 |                             "show_traceback": show_traceback,
65 |                             "message": message
66 |                         },
67 |                         "data" : exc.payload if hasattr(exc, "payload") else {}
68 |                     },
69 |                     "version": __version__
70 |                 }, output_file, indent=4)
71 |                 output_file.write("\n")
72 | 
73 |         # Output exception to stderr
74 |         if "ansi" in self.outputs or "html" in self.outputs:
75 |             if show_traceback:
76 |                 for line in formatted_traceback:
77 |                     termcolor.cprint(line, end="", file=sys.stderr)
78 |             termcolor.cprint(message, "red", file=sys.stderr)
79 | 
80 |         sys.exit(1)
81 | 
82 |     @classmethod
83 |     def initialize(cls, *args, **kwargs):
84 |         sys.excepthook = cls(*args, **kwargs)
85 | 


--------------------------------------------------------------------------------
/check50/_simple.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Functions that compile "simple" YAML checks into our standard python checks
  3 | """
  4 | 
  5 | import re
  6 | 
  7 | 
  8 | def compile(checks):
  9 |     """Returns compiled check50 checks from simple YAML checks in path."""
 10 | 
 11 |     out = ["import check50"]
 12 | 
 13 |     for name, check in checks.items():
 14 |         out.append(_compile_check(name, check))
 15 | 
 16 |     return "\n\n".join(out)
 17 | 
 18 | 
 19 | def _run(arg):
 20 |     return f'.run("{arg}")'
 21 | 
 22 | 
 23 | def _stdin(arg):
 24 |     if isinstance(arg, list):
 25 |         arg = r"\n".join(str(a) for a in arg)
 26 | 
 27 |     arg = str(arg).replace("\n", r"\n").replace("\t", r"\t").replace('"', '\"')
 28 |     return f'.stdin("{arg}", prompt=False)'
 29 | 
 30 | 
 31 | def _stdout(arg):
 32 |     if isinstance(arg, list):
 33 |         arg = r"\n".join(str(a) for a in arg)
 34 |     arg = str(arg).replace("\n", r"\n").replace("\t", r"\t").replace('"', '\"')
 35 |     return f'.stdout("{arg}", regex=False)'
 36 | 
 37 | 
 38 | def _exit(arg):
 39 |     if arg is None:
 40 |         return ".exit()"
 41 | 
 42 |     try:
 43 |         arg = int(arg)
 44 |     except ValueError:
 45 |         raise InvalidArgument(f"exit command only accepts integers, not {arg}")
 46 |     return f'.exit({arg})'
 47 | 
 48 | 
 49 | COMMANDS = {"run": _run, "stdin": _stdin, "stdout": _stdout, "exit": _exit}
 50 | 
 51 | 
 52 | def _compile_check(name, check):
 53 |     indent = " " * 4
 54 | 
 55 |     # Allow check names to contain spaces/dashes, but replace them with underscores
 56 |     check_name = name.replace(' ', '_').replace('-', '_')
 57 | 
 58 |     # Allow check names to start with numbers by prepending an _ to them
 59 |     if check_name[0].isdigit():
 60 |         check_name = f"_{check_name}"
 61 | 
 62 |     if not re.match("\w+", check_name):
 63 |         raise CompileError(
 64 |                 _("{} is not a valid name for a check; check names should consist only of alphanumeric characters, underscores, and spaces").format(name))
 65 | 
 66 |     out = ["@check50.check()",
 67 |            f"def {check_name}():",
 68 |            f'{indent}"""{name}"""']
 69 | 
 70 |     for run in check:
 71 |         _validate(name, run)
 72 | 
 73 |         # Append exit with no args if unspecified
 74 |         if "exit" not in run:
 75 |             run["exit"] = None
 76 | 
 77 |         line = [f"{indent}check50"]
 78 | 
 79 |         for command_name in COMMANDS:
 80 |             if command_name in run:
 81 |                 line.append(COMMANDS[command_name](run[command_name]))
 82 |         out.append("".join(line))
 83 | 
 84 |     return "\n".join(out)
 85 | 
 86 | 
 87 | def _validate(name, run):
 88 |     if run == "run":
 89 |         raise CompileError(_("You forgot a - in front of run"))
 90 | 
 91 |     for key in run:
 92 |         if key not in COMMANDS:
 93 |             raise UnsupportedCommand(
 94 |                 _("{} is not a valid command in check {}, use only: {}").format(key, name, COMMANDS))
 95 | 
 96 |     for required_command in ["run"]:
 97 |         if required_command not in run:
 98 |             raise MissingCommand(_("Missing {} in check {}").format(required_name, name))
 99 | 
100 | 
101 | class CompileError(Exception):
102 |     pass
103 | 
104 | 
105 | class UnsupportedCommand(CompileError):
106 |     pass
107 | 
108 | 
109 | class MissingCommand(CompileError):
110 |     pass
111 | 
112 | 
113 | class InvalidArgument(CompileError):
114 |     pass
115 | 


--------------------------------------------------------------------------------
/check50/c.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import re
  3 | import tempfile
  4 | from pathlib import Path
  5 | import xml.etree.cElementTree as ET
  6 | 
  7 | from ._api import run, log, Failure
  8 | from . import internal
  9 | 
 10 | #: Default compiler for :func:`check50.c.compile`
 11 | CC = "clang"
 12 | 
 13 | #: Default CFLAGS for :func:`check50.c.compile`
 14 | CFLAGS = {"std": "c11", "ggdb": True, "lm": True}
 15 | 
 16 | 
 17 | def compile(*files, exe_name=None, cc=CC, max_log_lines=50, **cflags):
 18 |     """
 19 |     Compile C source files.
 20 | 
 21 |     :param files: filenames to be compiled
 22 |     :param exe_name: name of resulting executable
 23 |     :param cc: compiler to use (:data:`check50.c.CC` by default)
 24 |     :param cflags: additional flags to pass to the compiler
 25 |     :raises check50.Failure: if compilation failed (i.e., if the compiler returns a non-zero exit status).
 26 |     :raises RuntimeError: if no filenames are specified
 27 | 
 28 |     If ``exe_name`` is None, :func:`check50.c.compile` will default to the first
 29 |     file specified sans the ``.c`` extension::
 30 | 
 31 | 
 32 |         check50.c.compile("foo.c", "bar.c") # clang foo.c bar.c -o foo -std=c11 -ggdb -lm
 33 | 
 34 |     Additional CFLAGS may be passed as keyword arguments like so::
 35 | 
 36 |         check50.c.compile("foo.c", "bar.c", lcs50=True) # clang foo.c bar.c -o foo -std=c11 -ggdb -lm -lcs50
 37 | 
 38 |     In the same vein, the default CFLAGS may be overridden via keyword arguments::
 39 | 
 40 |         check50.c.compile("foo.c", "bar.c", std="c99", lm=False) # clang foo.c bar.c -o foo -std=c99 -ggdb
 41 |     """
 42 | 
 43 |     if not files:
 44 |         raise RuntimeError(_("compile requires at least one file"))
 45 | 
 46 |     if exe_name is None and files[0].endswith(".c"):
 47 |         exe_name = Path(files[0]).stem
 48 | 
 49 |     files = " ".join(files)
 50 | 
 51 |     flags = CFLAGS.copy()
 52 |     flags.update(cflags)
 53 |     flags = " ".join((f"-{flag}" + (f"={value}" if value is not True else "")).replace("_", "-")
 54 |                      for flag, value in flags.items() if value)
 55 | 
 56 |     out_flag = f" -o {exe_name} " if exe_name is not None else " "
 57 | 
 58 |     process = run(f"{cc} {files}{out_flag}{flags}")
 59 | 
 60 |     # Strip out ANSI codes
 61 |     stdout = re.sub(r"\x1B\[[0-?]*[ -/]*[@-~]", "",  process.stdout())
 62 | 
 63 |     # Log max_log_lines lines of output in case compilation fails
 64 |     if process.exitcode != 0:
 65 |         lines = stdout.splitlines()
 66 | 
 67 |         if len(lines) > max_log_lines:
 68 |             lines = lines[:max_log_lines // 2] + lines[-(max_log_lines // 2):]
 69 | 
 70 |         for line in lines:
 71 |             log(line)
 72 | 
 73 |         raise Failure("code failed to compile")
 74 | 
 75 | 
 76 | def valgrind(command, env={}):
 77 |     """Run a command with valgrind.
 78 | 
 79 |     :param command: command to be run
 80 |     :type command: str
 81 |     :param env: environment in which to run command
 82 |     :type env: str
 83 |     :raises check50.Failure: if, at the end of the check, valgrind reports any errors
 84 | 
 85 |     This function works exactly like :func:`check50.run`, with the additional effect that ``command`` is run through
 86 |     ``valgrind`` and ``valgrind``'s output is automatically reviewed at the end of the check for memory leaks and other
 87 |     bugs. If ``valgrind`` reports any issues, the check is failed and student-friendly messages are printed to the log.
 88 | 
 89 |     Example usage::
 90 | 
 91 |         check50.c.valgrind("./leaky").stdin("foo").stdout("bar").exit(0)
 92 | 
 93 |     .. note::
 94 |         It is recommended that the student's code is compiled with the `-ggdb`
 95 |         flag so that additional information, such as the file and line number at which
 96 |         the issue was detected can be included in the log as well.
 97 |     """
 98 |     xml_file = tempfile.NamedTemporaryFile()
 99 |     internal.register.after_check(lambda: _check_valgrind(xml_file))
100 | 
101 |     # Ideally we'd like for this whole command not to be logged.
102 |     return run(f"valgrind --show-leak-kinds=all --xml=yes --xml-file={xml_file.name} -- {command}", env=env)
103 | 
104 | 
105 | def _check_valgrind(xml_file):
106 |     """Log and report any errors encountered by valgrind."""
107 |     log(_("checking for valgrind errors..."))
108 | 
109 |     # Load XML file created by valgrind
110 |     xml = ET.ElementTree(file=xml_file)
111 | 
112 |     # Ensure that we don't get duplicate error messages.
113 |     reported = set()
114 |     for error in xml.iterfind("error"):
115 |         # Type of error valgrind encountered
116 |         kind = error.find("kind").text
117 | 
118 |         # Valgrind's error message
119 |         what = error.find("xwhat/text" if kind.startswith("Leak_") else "what").text
120 | 
121 |         # Error message that we will report
122 |         msg = ["\t", what]
123 | 
124 |         # Find first stack frame within student's code.
125 |         for frame in error.iterfind("stack/frame"):
126 |             obj = frame.find("obj")
127 |             if obj is not None and internal.run_dir in Path(obj.text).parents:
128 |                 file, line = frame.find("file"), frame.find("line")
129 |                 if file is not None and line is not None:
130 |                     msg.append(f": ({_('file')}: {file.text}, {_('line')}: {line.text})")
131 |                 break
132 | 
133 |         msg = "".join(msg)
134 |         if msg not in reported:
135 |             log(msg)
136 |             reported.add(msg)
137 | 
138 |     # Only raise exception if we encountered errors.
139 |     if reported:
140 |         raise Failure(_("valgrind tests failed; see log for more information."))
141 | 


--------------------------------------------------------------------------------
/check50/contextmanagers.py:
--------------------------------------------------------------------------------
1 | import contextlib
2 | 
3 | 
4 | @contextlib.contextmanager
5 | def nullcontext(entry_result=None):
6 |     """This is just contextlib.nullcontext but that function is only available in 3.7+."""
7 |     yield entry_result
8 | 


--------------------------------------------------------------------------------
/check50/flask.py:
--------------------------------------------------------------------------------
  1 | import pathlib
  2 | import re
  3 | import sys
  4 | import urllib.parse as url
  5 | import warnings
  6 | 
  7 | from bs4 import BeautifulSoup
  8 | 
  9 | from ._api import log, Failure
 10 | from . import internal
 11 | 
 12 | 
 13 | class app:
 14 |     """Spawn a Flask app.
 15 | 
 16 |     :param path: path to python file containing Flask app
 17 |     :param app_name: name of Flask app in file
 18 |     :type path: str
 19 |     :type env: str
 20 | 
 21 |     Example usage::
 22 | 
 23 |         check50.flask.app("application.py").get("/").status(200)
 24 |     """
 25 | 
 26 |     def __init__(self, path="application.py", app_name="app"):
 27 | 
 28 |         path = pathlib.Path(path).resolve()
 29 | 
 30 |         # Add directory of flask app to sys.path so we can import it properly
 31 |         prevpath = sys.path[0]
 32 |         try:
 33 |             sys.path[0] = str(path.parent)
 34 |             mod = internal.import_file(path.stem, path.name)
 35 |         except FileNotFoundError:
 36 |             raise Failure(_("could not find {}").format(path.name))
 37 |         finally:
 38 |             # Restore sys.path
 39 |             sys.path[0] = prevpath
 40 | 
 41 |         try:
 42 |             app = getattr(mod, app_name)
 43 |         except AttributeError:
 44 |             raise Failure(_("{} does not contain an app").format(file))
 45 | 
 46 |         app.testing = True
 47 |         # Initialize flask client
 48 |         self._client = app.test_client()
 49 | 
 50 |         self.response = None
 51 | 
 52 |     def get(self, route, data=None, params=None, follow_redirects=True):
 53 |         """Send GET request to app.
 54 | 
 55 |         :param route: route to send request to
 56 |         :type route: str
 57 |         :param data: form data to include in request
 58 |         :type data: dict
 59 |         :param params: URL parameters to include in request
 60 |         :param follow_redirects: enable redirection (defaults to ``True``)
 61 |         :type follow_redirects: bool
 62 |         :returns: ``self``
 63 |         :raises check50.Failure: if Flask application throws an uncaught exception
 64 | 
 65 |         Example usage::
 66 | 
 67 |             check50.flask.app("application.py").get("/buy", params={"q": "02138"}).content()
 68 |         """
 69 |         return self._send("GET", route, data, params, follow_redirects=follow_redirects)
 70 | 
 71 |     def post(self, route, data=None, params=None, follow_redirects=True):
 72 |         """Send POST request to app.
 73 | 
 74 |         :param route: route to send request to
 75 |         :type route: str
 76 |         :param data: form data to include in request
 77 |         :type data: dict
 78 |         :param params: URL parameters to include in request
 79 |         :param follow_redirects: enable redirection (defaults to ``True``)
 80 |         :type follow_redirects: bool
 81 |         :raises check50.Failure: if Flask application throws an uncaught exception
 82 | 
 83 |         Example usage::
 84 | 
 85 |             check50.flask.app("application.py").post("/buy", data={"symbol": "GOOG", "shares": 10}).status(200)
 86 |         """
 87 | 
 88 |         return self._send("POST", route, data, params, follow_redirects=follow_redirects)
 89 | 
 90 |     def status(self, code=None):
 91 |         """Check status code in response returned by application.
 92 |         If ``code`` is not None, assert that ``code`` is returned by application,
 93 |         else simply return the status code.
 94 | 
 95 |         :param code: ``code`` to assert that application returns
 96 |         :type code: int
 97 | 
 98 |         Example usage::
 99 | 
100 |             check50.flask.app("application.py").status(200)
101 | 
102 |             status = check50.flask.app("application.py").get("/").status()
103 |             if status != 200:
104 |                 raise check50.Failure(f"expected status code 200, but got {status}")
105 | 
106 |         """
107 |         if code is None:
108 |             return self.response.status_code
109 | 
110 |         log(_("checking that status code {} is returned...").format(code))
111 |         if code != self.response.status_code:
112 |             raise Failure(_("expected status code {}, but got {}").format(
113 |                 code, self.response.status_code))
114 |         return self
115 | 
116 |     def raw_content(self, output=None, str_output=None):
117 |         """Searches for `output` regex match within content of page, regardless of mimetype."""
118 |         return self._search_page(output, str_output, self.response.data, lambda regex, content: regex.search(content.decode()))
119 | 
120 |     def content(self, output=None, str_output=None, **kwargs):
121 |         """Searches for `output` regex within HTML page. kwargs are passed to BeautifulSoup's find function to filter for tags."""
122 |         if self.response.mimetype != "text/html":
123 |             raise Failure(_("expected request to return HTML, but it returned {}").format(
124 |                 self.response.mimetype))
125 | 
126 |         # TODO: Remove once beautiful soup updates to accommodate python 3.7
127 |         with warnings.catch_warnings():
128 |             warnings.filterwarnings("ignore", category=DeprecationWarning)
129 |             content = BeautifulSoup(self.response.data, "html.parser")
130 | 
131 |         return self._search_page(
132 |             output,
133 |             str_output,
134 |             content,
135 |             lambda regex, content: any(regex.search(str(tag)) for tag in content.find_all(**kwargs)))
136 | 
137 |     def _send(self, method, route, data, params, **kwargs):
138 |         """Send request of type `method` to `route`."""
139 |         route = self._fmt_route(route, params)
140 |         log(_("sending {} request to {}").format(method.upper(), route))
141 |         try:
142 |             self.response = getattr(self._client, method.lower())(route, data=data, **kwargs)
143 |         except BaseException as e:  # Catch all exceptions thrown by app
144 |             log(_("exception raised in application: {}: {}").format(type(e).__name__, e))
145 |             raise Failure(_("application raised an exception (see the log for more details)"))
146 |         return self
147 | 
148 |     def _search_page(self, output, str_output, content, match_fn, **kwargs):
149 |         if output is None:
150 |             return content
151 | 
152 |         if str_output is None:
153 |             str_output = output
154 | 
155 |         log(_("checking that \"{}\" is in page").format(str_output))
156 | 
157 |         regex = re.compile(output)
158 | 
159 |         if not match_fn(regex, content):
160 |             raise Failure(
161 |                 _("expected to find \"{}\" in page, but it wasn't found").format(str_output))
162 | 
163 |         return self
164 | 
165 |     @staticmethod
166 |     def _fmt_route(route, params):
167 |         parsed = url.urlparse(route)
168 | 
169 |         # Convert params dict into urlencoded string
170 |         params = url.urlencode(params) if params else ""
171 | 
172 |         # Concatenate params
173 |         param_str = "&".join((ps for ps in [params, parsed.query] if ps))
174 |         if param_str:
175 |             param_str = "?" + param_str
176 | 
177 |         # Only display netloc if it isn't localhost
178 |         return "".join([parsed.netloc if parsed.netloc != "localhost" else "", parsed.path, param_str])
179 | 


--------------------------------------------------------------------------------
/check50/internal.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Additional check50 internals exposed to extension writers in addition to the standard API
  3 | """
  4 | 
  5 | from pathlib import Path
  6 | import importlib
  7 | import json
  8 | import sys
  9 | import termcolor
 10 | import traceback
 11 | 
 12 | import lib50
 13 | 
 14 | from . import _simple, _exceptions
 15 | 
 16 | #: Directory containing the check and its associated files
 17 | check_dir = None
 18 | 
 19 | #: Temporary directory in which the current check is being run
 20 | run_dir = None
 21 | 
 22 | #: Temporary directory that is the root (parent) of all run_dir(s)
 23 | run_root_dir = None
 24 | 
 25 | #: Directory check50 was run from
 26 | student_dir = None
 27 | 
 28 | #: Boolean that indicates if a check is currently running
 29 | check_running = False
 30 | 
 31 | #: The user specified slug used to indentifies the set of checks
 32 | slug = None
 33 | 
 34 | #: ``lib50`` config loader
 35 | CONFIG_LOADER = lib50.config.Loader("check50")
 36 | CONFIG_LOADER.scope("files", "include", "exclude", "require")
 37 | 
 38 | 
 39 | class Register:
 40 |     """
 41 |     Class with which functions can be registered to run before / after checks.
 42 |     :data:`check50.internal.register` should be the sole instance of this class.
 43 |     """
 44 |     def __init__(self):
 45 |         self._before_everies = []
 46 |         self._after_everies = []
 47 |         self._after_checks = []
 48 | 
 49 |     def after_check(self, func):
 50 |         """Run func once at the end of the check, then discard func.
 51 | 
 52 |         :param func: callback to run after check
 53 |         :raises check50.internal.Error: if called when no check is being run"""
 54 |         if not check_running:
 55 |             raise _exceptions.Error("cannot register callback to run after check when no check is running")
 56 |         self._after_checks.append(func)
 57 | 
 58 |     def after_every(self, func):
 59 |         """Run func at the end of every check.
 60 | 
 61 |         :param func: callback to be run after every check
 62 |         :raises check50.internal.Error: if called when a check is being run"""
 63 |         if check_running:
 64 |             raise _exceptions.Error("cannot register callback to run after every check when check is running")
 65 |         self._after_everies.append(func)
 66 | 
 67 |     def before_every(self, func):
 68 |         """Run func at the start of every check.
 69 | 
 70 |         :param func: callback to be run before every check
 71 |         :raises check50.internal.Error: if called when a check is being run"""
 72 | 
 73 |         if check_running:
 74 |             raise _exceptions.Error("cannot register callback to run before every check when check is running")
 75 |         self._before_everies.append(func)
 76 | 
 77 |     def __enter__(self):
 78 |         for f in self._before_everies:
 79 |             f()
 80 | 
 81 |     def __exit__(self, exc_type, exc_val, exc_tb):
 82 |         # Only run 'afters' when check has passed
 83 |         if exc_type is not None:
 84 |             return
 85 | 
 86 |         # Run and remove all checks registered to run after a single check
 87 |         while self._after_checks:
 88 |             self._after_checks.pop()()
 89 | 
 90 |         for f in self._after_everies:
 91 |             f()
 92 | 
 93 | 
 94 | #: Sole instance of the :class:`check50.internal.Register` class
 95 | register = Register()
 96 | 
 97 | 
 98 | def load_config(check_dir):
 99 |     """
100 |     Load configuration file from ``check_dir / ".cs50.yaml"``, applying
101 |     defaults to unspecified values.
102 | 
103 |     :param check_dir: directory from which to load config file
104 |     :type check_dir: str / Path
105 |     :rtype: dict
106 |     """
107 | 
108 |     # Defaults for top-level keys
109 |     options = {
110 |         "checks": "__init__.py",
111 |         "dependencies": None,
112 |         "translations": None
113 |     }
114 | 
115 |     # Defaults for translation keys
116 |     translation_options = {
117 |         "localedir": "locale",
118 |         "domain": "messages",
119 |     }
120 | 
121 |     # Get config file
122 |     try:
123 |         config_file = lib50.config.get_config_filepath(check_dir)
124 |     except lib50.Error:
125 |         raise _exceptions.Error(_("Invalid slug for check50. Did you mean something else?"))
126 | 
127 |     # Load config
128 |     with open(config_file) as f:
129 |         try:
130 |             config = CONFIG_LOADER.load(f.read())
131 |         except lib50.InvalidConfigError:
132 |             raise _exceptions.Error(_("Invalid slug for check50. Did you mean something else?"))
133 | 
134 |     # Update the config with defaults
135 |     if isinstance(config, dict):
136 |         options.update(config)
137 | 
138 |     # Apply translations
139 |     if options["translations"]:
140 |         if isinstance(options["translations"], dict):
141 |             translation_options.update(options["translations"])
142 |         options["translations"] = translation_options
143 | 
144 |     return options
145 | 
146 | 
147 | def compile_checks(checks, prompt=False, out_file="__init__.py"):
148 |     """
149 |     Compile YAML checks to a Python file
150 | 
151 |     :param checks: YAML checks read from config
152 |     :type checkcs: dict
153 |     :param prompt: prompt user if ``out_file`` already exists
154 |     :type prompt: bool
155 |     :param out_file: file to write compiled checks
156 |     :type out_file: str
157 |     :returns: ``out_file``
158 |     :rtype: str
159 |     """
160 | 
161 |     file_path = check_dir / out_file
162 |     # Prompt to replace __init__.py (compile destination)
163 |     if prompt and file_path.exists():
164 |         if not _yes_no_prompt("check50 will compile the YAML checks to __init__.py, are you sure you want to overwrite its contents?"):
165 |             raise _exceptions.Error("Aborting: could not overwrite to __init__.py")
166 | 
167 |     # Compile simple checks
168 |     with open(check_dir / out_file, "w") as f:
169 |         f.write(_simple.compile(checks))
170 | 
171 |     return out_file
172 | 
173 | 
174 | def import_file(name, path):
175 |     """
176 |     Import a file given a raw file path.
177 | 
178 |     :param name: Name of module to be imported
179 |     :type name: str
180 |     :param path: Path to Python file
181 |     :type path: str / Path
182 |     """
183 |     spec = importlib.util.spec_from_file_location(name, path)
184 |     mod = importlib.util.module_from_spec(spec)
185 |     spec.loader.exec_module(mod)
186 |     return mod
187 | 
188 | 
189 | def _yes_no_prompt(prompt):
190 |     """
191 |     Raise a prompt, returns True if yes is entered, False if no is entered.
192 |     Will reraise prompt in case of any other reply.
193 |     """
194 |     return _("yes").startswith(input(_("{} [Y/n] ").format(prompt)).lower())
195 | 


--------------------------------------------------------------------------------
/check50/py.py:
--------------------------------------------------------------------------------
 1 | import importlib
 2 | from pathlib import Path
 3 | import py_compile
 4 | 
 5 | from . import internal
 6 | from ._api import Failure, exists, log
 7 | 
 8 | 
 9 | def append_code(original, codefile):
10 |     """Append the contents of one file to another.
11 | 
12 |     :param original: name of file that will be appended to
13 |     :type original: str
14 |     :param codefile: name of file that will be appende
15 |     :type codefile: str
16 | 
17 |     This function is particularly useful when one wants to replace a function
18 |     in student code with their own implementation of one. If two functions are
19 |     defined with the same name in Python, the latter definition is taken so overwriting
20 |     a function is as simple as writing it to a file and then appending it to the
21 |     student's code.
22 | 
23 |     Example usage::
24 | 
25 |         # Include a file containing our own implementation of a lookup function.
26 |         check50.include("lookup.py")
27 | 
28 |         # Overwrite the lookup function in helpers.py with our own implementation.
29 |         check50.py.append_code("helpers.py", "lookup.py")
30 |     """
31 |     with open(codefile) as code, open(original, "a") as o:
32 |         o.write("\n")
33 |         o.writelines(code)
34 | 
35 | 
36 | def import_(path):
37 |     """Import a Python program given a raw file path
38 | 
39 |     :param path: path to python file to be imported
40 |     :type path: str
41 |     :raises check50.Failure: if ``path`` doesn't exist, or if the Python file at ``path`` throws an exception when imported.
42 |     """
43 |     exists(path)
44 |     log(_("importing {}...").format(path))
45 |     name = Path(path).stem
46 |     try:
47 |         return internal.import_file(name, path)
48 |     except Exception as e:
49 |         raise Failure(str(e))
50 | 
51 | def compile(file):
52 |     """
53 |     Compile a Python program into byte code
54 | 
55 |     :param file: file to be compiled
56 |     :raises check50.Failure: if compilation fails e.g. if there is a SyntaxError
57 |     """
58 |     log(_("compiling {} into byte code...").format(file))
59 | 
60 |     try:
61 |         py_compile.compile(file, doraise=True)
62 |     except py_compile.PyCompileError as e:
63 |         log(_("Exception raised: "))
64 |         for line in e.msg.splitlines():
65 |             log(line)
66 | 
67 |         raise Failure(_("{} raised while compiling {} (see the log for more details)").format(e.exc_type_name, file))
68 | 


--------------------------------------------------------------------------------
/check50/regex.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | 
 3 | 
 4 | def decimal(number):
 5 |     """
 6 |     Create a regular expression to match the number exactly:
 7 | 
 8 |     In case of a positive number::
 9 | 
10 |         (?<![\d\-])number(?!(\.?\d))
11 | 
12 |     In case of a negative number::
13 | 
14 |         number(?!(\.?\d))
15 | 
16 |     :code:`(?<![\d\-])` = negative lookbehind, \
17 |         asserts that there are no digits and no - in front of the number.
18 | 
19 |     :code:`(?!(\.?\d))` = negative lookahead, \
20 |         asserts that there are no digits and no additional . followed by digits after the number.
21 | 
22 |     :param number: the number to match in the regex
23 |     :type number: any numbers.Number (such as int, float, ...)
24 |     :rtype: str
25 | 
26 |     Example usage::
27 | 
28 |         # Check that 7.0000 is printed with 5 significant figures
29 |         check50.run("./prog").stdout(check50.regex.decimal("7.0000"))
30 | 
31 |     """
32 |     negative_lookbehind = fr"(?<![\d\-])" if number >= 0 else ""
33 |     return fr"{negative_lookbehind}{re.escape(str(number))}(?!(\.?\d))"
34 | 


--------------------------------------------------------------------------------
/check50/renderer/__init__.py:
--------------------------------------------------------------------------------
1 | from ._renderers import to_ansi, to_html, to_json
2 | 


--------------------------------------------------------------------------------
/check50/renderer/_renderers.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import pathlib
 3 | 
 4 | import jinja2
 5 | import termcolor
 6 | 
 7 | from importlib.resources import files
 8 | 
 9 | TEMPLATES = pathlib.Path(files("check50.renderer").joinpath("templates"))
10 | 
11 | 
12 | def to_html(slug, results, version):
13 |     with open(TEMPLATES / "results.html") as f:
14 |         content = f.read()
15 | 
16 |     template = jinja2.Template(
17 |         content, autoescape=jinja2.select_autoescape(enabled_extensions=("html",)))
18 |     html = template.render(slug=slug, results=results, version=version)
19 | 
20 |     return html
21 | 
22 | 
23 | def to_json(slug, results, version):
24 |     return json.dumps({"slug": slug, "results": results, "version": version}, indent=4)
25 | 
26 | 
27 | def to_ansi(slug, results, version, _log=False):
28 |     lines = [termcolor.colored(_("Results for {} generated by check50 v{}").format(slug, version), "white", attrs=["bold"])]
29 |     for result in results:
30 |         if result["passed"]:
31 |             lines.append(termcolor.colored(f":) {result['description']}", "green"))
32 |         elif result["passed"] is None:
33 |             lines.append(termcolor.colored(f":| {result['description']}", "yellow"))
34 |             lines.append(termcolor.colored(f"    {result['cause'].get('rationale') or _('check skipped')}", "yellow"))
35 |             if result["cause"].get("error") is not None:
36 |                 lines.append(f"    {result['cause']['error']['type']}: {result['cause']['error']['value']}")
37 |                 lines += (f"    {line.rstrip()}" for line in result["cause"]["error"]["traceback"])
38 |         else:
39 |             lines.append(termcolor.colored(f":( {result['description']}", "red"))
40 |             if result["cause"].get("rationale") is not None:
41 |                 lines.append(termcolor.colored(f"    {result['cause']['rationale']}", "red"))
42 |             if result["cause"].get("help") is not None:
43 |                 lines.append(termcolor.colored(f"    {result['cause']['help']}", "red"))
44 | 
45 |         if _log:
46 |             lines += (f"    {line}" for line in result["log"])
47 |     return "\n".join(lines)
48 | 
49 | 


--------------------------------------------------------------------------------
/check50/renderer/templates/results.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 |     <head>
  4 |         <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
  5 |         <title>This is check50.</title>
  6 |     </head>
  7 |     <body>
  8 |         <div class="container">
  9 |             <div class="row">
 10 | 
 11 |             <h1>check50</h1>
 12 |             <h2>{{ slug }}</h2>
 13 |             <hr>
 14 |               {% for check in results %}
 15 |                   {% if check.passed == True %}
 16 |                       <h3 style="color:green">:) {{ check.description }}</h3>
 17 |                   {% elif check.passed == False %}
 18 |                       <h3 style="color:red">:( {{ check.description }}</h3>
 19 |                   {% else %}
 20 |                       <h3 style="color:orange">:| {{ check.description }}</h3>
 21 |                   {% endif %}
 22 | 
 23 |                   {% if check.cause or check.log %}
 24 |                       <div class="well well-sm">
 25 |                           {# Reason why the check received its status #}
 26 |                           {% if check.cause %}
 27 |                           <p>
 28 |                               <b>Cause</b><br/>
 29 |                               {% autoescape false %}
 30 |                                   {{ check.cause.rationale | e | replace(" ", "&nbsp;") }}
 31 |                               {% endautoescape %}
 32 |                               <br/>
 33 |                               {% autoescape false %}
 34 |                                   {% if check.cause.help %}
 35 |                                       {{ check.cause.help | e | replace(" ", "&nbsp;") }}
 36 |                                       <br/>
 37 |                                   {% endif %}
 38 |                               {% endautoescape %}
 39 |                           </p>
 40 |                           {% endif %}
 41 | 
 42 |                           {# Log information #}
 43 |                           {% if check.log %}
 44 |                               <samp>
 45 |                                   <b>Log</b><br/>
 46 |                                   <div style="border:2px solid; padding:2px; max-height:20em; overflow:scroll;">
 47 |                                       {% for item in check.log %}
 48 |                                           {{ item }}
 49 |                                           <br/>
 50 |                                       {% endfor %}
 51 |                                   </div>
 52 |                               </samp>
 53 |                           {% endif %}
 54 | 
 55 |                           {% if check.cause and check.cause.error %}
 56 |                               <div class="row">
 57 |                                   <div class="col-md-12">
 58 |                                       <br/>
 59 |                                       <b>Error</b>
 60 |                                       <div style="background-color:black; color:white;">
 61 |                                           <samp>
 62 |                                               {{check.cause.error.type}}: {{check.cause.error.value}}
 63 |                                               <br/>
 64 |                                               {% for line in check.cause.error.traceback%}
 65 |                                                   &nbsp;&nbsp;&nbsp;&nbsp;{{line}}
 66 |                                                   <br/>
 67 |                                               {% endfor %}
 68 |                                           </samp>
 69 |                                       </div>
 70 |                                  </div>
 71 |                               </div>
 72 |                           {% endif %}
 73 | 
 74 |                           {# Mismatch if there was one #}
 75 |                           {% if check.cause and "actual" in check.cause and "expected" in check.cause %}
 76 |                               <br/>
 77 |                               <div class="row">
 78 |                                   <div class="col-md-6">
 79 |                                       <b>Expected Output:</b>
 80 |                                       <br/>
 81 |                                       <div style="background-color:black; color:white;">
 82 |                                       <samp>
 83 |                                           {% autoescape false %}
 84 |                                               {% if check.cause.expected is not string %}
 85 |                                                   {% set expected = check.cause.expected | join('\n') | e %}
 86 |                                               {% else %}
 87 |                                                   {% set expected = check.cause.expected | e %}
 88 |                                               {% endif %}
 89 |                                               {{ expected | replace(" ", "&nbsp;") | replace("\n", "<br/>") }}
 90 |                                           {% endautoescape %}
 91 |                                       </samp>
 92 |                                       </div>
 93 |                                   </div>
 94 |                                   <div class="col-md-6">
 95 |                                       <b>Actual Output:</b>
 96 |                                       <br/>
 97 |                                       <div style="background-color:black; color:white;">
 98 |                                       <samp>
 99 |                                           {% autoescape false %}
100 |                                               {% if check.cause.actual is not string %}
101 |                                                   {% set actual = check.cause.actual | join('\n') | e %}
102 |                                               {% else %}
103 |                                                   {% set actual = check.cause.actual | e %}
104 |                                               {% endif %}
105 |                                               {{ actual | replace(" ", "&nbsp;") | replace("\n", "<br/>") }}
106 |                                           {% endautoescape %}
107 |                                       </samp>
108 |                                       </div>
109 |                                   </div>
110 |                               </div>
111 |                           {% endif %}
112 | 
113 |                           {# Missing if there was one #}
114 |                           {% if check.cause and "missing_item" in check.cause and "collection" in check.cause %}
115 |                               <br/>
116 | 
117 |                               <div class="row">
118 |                                   <div class="col-md-6">
119 |                                       <b>Could not find the following in the output:</b>
120 |                                       <br/>
121 |                                       <div style="background-color:black; color:#90ee90;">
122 |                                       <samp>
123 |                                           {% autoescape false %}
124 |                                               {% set item = check.cause.missing_item | e %}
125 |                                               {{ item | replace(" ", "&nbsp;") | replace("\n", "<br/>") }}
126 |                                           {% endautoescape %}
127 |                                       </samp>
128 |                                       </div>
129 |                                   </div>
130 |                                   <div class="col-md-6">
131 |                                       <b>Actual Output:</b>
132 |                                       <br/>
133 |                                       <div style="background-color:black; color:white;">
134 |                                       <samp>
135 |                                           {% autoescape false %}
136 |                                               {% set collection = check.cause.collection | e %}
137 |                                               {{ collection | replace(" ", "&nbsp;") | replace("\n", "<br/>") }}
138 |                                           {% endautoescape %}
139 |                                       </samp>
140 |                                       </div>
141 |                                   </div>
142 |                               </div>
143 |                           {% endif %}
144 | 
145 |                       </div>
146 |                   {% endif %}
147 |               {% endfor %}
148 | 
149 |             </div>
150 |         </div>
151 |     </body>
152 | </html>
153 | 


--------------------------------------------------------------------------------
/check50/runner.py:
--------------------------------------------------------------------------------
  1 | import collections
  2 | from contextlib import contextmanager
  3 | import concurrent.futures as futures
  4 | import enum
  5 | import functools
  6 | import inspect
  7 | import importlib
  8 | import gettext
  9 | import multiprocessing
 10 | import os
 11 | from pathlib import Path
 12 | import pickle
 13 | import shutil
 14 | import signal
 15 | import sys
 16 | import tempfile
 17 | import traceback
 18 | 
 19 | import attr
 20 | import lib50
 21 | 
 22 | from . import internal, _exceptions, __version__
 23 | from ._api import log, Failure, _copy, _log, _data
 24 | 
 25 | _check_names = []
 26 | 
 27 | 
 28 | @attr.s(slots=True)
 29 | class CheckResult:
 30 |     """Record returned by each check"""
 31 |     name = attr.ib()
 32 |     description = attr.ib()
 33 |     passed = attr.ib(default=None)
 34 |     log = attr.ib(default=attr.Factory(list))
 35 |     cause = attr.ib(default=None)
 36 |     data = attr.ib(default=attr.Factory(dict))
 37 |     dependency = attr.ib(default=None)
 38 | 
 39 |     @classmethod
 40 |     def from_check(cls, check, *args, **kwargs):
 41 |         """Create a check_result given a check function, automatically recording the name,
 42 |         the dependency, and the (translated) description.
 43 |         """
 44 |         return cls(name=check.__name__, description=_(check.__doc__ if check.__doc__ else check.__name__.replace("_", " ")),
 45 |                    dependency=check._check_dependency.__name__ if check._check_dependency else None,
 46 |                    *args,
 47 |                    **kwargs)
 48 | 
 49 |     @classmethod
 50 |     def from_dict(cls, d):
 51 |         """Create a CheckResult given a dict. Dict must contain at least the fields in the CheckResult.
 52 |         Throws a KeyError if not."""
 53 |         return cls(**{field.name: d[field.name] for field in attr.fields(cls)})
 54 | 
 55 | 
 56 | 
 57 | class Timeout(Failure):
 58 |     def __init__(self, seconds):
 59 |         super().__init__(rationale=_("check timed out after {} seconds").format(seconds))
 60 | 
 61 | 
 62 | @contextmanager
 63 | def _timeout(seconds):
 64 |     """Context manager that runs code block until timeout is reached.
 65 | 
 66 |     Example usage::
 67 | 
 68 |         try:
 69 |             with _timeout(10):
 70 |                 do_stuff()
 71 |         except Timeout:
 72 |             print("do_stuff timed out")
 73 |     """
 74 | 
 75 |     def _handle_timeout(*args):
 76 |         raise Timeout(seconds)
 77 | 
 78 |     signal.signal(signal.SIGALRM, _handle_timeout)
 79 |     signal.alarm(seconds)
 80 |     try:
 81 |         yield
 82 |     finally:
 83 |         signal.alarm(0)
 84 |         signal.signal(signal.SIGALRM, signal.SIG_DFL)
 85 | 
 86 | 
 87 | def check(dependency=None, timeout=60, max_log_lines=100):
 88 |     """Mark function as a check.
 89 | 
 90 |     :param dependency: the check that this check depends on
 91 |     :type dependency: function
 92 |     :param timeout: maximum number of seconds the check can run
 93 |     :type timeout: int
 94 |     :param max_log_lines: maximum number of lines that can appear in the log
 95 |     :type max_log_lines: int
 96 | 
 97 |     When a check depends on another, the former will only run if the latter passes.
 98 |     Additionally, the dependent check will inherit the filesystem of its dependency.
 99 |     This is particularly useful when writing e.g., a ``compiles`` check that compiles a
100 |     student's program (and checks that it compiled successfully). Any checks that run the
101 |     student's program will logically depend on this check, and since they inherit the
102 |     resulting filesystem of the check, they will immediately have access to the compiled
103 |     program without needing to recompile.
104 | 
105 |     Example usage::
106 | 
107 |         @check50.check() # Mark 'exists' as a check
108 |         def exists():
109 |             \"""hello.c exists\"""
110 |             check50.exists("hello.c")
111 | 
112 |         @check50.check(exists) # Mark 'compiles' as a check that depends on 'exists'
113 |         def compiles():
114 |             \"""hello.c compiles\"""
115 |             check50.c.compile("hello.c")
116 | 
117 |         @check50.check(compiles)
118 |         def prints_hello():
119 |             \"""prints "Hello, world!\\\\n\"""
120 |             # Since 'prints_hello', depends on 'compiles' it inherits the compiled binary
121 |             check50.run("./hello").stdout("[Hh]ello, world!?\\n", "hello, world\\n").exit()
122 | 
123 |     """
124 |     def decorator(check):
125 | 
126 |         # Modules are evaluated from the top of the file down, so _check_names will
127 |         # contain the names of the checks in the order in which they are declared
128 |         _check_names.append(check.__name__)
129 |         check._check_dependency = dependency
130 | 
131 |         @functools.wraps(check)
132 |         def wrapper(run_root_dir, dependency_state):
133 |             # Result template
134 |             result = CheckResult.from_check(check)
135 |             # Any shared (returned) state
136 |             state = None
137 | 
138 |             try:
139 |                 # Setup check environment, copying disk state from dependency
140 |                 internal.run_dir = run_root_dir / check.__name__
141 |                 src_dir = run_root_dir / (dependency.__name__ if dependency else "-")
142 |                 shutil.copytree(src_dir, internal.run_dir)
143 |                 os.chdir(internal.run_dir)
144 | 
145 |                 # Run registered functions before/after running check and set timeout
146 |                 with internal.register, _timeout(seconds=timeout):
147 |                     args = (dependency_state,) if inspect.getfullargspec(check).args else ()
148 |                     state = check(*args)
149 |             except Failure as e:
150 |                 result.passed = False
151 |                 result.cause = e.payload
152 |             except BaseException as e:
153 |                 result.passed = None
154 |                 result.cause = {"rationale": _("check50 ran into an error while running checks!"),
155 |                                 "error": {
156 |                                     "type": type(e).__name__,
157 |                                     "value": str(e),
158 |                                     "traceback": traceback.format_tb(e.__traceback__),
159 |                                     "data" : e.payload if hasattr(e, "payload") else {}
160 |                                 }}
161 |             else:
162 |                 result.passed = True
163 |             finally:
164 |                 result.log = _log if len(_log) <= max_log_lines else ["..."] + _log[-max_log_lines:]
165 |                 result.data = _data
166 |                 return result, state
167 |         return wrapper
168 |     return decorator
169 | 
170 | 
171 | class CheckRunner:
172 |     def __init__(self, checks_path, included_files):
173 |         self.checks_path = checks_path
174 |         self.included_files = included_files
175 | 
176 |     def run(self, targets=None):
177 |         """
178 |         Run checks concurrently.
179 |         Returns a list of CheckResults ordered by declaration order of the checks in the imported module
180 |         targets allows you to limit which checks run. If targets is false-y, all checks are run.
181 |         """
182 |         graph = self.build_subgraph(targets) if targets else self.dependency_graph
183 | 
184 |         # Ensure that dictionary is ordered by check declaration order (via self.check_names)
185 |         # NOTE: Requires CPython 3.6. If we need to support older versions of Python, replace with OrderedDict.
186 |         results = {name: None for name in self.check_names}
187 | 
188 |         try:
189 |             max_workers = int(os.environ.get("CHECK50_WORKERS"))
190 |         except (ValueError, TypeError):
191 |             max_workers = None
192 | 
193 |         with futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
194 |             # Start all checks that have no dependencies
195 |             not_done = set(executor.submit(run_check(name, self.checks_spec))
196 |                            for name in graph[None])
197 |             not_passed = []
198 | 
199 |             while not_done:
200 |                 done, not_done = futures.wait(not_done, return_when=futures.FIRST_COMPLETED)
201 |                 for future in done:
202 |                     # Get result from completed check
203 |                     result, state = future.result()
204 |                     results[result.name] = result
205 |                     if result.passed:
206 |                         # Dispatch dependent checks
207 |                         for child_name in graph[result.name]:
208 |                             not_done.add(executor.submit(
209 |                                 run_check(child_name, self.checks_spec, state)))
210 |                     else:
211 |                         not_passed.append(result.name)
212 | 
213 |         for name in not_passed:
214 |             self._skip_children(name, results)
215 | 
216 |         # Don't include checks we don't have results for (i.e. in the case that targets != None) in the list.
217 |         return list(filter(None, results.values()))
218 | 
219 | 
220 |     def build_subgraph(self, targets):
221 |         """
222 |         Build minimal subgraph of self.dependency_graph that contains each check in targets
223 |         """
224 |         checks = self.dependencies_of(targets)
225 |         subgraph = collections.defaultdict(set)
226 |         for dep, children in self.dependency_graph.items():
227 |             # If dep is not a dependency of any target,
228 |             # none of its children will be either, may as well skip.
229 |             if dep is not None and dep not in checks:
230 |                 continue
231 |             for child in children:
232 |                 if child in checks:
233 |                     subgraph[dep].add(child)
234 |         return subgraph
235 | 
236 | 
237 |     def dependencies_of(self, targets):
238 |         """Get all unique dependencies of the targeted checks (tartgets)."""
239 |         inverse_graph = self._create_inverse_dependency_graph()
240 |         deps = set()
241 |         for target in targets:
242 |             if target not in inverse_graph:
243 |                 raise _exceptions.Error(_("Unknown check: {}").format(e.args[0]))
244 |             curr_check = target
245 |             while curr_check is not None and curr_check not in deps:
246 |                 deps.add(curr_check)
247 |                 curr_check = inverse_graph[curr_check]
248 |         return deps
249 | 
250 | 
251 |     def _create_inverse_dependency_graph(self):
252 |         """Build an inverse dependency map, from a check to its dependency."""
253 |         inverse_dependency_graph = {}
254 |         for check_name, dependents in self.dependency_graph.items():
255 |             for dependent_name in dependents:
256 |                 inverse_dependency_graph[dependent_name] = check_name
257 |         return inverse_dependency_graph
258 | 
259 | 
260 |     def _skip_children(self, check_name, results):
261 |         """
262 |         Recursively skip the children of check_name (presumably because check_name
263 |         did not pass).
264 |         """
265 |         for name in self.dependency_graph[check_name]:
266 |             if results[name] is None:
267 |                 results[name] = CheckResult(name=name, description=self.check_descriptions[name],
268 |                                             passed=None,
269 |                                             dependency=check_name,
270 |                                             cause={"rationale": _("can't check until a frown turns upside down")})
271 |                 self._skip_children(name, results)
272 | 
273 | 
274 |     def __enter__(self):
275 |         # Remember the student's directory
276 |         internal.student_dir = Path.cwd()
277 | 
278 |         # Set up a temp dir for the checks
279 |         self._working_area_manager = lib50.working_area(self.included_files, name='-')
280 |         internal.run_root_dir = self._working_area_manager.__enter__().parent
281 | 
282 |         # Change current working dir to the temp dir
283 |         self._cd_manager = lib50.cd(internal.run_root_dir)
284 |         self._cd_manager.__enter__()
285 | 
286 |         # TODO: Naming the module "checks" is arbitrary. Better name?
287 |         self.checks_spec = importlib.util.spec_from_file_location("checks", self.checks_path)
288 | 
289 |         # Clear check_names, import module, then save check_names. Not thread safe.
290 |         # Ideally, there'd be a better way to extract declaration order than @check mutating global state,
291 |         # but there are a lot of subtleties with using `inspect` or similar here
292 |         _check_names.clear()
293 |         check_module = importlib.util.module_from_spec(self.checks_spec)
294 |         self.checks_spec.loader.exec_module(check_module)
295 |         self.check_names = _check_names.copy()
296 |         _check_names.clear()
297 | 
298 |         # Grab all checks from the module
299 |         checks = inspect.getmembers(check_module, lambda f: hasattr(f, "_check_dependency"))
300 | 
301 |         # Map each check to tuples containing the names of the checks that depend on it
302 |         self.dependency_graph = collections.defaultdict(set)
303 |         for name, check in checks:
304 |             dependency = None if check._check_dependency is None else check._check_dependency.__name__
305 |             self.dependency_graph[dependency].add(name)
306 | 
307 |         # Map each check name to its description
308 |         self.check_descriptions = {name: check.__doc__ for name, check in checks}
309 | 
310 |         return self
311 | 
312 | 
313 |     def __exit__(self, type, value, tb):
314 |         # Destroy the temporary directory for the checks
315 |         self._working_area_manager.__exit__(type, value, tb)
316 | 
317 |         # cd back to the directory check50 was called from
318 |         self._cd_manager.__exit__(type, value, tb)
319 | 
320 | 
321 | class run_check:
322 |     """
323 |     Check job that runs in a separate process.
324 |     This is only a class to get around the fact that `pickle` can't serialize closures.
325 |     This class is essentially a function that reimports the check module and runs the check.
326 |     """
327 | 
328 |     # All attributes shared between check50's main process and each checks' process
329 |     # Required for "spawn": https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
330 |     CROSS_PROCESS_ATTRIBUTES = (
331 |         "internal.check_dir",
332 |         "internal.slug",
333 |         "internal.student_dir",
334 |         "internal.run_root_dir",
335 |         "sys.excepthook",
336 |         "__version__"
337 |     )
338 | 
339 |     def __init__(self, check_name, spec, state=None):
340 |         self.check_name = check_name
341 |         self.spec = spec
342 |         self.state = state
343 |         self._store_attributes()
344 | 
345 |     def _store_attributes(self):
346 |         """"
347 |         Store all values from the attributes from run_check.CROSS_PROCESS_ATTRIBUTES on this object,
348 |         in case multiprocessing is using spawn as its starting method. 
349 |         """
350 | 
351 |         # Attributes only need to be passed explicitly to child processes when using spawn
352 |         if multiprocessing.get_start_method() != "spawn":
353 |            return
354 | 
355 |         self._attribute_values = [eval(name) for name in self.CROSS_PROCESS_ATTRIBUTES]
356 |         
357 |         # Replace all unpickle-able values with nothing, assuming they've been set externally,
358 |         # and will be set again upon re-importing the checks module
359 |         # https://github.com/cs50/check50/issues/235
360 |         for i, value in enumerate(self._attribute_values):
361 |             try:
362 |                 pickle.dumps(value)
363 |             except (pickle.PicklingError, AttributeError):
364 |                 self._attribute_values[i] = None
365 |                 
366 |         self._attribute_values = tuple(self._attribute_values)
367 | 
368 | 
369 |     def _set_attributes(self):
370 |         """
371 |         If the parent process set any values in self._attribute_values, 
372 |         restore them in the child process.
373 |         """
374 |         if not hasattr(self, "_attribute_values"):
375 |            return
376 | 
377 |         for name, val in zip(self.CROSS_PROCESS_ATTRIBUTES, self._attribute_values):
378 |             self._set_attribute(name, val)
379 | 
380 | 
381 |     @staticmethod
382 |     def _set_attribute(name, value):
383 |         """Get an attribute from a name in global scope and set its value."""
384 |         parts = name.split(".")
385 | 
386 |         obj = sys.modules[__name__]
387 |         for part in parts[:-1]:
388 |             obj = getattr(obj, part)
389 | 
390 |         setattr(obj, parts[-1], value)
391 | 
392 | 
393 |     def __call__(self):
394 |         # Restore any attributes from the parent process
395 |         self._set_attributes()
396 | 
397 |         # Create the checks module
398 |         mod = importlib.util.module_from_spec(self.spec)
399 | 
400 |         # Execute (effectively import) the checks module
401 |         self.spec.loader.exec_module(mod)
402 | 
403 |         # Run just the check named self.check_name
404 |         internal.check_running = True
405 |         try:
406 |             return getattr(mod, self.check_name)(internal.run_root_dir, self.state)
407 |         finally:
408 |             internal.check_running = False
409 | 


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line.
 5 | SPHINXOPTS    =
 6 | SPHINXBUILD = sphinx-build
 7 | SPHINXPROJ    = check50
 8 | SOURCEDIR     = source
 9 | BUILDDIR      = build
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 | 


--------------------------------------------------------------------------------
/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | sphinx
2 | sphinx-autobuild
3 | sphinx_rtd_theme
4 | 


--------------------------------------------------------------------------------
/docs/source/ansi_output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/ansi_output.png


--------------------------------------------------------------------------------
/docs/source/api.rst:
--------------------------------------------------------------------------------
 1 | .. _api:
 2 | 
 3 | API docs
 4 | ========
 5 | 
 6 | .. _check50:
 7 | 
 8 | 
 9 | check50
10 | *******
11 | 
12 | .. automodule:: check50
13 |    :members:
14 | 
15 | 
16 | check50.c
17 | **********
18 | 
19 | .. automodule:: check50.c
20 |   :members:
21 | 
22 | 
23 | check50.flask
24 | ****************
25 | 
26 | .. automodule:: check50.flask
27 |    :members:
28 | 
29 | 
30 | check50.py
31 | ***********
32 | .. automodule:: check50.py
33 |    :members:
34 | 
35 | 
36 | check50.regex
37 | **************
38 | .. automodule:: check50.regex
39 |   :members:
40 | 
41 | 
42 | check50.internal
43 | *****************
44 | 
45 | .. automodule:: check50.internal
46 |    :members:
47 | 


--------------------------------------------------------------------------------
/docs/source/check50_user.rst:
--------------------------------------------------------------------------------
  1 | .. _check50_user:
  2 | 
  3 | Running check50
  4 | ======================
  5 | 
  6 | Slug
  7 | **********************
  8 | check50 requires one positional argument, a so called slug. Something like this:
  9 | 
 10 | .. code-block:: bash
 11 | 
 12 |     check50 cs50/problems/2018/x/caesar
 13 | 
 14 | Why? Well, anyone can write checks for check50 without needing to ask for permission. To achieve this goal check50, the tool, is decoupled from any of its checks. The checks themselves are hosted anywhere on a popular hosting platform for code, GitHub. All you need to do to add your own checks is to put them on anywhere on GitHub. With this flexibility comes a price, as check50 does not know where your checks live on GitHub. In order to uniquely identify a collection of checks on GitHub check50 needs the following:
 15 | 
 16 | * org
 17 | * repository
 18 | * branch
 19 | * path to problem
 20 | 
 21 | These four pieces of information separated by a / is what check50 calls a slug, a string that uniquely identifies a set of checks. For instance the slug :code:`cs50/problems/2018/x/caesar` uniquely identifies the org :code:`cs50`, the repository :code:`problems`, the branch :code:`2018/x` and the path :code:`caesar`.
 22 | 
 23 | 
 24 | Operation modes
 25 | **********************
 26 | Check50 can run in four mutually exclusive modes of operation.
 27 | 
 28 | **********************
 29 | online
 30 | **********************
 31 | By default check50 runs the checks remotely and then waits for the results to come back.
 32 | 
 33 | **********************
 34 | local
 35 | **********************
 36 | To run checks locally, pass the :code:`--local` flag. The checks are still fetched remotely from GitHub.
 37 | 
 38 | **********************
 39 | offline
 40 | **********************
 41 | Running with :code:`--offline` runs the checks locally and has check50 look for checks locally. check50 will not try to fetch checks remotely in this mode.
 42 | 
 43 | **********************
 44 | dev
 45 | **********************
 46 | The :code:`--dev` flag signals check50 to run in developer mode. This implies :code:`--offline`. More on this mode in :ref:`check_writer`.
 47 | 
 48 | 
 49 | Additional output
 50 | **********************
 51 | By default check50 will try to keep its output concise in its :code:`ansi` output mode. For each check you will see at most its description and rationale/help on why the check failed. check50 will by default hide tracebacks and log output. You can show both by running check50 with the :code:`--verbose` flag, or just the log with the :code:`--log` flag.
 52 | 
 53 | **********************
 54 | verbose
 55 | **********************
 56 | Running with :code:`--verbose` lets check50 output any tracebacks in the :code:`ansi` output mode.
 57 | 
 58 | **********************
 59 | log-level
 60 | **********************
 61 | Running check50 with :code:`--log-level INFO` will display any git commands run. :code:`--log-level DEBUG` adds all output of any git commands run.
 62 | 
 63 | 
 64 | Targeting checks
 65 | **********************
 66 | Check50 lets you target specific checks by name with the :code:`--target` flags. This will have check50 run just these checks and their dependencies.
 67 | 
 68 | **********************
 69 | target
 70 | **********************
 71 | With :code:`--target` you can target checks from a larger body of checks by name. check50 will only run and show these checks and their dependencies.
 72 | 
 73 | 
 74 | Output modes
 75 | **********************
 76 | check50 supports three output modes: :code:`ansi`, :code:`html` and :code:`json`. In short, the :code:`ansi` output mode is text-based output meant to be displayed in a terminal. :code:`html` is an extension of :code:`ansi` showing the same results but in a webpage. This allows for visual comparisons and more information to be displayed in general. Finally, the :code:`json` output mode provides a machine readable form of output, that can for instance be used for automatic grading.
 77 | 
 78 | The output modes can be mixed and matched through the :code:`--output` or :code:`-o` flag.
 79 | 
 80 | .. code-block:: bash
 81 | 
 82 |     check50 <slug> -o ansi
 83 |     check50 <slug> -o ansi html # equivalent to check50 <slug>
 84 |     check50 <slug> -o json
 85 | 
 86 | By default check50 shows both :code:`ansi` and :code:`html` output.
 87 | 
 88 | **********************
 89 | ansi
 90 | **********************
 91 | The :code:`ansi` output mode will have check50 print the results from the checks to stdout. This output mode keeps students in the terminal, the environment in which they are running check50 in the first place. Limited by its nature, check50's :code:`ansi` output mode is not suited for large pieces of text or visual comparisons. The output format is sufficient for showing which checks passed and failed, and offering short text based help or explanation on those checks.
 92 | 
 93 | .. image:: ansi_output.png
 94 | 
 95 | **********************
 96 | html
 97 | **********************
 98 | In addition to :code:`ansi`, check50 comes with a :code:`html` output mode. This output mode allows check50 to show results side by side and to display more verbose information like the log by default. check50 creates a local self contained static html file in :code:`/tmp` and will output the path to file in stdout.
 99 | 
100 | .. image:: html_output.png
101 | 
102 | **********************
103 | json
104 | **********************
105 | check50 can provide machine readable output in the form of :code:`json`. By default this output mode will print to stdout, but like any other form of output check50 can write to a file with the :code:`--output-file` command line option. For a complete overview of the :code:`json` output please refer to the :ref:`json_specification`.
106 | 
107 | .. code-block:: json
108 | 
109 |     {
110 |         "slug": "cs50/problems/2018/x/caesar",
111 |         "results": [
112 |             {
113 |                 "name": "exists",
114 |                 "description": "caesar.c exists.",
115 |                 "passed": true,
116 |                 "log": [
117 |                     "checking that caesar.c exists..."
118 |                 ],
119 |                 "cause": null,
120 |                 "data": {},
121 |                 "dependency": null
122 |             },
123 |             {
124 |                 "name": "compiles",
125 |                 "description": "caesar.c compiles.",
126 |                 "passed": false,
127 |                 "log": [
128 |                     "running clang caesar.c -o caesar -std=c11 -ggdb -lm -lcs50...",
129 |                     "caesar.c:24:5: warning: implicit declaration of function 'f' is invalid in C99",
130 |                     "      [-Wimplicit-function-declaration]",
131 |                     "    f (argc != 2)",
132 |                     "    ^",
133 |                     "caesar.c:24:18: error: expected ';' after expression",
134 |                     "    f (argc != 2)",
135 |                     "                 ^",
136 |                     "                 ;",
137 |                     "1 warning and 1 error generated."
138 |                 ],
139 |                 "cause": {
140 |                     "rationale": "code failed to compile",
141 |                     "help": null
142 |                 },
143 |                 "data": {},
144 |                 "dependency": "exists"
145 |             },
146 |             {
147 |                 "name": "encrypts_a_as_b",
148 |                 "description": "encrypts \"a\" as \"b\" using 1 as key",
149 |                 "passed": null,
150 |                 "log": [],
151 |                 "cause": {
152 |                     "rationale": "can't check until a frown turns upside down"
153 |                 },
154 |                 "data": {},
155 |                 "dependency": "compiles"
156 |             },
157 |             {
158 |                 "name": "encrypts_barfoo_as_yxocll",
159 |                 "description": "encrypts \"barfoo\" as \"yxocll\" using 23 as key",
160 |                 "passed": null,
161 |                 "log": [],
162 |                 "cause": {
163 |                     "rationale": "can't check until a frown turns upside down"
164 |                 },
165 |                 "data": {},
166 |                 "dependency": "compiles"
167 |             },
168 |             {
169 |                 "name": "encrypts_BARFOO_as_EDUIRR",
170 |                 "description": "encrypts \"BARFOO\" as \"EDUIRR\" using 3 as key",
171 |                 "passed": null,
172 |                 "log": [],
173 |                 "cause": {
174 |                     "rationale": "can't check until a frown turns upside down"
175 |                 },
176 |                 "data": {},
177 |                 "dependency": "compiles"
178 |             },
179 |             {
180 |                 "name": "encrypts_BaRFoo_FeVJss",
181 |                 "description": "encrypts \"BaRFoo\" as \"FeVJss\" using 4 as key",
182 |                 "passed": null,
183 |                 "log": [],
184 |                 "cause": {
185 |                     "rationale": "can't check until a frown turns upside down"
186 |                 },
187 |                 "data": {},
188 |                 "dependency": "compiles"
189 |             },
190 |             {
191 |                 "name": "encrypts_barfoo_as_onesbb",
192 |                 "description": "encrypts \"barfoo\" as \"onesbb\" using 65 as key",
193 |                 "passed": null,
194 |                 "log": [],
195 |                 "cause": {
196 |                     "rationale": "can't check until a frown turns upside down"
197 |                 },
198 |                 "data": {},
199 |                 "dependency": "compiles"
200 |             },
201 |             {
202 |                 "name": "checks_for_handling_non_alpha",
203 |                 "description": "encrypts \"world, say hello!\" as \"iadxp, emk tqxxa!\" using 12 as key",
204 |                 "passed": null,
205 |                 "log": [],
206 |                 "cause": {
207 |                     "rationale": "can't check until a frown turns upside down"
208 |                 },
209 |                 "data": {},
210 |                 "dependency": "compiles"
211 |             },
212 |             {
213 |                 "name": "handles_no_argv",
214 |                 "description": "handles lack of argv[1]",
215 |                 "passed": null,
216 |                 "log": [],
217 |                 "cause": {
218 |                     "rationale": "can't check until a frown turns upside down"
219 |                 },
220 |                 "data": {},
221 |                 "dependency": "compiles"
222 |             }
223 |         ],
224 |         "version": "3.0.0"
225 |     }
226 | 


--------------------------------------------------------------------------------
/docs/source/check_writer.rst:
--------------------------------------------------------------------------------
  1 | .. _check_writer:
  2 | 
  3 | Writing check50 checks
  4 | ======================
  5 | 
  6 | check50 checks live in a git repo on Github. check50 finds the git repo based on the slug that is passed to check50. For instance, consider the following execution of check50:
  7 | 
  8 | .. code-block:: bash
  9 | 
 10 |     check50 cs50/problems/2018/x/hello
 11 | 
 12 | check50 will look for an owner called `cs50`, a repo called `problems`, a branch called `2018` or `2018/x` and a problem called `x/hello` or `hello`. The slug is thus parsed like so:
 13 | 
 14 | .. code-block:: bash
 15 | 
 16 |     check50 <owner>/<repo>/<branch>/<problem>
 17 | 
 18 | Creating a git repo
 19 | *******************
 20 | 
 21 | To get you started, the first thing you need to do is |register_github|. Once you have done so, or if you already have an account with Github, |create_repo|. Make sure to think of a good name for your repo, as this is what students will be typing. Also make sure your repo is set to public, it is initialised with a `README`, and finally add a Python `.gitignore`. Ultimately you should have something looking like this:
 22 | 
 23 | .. |register_github| raw:: html
 24 | 
 25 |    <a href="https://github.com/join" target="_blank">register with Github</a>
 26 | 
 27 | .. |create_repo| raw:: html
 28 | 
 29 |    <a href="https://github.com/new" target="_blank">create a new git repo</a>
 30 | 
 31 | .. image:: repo.png
 32 | 
 33 | Creating a check and running it
 34 | *******************************
 35 | 
 36 | Your new repo should live at ``https://github.com/<user>/<repo>``, that is ``https://github.com/cs50/example_checks`` in our example. Once you have created your new repo, create a new file by clicking the `Create new file` button:
 37 | 
 38 | .. image:: new_file.png
 39 | 
 40 | Then continue by creating the following `.cs50.yaml` file. All indentation is done by 2 spaces, as per YAML syntax.
 41 | 
 42 | .. image:: new_yaml.png
 43 | 
 44 | Or in text, if you want to quickly copy-paste:
 45 | 
 46 | .. code-block:: yaml
 47 |     :linenos:
 48 |     :caption: **.cs50.yaml**
 49 | 
 50 |     check50:
 51 |       checks:
 52 |         hello world:
 53 |           - run: python3 hello.py
 54 |             stdout: Hello, world!
 55 |             exit: 0
 56 | 
 57 | Note that you should create a directory like in the example above by typing: `example/.cs50.yaml`. Once you have populated the file with the code above. Scroll down the page and hit the commit button:
 58 | 
 59 | .. image:: commit.png
 60 | 
 61 | That's it! You know have a repo that check50 can use to check whether a python file called `hello.py` prints ``Hello, world!`` and exits with a ``0`` as exit code. To try it, simply execute:
 62 | 
 63 | .. code-block:: bash
 64 | 
 65 |     check50 <owner>/<repo>/master/example --local
 66 | 
 67 | Where you substitute `<owner>` for your own username, `<repo>` for the repo you've just created. Given that a file called `hello.py` is in your current working directory, and it actually prints ``Hello, world!`` when run, you should now see the following:
 68 | 
 69 | .. code-block:: bash
 70 | 
 71 |     :) hello world
 72 | 
 73 | Simple YAML checks
 74 | ******************
 75 | 
 76 | To get you started, and to cover the basics of input/output checking, check50 lets you write simple checks in YAML syntax. Under the hood, check50 compiles these YAML checks to Python checks that check50 then runs.
 77 | 
 78 | YAML checks in check50 all live in `.cs50.yaml` and start with a top-level key called ``check50``, that specifies a ``checks``. The ``checks`` record contains all checks, where the name of the check is the name of the YAML record. Like so:
 79 | 
 80 | .. code-block:: yaml
 81 |     :linenos:
 82 |     :caption: **.cs50.yaml**
 83 | 
 84 |     check50:
 85 |       checks:
 86 |         hello world: # define a check named hello world
 87 |           # check code
 88 |         foo: # define a check named foo
 89 |           # check code
 90 |         bar: # define a check named bar
 91 |           # check code
 92 | 
 93 | 
 94 | This code snippet defines three checks, named ``hello world``, ``foo`` and ``bar`` respectively. These checks should contain a list of ``run`` records, that can each contain a combination of ``stdin``, ``stdout`` and ``exit``. See below:
 95 | 
 96 | .. code-block:: yaml
 97 |     :linenos:
 98 |     :caption: **.cs50.yaml**
 99 | 
100 |     check50:
101 |       checks:
102 |         hello world:
103 |           - run: python3 hello.py # run python3 hello.py
104 |             stdout: Hello, world! # expect Hello, world! in stdout
105 |             exit: 0 # expect program to exit with exitcode 0
106 |         foo:
107 |           - run: python3 foo.py # run python3 foo.py
108 |             stdin: baz # insert baz into stdin
109 |             stdout: baz # expect baz in stdout
110 |             exit: 0 # expect program to exit with exitcode 0
111 |         bar:
112 |           - run: python3 bar.py # run python3 bar.py
113 |             stdin: baz # insert baz into stdin
114 |             stdout: bar baz # expect bar baz in stdout
115 |           - run: python3 bar.py # run python3 bar.py
116 |             stdin:
117 |               - baz # insert baz into stdin
118 |               - qux # insert qux into stdin
119 |             stdout:
120 |               - bar baz # first expect bar baz in stdout
121 |               - bar qux # then expect bar qux in stdout
122 | 
123 | The code snippet above again defines three checks: `hello world`, `foo` and `bar`.
124 | 
125 | The ``hello world`` check runs ``python3 hello.py`` in the terminal, expects ``Hello, world!`` to be outputted in stdout, and then expects the program to exit with exitcode ``0``.
126 | 
127 | The ``foo`` check runs ``python3 foo.py`` in the terminal, inserts ``baz`` into stdin, expects ``baz`` to be outputted in stdout, and finally expects the program to exit with exitcode ``0``.
128 | 
129 | The ``bar`` check runs two commands in order in the terminal. First ``python3 bar.py`` gets run, ``baz`` gets put in stdin and ``bar baz`` is expected in stdout. There is no mention of ``exit`` here, so the exitcode is not checked. Secondly, ``python3 bar.py`` gets run, ``baz`` and ``qux`` get put into stdin, and first ``bar baz`` is expected in stdout, then ``bar qux``.
130 | 
131 | We encourage you to play around with the example above by copying its code to your checks git repo. Then try to write a `bar.py` and `foo.py` that make you pass these tests.
132 | 
133 | In case you want to check for multiline input, you can make use of YAML's ``|`` operator like so:
134 | 
135 | .. code-block:: yaml
136 |     :linenos:
137 |     :caption: **.cs50.yaml**
138 | 
139 |     check50:
140 |       checks:
141 |         multiline hello world:
142 |           - run: python3 multi_hello.py
143 |             stdout: | # expect Hello\nWorld!\n in stdout
144 |               Hello
145 |               World!
146 |             exit: 0
147 | 
148 | Developing locally
149 | ******************
150 | 
151 | To write checks on your own machine, rather than on the Github webpage, you can clone the repo via:
152 | 
153 | .. code-block:: bash
154 | 
155 |     git clone https://github.com/<owner>/<repo>
156 | 
157 | Where ``<owner>`` is your Github username, and ``<repo>`` is the name of your checks repository. Head on over to the new directory that git just created, and open up `.cs50.yaml` with your favorite editor.
158 | 
159 | To run the cloned checks locally, check50 comes with a ``--dev`` mode. That will let you target a local checks repo, rather than a github repo. So if your checks live in ``/Users/cs50/Documents/example_checks``, you would execute check50 like so:
160 | 
161 | .. code-block:: bash
162 | 
163 |     check50 --dev /Users/cs50/Documents/example_checks/example
164 | 
165 | This runs the `example` check from ``/Users/cs50/Documents/example_checks``. You can also specify a relative path, so if your current working directory is ``/Users/cs50/Documents/solutions``, you can execute check50 like so:
166 | 
167 | .. code-block:: bash
168 | 
169 |     check50 --dev ../example_checks/example
170 | 
171 | Now you're all set to develop new checks locally. Just remember to ``git add``, ``git commit`` and ``git push`` when you're done writing checks. Quick refresher:
172 | 
173 | .. code-block:: bash
174 | 
175 |     git add .cs50.yaml
176 |     git commit -m "wrote some awesome new checks!"
177 |     git push
178 | 
179 | Getting started with Python checks
180 | **********************************
181 | 
182 | If you need a little more than strict input / output testing, check50 lets you write checks in Python. A good starting point is the result of the compilation of the YAML checks. To get these, please make sure you have cloned the repo (via ``git clone`` ), and thus have the checks locally. First we need to run the .YAML checks once, so that check50 compiles the checks to Python. To do this execute:
183 | 
184 | .. code-block:: bash
185 | 
186 |     check50 --dev <checks_dir>/<check>
187 | 
188 | Where ``<checks_dir>`` is the local git repo of your checks, and ``<check>`` is the directory in which ``.cs50.yaml`` lives. Alternatively you could navigate to this directory and simply call:
189 | 
190 | .. code-block:: bash
191 | 
192 |     check50 --dev .
193 | 
194 | As a result you should now find a file called ``__init__.py`` in the check directory. This is the result of check50's compilation from YAML to Python. For instance, if your ``.cs50.yaml`` contains the following:
195 | 
196 | .. code-block:: yaml
197 |     :linenos:
198 |     :caption: **.cs50.yaml**
199 | 
200 |     check50:
201 |       checks:
202 |         hello world:
203 |           - run: python3 hello.py
204 |             stdout: Hello, world!
205 |             exit: 0
206 | 
207 | You should now find the following ``__init__.py``:
208 | 
209 | .. code-block:: python
210 |     :linenos:
211 |     :caption: **__init__.py**
212 | 
213 |     import check50
214 | 
215 |     @check50.check()
216 |     def hello_world():
217 |         """hello world"""
218 |         check50.run("python3 hello.py").stdout("Hello, world!", regex=False).exit(0)
219 | 
220 | check50 will by default ignore and overwrite what is in ``__init__.py`` for as long as there are checks in ``.cs50.yaml``. To change this you have to edit ``.cs50.yaml`` to:
221 | 
222 | .. code-block:: yaml
223 |     :caption: **.cs50.yaml**
224 | 
225 |     check50: true
226 | 
227 | If the ``checks`` key is not specified (as is the case above), ``check50`` will look for Python checks written in a file called ``__init__.py``. If you would like to write the Python checks in a file called ``foo.py`` instead, you could specify it like so:
228 | 
229 | .. code-block:: yaml
230 |     :caption: **.cs50.yaml**
231 | 
232 |     check50:
233 |         checks: foo.py
234 | 
235 | 
236 | To test whether everything is still in working order, run check50 again with:
237 | 
238 | .. code-block:: bash
239 | 
240 |     check50 --dev <checks_dir>/<check>
241 | 
242 | You should see the same results as the YAML checks gave you. Now that there are no YAML checks in ``.cs50.yaml`` and check50 knows where to look for Python checks, you can start writing Python checks. You can find documentation in :ref:`api`, and examples of Python checks below.
243 | 
244 | Python check specification
245 | **************************
246 | 
247 | A Python check is made up as follows:
248 | 
249 | .. code-block:: Python
250 |     :linenos:
251 |     :caption: **__init__.py**
252 | 
253 |     import check50 # import the check50 module
254 | 
255 |     @check50.check() # tag the function below as check50 check
256 |     def exists(): # the name of the check
257 |         """description""" # this is what you will see when running check50
258 |         check50.exists("hello.py") # the actual check
259 | 
260 |     @check50.check(exists) # only run this check if the exists check has passed
261 |     def prints_hello():
262 |         """prints "hello, world\\n" """
263 |         check50.run("python3 hello.py").stdout("[Hh]ello, world!?\n", regex=True).exit(0)
264 | 
265 | check50 uses its check decorator to tag functions as checks. You can pass another check as argument to specify a dependency. Docstrings are used as check descriptions, this is what will ultimately be shown when running check50. The checks themselves are just Python code. check50 comes with a simple API to run programs, send input to stdin, and check or retrieve output from stdout. A check fails if a ``check50.Failure`` exception or an exception inheriting from ``check50.Failure`` like ``check50.Mismatch`` is thrown. This allows you to write your own custom check code like so:
266 | 
267 | .. code-block:: Python
268 |     :linenos:
269 |     :caption: **__init__.py**
270 | 
271 |     import check50
272 | 
273 |     @check50.check()
274 |     def prints_hello():
275 |         """prints "hello, world\\n" """
276 |         from re import match
277 | 
278 |         expected = "[Hh]ello, world!?\n"
279 |         actual = check50.run("python3 hello.py").stdout()
280 |         if not match(expected, actual):
281 |             help = None
282 |             if match(expected[:-1], actual):
283 |                 help = r"did you forget a newline ('\n') at the end of your printf string?"
284 |             raise check50.Mismatch("hello, world\n", actual, help=help)
285 | 
286 | The above check breaks out of check50's API by calling ``stdout()`` on line 9 with no args, effectively retrieving all output from stdout in a string. Then there is some plain Python code, matching the output through Python's builtin regex module ``re`` against a regular expression with the expected outcome. If it doesn't match, a help message is provided only if there is a newline missing at the end. This help message is provided through an optional argument ``help`` passed to check50's ``Mismatch`` exception.
287 | 
288 | You can share state between checks if you make them dependent on each other. By default file state is shared, allowing you to for instance test compilation in one check, and then depend on the result of the compilation in dependent checks.
289 | 
290 | .. code-block:: Python
291 |     :linenos:
292 |     :caption: **__init__.py**
293 | 
294 |     import check50
295 |     import check50.c
296 | 
297 |     @check50.check()
298 |     def compiles():
299 |         """hello.c compiles"""
300 |         check50.c.compile("hello.c")
301 | 
302 |     @check50.check(compiles)
303 |     def prints_hello():
304 |         """prints "hello, world\\n" """
305 |         check50.run("./hello").stdout("[Hh]ello, world!?\n", regex=True).exit(0)
306 | 
307 | You can also share Python state between checks by returning what you want to share from a check. It's dependent can accept this by accepting an additional argument.
308 | 
309 | .. code-block:: Python
310 |     :linenos:
311 |     :caption: **__init__.py**
312 | 
313 |     import check50
314 | 
315 |     @check50.check()
316 |     def foo():
317 |         return 1
318 | 
319 |     @check50.check(foo)
320 |     def bar(state)
321 |         print(state) # prints 1
322 | 
323 | Python check examples
324 | *********************
325 | 
326 | Below you will find examples of Python checks. Don't forget to |cs50_checks| for more examples. You can try them yourself by copying them to ``__init__.py`` and running:
327 | 
328 | .. |cs50_checks| raw:: html
329 | 
330 |    <a href="https://github.com/cs50/problems" target="_blank">checkout CS50's own checks</a>
331 | 
332 | .. code-block:: bash
333 | 
334 |     check50 --dev <checks_dir>/<check>
335 | 
336 | Check whether a file exists:
337 | 
338 | .. code-block:: python
339 |     :linenos:
340 |     :caption: **__init__.py**
341 | 
342 |     import check50
343 | 
344 |     @check50.check()
345 |     def exists():
346 |         """hello.py exists"""
347 |         check50.exists("hello.py")
348 | 
349 | Check stdout for an exact string:
350 | 
351 | .. code-block:: python
352 |     :linenos:
353 |     :caption: **__init__.py**
354 | 
355 |     @check50.check(exists)
356 |     def prints_hello_world():
357 |         """prints Hello, world!"""
358 |         check50.run("python3 hello.py").stdout("Hello, world!", regex=False).exit(0)
359 | 
360 | Check stdout for a rough match:
361 | 
362 | .. code-block:: python
363 |     :linenos:
364 |     :caption: **__init__.py**
365 | 
366 |     @check50.check(exists)
367 |     def prints_hello():
368 |         """prints "hello, world\\n" """
369 |         # regex=True by default :)
370 |         check50.run("python3 hello.py").stdout("[Hh]ello, world!?\n").exit(0)
371 | 
372 | Put something in stdin, expect it in stdout:
373 | 
374 | .. code-block:: python
375 |     :linenos:
376 |     :caption: **__init__.py**
377 | 
378 |     import check50
379 | 
380 |     @check50.check()
381 |     def id():
382 |         """id.py prints what you give it"""
383 |         check50.run("python3 hello.py").stdin("foo").stdout("foo").stdin("bar").stdout("bar")
384 | 
385 | Be helpful, check for common mistakes:
386 | 
387 | .. code-block:: python
388 |     :linenos:
389 |     :caption: **__init__.py**
390 | 
391 |     import check50
392 |     import re
393 | 
394 |     def coins(num):
395 |         # regex that matches `num` not surrounded by any other numbers
396 |         # (so coins(2) won't match e.g. 123)
397 |         return fr"(?<!\d){num}(?!\d)"
398 | 
399 |     @check50.check()
400 |     def test420():
401 |         """input of 4.2 yields output of 18"""
402 |         expected = "18\n"
403 |         actual = check50.run("python3 cash.py").stdin("4.2").stdout()
404 |         if not re.search(coins(18), actual):
405 |             help = None
406 |             if re.search(coins(22), actual):
407 |                 help = "did you forget to round your input to the nearest cent?"
408 |             raise check50.Mismatch(expected, actual, help=help)
409 | 
410 | Create your own assertions:
411 | 
412 | .. code-block:: python
413 |     :linenos:
414 |     :caption: **__init__.py**
415 | 
416 |     import check50
417 | 
418 |     @check50.check()
419 |     def atleast_one_match()
420 |         """matches either foo, bar or baz"""
421 |         output = check50.run("python3 qux.py").stdout()
422 |         if not any(answer in output for answer in ["foo", "bar", "baz"]):
423 |             raise check50.Failure("no match found")
424 | 
425 | 
426 | Configuring check50
427 | *******************
428 | 
429 | Check50, and other CS50 tools like submit50 and lab50, use a special configuration file called ``.cs50.yaml``. Here is how you can configure check50 via ``.cs50.yaml``.
430 | 
431 | *******
432 | checks:
433 | *******
434 | 
435 | ``checks:`` takes a filename specifying a file containing check50 Python checks, or a record of check50 YAML checks. If not specified, it will default to ``__init__.py``.
436 | 
437 | .. code-block:: YAML
438 |     :linenos:
439 |     :caption: **.cs50.yaml**
440 | 
441 |     check50: 
442 |       checks: checks.py
443 | 
444 | Only specifies that this is a valid slug for check50. This configuration will allow you to run ``check50 <slug>``, by default ``check50`` will look for an ``__init__.py`` containing Python checks.
445 | 
446 | .. code-block:: YAML
447 |     :linenos:
448 |     :caption: **.cs50.yaml**
449 | 
450 |     check50:
451 |       checks: "my_filename.py"
452 | 
453 | Specifies that this is a valid slug for check50, and has check50 look for ``my_filename.py`` instead of ``__init__.py``.
454 | 
455 | .. code-block:: YAML
456 |     :linenos:
457 |     :caption: **.cs50.yaml**
458 | 
459 |     check50:
460 |       checks:
461 |         hello world:
462 |           - run: python3 hello.py
463 |             stdout: Hello, world!
464 |             exit: 0
465 | 
466 | Specifies that this is a valid slug for check50, and has check50 compile and run the YAML check. For more on YAML checks in check50 see :ref:``check_writer``.
467 | 
468 | 
469 | ******
470 | files:
471 | ******
472 | 
473 | ``files:`` takes a list of files/patterns. Every item in the list must be tagged by either ``!include``, ``!exclude`` or ``!require``. All files matching a pattern tagged with ``!include`` are included and likewise for ``!exclude``. ``!require`` is similar to ``!include``, however it does not accept patterns, only filenames, and will cause ``check50`` to display an error if that file is missing. The list that is given to ``files:`` is processed top to bottom. Later items in ``files:`` win out over earlier items.
474 | 
475 | The patterns that ``!include`` and ``!exclude`` accept are globbed, any matching files are added. check50 introduces one exception for convenience, similarly to how git treats .gitignore: If and only if a pattern does not contain a ``/``, and starts with a ``*``, it is considered recursive in such a way that ``*.o`` will exclude all files in any directory ending with ``.o``. This special casing is just for convenience. Alternatively you could write ``**/*.o`` that is functionally identical to ``*.o``, or write ``./*.o`` if you only want to exclude files ending with ``.o`` from the top-level directory.
476 | 
477 | .. code-block:: YAML
478 |     :linenos:
479 |     :caption: **.cs50.yaml**
480 | 
481 |     check50:
482 |       files:
483 |         - !exclude "*.pyc"
484 | 
485 | Excludes all files ending with ``.pyc``.
486 | 
487 | .. code-block:: YAML
488 |     :linenos:
489 |     :caption: **.cs50.yaml**
490 | 
491 |     check50:
492 |       files:
493 |         - !exclude "*"
494 |         - !include "*.py"
495 | 
496 | Exclude all files, but include all files ending with ``.py``. Note that order is important here, if you would inverse the two lines it would read: include all files ending with ``.py``, exclude everything. Effectively excluding everything!
497 | 
498 | .. code-block:: YAML
499 |     :linenos:
500 |     :caption: **.cs50.yaml**
501 | 
502 |     check50:
503 |       files:
504 |         - !exclude "*"
505 |         - !include "source/"
506 | 
507 | Exclude all files, but include all files in the source directory.
508 | 
509 | .. code-block:: YAML
510 |     :linenos:
511 |     :caption: **.cs50.yaml**
512 | 
513 |     check50:
514 |       files:
515 |         - !exclude "build/"
516 |         - !exclude "docs/"
517 | 
518 | Include everything, but exclude everything in the build and docs directories.
519 | 
520 | .. code-block:: YAML
521 |     :linenos:
522 |     :caption: **.cs50.yaml**
523 | 
524 |     check50:
525 |       files:
526 |         - !exclude "*"
527 |         - !include "source/"
528 |         - !exclude "*.pyc"
529 | 
530 | Exclude everything, include everything from the source directory, but exclude all files ending with ``.pyc``.
531 | 
532 | .. code-block:: YAML
533 |     :linenos:
534 |     :caption: **.cs50.yaml**
535 | 
536 |     check50:
537 |       files:
538 |         - !exclude "source/**/*.pyc"
539 | 
540 | Include everything, but any files ending on ``.pyc`` within the source directory. The ``**`` here pattern matches any directory.
541 | 
542 | .. code-block:: YAML
543 |     :linenos:
544 |     :caption: **.cs50.yaml**
545 | 
546 |     check50:
547 |       files:
548 |         - !require "foo.py"
549 |         - !require "bar.c"
550 | 
551 | Require that both foo.py and bar.c are present and include them.
552 | 
553 | .. code-block:: YAML
554 |     :linenos:
555 |     :caption: **.cs50.yaml**
556 | 
557 |     check50:
558 |       files:
559 |         - !exclude "*"
560 |         - !include "*.py"
561 |         - !require "foo.py"
562 |         - !require "bar.c"
563 | 
564 | Exclude everything, include all files ending with ``.py`` and require (and include) both foo.py and bar.c. It is generally recommended to place any ``!require``d files at the end of the ``files:``, this ensures they are always included. 
565 | 
566 | *************
567 | dependencies:
568 | *************
569 | 
570 | ``dependencies:`` is a list of ``pip`` installable dependencies that check50 will install.
571 | 
572 | .. code-block:: YAML
573 |     :linenos:
574 |     :caption: **.cs50.yaml**
575 | 
576 |     check50:
577 |       dependencies:
578 |         - pyyaml
579 |         - flask
580 | 
581 | Has check50 install both ``pyyaml`` and ``flask`` via ``pip``.
582 | 
583 | .. code-block:: YAML
584 |     :linenos:
585 |     :caption: **.cs50.yaml**
586 | 
587 |     check50:
588 |       dependencies:
589 |         - git+https://github.com/cs50/submit50#egg=submit50
590 | 
591 | Has check50 ``pip install`` submit50 from GitHub, especially useful for projects that are not hosted on PyPi. See https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support for more info on installing from a VCS.
592 | 
593 | 
594 | Internationalizing checks
595 | *************************
596 | TODO
597 | 


--------------------------------------------------------------------------------
/docs/source/commit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/commit.png


--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import sys
 3 | import time
 4 | 
 5 | _tool = "check50"
 6 | 
 7 | # Add path to module for autodoc
 8 | sys.path.insert(0, os.path.abspath(f'../../{_tool}'))
 9 | 
10 | # Include __init__ in autodoc
11 | # Source: https://stackoverflow.com/questions/5599254/how-to-use-sphinxs-autodoc-to-document-a-classs-init-self-method
12 | def skip(app, what, name, obj, would_skip, options):
13 |     if name == "__init__":
14 |         return False
15 |     return would_skip
16 | 
17 | def setup(app):
18 |     app.connect("autodoc-skip-member", skip)
19 | 
20 | extensions = ['sphinx.ext.autodoc']
21 | 
22 | html_css_files = ["https://cs50.readthedocs.io/_static/custom.css?" + str(round(time.time()))]
23 | html_js_files = ["https://cs50.readthedocs.io/_static/custom.js?" + str(round(time.time()))]
24 | html_theme = "sphinx_rtd_theme"
25 | html_theme_options = {
26 |     "display_version": False,
27 |     "prev_next_buttons_location": False,
28 |     "sticky_navigation": False
29 | }
30 | html_title = f'{_tool} Docs'
31 | 
32 | project = f'{_tool}'
33 | 


--------------------------------------------------------------------------------
/docs/source/extension_writer.rst:
--------------------------------------------------------------------------------
 1 | .. _extension_writer:
 2 | 
 3 | Writing check50 extensions
 4 | ==========================
 5 | 
 6 | Core to check50's design is extensibility. Not only in checks, but also in the tool itself. We designed :code:`check50.c`, :code:`check50.py` and :code:`check50.flask` to be extensions of check50 that ship with check50. By design these three modules are all standalone and only depend on the core of check50, and no other part of check50 depends on them. In other words, you can remove or add any of these modules and check50 would still function as expected.
 7 | 
 8 | We ship check50 with three extensions because these extensions are core to the material cs50 teaches. But different courses have different needs, and we realize we cannot predict and cater for every usecase. This is why check50 comes with the ability to `pip install` other Python packages. You can configure this via the :code:`dependencies` key in :code:`.cs50.yaml`. Through this mechanism you can write your own extension and then have check50 install it for you whenever check50 runs. Host your extension anywhere `pip` can install from, for instance GitHub or PyPi. And all you have to do then is to fill in the :code:`dependencies` key of :code:`.cs50.yaml` with the location of your extension. check50 will make sure your extension is always there when the checks are run.
 9 | 
10 | 
11 | check50.internal
12 | *******************************
13 | 
14 | In addition to all the functionality check50 exposes, we expose an extra API for extensions in :code:`check50.internal`. You can find the documentation in :ref:`api`.
15 | 
16 | 
17 | Example: a JavaScript extension
18 | *******************************
19 | Out of the box check50 does not ship with any JavaScript specific functionality. You can use check50's generic API and run a :code:`.js` file through an interpreter such as :code:`node`: :code:`check50.run('node <student_file.js>')`. But we realize most JavaScript classes are not about writing command-line scripts, and we do need a way to call functions. This is why we wrote a small javascript extension for check50 dubbed check50_js at
20 | https://github.com/cs50/check50/tree/sample-extension.
21 | 
22 | 
23 | *******************************
24 | check50_js
25 | *******************************
26 | The challenge in writing this extension is that check50 itself is written in Python, so we need an interface between the two languages. This could be as simple as an intermediate JavaScript script that runs the students function and then outputs the results to stdout for check50 to read. But this approach does create indirection and creates quite some clutter in the checks codebase. Luckily, in case of Python and JavaScript (and PHP and Perl) we have access to a Python package called :code:`python-bond`. This package lets us "bond" two languages together, and lets you evaluate code in another language's interpreter. Effectively creating a channel through which you can evaluate code and call functions from the other language. This is what we ended up doing for our JavaScript extension.
27 | 
28 | You can find example checks using check50_js and their solutions at:
29 | 
30 | * **hello.js**: `checks <https://github.com/cs50/check50/tree/examples/js/hello>`__ `solution <https://github.com/cs50/check50/tree/examples/solutions/hello_js/hello.js>`__
31 | * **line.js**: `checks <https://github.com/cs50/check50/tree/examples/js/line>`__ `solution <https://github.com/cs50/check50/tree/examples/solutions/line_js/line.js>`__
32 | * **addition.js**: `checks <https://github.com/cs50/check50/tree/examples/js/addition>`__ `solution <https://github.com/cs50/check50/tree/examples/solutions/addition_js/addition.js>`__
33 | 
34 | 
35 | To try any of these examples for yourself, simply run:
36 | 
37 | * **hello.js**:
38 | 
39 | .. code-block:: bash
40 | 
41 |   wget https://raw.githubusercontent.com/cs50/check50/examples/solutions/hello_js/hello.js
42 |   check50 cs50/check50/examples/js/hello
43 | 
44 | * **line.js**:
45 | 
46 | .. code-block:: bash
47 | 
48 |   wget https://raw.githubusercontent.com/cs50/check50/examples/solutions/line_js/line.js
49 |   check50 cs50/check50/examples/js/line
50 | 
51 | * **addition.js**:
52 | 
53 | .. code-block:: bash
54 | 
55 |   wget https://raw.githubusercontent.com/cs50/check50/examples/solutions/addition_js/addition.js
56 |   check50 cs50/check50/examples/js/addition
57 | 


--------------------------------------------------------------------------------
/docs/source/html_output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/html_output.png


--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
  1 | ``check50``
  2 | ===========
  3 | 
  4 | .. toctree::
  5 |    :hidden:
  6 |    :maxdepth: 3
  7 |    :caption: Contents:
  8 | 
  9 |    check50_user.rst
 10 |    api.rst
 11 |    json_specification.rst
 12 |    check_writer.rst
 13 |    extension_writer.rst
 14 | 
 15 | .. Indices and tables
 16 | .. ==================
 17 | 
 18 | .. * :ref:`genindex`
 19 | .. * :ref:`api`
 20 | .. * :ref:`modindex`
 21 | .. * :ref:`search`
 22 | 
 23 | check50 is a tool for checking student code. As a student you can use check50 to check your CS50 problem sets or any other Problem sets for which check50 checks exist. check50 allows teachers to automatically grade code on correctness and to provide automatic feedback while students are coding.
 24 | 
 25 | Installation
 26 | ************
 27 | 
 28 | First make sure you have Python 3.6 or higher installed. You can download Python |download_python|.
 29 | 
 30 | .. |download_python| raw:: html
 31 | 
 32 |    <a href="https://www.python.org/downloads/" target="_blank">here</a>
 33 | 
 34 | check50 has a dependency on git, please make sure to |install_git| if git is not already installed.
 35 | 
 36 | .. |install_git| raw:: html
 37 | 
 38 |    <a href="https://git-scm.com/book/en/v2/Getting-Started-Installing-Git" target="_blank">install git</a>
 39 | 
 40 | To install check50 under Linux / OS X:
 41 | 
 42 | .. code-block:: bash
 43 | 
 44 |     pip install check50
 45 | 
 46 | Under Windows, please |install_windows_sub|. Then install check50 within the subsystem.
 47 | 
 48 | .. |install_windows_sub| raw:: html
 49 | 
 50 |    <a href="https://docs.microsoft.com/en-us/windows/wsl/install-win10" target="_blank">install the Linux subsystem</a>
 51 | 
 52 | Usage
 53 | *******
 54 | 
 55 | To use check50 to check a problem, execute check50 like so:
 56 | 
 57 | .. code-block:: bash
 58 | 
 59 |     check50 <owner>/<repo>/<branch>/<check>
 60 | 
 61 | For instance, if you want to check |2018x_caesar| you call:
 62 | 
 63 | .. |2018x_caesar| raw:: html
 64 | 
 65 |    <a href="https://github.com/cs50/problems/tree/2018/x/caesar" target="_blank">CS50's Caesar problem from edX 2018</a>
 66 | 
 67 | .. code-block:: bash
 68 | 
 69 |     check50 cs50/problems/2018/x/caesar
 70 | 
 71 | You can choose to run checks locally by passing the ``--local`` flag like so:
 72 | 
 73 | .. code-block:: bash
 74 | 
 75 |     check50 --local <owner>/<repo>/<branch>/<check>
 76 | 
 77 | For an overview of all flags run:
 78 | 
 79 | .. code-block:: bash
 80 | 
 81 |     check50 --help
 82 | 
 83 | Design
 84 | *******
 85 | 
 86 | * **Write checks for code in code** check50 uses pure Python for checks and exposes a small Python api for common functionality.
 87 | * **Extensibility in checks** Anyone can add checks to check50 without asking for permission. In fact, here is a tutorial to get you started: :ref:`check_writer`
 88 | * **Extensibility in the tool itself** We cannot predict everything you need, nor can we cater for every use-case out of the box. This is why check50 provides you with a mechanism for adding your own code to the tool, once more without asking for permission. This lets you support different programming languages and add new functionality. Jump to :ref:`extension_writer` to learn more.
 89 | * **PaaS** check50 can run online. This guarantees a consistent environment and lets you check code for correctness without introducing your own hardware.
 90 | 
 91 | Checks
 92 | *******
 93 | In check50 the checks are decoupled from the tool. You can find CS50's set of checks for CS50 problem sets at |cs50_checks|. If you would like to develop your own set of checks such that you can use check50 in your own course jump to  :ref:`check_writer`.
 94 | 
 95 | .. |cs50_checks| raw:: html
 96 | 
 97 |    <a href="https://github.com/cs50/problems" target="_blank">/cs50/problems</a>
 98 | 
 99 | Under the hood, checks are naked Python functions decorated with the ``@check50.check`` decorator. check50 exposes several functions, see :ref:`api`, that allow you to easily write checks for input/output testing. check50 comes with three builtin extensions: ``c``, ``py`` and ``flask``. These extensions add extra functionality for C, Python and Python's Flask framework to check50's core.
100 | 
101 | By design check50 is extensible. If you want to add support for other programming languages / frameworks and you are comfortable with Python please check out :ref:`extension_writer`.
102 | 


--------------------------------------------------------------------------------
/docs/source/json_specification.rst:
--------------------------------------------------------------------------------
  1 | .. _json_specification:
  2 | 
  3 | JSON specification
  4 | ==========================
  5 | Check50 can create a machine readable output in the form of `json`. For instance, running check50 with `-o json` on a non compiling implementation of one of our problems called caesar:
  6 | 
  7 | .. code-block:: bash
  8 | 
  9 |     check50 cs50/problems/2018/x/caesar -o json
 10 | 
 11 | Produces the following:
 12 | 
 13 | .. code-block:: json
 14 | 
 15 |     {
 16 |         "slug": "cs50/problems/2018/x/caesar",
 17 |         "results": [
 18 |             {
 19 |                 "name": "exists",
 20 |                 "description": "caesar.c exists.",
 21 |                 "passed": true,
 22 |                 "log": [
 23 |                     "checking that caesar.c exists..."
 24 |                 ],
 25 |                 "cause": null,
 26 |                 "data": {},
 27 |                 "dependency": null
 28 |             },
 29 |             {
 30 |                 "name": "compiles",
 31 |                 "description": "caesar.c compiles.",
 32 |                 "passed": false,
 33 |                 "log": [
 34 |                     "running clang caesar.c -o caesar -std=c11 -ggdb -lm -lcs50...",
 35 |                     "caesar.c:24:5: warning: implicit declaration of function 'f' is invalid in C99",
 36 |                     "      [-Wimplicit-function-declaration]",
 37 |                     "    f (argc != 2)",
 38 |                     "    ^",
 39 |                     "caesar.c:24:18: error: expected ';' after expression",
 40 |                     "    f (argc != 2)",
 41 |                     "                 ^",
 42 |                     "                 ;",
 43 |                     "1 warning and 1 error generated."
 44 |                 ],
 45 |                 "cause": {
 46 |                     "rationale": "code failed to compile",
 47 |                     "help": null
 48 |                 },
 49 |                 "data": {},
 50 |                 "dependency": "exists"
 51 |             },
 52 |             {
 53 |                 "name": "encrypts_a_as_b",
 54 |                 "description": "encrypts \"a\" as \"b\" using 1 as key",
 55 |                 "passed": null,
 56 |                 "log": [],
 57 |                 "cause": {
 58 |                     "rationale": "can't check until a frown turns upside down"
 59 |                 },
 60 |                 "data": {},
 61 |                 "dependency": "compiles"
 62 |             }
 63 |         ],
 64 |         "version": "3.0.0"
 65 |     }
 66 | 
 67 | Top level
 68 | *********
 69 | Assuming `check50` is able to run successfully, you will find three keys at the top level of the json output: `slug`, `results` and `version`.
 70 | 
 71 | * **slug** (`string`) is the slug with which check50 was run, `cs50/problems/2018/x/caesar` in the above example.
 72 | * **results** (`[object]`) is a list containing the results of each run check. More on this key below.
 73 | * **version** (`string`) is the version of check50 used to run the checks.
 74 | 
 75 | If check50 encounters an error while running, e.g. due to an invalid slug, the `results` key will be replaced by an `error` key containing information about the error encountered.
 76 | 
 77 | results
 78 | *******
 79 | If the results key exists (that is, check50 was able to run the checks successfully), it will contain a list of objects each corresponding to a check. The order of these objects corresponds to the order the checks appear in the file in which they were written. Each object will contain the following fields:
 80 | 
 81 | * **name** (`string`) is the unique name of the check (the literal name of the Python function specifying the check).
 82 | * **description** (`string`) is a description of the check.
 83 | * **passed** (`bool`, nullable) is `true` if the check passed, `false` if the check failed, or `null` if the check was skipped (either because the check's dependency did not pass or because the check threw some unexpected error).
 84 | * **log** (`[string]`) contains the log accrewed during the execution of the check. Each element of the list is a line from the log.
 85 | * **cause** (`object`, nullable) contains the reason that a check did not pass. If `passed` is `true`, `cause` will be `null` and `cause` will never be `null` if `passed` is not `true`. More detail about keys that may appear within `cause` below.
 86 | * **data** (`object`) contains arbitrary data communicated by the check via the `check50.data` API call. Checks could use this to add additional information such as memory usage to the results, but check50 itself does not add anything to `data` by default.
 87 | * **dependency** (`string`, nullable) is the name of the check upon which this check depends, or `null` if the check has no dependency.
 88 | 
 89 | *****
 90 | cause
 91 | *****
 92 | The cause key is `null` if the check passed and non-null. This key is by design an open-ended object. Everything in the `.payload` attribute of a `check50.Failure` will be put in the `cause` key. Through this mechanism you can communicate any information you want from a failing check to the results. Depending on what occurred, check50 adds the following keys to `cause`:
 93 | 
 94 | * **rationale** (`string`) is a stduent-facing explanation of why the check did not pass (e.g. the student's program did not output what was expected).
 95 | * **help** (`string`) is an additional help message that may appear alongside the rationale giving additional context.
 96 | * **expected** (`string`) and **actual** (`string`) are keys that always appear in a pair. In case you are expecting X as output, but Y was found instead, you will find these keys containing X and Y in the `cause` field. These appear when a check raises a `check50.Mismatch` exception.
 97 | * **error** (`object`) appears in `cause` when an unexpected error occurred during a check. It will contain the keys `type`, `value`, `traceback` and `data` with the same properties as in the top-level `error` key described below.
 98 | 
 99 | 
100 | 
101 | error
102 | *****
103 | If check50 encounters an unexpected error, the `error` key will replace the `results` key in the JSON output. It will contain the following keys:
104 | 
105 | * **type** (`string`) contains the type name of the thrown exception.
106 | * **value** (`string`) contains the result of converting the exception to a string.
107 | * **traceback** (`[string]`) contains the stack trace of the thrown exception.
108 | * **data** (`object`) contains any additional data the exception may carry in its `payload` attribute.
109 | 
110 | 


--------------------------------------------------------------------------------
/docs/source/new_file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/new_file.png


--------------------------------------------------------------------------------
/docs/source/new_yaml.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/new_yaml.png


--------------------------------------------------------------------------------
/docs/source/repo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/docs/source/repo.png


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
 1 | [compile_catalog]
 2 | domain = check50
 3 | directory = check50/locale/
 4 | 
 5 | [extract_messages]
 6 | keywords = _ gettext ngettext
 7 | width = 100
 8 | output_file = check50/locale/check50.pot
 9 | 
10 | [update_catalog]
11 | input_file = check50/locale/check50.pot
12 | domain = check50
13 | output_dir = check50/locale/
14 | 
15 | [init_catalog]
16 | input_file = check50/locale/check50.pot
17 | domain = check50
18 | output_dir = check50/locale
19 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | if __import__("os").name == "nt":
 2 |     raise RuntimeError("check50 does not support Windows directly. Instead, you should install the Windows Subsystem for Linux (https://docs.microsoft.com/en-us/windows/wsl/install-win10) and then install check50 within that.")
 3 | 
 4 | from setuptools import setup
 5 | 
 6 | setup(
 7 |     author="CS50",
 8 |     author_email="sysadmins@cs50.harvard.edu",
 9 |     classifiers=[
10 |         "Intended Audience :: Education",
11 |         "Programming Language :: Python :: 3",
12 |         "Topic :: Education",
13 |         "Topic :: Utilities"
14 |     ],
15 |     description="This is check50, with which you can check solutions to problems for CS50.",
16 |     long_description=open("README.md").read(),
17 |     license="GPLv3",
18 |     message_extractors = {
19 |         'check50': [('**.py', 'python', None),],
20 |     },
21 |     install_requires=["attrs>=18", "beautifulsoup4>=0", "lib50>=3,<4", "packaging", "pexpect>=4.6", "pyyaml>6,<7", "requests>=2.19", "setuptools", "termcolor>=1.1", "jinja2>=2.10"],
22 |     extras_require = {
23 |         "develop": ["sphinx", "sphinx-autobuild", "sphinx_rtd_theme"]
24 |     },
25 |     keywords=["check", "check50"],
26 |     name="check50",
27 |     packages=["check50", "check50.renderer"],
28 |     python_requires=">= 3.6",
29 |     entry_points={
30 |         "console_scripts": ["check50=check50.__main__:main"]
31 |     },
32 |     url="https://github.com/cs50/check50",
33 |     version="3.3.11",
34 |     include_package_data=True
35 | )
36 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/tests/__init__.py


--------------------------------------------------------------------------------
/tests/__main__.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | import sys
3 | 
4 | from . import *
5 | 
6 | suite = unittest.TestLoader().discover("tests", pattern="*_tests.py")
7 | result = unittest.TextTestRunner(verbosity=2).run(suite)
8 | sys.exit(bool(result.errors or result.failures))
9 | 


--------------------------------------------------------------------------------
/tests/api_tests.py:
--------------------------------------------------------------------------------
  1 | import unittest
  2 | import os
  3 | import pathlib
  4 | import shutil
  5 | import sys
  6 | import tempfile
  7 | 
  8 | import check50
  9 | import check50.internal
 10 | 
 11 | 
 12 | class Base(unittest.TestCase):
 13 |     def setUp(self):
 14 |         self.working_directory = tempfile.TemporaryDirectory()
 15 |         os.chdir(self.working_directory.name)
 16 | 
 17 |         self.filename = "foo.py"
 18 |         self.write("")
 19 | 
 20 |         self.process = None
 21 | 
 22 |     def tearDown(self):
 23 |         if self.process and self.process.process.isalive():
 24 |             self.process.kill()
 25 |         self.working_directory.cleanup()
 26 | 
 27 |     def write(self, source):
 28 |         with open(self.filename, "w") as f:
 29 |             f.write(source)
 30 | 
 31 |     def runpy(self):
 32 |         self.process = check50.run(f"python3 ./{self.filename}")
 33 | 
 34 | class TestInclude(Base):
 35 |     def setUp(self):
 36 |         super().setUp()
 37 |         self._old_check_dir = check50.internal.check_dir
 38 |         os.mkdir("bar")
 39 |         with open("./bar/baz.txt", "w") as f:
 40 |             pass
 41 |         check50.internal.check_dir = pathlib.Path("./bar").absolute()
 42 | 
 43 |     def tearDown(self):
 44 |         super().tearDown()
 45 |         check50.internal.check_dir = self._old_check_dir
 46 | 
 47 |     def test_include(self):
 48 |         check50.include("baz.txt")
 49 |         self.assertTrue((pathlib.Path(".").absolute() / "baz.txt").exists())
 50 |         self.assertTrue((check50.internal.check_dir / "baz.txt").exists())
 51 | 
 52 | class TestExists(Base):
 53 |     def test_file_does_not_exist(self):
 54 |         with self.assertRaises(check50.Failure):
 55 |             check50.exists("i_do_not_exist")
 56 | 
 57 |     def test_file_exists(self):
 58 |         check50.exists(self.filename)
 59 | 
 60 | 
 61 | class TestImportChecks(Base):
 62 |     def setUp(self):
 63 |         super().setUp()
 64 |         self._old_check_dir = check50.internal.check_dir
 65 |         os.mkdir("bar")
 66 |         check50.internal.check_dir = pathlib.Path(".").absolute()
 67 | 
 68 |     def tearDown(self):
 69 |         super().tearDown()
 70 |         check50.internal.check_dir = self._old_check_dir
 71 | 
 72 |     def test_simple_import(self):
 73 |         with open(".cs50.yaml", "w") as f:
 74 |             f.write("check50:\n")
 75 |             f.write("  checks: foo.py")
 76 |         mod = check50.import_checks(".")
 77 |         self.assertEqual(mod.__name__, pathlib.Path(self.working_directory.name).name)
 78 | 
 79 |     def test_relative_import(self):
 80 |         with open("./bar/baz.py", "w") as f:
 81 |             f.write("qux = 0")
 82 | 
 83 |         with open("./bar/.cs50.yaml", "w") as f:
 84 |             f.write("check50:\n")
 85 |             f.write("  checks: baz.py")
 86 | 
 87 |         mod = check50.import_checks("./bar")
 88 |         self.assertEqual(mod.__name__, "bar")
 89 |         self.assertEqual(mod.qux, 0)
 90 | 
 91 | 
 92 | class TestRun(Base):
 93 |     def test_returns_process(self):
 94 |         self.process = check50.run("python3 ./{self.filename}")
 95 | 
 96 | 
 97 | class TestProcessKill(Base):
 98 |     def test_kill(self):
 99 |         self.runpy()
100 |         self.assertTrue(self.process.process.isalive())
101 |         self.process.kill()
102 |         self.assertFalse(self.process.process.isalive())
103 | 
104 | class TestProcessStdin(Base):
105 |     def test_expect_prompt_no_prompt(self):
106 |         self.write("x = input()")
107 |         self.runpy()
108 |         with self.assertRaises(check50.Failure):
109 |             self.process.stdin("bar")
110 | 
111 |     def test_expect_prompt(self):
112 |         self.write("x = input('foo')")
113 |         self.runpy()
114 |         self.process.stdin("bar")
115 |         self.assertTrue(self.process.process.isalive())
116 | 
117 |     def test_no_prompt(self):
118 |         self.write("x = input()\n")
119 |         self.runpy()
120 |         self.process.stdin("bar", prompt=False)
121 |         self.assertTrue(self.process.process.isalive())
122 | 
123 | class TestProcessStdout(Base):
124 |     def test_no_out(self):
125 |         self.runpy()
126 |         out = self.process.stdout(timeout=1)
127 |         self.assertEqual(out, "")
128 |         self.assertFalse(self.process.process.isalive())
129 | 
130 |         self.write("print('foo')")
131 |         self.runpy()
132 |         out = self.process.stdout()
133 |         self.assertEqual(out, "foo\n")
134 |         self.assertFalse(self.process.process.isalive())
135 | 
136 |     def test_out(self):
137 |         self.runpy()
138 |         with self.assertRaises(check50.Failure):
139 |             self.process.stdout("foo")
140 |         self.assertFalse(self.process.process.isalive())
141 | 
142 |         self.write("print('foo')")
143 |         self.runpy()
144 |         self.process.stdout("foo\n")
145 | 
146 |     def test_outs(self):
147 |         self.write("print('foo')\nprint('bar')\n")
148 |         self.runpy()
149 |         self.process.stdout("foo\n")
150 |         self.process.stdout("bar")
151 |         self.process.stdout("\n")
152 | 
153 |     def test_out_regex(self):
154 |         self.write("print('foo')")
155 |         self.runpy()
156 |         self.process.stdout(".o.")
157 |         self.process.stdout("\n")
158 | 
159 |     def test_out_no_regex(self):
160 |         self.write("print('foo')")
161 |         self.runpy()
162 |         with self.assertRaises(check50.Failure):
163 |             self.process.stdout(".o.", regex=False)
164 |         self.assertFalse(self.process.process.isalive())
165 | 
166 |     def test_int(self):
167 |         self.write("print(123)")
168 |         self.runpy()
169 |         with self.assertRaises(check50.Failure):
170 |             self.process.stdout(1)
171 | 
172 |         self.write("print(21)")
173 |         self.runpy()
174 |         with self.assertRaises(check50.Failure):
175 |             self.process.stdout(1)
176 | 
177 |         self.write("print(1.0)")
178 |         self.runpy()
179 |         with self.assertRaises(check50.Failure):
180 |             self.process.stdout(1)
181 | 
182 |         self.write("print('a1b')")
183 |         self.runpy()
184 |         self.process.stdout(1)
185 | 
186 |         self.write("print(1)")
187 |         self.runpy()
188 |         self.process.stdout(1)
189 | 
190 |     def test_float(self):
191 |         self.write("print(1.01)")
192 |         self.runpy()
193 |         with self.assertRaises(check50.Failure):
194 |             self.process.stdout(1.0)
195 | 
196 |         self.write("print(21.0)")
197 |         self.runpy()
198 |         with self.assertRaises(check50.Failure):
199 |             self.process.stdout(1.0)
200 | 
201 |         self.write("print(1)")
202 |         self.runpy()
203 |         with self.assertRaises(check50.Failure):
204 |             self.process.stdout(1.0)
205 | 
206 |         self.write("print('a1.0b')")
207 |         self.runpy()
208 |         self.process.stdout(1.0)
209 | 
210 |         self.write("print(1.0)")
211 |         self.runpy()
212 |         self.process.stdout(1.0)
213 | 
214 |     def test_negative_number(self):
215 |         self.write("print(1)")
216 |         self.runpy()
217 |         with self.assertRaises(check50.Failure):
218 |             self.process.stdout(-1)
219 | 
220 |         self.write("print(-1)")
221 |         self.runpy()
222 |         with self.assertRaises(check50.Failure):
223 |             self.process.stdout(1)
224 | 
225 |         self.write("print('2-1')")
226 |         self.runpy()
227 |         self.process.stdout(-1)
228 | 
229 |         self.write("print(-1)")
230 |         self.runpy()
231 |         self.process.stdout(-1)
232 | 
233 | 
234 | class TestProcessStdoutFile(Base):
235 |     def setUp(self):
236 |         super().setUp()
237 |         self.txt_filename = "foo.txt"
238 |         with open(self.txt_filename, "w") as f:
239 |             f.write("foo")
240 | 
241 |     def test_file(self):
242 |         self.write("print('bar')")
243 |         self.runpy()
244 |         with open(self.txt_filename, "r") as f:
245 |             with self.assertRaises(check50.Failure):
246 |                 self.process.stdout(f, regex=False)
247 | 
248 |         self.write("print('foo')")
249 |         self.runpy()
250 |         with open(self.txt_filename, "r") as f:
251 |             self.process.stdout(f, regex=False)
252 | 
253 |     def test_file_regex(self):
254 |         self.write("print('bar')")
255 |         with open(self.txt_filename, "w") as f:
256 |             f.write(".a.")
257 |         self.runpy()
258 |         with open(self.txt_filename, "r") as f:
259 |             self.process.stdout(f)
260 | 
261 | class TestProcessExit(Base):
262 |     def test_exit(self):
263 |         self.write("sys.exit(1)")
264 |         self.runpy()
265 |         with self.assertRaises(check50.Failure):
266 |             self.process.exit(0)
267 |         self.process.kill()
268 | 
269 |         self.write("sys.exit(1)")
270 |         self.runpy()
271 |         self.process.exit(1)
272 | 
273 |     def test_no_exit(self):
274 |         self.write("sys.exit(1)")
275 |         self.runpy()
276 |         exit_code = self.process.exit()
277 |         self.assertEqual(exit_code, 1)
278 | 
279 | class TestProcessKill(Base):
280 |     def test_kill(self):
281 |         self.runpy()
282 |         self.process.kill()
283 |         self.assertFalse(self.process.process.isalive())
284 | 
285 | class TestProcessReject(Base):
286 |     def test_reject(self):
287 |         self.write("input()")
288 |         self.runpy()
289 |         self.process.reject()
290 |         self.process.stdin("foo", prompt=False)
291 |         with self.assertRaises(check50.Failure):
292 |             self.process.reject()
293 | 
294 |     def test_no_reject(self):
295 |         self.runpy()
296 |         with self.assertRaises(check50.Failure):
297 |             self.process.reject()
298 | 
299 | if __name__ == '__main__':
300 |     unittest.main()
301 | 


--------------------------------------------------------------------------------
/tests/c_tests.py:
--------------------------------------------------------------------------------
 1 | import pexpect
 2 | import unittest
 3 | import sys
 4 | import shutil
 5 | import os
 6 | import functools
 7 | import tempfile
 8 | import pathlib
 9 | import check50
10 | import check50.c
11 | import check50.internal
12 | 
13 | CLANG_INSTALLED = bool(shutil.which("clang"))
14 | VALGRIND_INSTALLED = bool(shutil.which("valgrind"))
15 | CHECKS_DIRECTORY = pathlib.Path(__file__).absolute().parent / "checks"
16 | 
17 | class Base(unittest.TestCase):
18 |     def setUp(self):
19 |         if not CLANG_INSTALLED:
20 |             raise unittest.SkipTest("clang not installed")
21 |         if not VALGRIND_INSTALLED:
22 |             raise unittest.SkipTest("valgrind not installed")
23 | 
24 |         self.working_directory = tempfile.TemporaryDirectory()
25 |         os.chdir(self.working_directory.name)
26 | 
27 |     def tearDown(self):
28 |         self.working_directory.cleanup()
29 | 
30 | class TestCompile(Base):
31 |     def test_compile_incorrect(self):
32 |         open("blank.c", "w").close()
33 | 
34 |         with self.assertRaises(check50.Failure):
35 |             check50.c.compile("blank.c")
36 | 
37 |     def test_compile_hello_world(self):
38 |         with open("hello.c", "w") as f:
39 |             src =   '#include <stdio.h>\n'\
40 |                     'int main() {\n'\
41 |                     '    printf("hello, world!\\n");\n'\
42 |                     '}'
43 |             f.write(src)
44 | 
45 |         check50.c.compile("hello.c")
46 | 
47 |         self.assertTrue(os.path.isfile("hello"))
48 |         check50.run("./hello").stdout("hello, world!", regex=False)
49 | 
50 | class TestValgrind(Base):
51 |     def setUp(self):
52 |         super().setUp()
53 |         if not (sys.platform == "linux" or sys.platform == "linux2"):
54 |             raise unittest.SkipTest("skipping valgrind checks under anything other than Linux due to false positives")
55 | 
56 |     def test_no_leak(self):
57 |         check50.internal.check_running = True
58 |         with open("foo.c", "w") as f:
59 |             src = 'int main() {}'
60 |             f.write(src)
61 | 
62 |         check50.c.compile("foo.c")
63 |         with check50.internal.register:
64 |             check50.c.valgrind("./foo").exit()
65 |         check50.internal.check_running = False
66 | 
67 |     def test_leak(self):
68 |         check50.internal.check_running = True
69 |         with open("leak.c", "w") as f:
70 |             src =   '#include <stdlib.h>\n'\
71 |                     'void leak() {malloc(sizeof(int));}\n'\
72 |                     'int main() {\n'\
73 |                     '    leak();\n'\
74 |                     '}'
75 |             f.write(src)
76 | 
77 |         check50.c.compile("leak.c")
78 |         with self.assertRaises(check50.Failure):
79 |             with check50.internal.register:
80 |                 check50.c.valgrind("./leak").exit()
81 |         check50.internal.check_running = False
82 | 
83 | 
84 | if __name__ == "__main__":
85 |     suite = unittest.TestLoader().loadTestsFromModule(module=sys.modules[__name__])
86 |     unittest.TextTestRunner(verbosity=2).run(suite)
87 | 


--------------------------------------------------------------------------------
/tests/check50_tests.py:
--------------------------------------------------------------------------------
  1 | import unittest
  2 | import json
  3 | import pexpect
  4 | import pathlib
  5 | import shutil
  6 | import subprocess
  7 | import os
  8 | import tempfile
  9 | 
 10 | CHECKS_DIRECTORY = pathlib.Path(__file__).absolute().parent / "checks"
 11 | 
 12 | class Base(unittest.TestCase):
 13 |     def setUp(self):
 14 |         self.working_directory = tempfile.TemporaryDirectory()
 15 |         os.chdir(self.working_directory.name)
 16 | 
 17 |     def tearDown(self):
 18 |         self.working_directory.cleanup()
 19 | 
 20 | 
 21 | class SimpleBase(Base):
 22 |     compiled_loc = None
 23 | 
 24 |     def setUp(self):
 25 |         super().setUp()
 26 |         if os.path.exists(self.compiled_loc):
 27 |             os.remove(self.compiled_loc)
 28 | 
 29 |     def tearDown(self):
 30 |         super().tearDown()
 31 |         if os.path.exists(self.compiled_loc):
 32 |             os.remove(self.compiled_loc)
 33 | 
 34 | 
 35 | class TestExists(Base):
 36 |     def test_no_file(self):
 37 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exists")
 38 |         process.expect_exact(":(")
 39 |         process.expect_exact("foo.py exists")
 40 |         process.expect_exact("foo.py not found")
 41 |         process.close(force=True)
 42 | 
 43 |     def test_with_file(self):
 44 |         open("foo.py", "w").close()
 45 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exists")
 46 |         process.expect_exact(":)")
 47 |         process.expect_exact("foo.py exists")
 48 |         process.close(force=True)
 49 | 
 50 | 
 51 | class TestExitPy(Base):
 52 |     def test_no_file(self):
 53 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exit_py")
 54 |         process.expect_exact(":(")
 55 |         process.expect_exact("foo.py exists")
 56 |         process.expect_exact("foo.py not found")
 57 |         process.expect_exact(":|")
 58 |         process.expect_exact("foo.py exits properly")
 59 |         process.expect_exact("can't check until a frown turns upside down")
 60 |         process.close(force=True)
 61 | 
 62 |     def test_with_correct_file(self):
 63 |         open("foo.py", "w").close()
 64 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exit_py")
 65 |         process.expect_exact(":)")
 66 |         process.expect_exact("foo.py exists")
 67 |         process.expect_exact(":)")
 68 |         process.expect_exact("foo.py exits properly")
 69 |         process.close(force=True)
 70 | 
 71 |     def test_with_incorrect_file(self):
 72 |         with open("foo.py", "w") as f:
 73 |             f.write("from sys import exit\nexit(1)")
 74 | 
 75 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/exit_py")
 76 |         process.expect_exact(":)")
 77 |         process.expect_exact("foo.py exists")
 78 |         process.expect_exact(":(")
 79 |         process.expect_exact("foo.py exits properly")
 80 |         process.expect_exact("expected exit code 0, not 1")
 81 |         process.close(force=True)
 82 | 
 83 | 
 84 | class TestStdoutPy(Base):
 85 |     def test_no_file(self):
 86 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdout_py")
 87 |         process.expect_exact(":(")
 88 |         process.expect_exact("foo.py exists")
 89 |         process.expect_exact("foo.py not found")
 90 |         process.expect_exact(":|")
 91 |         process.expect_exact("prints hello")
 92 |         process.expect_exact("can't check until a frown turns upside down")
 93 |         process.close(force=True)
 94 | 
 95 |     def test_with_empty_file(self):
 96 |         open("foo.py", "w").close()
 97 | 
 98 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdout_py")
 99 |         process.expect_exact(":)")
100 |         process.expect_exact("foo.py exists")
101 |         process.expect_exact(":(")
102 |         process.expect_exact("prints hello")
103 |         process.expect_exact("expected \"hello\", not \"\"")
104 |         process.close(force=True)
105 | 
106 | 
107 |     def test_with_correct_file(self):
108 |         with open("foo.py", "w") as f:
109 |             f.write('print("hello")')
110 | 
111 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdout_py")
112 |         process.expect_exact(":)")
113 |         process.expect_exact("foo.py exists")
114 |         process.expect_exact(":)")
115 |         process.expect_exact("prints hello")
116 |         process.close(force=True)
117 | 
118 | class TestStdoutTimeout(Base):
119 |     def test_stdout_timeout(self):
120 |         with open("foo.py", "w") as f:
121 |             f.write("while True: pass")
122 | 
123 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdout_py")
124 |         process.expect_exact(":)")
125 |         process.expect_exact("foo.py exists")
126 |         process.expect_exact(":(")
127 |         process.expect_exact("check50 waited 1 seconds for the output of the program")
128 |         process.close(force=True)
129 | 
130 | class TestStdinPy(Base):
131 |     def test_no_file(self):
132 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_py")
133 |         process.expect_exact(":(")
134 |         process.expect_exact("foo.py exists")
135 |         process.expect_exact("foo.py not found")
136 |         process.expect_exact(":|")
137 |         process.expect_exact("prints hello name")
138 |         process.expect_exact("can't check until a frown turns upside down")
139 |         process.close(force=True)
140 | 
141 |     def test_with_empty_file(self):
142 |         open("foo.py", "w").close()
143 | 
144 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_py")
145 |         process.expect_exact(":)")
146 |         process.expect_exact("foo.py exists")
147 |         process.expect_exact(":(")
148 |         process.expect_exact("prints hello name")
149 |         process.expect_exact("expected \"hello bar\", not \"\"")
150 |         process.close(force=True)
151 | 
152 |     def test_with_correct_file(self):
153 |         with open("foo.py", "w") as f:
154 |             f.write('name = input()\nprint("hello", name)')
155 | 
156 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_py")
157 |         process.expect_exact(":)")
158 |         process.expect_exact("foo.py exists")
159 |         process.expect_exact(":)")
160 |         process.expect_exact("prints hello name")
161 |         process.close(force=True)
162 | 
163 | 
164 | class TestStdinPromptPy(Base):
165 |     def test_no_file(self):
166 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_prompt_py")
167 |         process.expect_exact(":(")
168 |         process.expect_exact("prints hello name")
169 |         process.close(force=True)
170 | 
171 |     def test_with_empty_file(self):
172 |         open("foo.py", "w").close()
173 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_prompt_py")
174 |         process.expect_exact(":(")
175 |         process.expect_exact("prints hello name")
176 |         process.expect_exact("expected prompt for input, found none")
177 |         process.close(force=True)
178 | 
179 |     def test_with_incorrect_file(self):
180 |         with open("foo.py", "w") as f:
181 |             f.write('name = input()\nprint("hello", name)')
182 | 
183 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_prompt_py")
184 |         process.expect_exact(":(")
185 |         process.expect_exact("prints hello name")
186 |         process.expect_exact("expected prompt for input, found none")
187 |         process.close(force=True)
188 | 
189 |     def test_with_correct_file(self):
190 |         with open("foo.py", "w") as f:
191 |             f.write('name = input("prompt")\nprint("hello", name)')
192 | 
193 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_prompt_py")
194 |         process.expect_exact(":)")
195 |         process.expect_exact("prints hello name")
196 |         process.close(force=True)
197 | 
198 | 
199 | class TestStdinMultiline(Base):
200 |     def test_no_file(self):
201 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_multiline")
202 |         process.expect_exact(":(")
203 |         process.expect_exact("prints hello name (non chaining)")
204 |         process.expect_exact(":(")
205 |         process.expect_exact("prints hello name (non chaining) (prompt)")
206 |         process.expect_exact(":(")
207 |         process.expect_exact("prints hello name (chaining)")
208 |         process.expect_exact(":(")
209 |         process.expect_exact("prints hello name (chaining) (order)")
210 |         process.close(force=True)
211 | 
212 |     def test_with_empty_file(self):
213 |         open("foo.py", "w").close()
214 | 
215 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_multiline")
216 |         process.expect_exact(":(")
217 |         process.expect_exact("prints hello name (non chaining)")
218 |         process.expect_exact(":(")
219 |         process.expect_exact("prints hello name (non chaining) (prompt)")
220 |         process.expect_exact(":(")
221 |         process.expect_exact("prints hello name (chaining)")
222 |         process.expect_exact(":(")
223 |         process.expect_exact("prints hello name (chaining) (order)")
224 |         process.close(force=True)
225 | 
226 |     def test_with_incorrect_file(self):
227 |         with open("foo.py", "w") as f:
228 |             f.write('name = input()\nprint("hello", name)')
229 | 
230 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_multiline")
231 |         process.expect_exact(":(")
232 |         process.expect_exact("prints hello name (non chaining)")
233 |         process.expect_exact(":(")
234 |         process.expect_exact("prints hello name (non chaining) (prompt)")
235 |         process.expect_exact("expected prompt for input, found none")
236 |         process.expect_exact(":(")
237 |         process.expect_exact("prints hello name (chaining)")
238 |         process.expect_exact(":(")
239 |         process.expect_exact("prints hello name (chaining) (order)")
240 |         process.close(force=True)
241 | 
242 |     def test_with_correct_file(self):
243 |         with open("foo.py", "w") as f:
244 |             f.write('for _ in range(2):\n    name = input("prompt")\n    print("hello", name)')
245 | 
246 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_multiline")
247 |         process.expect_exact(":)")
248 |         process.expect_exact("prints hello name (non chaining)")
249 |         process.expect_exact(":)")
250 |         process.expect_exact("prints hello name (non chaining) (prompt)")
251 |         process.expect_exact(":)")
252 |         process.expect_exact("prints hello name (chaining)")
253 |         process.expect_exact(":)")
254 |         process.expect_exact("prints hello name (chaining) (order)")
255 |         process.close(force=True)
256 | 
257 | class TestStdinHumanReadable(Base):
258 |     def test_without_human_readable_string(self):
259 |         with open("foo.py", "w") as f:
260 |             f.write('name = input()\nprint("hello", name)')
261 | 
262 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_py")
263 |         process.expect_exact(":)")
264 |         process.expect_exact("foo.py exists")
265 |         process.expect_exact("checking that foo.py exists...")
266 |         process.expect_exact(":)")
267 |         process.expect_exact("prints hello name")
268 |         process.expect_exact("running python3 foo.py...")
269 |         process.expect_exact("sending input bar...")
270 |         process.expect_exact("checking for output \"hello bar\"...")
271 |         process.close(force=True)
272 | 
273 |     def test_with_human_readable_string(self):
274 |         with open("foo.py", "w") as f:
275 |             f.write('name = input("prompt")')
276 | 
277 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/stdin_human_readable_py")
278 |         process.expect_exact(":)")
279 |         process.expect_exact("foo.py exists")
280 |         process.expect_exact("checking that foo.py exists...")
281 |         process.expect_exact(":)")
282 |         process.expect_exact("takes input")
283 |         process.expect_exact("running python3 foo.py...")
284 |         process.expect_exact("sending input bbb...")
285 |         process.close(force=True)
286 | 
287 | 
288 | class TestCompileExit(SimpleBase):
289 |     compiled_loc = CHECKS_DIRECTORY / "compile_exit" / "__init__.py"
290 | 
291 |     def test_no_file(self):
292 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_exit")
293 |         process.expect_exact(":(")
294 |         process.expect_exact("exit")
295 |         process.close(force=True)
296 | 
297 |     def test_with_correct_file(self):
298 |         open("foo.py", "w").close()
299 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_exit")
300 |         process.expect_exact(":)")
301 |         process.expect_exact("exit")
302 |         process.close(force=True)
303 | 
304 | 
305 | class TestCompileStd(SimpleBase):
306 |     compiled_loc = CHECKS_DIRECTORY / "compile_std" / "__init__.py"
307 | 
308 |     def test_no_file(self):
309 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_std")
310 |         process.expect_exact(":(")
311 |         process.expect_exact("std")
312 |         process.close(force=True)
313 | 
314 |     def test_with_incorrect_stdout(self):
315 |         with open("foo.py", "w") as f:
316 |             f.write('name = input()\nprint("hello", name)')
317 | 
318 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_std")
319 |         process.expect_exact(":)")
320 |         process.expect_exact("std")
321 |         process.close(force=True)
322 | 
323 |     def test_correct(self):
324 |         with open("foo.py", "w") as f:
325 |             f.write('name = input()\nprint(name)')
326 | 
327 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_std")
328 |         process.expect_exact(":)")
329 |         process.expect_exact("std")
330 |         process.close(force=True)
331 | 
332 | 
333 | class TestCompilePrompt(SimpleBase):
334 |     compiled_loc = CHECKS_DIRECTORY / "compile_prompt" / "__init__.py"
335 | 
336 |     def test_prompt_dev(self):
337 |         with open("foo.py", "w"), open(self.compiled_loc, "w"):
338 |             pass
339 | 
340 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/compile_prompt")
341 |         process.expect_exact("check50 will compile the YAML checks to __init__.py")
342 |         process.close(force=True)
343 | 
344 | 
345 | class TestOutputModes(Base):
346 |     def test_json_output(self):
347 |         pexpect.run(f"check50 --dev -o json --output-file foo.json {CHECKS_DIRECTORY}/output")
348 |         with open("foo.json", "r") as f:
349 |             json_out = json.load(f)
350 |             self.assertEqual(json_out["results"][0]["name"], "exists")
351 | 
352 |     def test_ansi_output(self):
353 |         process = pexpect.spawn(f"check50 --dev -o ansi -- {CHECKS_DIRECTORY}/output")
354 |         process.expect_exact(":(")
355 |         process.close(force=True)
356 | 
357 |     def test_html_output(self):
358 |         process = pexpect.spawn(f"check50 --dev -o html -- {CHECKS_DIRECTORY}/output")
359 |         process.expect_exact("file://")
360 |         process.close(force=True)
361 | 
362 |     def test_multiple_outputs(self):
363 |         process = pexpect.spawn(f"check50 --dev -o html ansi -- {CHECKS_DIRECTORY}/output")
364 |         process.expect_exact("file://")
365 |         process.expect_exact(":(")
366 |         process.close(force=True)
367 | 
368 |     def test_default(self):
369 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/output")
370 |         process.expect_exact(":(")
371 |         process.expect_exact("file://")
372 |         process.close(force=True)
373 | 
374 | 
375 | class TestHiddenCheck(Base):
376 |     def test_hidden_check(self):
377 |         pexpect.run(f"check50 --dev -o json --output-file foo.json {CHECKS_DIRECTORY}/hidden")
378 |         expected = [{'name': 'check', 'description': "check", 'passed': False, 'log': [], 'cause': {"rationale": "foo", "help": None}, 'data': {}, 'dependency': None}]
379 |         with open("foo.json", "r") as f:
380 |             self.assertEqual(json.load(f)["results"], expected)
381 | 
382 | 
383 | class TestPayloadCheck(Base):
384 |     def test_payload_check(self):
385 |         pexpect.run(f"check50 --dev -o json --output-file foo.json {CHECKS_DIRECTORY}/payload")
386 |         with open("foo.json", "r") as f:
387 |             error = json.load(f)["error"]
388 |         self.assertEqual(error["type"], "MissingFilesError")
389 |         self.assertEqual(error["data"]["files"], ["missing.c"])
390 |         self.assertEqual(pathlib.Path(error["data"]["dir"]).stem, pathlib.Path(self.working_directory.name).stem)
391 | 
392 | 
393 | class TestTarget(Base):
394 |     def test_target(self):
395 |         open("foo.py", "w").close()
396 | 
397 |         pexpect.run(f"check50 --dev -o json --output-file foo.json --target exists1 -- {CHECKS_DIRECTORY}/target")
398 |         with open("foo.json", "r") as f:
399 |             output = json.load(f)
400 | 
401 |         self.assertEqual(len(output["results"]), 1)
402 |         self.assertEqual(output["results"][0]["name"], "exists1")
403 | 
404 | 
405 |     def test_target_with_dependency(self):
406 |         open("foo.py", "w").close()
407 | 
408 |         pexpect.run(f"check50 --dev -o json --output-file foo.json --target exists3 -- {CHECKS_DIRECTORY}/target")
409 |         with open("foo.json", "r") as f:
410 |             output = json.load(f)
411 | 
412 |         self.assertEqual(len(output["results"]), 2)
413 |         self.assertEqual(output["results"][0]["name"], "exists1")
414 |         self.assertEqual(output["results"][1]["name"], "exists3")
415 | 
416 | 
417 |     def test_two_targets(self):
418 |         open("foo.py", "w").close()
419 | 
420 |         pexpect.run(f"check50 --dev -o json --output-file foo.json --target exists1 exists2 -- {CHECKS_DIRECTORY}/target")
421 |         with open("foo.json", "r") as f:
422 |             output = json.load(f)
423 | 
424 |         self.assertEqual(len(output["results"]), 2)
425 |         self.assertEqual(output["results"][0]["name"], "exists1")
426 |         self.assertEqual(output["results"][1]["name"], "exists2")
427 | 
428 | 
429 |     def test_target_failing_dependency(self):
430 |         open("foo.py", "w").close()
431 | 
432 |         pexpect.run(f"check50 --dev -o json --output-file foo.json --target exists5 -- {CHECKS_DIRECTORY}/target")
433 |         with open("foo.json", "r") as f:
434 |             output = json.load(f)
435 | 
436 |         self.assertEqual(len(output["results"]), 2)
437 |         self.assertEqual(output["results"][0]["name"], "exists4")
438 |         self.assertEqual(output["results"][1]["name"], "exists5")
439 | 
440 | 
441 | class TestRemoteException(Base):
442 |     def test_no_traceback(self):
443 |         # Check that bar (part of traceback) is not shown
444 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_no_traceback")
445 |         self.assertRaises(pexpect.exceptions.EOF, lambda: process.expect("bar"))
446 | 
447 |         # Check that foo (the message) is shown
448 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_no_traceback")
449 |         process.expect("foo")
450 | 
451 |     def test_traceback(self):
452 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/remote_exception_traceback")
453 |         process.expect("bar")
454 |         process.expect("foo")
455 | 
456 | 
457 | class TestInternalDirectories(Base):
458 |     def test_directories_exist(self):
459 |         with open("foo.py", "w") as f:
460 |             f.write(os.getcwd())
461 | 
462 |         process = pexpect.spawn(f"check50 --dev {CHECKS_DIRECTORY}/internal_directories")
463 |         process.expect_exact(":)")
464 | 
465 | 
466 | class TestExitCode(Base):
467 |     def test_error_result_exit_code(self):
468 |         process = subprocess.run(
469 |             ["check50", "--dev", f"{CHECKS_DIRECTORY}/exit_code/error"],
470 |             stdout=subprocess.DEVNULL,
471 |             stderr=subprocess.DEVNULL,
472 |             timeout=2
473 |         )
474 |         self.assertEqual(process.returncode, 1)
475 | 
476 |     def test_failed_check_exit_code(self):
477 |         process = subprocess.run(
478 |             ["check50", "--dev", f"{CHECKS_DIRECTORY}/exit_code/failure"],
479 |             stdout=subprocess.DEVNULL,
480 |             stderr=subprocess.DEVNULL,
481 |             timeout=2
482 |         )
483 |         self.assertEqual(process.returncode, 1)
484 | 
485 |     def test_successful_exit(self):
486 |         process = subprocess.run(
487 |             ["check50", "--dev", f"{CHECKS_DIRECTORY}/exit_code/success"],
488 |             stdout=subprocess.DEVNULL,
489 |             stderr=subprocess.DEVNULL,
490 |             timeout=2
491 |         )
492 |         self.assertEqual(process.returncode, 0)
493 | 
494 | 
495 | if __name__ == "__main__":
496 |     unittest.main()
497 | 


--------------------------------------------------------------------------------
/tests/checks/compile_exit/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks:
3 |     exit:
4 |       - run: python3 foo.py
5 |         exit: 0
6 | 


--------------------------------------------------------------------------------
/tests/checks/compile_prompt/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks:
3 |     prompt:
4 |       - run: python3 foo.py
5 |         exit: 0
6 | 


--------------------------------------------------------------------------------
/tests/checks/compile_std/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks:
3 |     std:
4 |       - run: python3 foo.py
5 |         stdin: bar
6 |         stdout: bar
7 |         exit: 0
8 | 


--------------------------------------------------------------------------------
/tests/checks/exists/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/exists/check.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | def exists():
5 |     """foo.py exists"""
6 |     check50.exists("foo.py")
7 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/error/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/error/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | from check50._exceptions import RemoteCheckError
 3 | 
 4 | json = {
 5 |     "slug": "jelleas/foo/master",
 6 |     "error": {
 7 |         "type": "InvalidSlugError",
 8 |         "value": "foo",
 9 |         "traceback": [
10 |             "Traceback (most recent call last):\n",
11 |             "bar\n"
12 |         ],
13 |         "actions": {
14 |             "show_traceback": False,
15 |             "message": "foo"
16 |         },
17 |         "data": {}
18 |     },
19 |     "version": "3.1.1"
20 | }
21 | 
22 | raise RemoteCheckError(json)
23 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/failure/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/failure/__init__.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | def should_fail():
5 |     raise check50.Failure("BANG!")
6 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/success/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/exit_code/success/__init__.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | def should_pass():
5 |     pass
6 | 


--------------------------------------------------------------------------------
/tests/checks/exit_py/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/exit_py/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def exists():
 5 |     """foo.py exists"""
 6 |     check50.exists("foo.py")
 7 | 
 8 | @check50.check(exists)
 9 | def exits():
10 |     """foo.py exits properly"""
11 |     check50.run("python3 foo.py").exit(0)
12 | 


--------------------------------------------------------------------------------
/tests/checks/hidden/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/hidden/__init__.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | @check50.hidden("foo")
5 | def check():
6 |     check50.log("AHHHHHHHHHHHHHHHHHHH")
7 |     raise check50.Failure("AHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH")
8 | 


--------------------------------------------------------------------------------
/tests/checks/internal_directories/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/internal_directories/__init__.py:
--------------------------------------------------------------------------------
 1 | import pathlib
 2 | import check50
 3 | import check50.internal as internal
 4 | 
 5 | @check50.check()
 6 | def foo():
 7 |     """directories exist"""
 8 |     assert internal.run_dir.resolve() == pathlib.Path.cwd()
 9 |     assert internal.run_dir.parent == internal.run_root_dir
10 | 
11 | assert internal.run_root_dir.exists()
12 | assert internal.run_root_dir.resolve() == pathlib.Path.cwd()
13 | 
14 | assert internal.student_dir.exists()
15 | with open(internal.student_dir / "foo.py") as f:
16 |     student_dir = pathlib.Path(f.read().strip())
17 | assert internal.student_dir == student_dir
18 | 
19 | assert internal.check_dir.exists()
20 | assert internal.check_dir.name == "internal_directories"
21 | 


--------------------------------------------------------------------------------
/tests/checks/output/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/output/__init__.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | def exists():
5 |     """foo.py exists"""
6 |     check50.exists("foo.py")
7 | 


--------------------------------------------------------------------------------
/tests/checks/payload/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   files:
3 |     - !require missing.c
4 | 


--------------------------------------------------------------------------------
/tests/checks/payload/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/cs50/check50/ab2aa996b666f8141d78cf763d23dad422feef07/tests/checks/payload/__init__.py


--------------------------------------------------------------------------------
/tests/checks/remote_exception_no_traceback/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/remote_exception_no_traceback/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | from check50._exceptions import RemoteCheckError
 3 | 
 4 | json = {
 5 |     "slug": "jelleas/foo/master",
 6 |     "error": {
 7 |         "type": "InvalidSlugError",
 8 |         "value": "foo",
 9 |         "traceback": [
10 |             "Traceback (most recent call last):\n",
11 |             "bar\n"
12 |         ],
13 |         "actions": {
14 |             "show_traceback": False,
15 |             "message": "foo"
16 |         },
17 |         "data": {}
18 |     },
19 |     "version": "3.1.1"
20 | }
21 | 
22 | raise RemoteCheckError(json)
23 | 


--------------------------------------------------------------------------------
/tests/checks/remote_exception_traceback/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/remote_exception_traceback/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | from check50._exceptions import RemoteCheckError
 3 | 
 4 | json = {
 5 |     "slug": "jelleas/foo/master",
 6 |     "error": {
 7 |         "type": "InvalidSlugError",
 8 |         "value": "foo",
 9 |         "traceback": [
10 |             "Traceback (most recent call last):\n",
11 |             "bar\n"
12 |         ],
13 |         "actions": {
14 |             "show_traceback": True,
15 |             "message": "foo"
16 |         },
17 |         "data": {}
18 |     },
19 |     "version": "3.1.1"
20 | }
21 | 
22 | raise RemoteCheckError(json)
23 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_human_readable_py/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_human_readable_py/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def exists():
 5 |     """foo.py exists"""
 6 |     check50.exists("foo.py")
 7 | 
 8 | @check50.check(exists)
 9 | def takes_input():
10 |     """takes input"""
11 |     check50.run("python3 foo.py").stdin("aaa", prompt=False, str_line="bbb")
12 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_multiline/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_multiline/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def prints_hello_name_non_chaining():
 5 |     """prints hello name (non chaining)"""
 6 |     check50.run("python3 foo.py").stdin("bar\nbaz", prompt=False).stdout("hello bar").stdout("hello baz")
 7 | 
 8 | @check50.check()
 9 | def prints_hello_name_non_chaining_prompt():
10 |     """prints hello name (non chaining) (prompt)"""
11 |     check50.run("python3 foo.py").stdin("bar\nbaz").stdout("hello bar").stdout("hello baz")
12 | 
13 | @check50.check()
14 | def prints_hello_name_chaining():
15 |     """prints hello name (chaining)"""
16 |     check50.run("python3 foo.py").stdin("bar", prompt=False).stdout("hello bar").stdin("baz", prompt=False).stdout("hello baz")
17 | 
18 | @check50.check()
19 | def prints_hello_name_chaining_order():
20 |     """prints hello name (chaining) (order)"""
21 |     check50.run("python3 foo.py").stdin("bar", prompt=False).stdin("baz", prompt=False).stdout("hello bar").stdout("hello baz")
22 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_prompt_py/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_prompt_py/check.py:
--------------------------------------------------------------------------------
1 | import check50
2 | 
3 | @check50.check()
4 | def prints_hello_name():
5 |     """prints hello name"""
6 |     check50.run("python3 foo.py").stdin("bar", prompt=True).stdout("hello bar")
7 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_py/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/stdin_py/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def exists():
 5 |     """foo.py exists"""
 6 |     check50.exists("foo.py")
 7 | 
 8 | @check50.check(exists)
 9 | def prints_hello_name():
10 |     """prints hello name"""
11 |     check50.run("python3 foo.py").stdin("bar", prompt=False).stdout("hello bar")
12 | 


--------------------------------------------------------------------------------
/tests/checks/stdout_py/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   checks: check.py
3 | 


--------------------------------------------------------------------------------
/tests/checks/stdout_py/check.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def exists():
 5 |     """foo.py exists"""
 6 |     check50.exists("foo.py")
 7 | 
 8 | @check50.check(exists)
 9 | def prints_hello():
10 |     """prints hello"""
11 |     check50.run("python3 foo.py").stdout("hello", show_timeout=True, timeout=1)
12 | 


--------------------------------------------------------------------------------
/tests/checks/target/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50:
2 |   files:
3 |     - !require foo.py
4 | 


--------------------------------------------------------------------------------
/tests/checks/target/__init__.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | 
 3 | @check50.check()
 4 | def exists1():
 5 |     """foo.py exists"""
 6 |     check50.exists("foo.py")
 7 | 
 8 | @check50.check()
 9 | def exists2():
10 |     """foo.py exists"""
11 |     check50.exists("foo.py")
12 | 
13 | @check50.check(exists1)
14 | def exists3():
15 |     """foo.py exists"""
16 |     check50.exists("foo.py")
17 | 
18 | @check50.check()
19 | def exists4():
20 |     """foo.py exists"""
21 |     raise check50.Failure()
22 | 
23 | @check50.check(exists4)
24 | def exists5():
25 |     """foo.py exists"""
26 |     check50.exists("foo.py")
27 | 


--------------------------------------------------------------------------------
/tests/checks/unpicklable_attribute/.cs50.yaml:
--------------------------------------------------------------------------------
1 | check50: true
2 | 


--------------------------------------------------------------------------------
/tests/checks/unpicklable_attribute/__init__.py:
--------------------------------------------------------------------------------
1 | import check50
2 | import sys
3 | 
4 | # Set the excepthook to an unpicklable value, the run_check process will try to copy this
5 | sys.excepthook = lambda type, value, traceback: "bar"
6 | 
7 | @check50.check()
8 | def foo():
9 |     """foo"""


--------------------------------------------------------------------------------
/tests/flask_tests.py:
--------------------------------------------------------------------------------
  1 | import pathlib
  2 | import os
  3 | import sys
  4 | import tempfile
  5 | import unittest
  6 | import check50
  7 | import check50.flask
  8 | 
  9 | try:
 10 |     import flask
 11 |     FLASK_INSTALLED = True
 12 | except ModuleNotFoundError:
 13 |     FLASK_INSTALLED = False
 14 | CHECKS_DIRECTORY = pathlib.Path(__file__).absolute().parent / "checks"
 15 | 
 16 | class Base(unittest.TestCase):
 17 |     def setUp(self):
 18 |         if not FLASK_INSTALLED:
 19 |             raise unittest.SkipTest("flask not installed")
 20 | 
 21 |         self.working_directory = tempfile.TemporaryDirectory()
 22 |         os.chdir(self.working_directory.name)
 23 | 
 24 |     def tearDown(self):
 25 |         self.working_directory.cleanup()
 26 | 
 27 | class TestApp(Base):
 28 |     def test_app(self):
 29 |         src = \
 30 | """from flask import Flask
 31 | app = Flask(__name__)
 32 | 
 33 | @app.route('/')
 34 | def root():
 35 |     return ''"""
 36 |         with open("hello.py", "w") as f:
 37 |             f.write(src)
 38 | 
 39 |         app = check50.flask.app("hello.py")
 40 |         self.assertIsInstance(app, check50.flask.app)
 41 | 
 42 |     def test_no_app(self):
 43 |         with self.assertRaises(check50.Failure):
 44 |             check50.flask.app("doesnt_exist.py")
 45 | 
 46 | 
 47 | 
 48 | class TestFlask(Base):
 49 |     def test_status(self):
 50 |         src = \
 51 | """from flask import Flask
 52 | app = Flask(__name__)
 53 | 
 54 | @app.route('/')
 55 | def root():
 56 |     return ''"""
 57 |         with open("hello.py", "w") as f:
 58 |             f.write(src)
 59 | 
 60 |         app = check50.flask.app("hello.py")
 61 |         app.get("/").status(200)
 62 |         app.get("/foo").status(404)
 63 |         app.post("/").status(405)
 64 | 
 65 |     def test_raw_content(self):
 66 |         src = \
 67 | """from flask import Flask
 68 | app = Flask(__name__)
 69 | 
 70 | @app.route('/')
 71 | def hello():
 72 |     return 'hello world'"""
 73 |         with open("hello.py", "w") as f:
 74 |             f.write(src)
 75 | 
 76 |         app = check50.flask.app("hello.py")
 77 |         app.get("/").raw_content("hello world")
 78 | 
 79 |         with self.assertRaises(check50.Failure):
 80 |             app.get("/").raw_content("foo")
 81 | 
 82 |     def test_content(self):
 83 |         src = \
 84 | """from flask import Flask
 85 | app = Flask(__name__)
 86 | 
 87 | @app.route('/')
 88 | def hello():
 89 |     return '<head>foo</head><body>bar</body>'"""
 90 |         with open("hello.py", "w") as f:
 91 |             f.write(src)
 92 | 
 93 |         app = check50.flask.app("hello.py")
 94 |         app.get("/").content("foo")
 95 | 
 96 |         with self.assertRaises(check50.Failure):
 97 |             app.get("/").content("foo", name="body")
 98 | 
 99 |         app.get("/").content("bar")
100 | 
101 | if __name__ == "__main__":
102 |     suite = unittest.TestLoader().loadTestsFromModule(module=sys.modules[__name__])
103 |     unittest.TextTestRunner(verbosity=2).run(suite)
104 | 


--------------------------------------------------------------------------------
/tests/internal_tests.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | import check50
 3 | import check50.internal
 4 | 
 5 | class TestRegisterAfterCheck(unittest.TestCase):
 6 |     def test_after_check(self):
 7 |         check50.internal.check_running = True
 8 |         l = []
 9 |         check50.internal.register.after_check(lambda : l.append("foo"))
10 | 
11 |         with check50.internal.register:
12 |             self.assertEqual(l, [])
13 | 
14 |         self.assertEqual(l, ["foo"])
15 | 
16 |         with check50.internal.register:
17 |             self.assertEqual(l, ["foo"])
18 | 
19 |         self.assertEqual(l, ["foo"])
20 |         check50.internal.check_running = False
21 | 
22 | class TestRegisterAfterEvery(unittest.TestCase):
23 |     def test_after_every(self):
24 |         l = []
25 |         check50.internal.register.after_every(lambda : l.append("foo"))
26 | 
27 |         with check50.internal.register:
28 |             self.assertEqual(l, [])
29 | 
30 |         self.assertEqual(l, ["foo"])
31 | 
32 |         with check50.internal.register:
33 |             self.assertEqual(l, ["foo"])
34 | 
35 |         self.assertEqual(l, ["foo", "foo"])
36 | 
37 | class TestRegisterBeforeEvery(unittest.TestCase):
38 |     def test_before_every(self):
39 |         l = []
40 |         check50.internal.register.before_every(lambda : l.append("foo"))
41 | 
42 |         with check50.internal.register:
43 |             self.assertEqual(l, ["foo"])
44 | 
45 |         with check50.internal.register:
46 |             self.assertEqual(l, ["foo", "foo"])
47 | 
48 |         self.assertEqual(l, ["foo", "foo"])
49 | 
50 | 
51 | if __name__ == "__main__":
52 |     unittest.main()
53 | 


--------------------------------------------------------------------------------
/tests/py_tests.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | import os
 3 | import pathlib
 4 | import tempfile
 5 | 
 6 | import check50.py
 7 | import check50.internal
 8 | 
 9 | class Base(unittest.TestCase):
10 |     def setUp(self):
11 |         self.working_directory = tempfile.TemporaryDirectory()
12 |         os.chdir(self.working_directory.name)
13 | 
14 |         self.filename = "foo.py"
15 |         self.write("")
16 | 
17 |     def write(self, source):
18 |         with open(self.filename, "w") as f:
19 |             f.write(source)
20 | 
21 |     def tearDown(self):
22 |         self.working_directory.cleanup()
23 | 
24 |     def runpy(self):
25 |         self.process = check50.run(f"python3 ./{self.filename}")
26 | 
27 | class TestAppendCode(Base):
28 |     def setUp(self):
29 |         super().setUp()
30 |         self.other_filename = "bar.py"
31 |         with open(self.other_filename, "w") as f:
32 |             f.write("baz")
33 | 
34 |     def test_empty_append(self):
35 |         check50.py.append_code(self.filename, self.other_filename)
36 |         with open(self.filename, "r") as f1, open(self.other_filename, "r") as f2:
37 |             self.assertEqual(f1.read(), f"\n{f2.read()}")
38 | 
39 |     def test_append(self):
40 |         with open(self.other_filename, "r") as f:
41 |             old_content2 = f.read()
42 | 
43 |         self.write("qux")
44 |         check50.py.append_code(self.filename, self.other_filename)
45 |         with open(self.filename, "r") as f1, open(self.other_filename, "r") as f2:
46 |             content1 = f1.read()
47 |             content2 = f2.read()
48 | 
49 |         self.assertNotEqual(content1, content2)
50 |         self.assertEqual(content2, old_content2)
51 |         self.assertEqual(content1, "qux\nbaz")
52 | 
53 | 
54 | class TestImport_(Base):
55 |     def setUp(self):
56 |         super().setUp()
57 |         self._old_check_dir = check50.internal.check_dir
58 |         os.mkdir("bar")
59 |         with open("./bar/baz.py", "w") as f:
60 |             f.write("qux = 0")
61 |         check50.internal.check_dir = pathlib.Path(".").absolute()
62 | 
63 |     def tearDown(self):
64 |         super().tearDown()
65 |         check50.internal.check_dir = self._old_check_dir
66 | 
67 |     def test_simple_import(self):
68 |         mod = check50.py.import_("foo.py")
69 |         self.assertEqual(mod.__name__, "foo")
70 | 
71 |     def test_relative_import(self):
72 |         mod = check50.py.import_("./bar/baz.py")
73 |         self.assertEqual(mod.__name__, "baz")
74 |         self.assertEqual(mod.qux, 0)
75 | 
76 | 
77 | if __name__ == "__main__":
78 |     unittest.main()
79 | 


--------------------------------------------------------------------------------
/tests/runner_tests.py:
--------------------------------------------------------------------------------
 1 | import check50
 2 | import check50.runner
 3 | 
 4 | import importlib
 5 | import multiprocessing
 6 | import os
 7 | import pathlib
 8 | import pexpect
 9 | import sys
10 | import tempfile
11 | import unittest
12 | 
13 | 
14 | CHECKS_DIRECTORY = pathlib.Path(__file__).absolute().parent / "checks"
15 | CHECK50_SUPPORTED_START_METHODS = ("fork", "spawn")
16 | 
17 | 
18 | # Just test spawn under OS X due to a bug with "fork": https://bugs.python.org/issue33725
19 | if sys.platform == "darwin":
20 |     SUPPORTED_START_METHODS = ("spawn",)
21 | 
22 | # Don't test forkserver under linux, serves no usecase for check50
23 | else:
24 |     SUPPORTED_START_METHODS = tuple(set(CHECK50_SUPPORTED_START_METHODS) & set(multiprocessing.get_all_start_methods()))
25 | 
26 | 
27 | class TestMultiprocessingStartMethods(unittest.TestCase):
28 |     def setUp(self):
29 |         self.working_directory = tempfile.TemporaryDirectory()
30 |         os.chdir(self.working_directory.name)
31 | 
32 |         # Keep track of get_start_method
33 |         # This function gets monkey patched to ensure run_check is aware of the multiprocessing context, 
34 |         # without needing to explicitly pass the context to run_check.
35 |         # The same behavior can't be achieved by multiprocessing.set_start_method as that can only run once per program
36 |         # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods
37 |         self._get_start_method = multiprocessing.get_start_method()
38 | 
39 |     def tearDown(self):
40 |         self.working_directory.cleanup()
41 |         multiprocessing.get_start_method = self._get_start_method
42 | 
43 |     def test_unpicklable_attribute(self):
44 |         # Create the checks_spec and check_name needed for run_check
45 |         checks_path = CHECKS_DIRECTORY / "unpicklable_attribute/__init__.py"
46 |         check_name = "foo"
47 |         spec = importlib.util.spec_from_file_location("checks", checks_path)
48 | 
49 |         # Execute the module once in the main process, as the Runner does too
50 |         # This will set sys.excepthook to an unpicklable lambda
51 |         check_module = importlib.util.module_from_spec(spec)
52 |         spec.loader.exec_module(check_module)
53 | 
54 |         # For each available method
55 |         for start_method in SUPPORTED_START_METHODS:
56 |             
57 |             # Create a multiprocessing context for that method
58 |             ctx = multiprocessing.get_context(start_method)
59 | 
60 |             # Monkey patch get_start_method() used by run_check to check for its method
61 |             multiprocessing.get_start_method = lambda: start_method
62 | 
63 |             # Start and join each check process
64 |             p = ctx.Process(target=check50.runner.run_check(check_name, spec))
65 |             p.start()
66 |             p.join()
67 | 
68 | 
69 | if __name__ == "__main__":
70 |     unittest.main()


--------------------------------------------------------------------------------
/tests/simple_tests.py:
--------------------------------------------------------------------------------
  1 | import unittest
  2 | import os
  3 | import yaml
  4 | import pathlib
  5 | import tempfile
  6 | 
  7 | from check50 import _simple
  8 | 
  9 | class Base(unittest.TestCase):
 10 | 
 11 |     config_file = ".check50.yaml"
 12 | 
 13 |     def setUp(self):
 14 |         self.working_directory = tempfile.TemporaryDirectory()
 15 |         os.chdir(self.working_directory.name)
 16 | 
 17 |     def tearDown(self):
 18 |         self.working_directory.cleanup()
 19 | 
 20 |     def write(self, source):
 21 |         with open(self.config_file, "w") as f:
 22 |             f.write(source)
 23 | 
 24 | class TestCompile(Base):
 25 |     def test_one_complete_check(self):
 26 |         checks = yaml.safe_load(\
 27 | """checks:
 28 |   bar:
 29 |     - run: python3 foo.py
 30 |       stdin: baz
 31 |       stdout: baz
 32 |       exit: 0
 33 | """)["checks"]
 34 | 
 35 |         expectation = \
 36 | """import check50
 37 | 
 38 | @check50.check()
 39 | def bar():
 40 |     \"\"\"bar\"\"\"
 41 |     check50.run("python3 foo.py").stdin("baz", prompt=False).stdout("baz", regex=False).exit(0)"""
 42 | 
 43 |         result = _simple.compile(checks)
 44 |         self.assertEqual(result, expectation)
 45 | 
 46 |     def test_multiple_checks(self):
 47 |         checks = yaml.safe_load(\
 48 | """checks:
 49 |   bar:
 50 |     - run: python3 foo.py
 51 |       exit: 0
 52 |   baz:
 53 |     - run: python3 foo.py
 54 |       exit: 0
 55 | """)["checks"]
 56 | 
 57 |         expectation = \
 58 | """import check50
 59 | 
 60 | @check50.check()
 61 | def bar():
 62 |     \"\"\"bar\"\"\"
 63 |     check50.run("python3 foo.py").exit(0)
 64 | 
 65 | @check50.check()
 66 | def baz():
 67 |     \"\"\"baz\"\"\"
 68 |     check50.run("python3 foo.py").exit(0)"""
 69 | 
 70 |         result = _simple.compile(checks)
 71 |         self.assertEqual(result, expectation)
 72 | 
 73 |     def test_multi(self):
 74 |         checks = yaml.safe_load(\
 75 | """checks:
 76 |   bar:
 77 |     - run: python3 foo.py
 78 |       stdin:
 79 |         - foo
 80 |         - bar
 81 |       stdout:
 82 |         - baz
 83 |         - qux
 84 |       exit: 0
 85 | """)["checks"]
 86 | 
 87 |         expectation = \
 88 | """import check50
 89 | 
 90 | @check50.check()
 91 | def bar():
 92 |     \"\"\"bar\"\"\"
 93 |     check50.run("python3 foo.py").stdin("foo\\nbar", prompt=False).stdout("baz\\nqux", regex=False).exit(0)"""
 94 | 
 95 |         result = _simple.compile(checks)
 96 |         self.assertEqual(result, expectation)
 97 | 
 98 |     def test_multiline(self):
 99 |         checks = yaml.safe_load(\
100 | """checks:
101 |   bar:
102 |     - run: python3 foo.py
103 |       stdout: |
104 |         Hello
105 |         World!
106 |       exit: 0
107 | """)["checks"]
108 | 
109 |         expectation = \
110 | """import check50
111 | 
112 | @check50.check()
113 | def bar():
114 |     \"\"\"bar\"\"\"
115 |     check50.run("python3 foo.py").stdout("Hello\\nWorld!\\n", regex=False).exit(0)"""
116 | 
117 |         result = _simple.compile(checks)
118 |         self.assertEqual(result, expectation)
119 | 
120 |     def test_number_in_name(self):
121 |         checks = yaml.safe_load(\
122 | """checks:
123 |   0bar:
124 |     - run: python3 foo.py
125 | """)["checks"]
126 | 
127 |         result = _simple.compile(checks)
128 |         self.assertTrue("def _0bar" in result)
129 |         self.assertTrue("\"\"\"0bar\"\"\"" in result)
130 | 
131 |     def test_space_in_name(self):
132 |         checks = yaml.safe_load(\
133 | """checks:
134 |   bar baz:
135 |     - run: python3 foo.py
136 | """)["checks"]
137 | 
138 |         result = _simple.compile(checks)
139 |         self.assertTrue("def bar_baz" in result)
140 |         self.assertTrue("\"\"\"bar baz\"\"\"" in result)
141 | 
142 |     def test_dash_in_name(self):
143 |         checks = yaml.safe_load(\
144 | """checks:
145 |   bar-baz:
146 |     - run: python3 foo.py
147 | """)["checks"]
148 | 
149 |         result = _simple.compile(checks)
150 |         self.assertTrue("def bar_baz" in result)
151 |         self.assertTrue("\"\"\"bar-baz\"\"\"" in result)
152 | 
153 |     def test_missing_exit(self):
154 |         checks = yaml.safe_load(\
155 | """checks:
156 |   bar:
157 |     - run: python3 foo.py
158 | """)["checks"]
159 | 
160 |         result = _simple.compile(checks)
161 |         self.assertTrue(".exit()" in result)
162 | 
163 | 
164 | if __name__ == "__main__":
165 |     unittest.main()
166 | 


--------------------------------------------------------------------------------